From c5a4dd288289b410fd95cc89d0222c5a8a6935bb Mon Sep 17 00:00:00 2001 From: wtucker Date: Mon, 12 Nov 2018 14:45:56 +0000 Subject: [PATCH 01/65] Restoring code from outdated oauth integration branch. --- cog/plugins/globus/transfer.py | 38 ++++++++++++++++------------- cog/templates/cog/openid/login.html | 21 +++++++++++++++- cog/urls.py | 5 +++- cog/views/views_account.py | 18 +++++++++++--- cog/views/views_globus.py | 14 ++++++++++- settings.py | 26 ++++++++++++++++++++ 6 files changed, 99 insertions(+), 23 deletions(-) diff --git a/cog/plugins/globus/transfer.py b/cog/plugins/globus/transfer.py index 5b6d3e500..3911ec26d 100644 --- a/cog/plugins/globus/transfer.py +++ b/cog/plugins/globus/transfer.py @@ -30,9 +30,9 @@ def generateGlobusDownloadScript(download_map): return script -def activateEndpoint(api_client, endpoint, openid=None, password=None): - - if not openid or not password: +def activateEndpoint(api_client, endpoint, openid=None, password=None, cert=None, key=None): + print 'activateEndpoint', openid + if (not openid or not password) and (not openid or not cert): # Try to autoactivate the endpoint code, reason, result = api_client.endpoint_autoactivate(endpoint, if_expires_in=2880) print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) @@ -46,20 +46,24 @@ def activateEndpoint(api_client, endpoint, openid=None, password=None): code, reason, reqs = api_client.endpoint_activation_requirements(endpoint) # Activate the endpoint using an X.509 user credential stored by esgf-idp in /tmp/x509up__ - #cred_file = "/tmp/x509up_%s_%s" % (hostname, username) - #public_key = reqs.get_requirement_value("delegate_proxy", "public_key") - #try: - # proxy = x509_proxy.create_proxy_from_file(cred_file, public_key, lifetime_hours=72) - #except Exception as e: - # print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) - # return False - #reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) - - # Activate the endpoint using MyProxy server method - reqs.set_requirement_value("myproxy", "hostname", hostname) - reqs.set_requirement_value("myproxy", "username", username) - reqs.set_requirement_value("myproxy", "passphrase", password) - reqs.set_requirement_value("myproxy", "lifetime_in_hours", "168") + if cert and key: + cred_file = "/tmp/x509up_%s_%s" % (hostname, username) + with open(cred_file, 'w') as cred: + cred.write(cert) + cred.write(key) + public_key = reqs.get_requirement_value("delegate_proxy", "public_key") + try: + proxy = x509_proxy.create_proxy_from_file(cred_file, public_key, lifetime_hours=72) + except Exception as e: + print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) + return False + reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) + else: + # Activate the endpoint using MyProxy server method + reqs.set_requirement_value("myproxy", "hostname", hostname) + reqs.set_requirement_value("myproxy", "username", username) + reqs.set_requirement_value("myproxy", "passphrase", password) + reqs.set_requirement_value("myproxy", "lifetime_in_hours", "168") try: code, reason, result = api_client.endpoint_activate(endpoint, reqs) diff --git a/cog/templates/cog/openid/login.html b/cog/templates/cog/openid/login.html index 1b5850446..b180d8b77 100644 --- a/cog/templates/cog/openid/login.html +++ b/cog/templates/cog/openid/login.html @@ -49,6 +49,25 @@ } return true; } + + function submit_form() { + var idp = document.getElementById('openid_identifier').value.trim(); + var xmlhttp = new XMLHttpRequest(); + xmlhttp.onreadystatechange = function() { + if (this.readyState == 4 && this.status == 200) { + var response = JSON.parse(this.responseText); + var form = document.getElementById('login_form'); + if (response.auth == 'OAuth2') + form.action = "{% url 'social:begin' 'esgf' %}"; + else + form.action = "{% url 'openid-login' %}"; + form.submit(); + } + }; + xmlhttp.open("GET", "{% url 'auth-discover' %}?openid_identifier=" + idp, true); + xmlhttp.send(); + return false; + } function init() { @@ -145,7 +164,7 @@

OpenID Login

-
+ {% csrf_token %}
diff --git a/cog/urls.py b/cog/urls.py index 36faefca4..1fc63da4d 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -53,7 +53,10 @@ url(r'^openid/complete/$', cog.views.custom_login_complete, name='openid-complete'), url(r'^openid/logo.gif$', django_openid_auth.views.logo, name='openid-logo'), - + # add OAuth2 urls + url(r'', include('social_django.urls', namespace='social')), + url(r'^esgf-auth/discover/$', cog.views.auth_discover, name='auth-discover'), + # force redirection to login page after logout #url(r'^logout/$', 'django.contrib.auth.views.logout_then_login', name='logout'), # use next=... to redirect to previous page after logout diff --git a/cog/views/views_account.py b/cog/views/views_account.py index 3ab5e00c2..95cdc6ce9 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -9,7 +9,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.forms.models import modelformset_factory -from django.http import HttpResponseRedirect, HttpResponseNotAllowed, HttpResponseServerError +from django.http import HttpResponseRedirect, HttpResponseNotAllowed, HttpResponseServerError, JsonResponse from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django_openid_auth.models import UserOpenID @@ -34,6 +34,15 @@ def redirectToIdp(): return False +from cog.backends.esgf import discover +def auth_discover(request, **kwargs): + openid = request.GET.get('openid_identifier', None) + protocol = 'OAuth2'#discover(openid) + if protocol == 'OAuth2': + return JsonResponse({'auth': 'OAuth2'}) + return JsonResponse({'auth': 'OpenID'}) + + def custom_login(request, **kwargs): """ Overrides standard login view that checks whether the authenticated user has any missing information. @@ -41,7 +50,6 @@ def custom_login(request, **kwargs): :param kwargs: :return: """ - # authenticate user via standard login response = login(request, **kwargs) @@ -328,7 +336,11 @@ def user_detail(request, user_id): user_profile = UserProfile(user=user) user_profile.save() print "Created empty profile for user=%s" % user - + if request.user.is_authenticated(): + if request.user.social_auth.filter(provider='esgf'): + social = request.user.social_auth.get(provider='esgf') + setattr(user_profile, 'openids', [social.uid]) + setattr(user_profile, 'localOpenid', False) # retrieve map of (project, roles) for this user (projTuples, groupTuples) = get_all_shared_user_info(user) print "\nprojTuples=" diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index d13a5f78f..9720db22c 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -4,6 +4,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError from django.shortcuts import render from django.template import RequestContext +from social_django.utils import load_strategy import urllib from cog.utils import getJson from urlparse import urlparse @@ -230,8 +231,19 @@ def submit(request): The access token and files to download are retrieved from the session. ''' openid = request.user.profile.openid() + cert = None + key = None + if not openid: + social = request.user.social_auth.get(provider='esgf') + access_token = social.extra_data['access_token'] + strategy = load_strategy() + backend = social.get_backend_instance(strategy) + key, cert, cn = backend.get_certificate(access_token) + openid = social.uid + print 'SUBMIT social.uid', social.uid # get a password if authoactivation failed and a user was asked for a password password = request.POST.get(ESGF_PASSWORD) + print 'SUBMIT password', password # retrieve all data transfer request parameters from session username = request.session[GLOBUS_USERNAME] access_token = request.session[GLOBUS_ACCESS_TOKEN] @@ -247,7 +259,7 @@ def submit(request): # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(api_client, target_endpoint) for source_endpoint, source_files in download_map.items(): - status, message = activateEndpoint(api_client, source_endpoint, openid, password) + status, message = activateEndpoint(api_client, source_endpoint, openid, password, cert, key) if not status: return render(request, 'cog/globus/password.html', diff --git a/settings.py b/settings.py index 0e6d74925..46ff54b6f 100644 --- a/settings.py +++ b/settings.py @@ -194,6 +194,8 @@ 'django.template.context_processors.tz', 'django.contrib.messages.context_processors.messages', 'django.template.context_processors.request', + 'social_django.context_processors.backends', + 'social_django.context_processors.login_redirect', 'cog.context_processors.cog_settings', ], 'debug': DEBUG, @@ -233,6 +235,7 @@ 'django.contrib.staticfiles', 'captcha', 'layouts', + 'social_django', 'cog.apps.CogConfig', 'cog.templatetags', ) @@ -241,9 +244,32 @@ AUTHENTICATION_BACKENDS = ( 'django_openid_auth.auth.OpenIDBackend', + 'cog.backends.esgf.ESGFOAuth2', 'django.contrib.auth.backends.ModelBackend', ) +SOCIAL_AUTH_PIPELINE = ( + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'social_core.pipeline.user.get_username', + 'cog.backends.esgf.associate_by_openid', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'social_core.pipeline.user.user_details' +) + +SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] +SOCIAL_AUTH_ESGF_KEY = 'XceQW2eGHpXqTQQsVUtc1UVNiLgoJNdFTZx6cM73' +SOCIAL_AUTH_ESGF_SECRET = 'qqXpMFz6BuVjiA0QlxovdGUOBycGQhnIPvfyEEuVX81n1k8eNZrGQJC6F4xDzi27DGBqtq6nYp4cCFWG5fJzz3d6derWawGlOfHBKCFg9ZNq75LUyeSWhw0oNkESeztT' +SOCIAL_AUTH_SANITIZE_REDIRECTS = False +SOCIAL_AUTH_ESGF_AUTH_EXTRA_ARGUMENTS = { + 'access_type': 'offline', +} + + # Default is X_FRAME_OPTIONS='SAMEORIGIN' # Using X_FRAME_OPTIONS = DENY breaks the CKEditor file uploader. #X_FRAME_OPTIONS = 'DENY' From 6d4f1309cb437cfcbc7768f81913c74312e31e5a Mon Sep 17 00:00:00 2001 From: wtucker Date: Thu, 15 Nov 2018 11:27:49 +0000 Subject: [PATCH 02/65] Fixed edit to auth_discover view --- cog/views/views_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cog/views/views_account.py b/cog/views/views_account.py index 95cdc6ce9..c7687101e 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -37,7 +37,7 @@ def redirectToIdp(): from cog.backends.esgf import discover def auth_discover(request, **kwargs): openid = request.GET.get('openid_identifier', None) - protocol = 'OAuth2'#discover(openid) + protocol = discover(openid) if protocol == 'OAuth2': return JsonResponse({'auth': 'OAuth2'}) return JsonResponse({'auth': 'OpenID'}) From a406807e0f349fd27e6433dfdea9a4d1a8640400 Mon Sep 17 00:00:00 2001 From: wtucker Date: Fri, 16 Nov 2018 12:05:53 +0000 Subject: [PATCH 03/65] Added a view for storing the openid_identifier in session before oauth login. --- cog/forms/forms_account.py | 6 ++++++ cog/templates/cog/openid/login.html | 2 +- cog/urls.py | 1 + cog/views/views_account.py | 22 ++++++++++++++++++++-- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/cog/forms/forms_account.py b/cog/forms/forms_account.py index e10b7c78e..042b19a0d 100644 --- a/cog/forms/forms_account.py +++ b/cog/forms/forms_account.py @@ -52,6 +52,12 @@ class Meta: fields = ('claimed_id',) +class OAuth2LoginForm(Form): + + openid_identifier = CharField() + next = CharField(required=False) + + class PasswordResetForm(Form): openid = CharField(required=True, widget=TextInput(attrs={'size': '50'})) diff --git a/cog/templates/cog/openid/login.html b/cog/templates/cog/openid/login.html index b180d8b77..e4043a040 100644 --- a/cog/templates/cog/openid/login.html +++ b/cog/templates/cog/openid/login.html @@ -58,7 +58,7 @@ var response = JSON.parse(this.responseText); var form = document.getElementById('login_form'); if (response.auth == 'OAuth2') - form.action = "{% url 'social:begin' 'esgf' %}"; + form.action = "{% url 'oauth2-login' %}"; else form.action = "{% url 'openid-login' %}"; form.submit(); diff --git a/cog/urls.py b/cog/urls.py index 1fc63da4d..238c4f75f 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -47,6 +47,7 @@ # c) openid-only login url(r'^login/$', cog.views.custom_login, {'template_name': 'cog/openid/login.html'}, name='login'), url(r'^openid/login/$', django_openid_auth.views.login_begin, {'template_name': 'cog/openid/login.html'}, name='openid-login'), + url(r'^oauth2/login/$', cog.views.oauth2_login_view, name='oauth2-login'), # b) or c) #url(r'^openid/complete/$', 'django_openid_auth.views.login_complete', name='openid-complete'), diff --git a/cog/views/views_account.py b/cog/views/views_account.py index c7687101e..ac2da8631 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -1,7 +1,7 @@ import urllib from urlparse import urlparse -from django.contrib.auth import logout +from django.contrib.auth import logout, REDIRECT_FIELD_NAME from django.contrib.auth.decorators import login_required, user_passes_test from django.contrib.auth.hashers import is_password_usable from django.contrib.auth.views import login @@ -10,7 +10,7 @@ from django.core.urlresolvers import reverse from django.forms.models import modelformset_factory from django.http import HttpResponseRedirect, HttpResponseNotAllowed, HttpResponseServerError, JsonResponse -from django.shortcuts import get_object_or_404, render +from django.shortcuts import get_object_or_404, render, redirect from django.template import RequestContext from django_openid_auth.models import UserOpenID from django_openid_auth.views import login_complete @@ -111,6 +111,24 @@ def _custom_login(request, response): return response +def oauth2_login_view(request, form_class=OAuth2LoginForm): + + if request.POST: + + form = form_class(request.POST) + if form.is_valid(): + openid_identifier = form.cleaned_data['openid_identifier'] + redirect_to = form.cleaned_data['next'] + + # store social-auth fields in session + request.session['openid_identifier'] = openid_identifier + request.session[REDIRECT_FIELD_NAME] = redirect_to + + return redirect('social:begin', 'esgf') + + return redirect('login') + + def notifyAdminsOfUserRegistration(user,request): subject = "New User Registration" From d7af0a7e2db70965c2d8dccbc5ae4333ec0ce2e9 Mon Sep 17 00:00:00 2001 From: wtucker Date: Fri, 16 Nov 2018 12:13:37 +0000 Subject: [PATCH 04/65] Adding oauth backend from esgf-auth. --- cog/backends/__init__.py | 0 cog/backends/esgf.py | 186 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 cog/backends/__init__.py create mode 100644 cog/backends/esgf.py diff --git a/cog/backends/__init__.py b/cog/backends/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cog/backends/esgf.py b/cog/backends/esgf.py new file mode 100644 index 000000000..9a182251c --- /dev/null +++ b/cog/backends/esgf.py @@ -0,0 +1,186 @@ +from social_core.backends.oauth import BaseOAuth2 +from social_core.backends.open_id import OpenIdAuth, OPENID_ID_FIELD +from social_core.exceptions import AuthMissingParameter +from six.moves.urllib_parse import urlencode, unquote + +import os +from base64 import b64encode +from OpenSSL import crypto +from urlparse import urlparse + +from openid.yadis.services import getServiceEndpoints +from openid.yadis.discover import DiscoveryFailure + +class ESGFOAuth2(BaseOAuth2): + name = 'esgf' + AUTHORIZATION_URL = 'https://slcs.ceda.ac.uk/oauth/authorize' + ACCESS_TOKEN_URL = 'https://slcs.ceda.ac.uk/oauth/access_token' + CERTIFICATE_URL = 'https://slcs.ceda.ac.uk/oauth/certificate/' + DEFAULT_SCOPE = ['https://slcs.ceda.ac.uk/oauth/certificate/'] + REDIRECT_STATE = True + ACCESS_TOKEN_METHOD = 'POST' + EXTRA_DATA = [ + ('access_token', 'access_token', True), + ('expires_in', 'expires_in', True), + ('refresh_token', 'refresh_token', True), + ('openid', 'openid', True) + ] + + + def __init__(self, strategy=None, redirect_uri=None): + super(ESGFOAuth2, self).__init__(strategy, redirect_uri) + + # Get openid_identifier added to the session by PSA. Requires + # SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] + # set in settings.py + if strategy: + openid = self.strategy.session_get(OPENID_ID_FIELD, None) + self.set_urls(openid) + + + # get XRDS and extract authorize, access token, and certificate service URLs + def set_urls(self, openid): + try: + uri, endpoints = getServiceEndpoints(openid) + except DiscoveryFailure: + return + for e in endpoints: + if e.matchTypes(['urn:esg:security:oauth:endpoint:authorize']): + self.AUTHORIZATION_URL = e.uri + elif e.matchTypes(['urn:esg:security:oauth:endpoint:access']): + self.ACCESS_TOKEN_URL = e.uri + elif e.matchTypes(['urn:esg:security:oauth:endpoint:resource']): + self.CERTIFICATE_URL = e.uri + self.DEFAULT_SCOPE = [e.uri] + + + + def get_certificate(self, access_token): + """ Generate a new key pair """ + key_pair = crypto.PKey() + key_pair.generate_key(crypto.TYPE_RSA, 2048) + self.private_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, key_pair).decode('utf-8') + + """ Generate a certificate request """ + cert_request = crypto.X509Req() + cert_request.set_pubkey(key_pair) + cert_request.sign(key_pair, 'md5') + cert_request = crypto.dump_certificate_request(crypto.FILETYPE_ASN1, cert_request) + + headers = {'Authorization': 'Bearer {}'.format(access_token)} + url = self.CERTIFICATE_URL + + request_args = {'headers': headers, + 'data': {'certificate_request': b64encode(cert_request)} + } + try: + response = self.request(url, method='POST', **request_args) + response.raise_for_status() + except Exception, err: + print Exception, err + self.cert = response.text + cert = crypto.load_certificate(crypto.FILETYPE_PEM, response.text) + self.subject = cert.get_subject() + return (self.private_key, self.cert, self.subject.commonName) + + + # TO DO when OIDC is deployed on ESGF IdP nodes + # extract user info from id_token (OpenID Connect) +# def user_data(self, access_token, *args, **kwargs): +# print 'user_data' +# response = kwargs.get('response') +# id_token = response.get('id_token') +# try: +# decoded_id_token = jwt_decode(id_token, verify=False) +# except (DecodeError, ExpiredSignature) as de: +# raise AuthTokenError(self, de) +# for key in decoded_id_token: +# print '%s:%s' % (key, decoded_id_token[key]) +# +# return {'uid': decoded_id_token.get('sub'), +# 'username': decoded_id_token.get('preferred_username'), +# 'name': decoded_id_token.get('name'), +# 'email': decoded_id_token.get('email') +# } + + + # return values that will be stored in the database as: + # social_auth_usersocialauth.extra_data['openid'], + # auth_user.username + def get_user_details(self, response): + access_token = response.get('access_token') + pkey, cert, openid = self.get_certificate(access_token) + username = os.path.basename(urlparse(openid).path) + return {'openid': openid, + 'username': username, + } + + + # PSA compares returned value with social_auth_usersocialauth.uid (provider=='esgf') + # if it's new, PSA creates a new user + def get_user_id(self, details, response): + return details['openid'] + + +class ESGFOpenId(OpenIdAuth): + name = 'esgf-openid' + + # Extend original openid.openid_url() to a case where openid_identifier is set in the session only + def openid_url(self): + """Return service provider URL. + This base class is generic accepting a POST parameter that specifies + provider URL.""" + if self.URL: + return self.URL + elif OPENID_ID_FIELD in self.data: + return self.data[OPENID_ID_FIELD] + elif self.strategy.session_get(OPENID_ID_FIELD, None): + return self.strategy.session_get(OPENID_ID_FIELD, None) + else: + raise AuthMissingParameter(self, OPENID_ID_FIELD) + + + def get_user_details(self, response): + print response.identity_url + username = os.path.basename(urlparse(response.identity_url).path) + return {'openid': response.identity_url, + 'username': username, + } + + +def associate_by_openid(backend, details, user=None, *args, **kwargs): + """ + Associate current auth with a user with the same social.uid in the DB. + """ + + if user: + return None + + openid = details.get('openid') + if openid: + # Try to associate accounts registered with the same OpenId. + # Probably it is possible to get backend.strategy.storage from kwargs['storage']. + if backend.name == 'esgf': + social = backend.strategy.storage.user.get_social_auth(provider='esgf-openid', uid=kwargs['uid']) + elif backend.name == 'esgf-openid': + social = backend.strategy.storage.user.get_social_auth(provider='esgf', uid=kwargs['uid']) + if social: + return {'user': social.user, 'is_new': False} + return None + + +def discover(openid): + try: + uri, endpoints = getServiceEndpoints(openid) + except DiscoveryFailure: + return None + authorize = None + access = None + for e in endpoints: + if e.matchTypes(['urn:esg:security:oauth:endpoint:authorize']): + authorize = True + elif e.matchTypes(['urn:esg:security:oauth:endpoint:access']): + access = True + if authorize and access: + return 'OAuth2' + return 'OpenID' From 36f329a0272069807797a1ed0bac16a9e60b3727 Mon Sep 17 00:00:00 2001 From: Lukasz Lacinski Date: Wed, 21 Nov 2018 01:51:01 -0600 Subject: [PATCH 05/65] Set OAuth2 client key and secret dynamically --- cog/urls.py | 2 +- cog/views/views_account.py | 27 ++++++++++++++++++++++++++- settings.py | 7 ++++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/cog/urls.py b/cog/urls.py index 238c4f75f..1d052c5f1 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -56,7 +56,7 @@ # add OAuth2 urls url(r'', include('social_django.urls', namespace='social')), - url(r'^esgf-auth/discover/$', cog.views.auth_discover, name='auth-discover'), + url(r'^auth/discover/$', cog.views.auth_discover, name='auth-discover'), # force redirection to login page after logout #url(r'^logout/$', 'django.contrib.auth.views.logout_then_login', name='logout'), diff --git a/cog/views/views_account.py b/cog/views/views_account.py index ac2da8631..63a1a0b14 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -1,5 +1,7 @@ import urllib from urlparse import urlparse +import json +import traceback from django.contrib.auth import logout, REDIRECT_FIELD_NAME from django.contrib.auth.decorators import login_required, user_passes_test @@ -35,10 +37,33 @@ def redirectToIdp(): from cog.backends.esgf import discover + + +def get_oauth2_cred(openid_identifier): + """ + Get a key and secret pair from /esg/config/.esgf_oauth2.json + """ + parsed_openid = urlparse(openid_identifier) + with open(settings.ESGF_OAUTH2_SECRET_FILE, 'r') as f: + try: + creds = json.loads(f.read()) + cred = creds.get(parsed_openid.netloc) + if cred and cred.get('key') and cred.get('secret'): + return cred + except Exception: + traceback.print_exc() + print('Could not find an OAuth2 client key and secret for {} in {}' + .format(parsed_openid.netloc, settings.ESGF_OAUTH2_SECRET_FILE)) + return None + + def auth_discover(request, **kwargs): openid = request.GET.get('openid_identifier', None) protocol = discover(openid) - if protocol == 'OAuth2': + credential = get_oauth2_cred(openid) + if protocol == 'OAuth2' and credential: + settings.SOCIAL_AUTH_ESGF_KEY = credential['key'] + settings.SOCIAL_AUTH_ESGF_SECRET = credential['secret'] return JsonResponse({'auth': 'OAuth2'}) return JsonResponse({'auth': 'OpenID'}) diff --git a/settings.py b/settings.py index 46ff54b6f..6b0f1cc63 100644 --- a/settings.py +++ b/settings.py @@ -262,13 +262,14 @@ ) SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] -SOCIAL_AUTH_ESGF_KEY = 'XceQW2eGHpXqTQQsVUtc1UVNiLgoJNdFTZx6cM73' -SOCIAL_AUTH_ESGF_SECRET = 'qqXpMFz6BuVjiA0QlxovdGUOBycGQhnIPvfyEEuVX81n1k8eNZrGQJC6F4xDzi27DGBqtq6nYp4cCFWG5fJzz3d6derWawGlOfHBKCFg9ZNq75LUyeSWhw0oNkESeztT' SOCIAL_AUTH_SANITIZE_REDIRECTS = False SOCIAL_AUTH_ESGF_AUTH_EXTRA_ARGUMENTS = { 'access_type': 'offline', } - +ESGF_OAUTH2_SECRET_FILE = os.environ.get( + 'ESGF_OAUTH2_SECRET_FILE', + '/esg/config/.esgf_oauth2.json' +) # Default is X_FRAME_OPTIONS='SAMEORIGIN' # Using X_FRAME_OPTIONS = DENY breaks the CKEditor file uploader. From 7f8d712137536546e65adde2c7244de036262e1f Mon Sep 17 00:00:00 2001 From: wtucker Date: Wed, 21 Nov 2018 10:28:29 +0000 Subject: [PATCH 06/65] Renamed oauth2_login view. --- cog/urls.py | 2 +- cog/views/views_account.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cog/urls.py b/cog/urls.py index 1d052c5f1..7a75d34b3 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -47,7 +47,7 @@ # c) openid-only login url(r'^login/$', cog.views.custom_login, {'template_name': 'cog/openid/login.html'}, name='login'), url(r'^openid/login/$', django_openid_auth.views.login_begin, {'template_name': 'cog/openid/login.html'}, name='openid-login'), - url(r'^oauth2/login/$', cog.views.oauth2_login_view, name='oauth2-login'), + url(r'^oauth2/login/$', cog.views.oauth2_login, name='oauth2-login'), # b) or c) #url(r'^openid/complete/$', 'django_openid_auth.views.login_complete', name='openid-complete'), diff --git a/cog/views/views_account.py b/cog/views/views_account.py index 63a1a0b14..eb4d0630a 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -136,10 +136,11 @@ def _custom_login(request, response): return response -def oauth2_login_view(request, form_class=OAuth2LoginForm): +def oauth2_login(request, form_class=OAuth2LoginForm): if request.POST: + # parse the login form form = form_class(request.POST) if form.is_valid(): openid_identifier = form.cleaned_data['openid_identifier'] @@ -151,6 +152,7 @@ def oauth2_login_view(request, form_class=OAuth2LoginForm): return redirect('social:begin', 'esgf') + # fallback to login view return redirect('login') From 54e80deda9e9f7d6b52b50742ec2f007fd08d6de Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 14:32:51 -0700 Subject: [PATCH 07/65] Remove pysqlite from requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0c2cbf204..9e1c43f36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ south==1.0.2 psycopg2==2.5.2 python-openid==2.2.5 passlib==1.6.5 -pysqlite==2.8.1 django-contrib-comments==1.6.2 oauth2client==2.0.1 globus-sdk==1.7.1 From 4770817fd7c4fa54e85f049ce019ae52ef79dcb2 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 14:41:44 -0700 Subject: [PATCH 08/65] Run 2to3 on all Python code --- apache/wsgi.py | 16 ++-- cog/config/search/__init__.py | 2 +- cog/config/search/config_project_search.py | 14 ++-- .../django_openid_auth/0001_initial.py | 2 +- .../0002_auto_20160106_0812.py | 2 +- cog/forms/__init__.py | 20 ++--- cog/forms/forms_account.py | 2 +- cog/forms/forms_image.py | 2 +- cog/forms/forms_others.py | 4 +- cog/forms/forms_post.py | 2 +- cog/forms/forms_project.py | 2 +- cog/forms/forms_utils.py | 2 +- cog/installation/config.py | 14 ++-- cog/installation/install.py | 2 +- cog/installation/setup.py | 16 ++-- cog/management/commands/init_site.py | 2 +- cog/management/commands/sync_projects.py | 2 +- cog/management/commands/sync_users.py | 14 ++-- cog/middleware/init_middleware.py | 6 +- cog/middleware/login_middleware.py | 4 +- cog/middleware/password_middleware.py | 2 +- cog/migrations/0001_initial.py | 2 +- cog/migrations/0002_auto_20150706_1045.py | 2 +- .../0003_project_nodeswidgetenabled.py | 2 +- cog/migrations/0004_auto_20160106_0812.py | 2 +- cog/migrations/0005_project_shared.py | 2 +- cog/migrations/0006_auto_20160303_1043.py | 2 +- cog/migrations/0007_auto_20160303_1524.py | 2 +- cog/migrations/0008_auto_20160609_1504.py | 2 +- cog/migrations/0009_auto_20160826_0339.py | 2 +- cog/models/__init__.py | 68 ++++++++-------- cog/models/auth.py | 8 +- cog/models/bookmark.py | 2 +- cog/models/collaborator.py | 4 +- cog/models/communication_means.py | 4 +- cog/models/communication_means_member.py | 4 +- cog/models/constants.py | 4 +- cog/models/datacart.py | 4 +- cog/models/doc.py | 6 +- cog/models/external_url.py | 6 +- cog/models/folder.py | 10 +-- cog/models/funding_source.py | 4 +- cog/models/lock.py | 4 +- cog/models/logged_event.py | 8 +- cog/models/management_body.py | 8 +- cog/models/management_body_member.py | 4 +- cog/models/membership.py | 2 +- cog/models/navbar.py | 2 +- cog/models/news.py | 4 +- cog/models/organization.py | 4 +- cog/models/organizational_role.py | 4 +- cog/models/organizational_role_member.py | 4 +- cog/models/post.py | 8 +- cog/models/project.py | 14 ++-- cog/models/project_impact.py | 4 +- cog/models/project_tab.py | 6 +- cog/models/project_tag.py | 2 +- cog/models/project_topic.py | 6 +- cog/models/search.py | 50 ++++++------ cog/models/search_facet.py | 4 +- cog/models/search_group.py | 4 +- cog/models/search_profile.py | 4 +- cog/models/signals.py | 14 ++-- cog/models/topic.py | 2 +- cog/models/user_profile.py | 8 +- cog/models/user_url.py | 2 +- cog/models/utils.py | 80 +++++++++---------- cog/notification.py | 22 ++--- cog/plugins/esgf/registry.py | 32 ++++---- cog/plugins/esgf/security.py | 14 ++-- cog/plugins/globus/directory_transfer.py | 12 +-- cog/plugins/globus/download.py | 12 +-- cog/plugins/globus/transfer.py | 22 ++--- cog/project_manager.py | 24 +++--- cog/services/SolrSerializer.py | 8 +- cog/services/membership.py | 16 ++-- cog/services/registration/__init__.py | 4 +- cog/services/registration/registration.py | 4 +- cog/services/search.py | 10 +-- cog/site_manager.py | 8 +- cog/templatetags/cog_utils.py | 12 +-- cog/templatetags/search_utils.py | 2 +- cog/tests.py | 2 +- cog/tests/test_openid.py | 6 +- cog/tests/test_registration.py | 2 +- cog/util/thumbnails.py | 8 +- cog/utils.py | 12 +-- cog/views/__init__.py | 32 ++++---- cog/views/utils.py | 22 ++--- cog/views/views_aboutus.py | 14 ++-- cog/views/views_access_control.py | 12 +-- cog/views/views_account.py | 54 ++++++------- cog/views/views_admin.py | 2 +- cog/views/views_bookmarks.py | 24 +++--- cog/views/views_datacart.py | 16 ++-- cog/views/views_doc.py | 8 +- cog/views/views_external_urls.py | 8 +- cog/views/views_globus.py | 32 ++++---- cog/views/views_governance.py | 14 ++-- cog/views/views_membership.py | 4 +- cog/views/views_news.py | 6 +- cog/views/views_post.py | 12 +-- cog/views/views_project.py | 42 +++++----- cog/views/views_search.py | 78 +++++++++--------- cog/views/views_share.py | 8 +- filebrowser/actions.py | 10 +-- filebrowser/base.py | 4 +- filebrowser/decorators.py | 2 +- filebrowser/fields.py | 5 +- filebrowser/forms.py | 30 +++---- filebrowser/functions.py | 14 ++-- .../commands/fb_version_generate.py | 4 +- .../management/commands/fb_version_remove.py | 6 +- filebrowser/settings.py | 4 +- filebrowser/sites.py | 58 +++++++------- filebrowser/templatetags/fb_csrf.py | 6 +- filebrowser/templatetags/fb_pagination.py | 14 ++-- filebrowser/templatetags/fb_tags.py | 8 +- filebrowser/templatetags/fb_versions.py | 14 ++-- filebrowser/tests/sites.py | 8 +- filebrowser/widgets.py | 12 +-- resources/scripts/configure_search.py | 4 +- resources/scripts/fix_broken_links.py | 2 +- resources/scripts/insert_esgf_user.py | 4 +- resources/scripts/migrate_roles.py | 6 +- resources/scripts/migrate_users.py | 2 +- resources/scripts/send_email.py | 12 +-- settings.py | 20 ++--- setup.py | 2 +- urls.py | 4 +- 130 files changed, 699 insertions(+), 702 deletions(-) diff --git a/apache/wsgi.py b/apache/wsgi.py index 27733709c..c1afd57fe 100644 --- a/apache/wsgi.py +++ b/apache/wsgi.py @@ -28,18 +28,18 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") # print debugging information -print 'Using Python version: %s' % sys.version -print 'Using Python path: %s' % sys.path -print 'PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None) -print 'LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None) -print 'SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None) -print 'SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None) +print('Using Python version: %s' % sys.version) +print('Using Python path: %s' % sys.path) +print('PYTHONPATH=%s' % os.environ.get('PYTHONPATH', None)) +print('LD_LIBRARY_PATH=%s' % os.environ.get('LD_LIBRARY_PATH', None)) +print('SSL_CERT_DIR=%s' % os.environ.get('SSL_CERT_DIR', None)) +print('SSL_CERT_FILE=%s' % os.environ.get('SSL_CERT_FILE', None)) try: application = get_wsgi_application() - print 'WSGI without exception' + print('WSGI without exception') except Exception: - print 'handling WSGI exception' + print('handling WSGI exception') # Error loading applications if 'mod_wsgi' in sys.modules: traceback.print_exc() diff --git a/cog/config/search/__init__.py b/cog/config/search/__init__.py index 3c1e72fc5..a088cfd2e 100644 --- a/cog/config/search/__init__.py +++ b/cog/config/search/__init__.py @@ -1 +1 @@ -from config_project_search import SearchConfigParser \ No newline at end of file +from .config_project_search import SearchConfigParser \ No newline at end of file diff --git a/cog/config/search/config_project_search.py b/cog/config/search/config_project_search.py index 4a3592eb2..386402e2f 100644 --- a/cog/config/search/config_project_search.py +++ b/cog/config/search/config_project_search.py @@ -1,4 +1,4 @@ -import sys, os, ConfigParser +import sys, os, configparser from django.conf import settings from cog.models import Project, SearchGroup, SearchFacet from cog.utils import str2bool @@ -15,7 +15,7 @@ def __init__(self, project): def _getConfigParser(self): # read project configuration - configParser = ConfigParser.RawConfigParser() + configParser = configparser.RawConfigParser() # must set following line explicitly to preserve the case of configuration keys configParser.optionxform = str @@ -25,7 +25,7 @@ def _getConfigParser(self): def write(self): '''Writes the project search configuration to the file $COG_CONFIG_DIR/projects//search.cfg''' - print 'Writing search configuration for project=%s' % self.project.short_name + print('Writing search configuration for project=%s' % self.project.short_name) # load project search profile project = Project.objects.get(short_name=self.project.short_name) @@ -66,7 +66,7 @@ def write(self): def read(self): '''Reads the project search configuration from the file $COG_CONFIG_DIR/projects//search.cfg''' - print 'Reading search configuration for project=%s' % self.project.short_name + print('Reading search configuration for project=%s' % self.project.short_name) # load project search profile project = Project.objects.get(short_name=self.project.short_name) @@ -74,7 +74,7 @@ def read(self): # remove existing groups of facets for group in search_profile.groups.all(): - print 'Deleting search group=%s' % group + print('Deleting search group=%s' % group) group.delete() # read project configuration @@ -82,7 +82,7 @@ def read(self): try: projConfig.read( self.config_file_path ) except Exception as e: - print "Configuration file %s not found" % self.config_file_path + print("Configuration file %s not found" % self.config_file_path) raise e # loop over configuration sections @@ -110,7 +110,7 @@ def read(self): for option in projConfig.options(section): value = projConfig.get(section, option) - print section, option, value + print(section, option, value) parts = value.split("|") facet_order = int(option) facet_key = parts[0] diff --git a/cog/db_migrations/django_openid_auth/0001_initial.py b/cog/db_migrations/django_openid_auth/0001_initial.py index 8a4aba821..83d3a6a65 100644 --- a/cog/db_migrations/django_openid_auth/0001_initial.py +++ b/cog/db_migrations/django_openid_auth/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals + from django.db import models, migrations from django.conf import settings diff --git a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py index f2dcf4692..5b1d561fe 100644 --- a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py +++ b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-06 08:12 -from __future__ import unicode_literals + from django.db import migrations diff --git a/cog/forms/__init__.py b/cog/forms/__init__.py index 463f90a06..686b091bc 100644 --- a/cog/forms/__init__.py +++ b/cog/forms/__init__.py @@ -1,10 +1,10 @@ -from forms_image import * -from forms_account import * -from forms_bookmarks import * -from forms_governance import * -from forms_others import * -from forms_post import * -from forms_project import * -from forms_aboutus import * -from forms_search import * -from forms_access_control import * \ No newline at end of file +from .forms_image import * +from .forms_account import * +from .forms_bookmarks import * +from .forms_governance import * +from .forms_others import * +from .forms_post import * +from .forms_project import * +from .forms_aboutus import * +from .forms_search import * +from .forms_access_control import * \ No newline at end of file diff --git a/cog/forms/forms_account.py b/cog/forms/forms_account.py index e10b7c78e..0a93499e2 100644 --- a/cog/forms/forms_account.py +++ b/cog/forms/forms_account.py @@ -286,7 +286,7 @@ def validate_username(form, user_id): # once the openid is validated, choose the closest possible username _username = createUsername(username) - print 'Created username=%s from=%s' % (_username, username) + print('Created username=%s from=%s' % (_username, username)) cleaned_data['username'] = _username # override form data else: # django will automatically check that the username is unique in the CoG database diff --git a/cog/forms/forms_image.py b/cog/forms/forms_image.py index 2fc471e63..aedd1e2e1 100644 --- a/cog/forms/forms_image.py +++ b/cog/forms/forms_image.py @@ -23,6 +23,6 @@ def clean(self): self._errors["image"] = self.error_class(["Image size exceeds the maximum allowed."]) except OSError as e: # image not existing on disk - print e + print(e) return self.cleaned_data \ No newline at end of file diff --git a/cog/forms/forms_others.py b/cog/forms/forms_others.py index 9069155cd..037e20ba7 100644 --- a/cog/forms/forms_others.py +++ b/cog/forms/forms_others.py @@ -126,10 +126,10 @@ def clean(self): file_ext = str(os.path.splitext(thefile.name)[1]) mime_type = magic.from_buffer(thefile.read(1024), mime=True) - print "Validating file extension=%s, mime type=%s" % (file_ext, mime_type) + print("Validating file extension=%s, mime type=%s" % (file_ext, mime_type)) if not file_ext: self._errors["file"] = self.error_class(["File name must have an extension."]) - elif file_ext.lower() not in VALID_MIME_TYPES.keys(): + elif file_ext.lower() not in list(VALID_MIME_TYPES.keys()): self._errors["file"] = self.error_class(["File extension %s is not supported." % file_ext]) elif mime_type not in VALID_MIME_TYPES[file_ext.lower()]: self._errors["file"] = self.error_class(["File extension %s does not match its valid mime type." % diff --git a/cog/forms/forms_post.py b/cog/forms/forms_post.py index dc746213f..94b8a50c7 100644 --- a/cog/forms/forms_post.py +++ b/cog/forms/forms_post.py @@ -137,7 +137,7 @@ def clean(self): # validate "topic" # cannot set both 'topic' and 'newtopic' if topic and newtopic: - errmsg = u"Please either choose an existing topic OR create a new one" + errmsg = "Please either choose an existing topic OR create a new one" self._errors["topic"] = self.error_class([errmsg]) del cleaned_data["topic"] del cleaned_data["newtopic"] diff --git a/cog/forms/forms_project.py b/cog/forms/forms_project.py index aae51da08..3d3e3bb2a 100644 --- a/cog/forms/forms_project.py +++ b/cog/forms/forms_project.py @@ -156,7 +156,7 @@ def clean(self): features = self.cleaned_data.get('software_features') if not hasText(features): self._errors["software_features"] = self.error_class(["'SoftwareFeatures' must not be empty."]) - print 'error' + print('error') return self.cleaned_data diff --git a/cog/forms/forms_utils.py b/cog/forms/forms_utils.py index bba31ce3e..9e8ceedde 100644 --- a/cog/forms/forms_utils.py +++ b/cog/forms/forms_utils.py @@ -26,7 +26,7 @@ def validate_image(form, field_name): # validate image header try: image_type = imghdr.what(image) - print 'Validating image header: detected image type=%s' % image_type + print('Validating image header: detected image type=%s' % image_type) if image_type is None: form._errors[field_name] = form.error_class(["Invalid image type: %s" % image_type]) except Exception as e: diff --git a/cog/installation/config.py b/cog/installation/config.py index 97ab9551c..506eefd8b 100644 --- a/cog/installation/config.py +++ b/cog/installation/config.py @@ -23,15 +23,15 @@ ''' -import ConfigParser -import StringIO +import configparser +import io import collections import logging import os import time from django.utils.crypto import get_random_string -from constants import (SECTION_DEFAULT, SECTION_ESGF, SECTION_EMAIL, +from .constants import (SECTION_DEFAULT, SECTION_ESGF, SECTION_EMAIL, ESGF_PROPERTIES_FILE, ESGF_PASSWORD_FILE, IDP_WHITELIST, KNOWN_PROVIDERS, PEER_NODES, DEFAULT_PROJECT_SHORT_NAME) @@ -59,7 +59,7 @@ def _readCogConfig(self): '''Method that reads an existing COG configuration file, or create a new one if not existing.''' # initialize COG configuration file - self.cogConfig = ConfigParser.ConfigParser(allow_no_value=True, + self.cogConfig = configparser.ConfigParser(allow_no_value=True, dict_type=collections.OrderedDict) # must set following line explicitly to preserve the case of configuration keys self.cogConfig.optionxform = str @@ -78,7 +78,7 @@ def _readCogConfig(self): logging.info("Configuration file: %s not found, will create new one" % CONFIGFILEPATH ) except Exception as e: - print e + print(e) logging.error("Error reading configuration file: %s" % CONFIGFILEPATH) logging.error(e) @@ -87,14 +87,14 @@ def _readEsgfConfig(self): '''Method that reads local parameters from ESGF configuration file esgf.properties.''' # read ESGF configuration file, if available - self.esgfConfig = ConfigParser.ConfigParser() + self.esgfConfig = configparser.ConfigParser() # $esg_config_dir/esgf.properties try: with open(ESGF_PROPERTIES_FILE, 'r') as f: # transform Java properties file into python configuration file: must prepend a section config_string = '[%s]\n' % SECTION_DEFAULT + f.read() - config_file = StringIO.StringIO(config_string) + config_file = io.StringIO(config_string) self.esgfConfig.readfp(config_file) logging.info("Read ESGF configuration parameters from file: %s" % ESGF_PROPERTIES_FILE) except IOError: diff --git a/cog/installation/install.py b/cog/installation/install.py index 566e6f1d0..01ca4a954 100644 --- a/cog/installation/install.py +++ b/cog/installation/install.py @@ -164,7 +164,7 @@ def _createObjects(self): except ObjectDoesNotExist: site = Site.objects.create(name=idpHostname, domain=idpHostname) idpPeerSite = PeerSite.objects.create(site=site, enabled=True) - print '\tCreated IdP Peer site: %s with enabled=%s' % (idpPeerSite, idpPeerSite.enabled) + print('\tCreated IdP Peer site: %s with enabled=%s' % (idpPeerSite, idpPeerSite.enabled)) def _getRootAdminPassword(self): diff --git a/cog/installation/setup.py b/cog/installation/setup.py index f7ca458f3..e1951ea65 100644 --- a/cog/installation/setup.py +++ b/cog/installation/setup.py @@ -25,25 +25,25 @@ def finalize_options(self): def run(self): # 1) create/update cog_settings.cfg BEFORE Django is started - print '>>> 1) Executing CogConfig...' - from config import CogConfig + print('>>> 1) Executing CogConfig...') + from .config import CogConfig cogConfig = CogConfig(self.esgf) cogConfig.config() - print '<<< ...done with CogConfig' + print('<<< ...done with CogConfig') # 2) setup Django registry to initialize CoG application - print '>>> 2) Setting up Django applications registry' + print('>>> 2) Setting up Django applications registry') os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' import django django.setup() - print '<<< ... done with django.setup()' + print('<<< ... done with django.setup()') # 3) use cog_settings.cfg to install/upgrade CoG database - print '>>> 3) Executing CoGInstall...' - from install import CoGInstall + print('>>> 3) Executing CoGInstall...') + from .install import CoGInstall cogInstall = CoGInstall() cogInstall.install() - print '<<< ...done with CoGInstall' + print('<<< ...done with CoGInstall') if __name__ == '__main__': diff --git a/cog/management/commands/init_site.py b/cog/management/commands/init_site.py index 3b7c3d8ef..5bfea44cc 100644 --- a/cog/management/commands/init_site.py +++ b/cog/management/commands/init_site.py @@ -20,4 +20,4 @@ def handle(self, *ags, **options): current_site.domain = settings.SITE_DOMAIN current_site.save() - print 'Updated current site: name=%s domain=%s' % (current_site.name, current_site.domain) \ No newline at end of file + print('Updated current site: name=%s domain=%s' % (current_site.name, current_site.domain)) \ No newline at end of file diff --git a/cog/management/commands/sync_projects.py b/cog/management/commands/sync_projects.py index 712277565..8261f79c9 100644 --- a/cog/management/commands/sync_projects.py +++ b/cog/management/commands/sync_projects.py @@ -15,4 +15,4 @@ class Command(BaseCommand): def handle(self, *ags, **options): sites = projectManager.sync() - print 'sync_projects: time=%s synchronized projects from sites=%s' % (datetime.datetime.now(), sites) \ No newline at end of file + print('sync_projects: time=%s synchronized projects from sites=%s' % (datetime.datetime.now(), sites)) \ No newline at end of file diff --git a/cog/management/commands/sync_users.py b/cog/management/commands/sync_users.py index 269e717d5..6a532863b 100644 --- a/cog/management/commands/sync_users.py +++ b/cog/management/commands/sync_users.py @@ -7,7 +7,7 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User from cog.models.user_profile import isUserLocal -import urllib +import urllib.request, urllib.parse, urllib.error HTTP_STATUS_CODE_OK = 200 HTTP_STATUS_CODE_NOT_FOUND = 404 @@ -24,21 +24,21 @@ def handle(self, *ags, **options): # test user existence by accessing the profile page userProfileUrl = user.profile.getAbsoluteUrl() - print "\nChecking user at URL: %s" % userProfileUrl.decode("utf-8") + print("\nChecking user at URL: %s" % userProfileUrl.decode("utf-8")) try: - response = urllib.urlopen( userProfileUrl ) + response = urllib.request.urlopen( userProfileUrl ) if response.getcode()==HTTP_STATUS_CODE_NOT_FOUND: - print '\tUser not found on remote node %s, deleting from local database...' % user.profile.site.domain + print('\tUser not found on remote node %s, deleting from local database...' % user.profile.site.domain) # delete this user from local database user.delete() else: - print '\tUser found.' + print('\tUser found.') # error checking this user except Exception as exception: - print 'Error checking URL: %s skipping... ' % userProfileUrl - print exception + print('Error checking URL: %s skipping... ' % userProfileUrl) + print(exception) pass \ No newline at end of file diff --git a/cog/middleware/init_middleware.py b/cog/middleware/init_middleware.py index 31595f0a7..324ce3852 100644 --- a/cog/middleware/init_middleware.py +++ b/cog/middleware/init_middleware.py @@ -10,14 +10,14 @@ class InitMiddleware(object): def __init__(self): - print 'Executing CoG initialization tasks' + print('Executing CoG initialization tasks') # update name, domain of current site into database current_site = Site.objects.get_current() current_site.name = settings.SITE_NAME current_site.domain = settings.SITE_DOMAIN current_site.save() - print 'Updated current site: name=%s domain=%s' % (current_site.name, current_site.domain) + print('Updated current site: name=%s domain=%s' % (current_site.name, current_site.domain)) # update list of ESGF peers into database filepath = siteManager.get('PEER_NODES') @@ -30,5 +30,5 @@ def __init__(self): raise MiddlewareNotUsed('Do not invoke ever again') def process_request(self, request): - print 'This line should never be printed...' + print('This line should never be printed...') return None \ No newline at end of file diff --git a/cog/middleware/login_middleware.py b/cog/middleware/login_middleware.py index 407a80b76..db3335e92 100644 --- a/cog/middleware/login_middleware.py +++ b/cog/middleware/login_middleware.py @@ -80,8 +80,8 @@ def process_response(self, request, response): if request.method=='POST' and not request.user.is_authenticated(): if response.status_code == 500: - print 'Authentication Error' - print response + print('Authentication Error') + print(response) if 'OpenID discovery error' in response.content: return HttpResponseRedirect(reverse('openid-login')+"?message=openid_discovery_error&next=%s&openid=%s" % (next, openid_identifier) ) diff --git a/cog/middleware/password_middleware.py b/cog/middleware/password_middleware.py index 01865c82a..7d7dd4387 100644 --- a/cog/middleware/password_middleware.py +++ b/cog/middleware/password_middleware.py @@ -19,7 +19,7 @@ def process_request(self, request): if not any(url in request.path for url in EXEMPT_URLS): try: if request.user.is_authenticated() and request.user.profile.type==1 and request.user.profile.hasPasswordExpired(): - print 'Password for user %s has expired, forcing mandatory change.' % request.user + print('Password for user %s has expired, forcing mandatory change.' % request.user) return HttpResponseRedirect(reverse('password_update', kwargs={'user_id':request.user.id})+"?message=password_expired&next=%s" % request.path) except ObjectDoesNotExist: diff --git a/cog/migrations/0001_initial.py b/cog/migrations/0001_initial.py index 038e8dc77..19a353034 100644 --- a/cog/migrations/0001_initial.py +++ b/cog/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals + from django.db import models, migrations import django.db.models.deletion diff --git a/cog/migrations/0002_auto_20150706_1045.py b/cog/migrations/0002_auto_20150706_1045.py index 2b5dc67a3..9bc18ac83 100644 --- a/cog/migrations/0002_auto_20150706_1045.py +++ b/cog/migrations/0002_auto_20150706_1045.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals + from django.db import models, migrations import cog.models.dbutils diff --git a/cog/migrations/0003_project_nodeswidgetenabled.py b/cog/migrations/0003_project_nodeswidgetenabled.py index 591b92c52..89c039ce1 100644 --- a/cog/migrations/0003_project_nodeswidgetenabled.py +++ b/cog/migrations/0003_project_nodeswidgetenabled.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals + from django.db import models, migrations diff --git a/cog/migrations/0004_auto_20160106_0812.py b/cog/migrations/0004_auto_20160106_0812.py index a708531fb..39abc04c1 100644 --- a/cog/migrations/0004_auto_20160106_0812.py +++ b/cog/migrations/0004_auto_20160106_0812.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-06 08:12 -from __future__ import unicode_literals + from django.db import migrations, models diff --git a/cog/migrations/0005_project_shared.py b/cog/migrations/0005_project_shared.py index 0bd618ff4..50a3145aa 100644 --- a/cog/migrations/0005_project_shared.py +++ b/cog/migrations/0005_project_shared.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-22 01:52 -from __future__ import unicode_literals + from django.db import migrations, models diff --git a/cog/migrations/0006_auto_20160303_1043.py b/cog/migrations/0006_auto_20160303_1043.py index 5990789e1..d4d7a4ec1 100644 --- a/cog/migrations/0006_auto_20160303_1043.py +++ b/cog/migrations/0006_auto_20160303_1043.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-03-03 10:43 -from __future__ import unicode_literals + from django.db import migrations, models diff --git a/cog/migrations/0007_auto_20160303_1524.py b/cog/migrations/0007_auto_20160303_1524.py index c943da26d..be721d1cd 100644 --- a/cog/migrations/0007_auto_20160303_1524.py +++ b/cog/migrations/0007_auto_20160303_1524.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-03-03 15:24 -from __future__ import unicode_literals + from django.db import migrations, models diff --git a/cog/migrations/0008_auto_20160609_1504.py b/cog/migrations/0008_auto_20160609_1504.py index 4a7276989..27258b99a 100644 --- a/cog/migrations/0008_auto_20160609_1504.py +++ b/cog/migrations/0008_auto_20160609_1504.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.4 on 2016-06-09 15:04 -from __future__ import unicode_literals + from django.db import migrations, models diff --git a/cog/migrations/0009_auto_20160826_0339.py b/cog/migrations/0009_auto_20160826_0339.py index 274ac56dd..2f37a86f2 100644 --- a/cog/migrations/0009_auto_20160826_0339.py +++ b/cog/migrations/0009_auto_20160826_0339.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.4 on 2016-08-26 03:39 -from __future__ import unicode_literals + from django.db import migrations diff --git a/cog/models/__init__.py b/cog/models/__init__.py index e16ef3fda..b69e39e20 100644 --- a/cog/models/__init__.py +++ b/cog/models/__init__.py @@ -1,59 +1,59 @@ # Users, Groups -from user_profile import UserProfile, isUserValid, isUserLocal, isUserRemote, discoverSiteForUser, getDataCartsForUser, isOpenidLocal -from user_url import UserUrl -from membership import MembershipRequest -from collaborator import Collaborator +from .user_profile import UserProfile, isUserValid, isUserLocal, isUserRemote, discoverSiteForUser, getDataCartsForUser, isOpenidLocal +from .user_url import UserUrl +from .membership import MembershipRequest +from .collaborator import Collaborator # Data Cart -from datacart import DataCart, DataCartItem, DataCartItemMetadataKey, DataCartItemMetadataValue +from .datacart import DataCart, DataCartItem, DataCartItemMetadataKey, DataCartItemMetadataValue # Projects -from topic import Topic -from project import * -from project_topic import ProjectTopic -from project_tab import * -from news import News -from project_tag import ProjectTag, MAX_PROJECT_TAG_LENGTH -from project_impact import ProjectImpact +from .topic import Topic +from .project import * +from .project_topic import ProjectTopic +from .project_tab import * +from .news import News +from .project_tag import ProjectTag, MAX_PROJECT_TAG_LENGTH +from .project_impact import ProjectImpact # Peers -from peer_site import PeerSite, getPeerSites +from .peer_site import PeerSite, getPeerSites # Posts -from doc import Doc -from post import Post -from external_url import ExternalUrl +from .doc import Doc +from .post import Post +from .external_url import ExternalUrl # Bookmarks -from folder import Folder, getTopFolder, getTopSubFolders, TOP_FOLDER, TOP_SUB_FOLDERS -from bookmark import Bookmark +from .folder import Folder, getTopFolder, getTopSubFolders, TOP_FOLDER, TOP_SUB_FOLDERS +from .bookmark import Bookmark # Search -from search_profile import SearchProfile -from search_group import SearchGroup -from search_facet import SearchFacet +from .search_profile import SearchProfile +from .search_group import SearchGroup +from .search_facet import SearchFacet # Governance -from funding_source import FundingSource -from organization import Organization -from management_body import ManagementBody, getManagementBodies, ManagementBodyPurpose, initManagementBodyPurpose -from management_body_member import ManagementBodyMember -from communication_means import CommunicationMeans -from communication_means_member import CommunicationMeansMember -from organizational_role import OrganizationalRole, getLeadOrganizationalRoles, getMemberOrganizationalRoles, getOrganizationalRoles -from organizational_role_member import OrganizationalRoleMember +from .funding_source import FundingSource +from .organization import Organization +from .management_body import ManagementBody, getManagementBodies, ManagementBodyPurpose, initManagementBodyPurpose +from .management_body_member import ManagementBodyMember +from .communication_means import CommunicationMeans +from .communication_means_member import CommunicationMeansMember +from .organizational_role import OrganizationalRole, getLeadOrganizationalRoles, getMemberOrganizationalRoles, getOrganizationalRoles +from .organizational_role_member import OrganizationalRoleMember # Search -from search import * +from .search import * # Logging -from logged_event import LoggedEvent +from .logged_event import LoggedEvent # Locks -from lock import Lock, getLock, createLock, deleteLock, isLockedOut +from .lock import Lock, getLock, createLock, deleteLock, isLockedOut # global function involving multiple objects -from utils import * +from .utils import * # model signals -from signals import account_created_receiver, update_user_projects, update_user_tags \ No newline at end of file +from .signals import account_created_receiver, update_user_projects, update_user_tags \ No newline at end of file diff --git a/cog/models/auth.py b/cog/models/auth.py index 420ae041c..efe08e079 100644 --- a/cog/models/auth.py +++ b/cog/models/auth.py @@ -3,14 +3,14 @@ from django.contrib.auth.models import User, Permission, Group from django.contrib.contenttypes.models import ContentType -from constants import APPLICATION_LABEL, ROLE_ADMIN, ROLE_CONTRIBUTOR, ROLE_USER +from .constants import APPLICATION_LABEL, ROLE_ADMIN, ROLE_CONTRIBUTOR, ROLE_USER # GROUPS def createGroup(group_name): group = Group(name=group_name) group.save() - print "Created group: %s" % group.name + print("Created group: %s" % group.name) return group # method to load a named group from the database, or create a new one if not existing already @@ -40,11 +40,11 @@ def createProjectPermission(pDesc, pCodeName, groups): projectContenType = ContentType.objects.get(app_label=APPLICATION_LABEL, model='project') permission = Permission(name=pDesc, codename=pCodeName, content_type=projectContenType) permission.save() - print 'Created permission=%s...' % permission.codename + print('Created permission=%s...' % permission.codename) for group in groups: group.permissions.add(permission) group.save() - print '...and associated to group=%s' % group.name + print('...and associated to group=%s' % group.name) return permission # method to return a named permission from the database, diff --git a/cog/models/bookmark.py b/cog/models/bookmark.py index caf3e1942..d6c5abe6c 100644 --- a/cog/models/bookmark.py +++ b/cog/models/bookmark.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from cog.models import Post diff --git a/cog/models/collaborator.py b/cog/models/collaborator.py index a3a184874..7c041b145 100644 --- a/cog/models/collaborator.py +++ b/cog/models/collaborator.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL, RESEARCH_KEYWORDS_MAX_CHARS -from project import Project +from .constants import APPLICATION_LABEL, RESEARCH_KEYWORDS_MAX_CHARS +from .project import Project class Collaborator(models.Model): diff --git a/cog/models/communication_means.py b/cog/models/communication_means.py index e39d53f3c..184189b30 100644 --- a/cog/models/communication_means.py +++ b/cog/models/communication_means.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL, PURPOSE_CV, COMMUNICATION_CV, MEMBERSHIP_CV -from project import Project +from .constants import APPLICATION_LABEL, PURPOSE_CV, COMMUNICATION_CV, MEMBERSHIP_CV +from .project import Project class CommunicationMeans(models.Model): diff --git a/cog/models/communication_means_member.py b/cog/models/communication_means_member.py index ef7760ff8..1989f1b82 100644 --- a/cog/models/communication_means_member.py +++ b/cog/models/communication_means_member.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User -from communication_means import CommunicationMeans +from .communication_means import CommunicationMeans class CommunicationMeansMember(models.Model): diff --git a/cog/models/constants.py b/cog/models/constants.py index f93661dd4..e2c052d3a 100644 --- a/cog/models/constants.py +++ b/cog/models/constants.py @@ -57,7 +57,7 @@ # create and merge a combined dictionary of organizational roles LEAD_ORGANIZATIONAL_ROLES_DICT = dict(LEAD_ORGANIZATIONAL_ROLES) MEMBER_ORGANIZATIONAL_ROLES_DICT = dict(MEMBER_ORGANIZATIONAL_ROLES) -ORGANIZATIONAL_ROLES_DICT = dict( LEAD_ORGANIZATIONAL_ROLES_DICT.items() + MEMBER_ORGANIZATIONAL_ROLES_DICT.items() ) +ORGANIZATIONAL_ROLES_DICT = dict( list(LEAD_ORGANIZATIONAL_ROLES_DICT.items()) + list(MEMBER_ORGANIZATIONAL_ROLES_DICT.items()) ) ROLE_CATEGORY_LEAD = 'Lead' ROLE_CATEGORY_MEMBER = 'Member' @@ -93,7 +93,7 @@ # create and merge a combined dictionary of management bodies STRATEGIC_MANAGEMENT_BODY_DICT = dict(STRATEGIC_MANAGEMENT_BODIES) OPERATIONAL_MANAGEMENT_BODY_DICT = dict(OPERATIONAL_MANAGEMENT_BODIES) -MANAGEMENT_BODY_DICT = dict( STRATEGIC_MANAGEMENT_BODY_DICT.items() + OPERATIONAL_MANAGEMENT_BODY_DICT.items() ) +MANAGEMENT_BODY_DICT = dict( list(STRATEGIC_MANAGEMENT_BODY_DICT.items()) + list(OPERATIONAL_MANAGEMENT_BODY_DICT.items()) ) MEMBERSHIP_TYPES = ('Open','Closed','By Invitation') diff --git a/cog/models/datacart.py b/cog/models/datacart.py index 149faf7a5..394d78856 100644 --- a/cog/models/datacart.py +++ b/cog/models/datacart.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User from cog.models.search import Record import json @@ -51,7 +51,7 @@ def create(datacart, id, metadata): item.save() # save additional metadata - for key, values in metadata.items(): + for key, values in list(metadata.items()): itemKey = DataCartItemMetadataKey(item=item, key=key) itemKey.save() for value in values: diff --git a/cog/models/doc.py b/cog/models/doc.py index 221d91ded..52ba48ffb 100644 --- a/cog/models/doc.py +++ b/cog/models/doc.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from project import Project +from .constants import APPLICATION_LABEL +from .project import Project from django.contrib.auth.models import User import os from django.conf import settings @@ -21,7 +21,7 @@ class OverridingFileStorage(FileSystemStorage): def save(self, name, content, max_length=None): # must delete current file first if self.exists(name): - print 'Deleting existing file=%s' % name + print('Deleting existing file=%s' % name) self.delete(name) # also, look for Doc objects for the same named file prefix = getattr(settings, "FILEBROWSER_DIRECTORY", "") diff --git a/cog/models/external_url.py b/cog/models/external_url.py index 2818bccc6..67d42ae17 100644 --- a/cog/models/external_url.py +++ b/cog/models/external_url.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL -from external_url_conf import externalUrlManager -from project import Project +from .constants import APPLICATION_LABEL +from .external_url_conf import externalUrlManager +from .project import Project # A reference to an external URL diff --git a/cog/models/folder.py b/cog/models/folder.py index e3954abc3..3f7d88aa1 100644 --- a/cog/models/folder.py +++ b/cog/models/folder.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from project import Project +from .constants import APPLICATION_LABEL +from .project import Project from collections import OrderedDict from cog.models.dbutils import UnsavedForeignKey @@ -47,7 +47,7 @@ def topParent(self): return self.parent.topParent() # recursion def isPredefined(self): - return self.parent is None or self.name in TOP_SUB_FOLDERS.values() + return self.parent is None or self.name in list(TOP_SUB_FOLDERS.values()) class Meta: unique_together = (("project", "name"),) @@ -63,7 +63,7 @@ def getTopFolder(project): # name = "%s %s" % (project.short_name, TOP_FOLDER) folder, created = Folder.objects.get_or_create(name=TOP_FOLDER, parent=None, project=project, active=True) if created: - print 'Project=%s: created Top-Level folder=%s' % (project.short_name, folder.name) + print('Project=%s: created Top-Level folder=%s' % (project.short_name, folder.name)) return folder @@ -80,7 +80,7 @@ def getTopSubFolders(project): _folders = [] # select pre-defined folders for folder in folders: - if folder.name in TOP_SUB_FOLDERS.values(): + if folder.name in list(TOP_SUB_FOLDERS.values()): _folders.append(folder) return _folders \ No newline at end of file diff --git a/cog/models/funding_source.py b/cog/models/funding_source.py index b8a313dfc..780bda767 100644 --- a/cog/models/funding_source.py +++ b/cog/models/funding_source.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL, UPLOAD_DIR_LOGOS -from project import Project +from .constants import APPLICATION_LABEL, UPLOAD_DIR_LOGOS +from .project import Project class FundingSource(models.Model): diff --git a/cog/models/lock.py b/cog/models/lock.py index c3a220d03..9c74f8345 100644 --- a/cog/models/lock.py +++ b/cog/models/lock.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User -from project_tab import ProjectTab +from .project_tab import ProjectTab from datetime import datetime, timedelta # default lock lifetime: 30 minutes diff --git a/cog/models/logged_event.py b/cog/models/logged_event.py index 4b41dfcc7..9e5eddcc4 100644 --- a/cog/models/logged_event.py +++ b/cog/models/logged_event.py @@ -1,13 +1,13 @@ from django.db import models -from constants import APPLICATION_LABEL, SIGNAL_OBJECT_CREATED, SIGNAL_OBJECT_UPDATED, SIGNAL_OBJECT_DELETED +from .constants import APPLICATION_LABEL, SIGNAL_OBJECT_CREATED, SIGNAL_OBJECT_UPDATED, SIGNAL_OBJECT_DELETED from django.contrib.auth.models import User from django.db.models.signals import post_save from django.core.signals import request_finished from django.core.urlresolvers import reverse from django.dispatch import receiver -from post import Post, post_signal -from doc import Doc -from news import News +from .post import Post, post_signal +from .doc import Doc +from .news import News class LoggedEvent(models.Model): diff --git a/cog/models/management_body.py b/cog/models/management_body.py index c0732966d..27ea5e76a 100644 --- a/cog/models/management_body.py +++ b/cog/models/management_body.py @@ -91,18 +91,18 @@ def getManagementBodies(project, category): def initManagementBodyPurpose(): """Function to populate the database with the necessary instances of ManagementBodyPurpose.""" - for purpose, order in STRATEGIC_MANAGEMENT_BODY_DICT.items(): + for purpose, order in list(STRATEGIC_MANAGEMENT_BODY_DICT.items()): try: mbp = ManagementBodyPurpose.objects.get(purpose=purpose, category=MANAGEMENT_BODY_CATEGORY_STRATEGIC) except ManagementBodyPurpose.DoesNotExist: mbp = ManagementBodyPurpose(purpose=purpose, order=order, category=MANAGEMENT_BODY_CATEGORY_STRATEGIC) mbp.save() - print "Created management body purpose: %s" % mbp + print("Created management body purpose: %s" % mbp) - for purpose, order in OPERATIONAL_MANAGEMENT_BODY_DICT.items(): + for purpose, order in list(OPERATIONAL_MANAGEMENT_BODY_DICT.items()): try: mbp = ManagementBodyPurpose.objects.get(purpose=purpose, category=MANAGEMENT_BODY_CATEGORY_OPERATIONAL) except ManagementBodyPurpose.DoesNotExist: mbp = ManagementBodyPurpose(purpose=purpose, order=order, category=MANAGEMENT_BODY_CATEGORY_OPERATIONAL) mbp.save() - print "Created management body purpose: %s" % mbp \ No newline at end of file + print("Created management body purpose: %s" % mbp) \ No newline at end of file diff --git a/cog/models/management_body_member.py b/cog/models/management_body_member.py index d5af246a2..be1decf53 100644 --- a/cog/models/management_body_member.py +++ b/cog/models/management_body_member.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User -from management_body import ManagementBody +from .management_body import ManagementBody class ManagementBodyMember(models.Model): diff --git a/cog/models/membership.py b/cog/models/membership.py index 0c8f260a6..99d1e2d6b 100644 --- a/cog/models/membership.py +++ b/cog/models/membership.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User, Group import django.dispatch diff --git a/cog/models/navbar.py b/cog/models/navbar.py index 3fe33b813..074b6729e 100644 --- a/cog/models/navbar.py +++ b/cog/models/navbar.py @@ -3,7 +3,7 @@ i.e. for the project pre-defined pages. """ -from external_url_conf import EXTERNAL_URL_TYPES, externalUrlManager +from .external_url_conf import EXTERNAL_URL_TYPES, externalUrlManager # dictionary containing (page key, page URL) TABS = {"ABOUTUS": "aboutus", "MISSION": "mission", diff --git a/cog/models/news.py b/cog/models/news.py index 33fd27ab3..df78f3d72 100644 --- a/cog/models/news.py +++ b/cog/models/news.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User -from project import Project +from .project import Project # A piece of News about a Project class News(models.Model): diff --git a/cog/models/organization.py b/cog/models/organization.py index c05c47e2c..b82a46468 100644 --- a/cog/models/organization.py +++ b/cog/models/organization.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL, UPLOAD_DIR_LOGOS -from project import Project +from .constants import APPLICATION_LABEL, UPLOAD_DIR_LOGOS +from .project import Project class Organization(models.Model): diff --git a/cog/models/organizational_role.py b/cog/models/organizational_role.py index 180bea883..f85647aa1 100644 --- a/cog/models/organizational_role.py +++ b/cog/models/organizational_role.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL, ORGANIZATIONAL_ROLE_CV, ORGANIZATIONAL_ROLE_CATEGORIES_CV -from project import Project +from .constants import APPLICATION_LABEL, ORGANIZATIONAL_ROLE_CV, ORGANIZATIONAL_ROLE_CATEGORIES_CV +from .project import Project from django.contrib.auth.models import User from cog.models.constants import ROLE_CATEGORY_LEAD, ROLE_CATEGORY_MEMBER, ORGANIZATIONAL_ROLES_DICT, LEAD_ORGANIZATIONAL_ROLES_DICT diff --git a/cog/models/organizational_role_member.py b/cog/models/organizational_role_member.py index c9681343c..c04a560c8 100644 --- a/cog/models/organizational_role_member.py +++ b/cog/models/organizational_role_member.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from django.contrib.auth.models import User -from organizational_role import OrganizationalRole +from .organizational_role import OrganizationalRole class OrganizationalRoleMember(models.Model): diff --git a/cog/models/post.py b/cog/models/post.py index a205ae00e..6f6c99ef1 100644 --- a/cog/models/post.py +++ b/cog/models/post.py @@ -1,9 +1,9 @@ from django.db import models -from constants import APPLICATION_LABEL, SIGNAL_OBJECT_CREATED, SIGNAL_OBJECT_UPDATED, SIGNAL_OBJECT_DELETED -from project import Project +from .constants import APPLICATION_LABEL, SIGNAL_OBJECT_CREATED, SIGNAL_OBJECT_UPDATED, SIGNAL_OBJECT_DELETED +from .project import Project from django.contrib.auth.models import User -from doc import Doc -from topic import Topic +from .doc import Doc +from .topic import Topic import django.dispatch # A web site post, which can be of different types diff --git a/cog/models/project.py b/cog/models/project.py index e95d00732..f569c3a60 100644 --- a/cog/models/project.py +++ b/cog/models/project.py @@ -1,18 +1,18 @@ from cog.utils import smart_truncate -from constants import * -from navbar import * +from .constants import * +from .navbar import * from django.conf import settings from django.contrib.auth.models import User, Permission, Group from django.db import models from django.db.models import Q from django.forms import Textarea -from membership import MembershipRequest +from .membership import MembershipRequest from os.path import basename from cog.models.user_profile import UserProfile from cog.models.topic import Topic from cog.models.project_tag import ProjectTag -from urllib import quote, unquote +from urllib.parse import quote, unquote import os import sys import re @@ -237,7 +237,7 @@ def getPublicUsers(self): pubUsers = [] for user in users: try: - print 'Checking user=%s' % user # TODO:FixME + print('Checking user=%s' % user) # TODO:FixME if not user.profile.private: pubUsers.append(user) except ObjectDoesNotExist: @@ -430,7 +430,7 @@ def getProjectsAndRolesForUsers(user, includeRemote=True): elif group.name.endswith('_users'): projects[project.short_name].append(ROLE_USER) except ObjectDoesNotExist: - print "WARNING: cannot retrieve project for group=%s" % group + print("WARNING: cannot retrieve project for group=%s" % group) pass return projects @@ -463,6 +463,6 @@ def create_upload_directory(project): fb_upload_dir = os.path.join(settings.MEDIA_ROOT, settings.FILEBROWSER_DIRECTORY, project.short_name.lower()) if not os.path.exists(fb_upload_dir): os.makedirs(fb_upload_dir) - print 'Project Upload directory created: %s' % fb_upload_dir + print('Project Upload directory created: %s' % fb_upload_dir) diff --git a/cog/models/project_impact.py b/cog/models/project_impact.py index d3f970549..ba2484324 100644 --- a/cog/models/project_impact.py +++ b/cog/models/project_impact.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from project import Project +from .constants import APPLICATION_LABEL +from .project import Project class ProjectImpact(models.Model): diff --git a/cog/models/project_tab.py b/cog/models/project_tab.py index 658d6239f..0d1b7ecab 100644 --- a/cog/models/project_tab.py +++ b/cog/models/project_tab.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL -from navbar import PROJECT_PAGES, DEFAULT_TABS -from project import Project +from .constants import APPLICATION_LABEL +from .navbar import PROJECT_PAGES, DEFAULT_TABS +from .project import Project from django.core.urlresolvers import reverse from cog.models.dbutils import UnsavedForeignKey diff --git a/cog/models/project_tag.py b/cog/models/project_tag.py index 6d7382269..e1a5e399b 100644 --- a/cog/models/project_tag.py +++ b/cog/models/project_tag.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL MAX_PROJECT_TAG_LENGTH = 20 diff --git a/cog/models/project_topic.py b/cog/models/project_topic.py index 641643c6a..e0c245ce7 100644 --- a/cog/models/project_topic.py +++ b/cog/models/project_topic.py @@ -1,7 +1,7 @@ from django.db import models -from constants import APPLICATION_LABEL -from topic import Topic -from project import Project +from .constants import APPLICATION_LABEL +from .topic import Topic +from .project import Project # intermediate model for Project-Topic association class ProjectTopic(models.Model): diff --git a/cog/models/search.py b/cog/models/search.py index 5c85cd14c..0ed87c861 100644 --- a/cog/models/search.py +++ b/cog/models/search.py @@ -1,5 +1,5 @@ from django.db import models -import ConfigParser +import configparser from datetime import datetime import os @@ -35,9 +35,9 @@ def getSortedValues(self): return [(value, self.values[value]) for value in sorted(self.values.keys())] def printme(self): - print "Facet key=%s label=%s" % (self.key, self.label) - for value, counts in self.values.items(): - print "\tValue=%s counts=%d" % (value, counts) + print("Facet key=%s label=%s" % (self.key, self.label)) + for value, counts in list(self.values.items()): + print("\tValue=%s counts=%d" % (value, counts)) class SearchInput: @@ -58,7 +58,7 @@ def addConstraint(self, name, value): self.constraints[name].append(value) except KeyError: self.constraints[name] = [value] - print "constraint name=%s value(s)=%s" % (name, self.constraints[name]) + print("constraint name=%s value(s)=%s" % (name, self.constraints[name])) def setConstraint(self, name, values): self.constraints[name] = values @@ -98,11 +98,11 @@ def isEmpty(self): return self.query == '' and len(self.constraints)==0 def printme(self): - print "Search Input" - print "\t Query=%s Type=%s Offset=%d Limit=%d Max Version=%s Min Version=%s" % (self.query, self.type, self.offset, self.limit, - self.max_version, self.min_version) - for key, values in self.constraints.items(): - print "\t Constraint key=%s value(s)=%s" % (key, values) + print("Search Input") + print("\t Query=%s Type=%s Offset=%d Limit=%d Max Version=%s Min Version=%s" % (self.query, self.type, self.offset, self.limit, + self.max_version, self.min_version)) + for key, values in list(self.constraints.items()): + print("\t Constraint key=%s value(s)=%s" % (key, values)) class SearchOutput: @@ -116,8 +116,8 @@ def setFacet(self, facet): self.facets[facet.key] = facet def printme(self): - print "Search Output: total number of results=%d" % self.counts - for facet in self.facets.values(): + print("Search Output: total number of results=%d" % self.counts) + for facet in list(self.facets.values()): facet.printme() for record in self.results: record.printme() @@ -136,9 +136,9 @@ def addField(self, name, value): self.fields[name] = [value] def printme(self): - print "Record id=%s" % self.id - for name, values in self.fields.items(): - print "\tField name=%s values=%s" % (name, values) + print("Record id=%s" % self.id) + for name, values in list(self.fields.items()): + print("\tField name=%s values=%s" % (name, values)) class FacetProfile: @@ -156,7 +156,7 @@ def __init__(self, facetGroups): self.map = {} for group in self.facetGroups: self.keys = self.keys + group.keys - self.map = dict( self.map.items() + group.map.items() ) + self.map = dict( list(self.map.items()) + list(group.map.items()) ) def getAllKeys(self): """Returns a list of keys over all its facet groups.""" @@ -217,14 +217,14 @@ def __init__(self, facetProfile, fixedConstraints, searchService, self.localFlag = localFlag def printme(self): - print 'Search Configuration Service:%s' % self.searchService - print 'Search Configuration Facets:' + print('Search Configuration Service:%s' % self.searchService) + print('Search Configuration Facets:') for facetGroup in self.facetProfile.facetGroups: - print "\tFacet Group=%s" % facetGroup.name + print("\tFacet Group=%s" % facetGroup.name) for key in facetGroup.getKeys(): - print "\t\tFacet key=%s, label=%s" % (key, facetGroup.getLabel(key)) - print 'Search Configuration Fixed Constraints=%s' % self.fixedConstraints - print 'Search Configuration options: show replica checkbox: %s, show latest checkbox: %s, show local checkbox:%s' % (self.replicaFlag, self.latestFlag, self.localFlag) + print("\t\tFacet key=%s, label=%s" % (key, facetGroup.getLabel(key))) + print('Search Configuration Fixed Constraints=%s' % self.fixedConstraints) + print('Search Configuration options: show replica checkbox: %s, show latest checkbox: %s, show local checkbox:%s' % (self.replicaFlag, self.latestFlag, self.localFlag)) class SearchMappings(object): """Class that reads facet option mappings from a local configuration file, @@ -237,7 +237,7 @@ def __init__(self): CONFIGFILEPATH = os.path.join(cog_config_dir, 'cog_search.cfg') self.mappings = {} - config = ConfigParser.RawConfigParser() + config = configparser.RawConfigParser() try: config.read( CONFIGFILEPATH ) for facet_key in config.sections(): @@ -249,8 +249,8 @@ def __init__(self): self.mappings[facet_key] = fmap #print 'Loaded search mappinsg from file: %s' % filepath except Exception as e: - print "Search mappings file not found" - print e + print("Search mappings file not found") + print(e) def getFacetOptionLabel(self, facet_key, facet_option): """Returns the facet_option for the given facet_key if found, diff --git a/cog/models/search_facet.py b/cog/models/search_facet.py index f19e9a8c0..d226cf7ce 100644 --- a/cog/models/search_facet.py +++ b/cog/models/search_facet.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from search_group import SearchGroup +from .constants import APPLICATION_LABEL +from .search_group import SearchGroup # Search facet displayed in user interface class SearchFacet(models.Model): diff --git a/cog/models/search_group.py b/cog/models/search_group.py index 38147eb15..0a0c4d1e4 100644 --- a/cog/models/search_group.py +++ b/cog/models/search_group.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from search_profile import SearchProfile +from .constants import APPLICATION_LABEL +from .search_profile import SearchProfile # Group of search facets class SearchGroup(models.Model): diff --git a/cog/models/search_profile.py b/cog/models/search_profile.py index e8ee72edb..2fa20149e 100644 --- a/cog/models/search_profile.py +++ b/cog/models/search_profile.py @@ -1,6 +1,6 @@ from django.db import models -from constants import APPLICATION_LABEL -from project import Project +from .constants import APPLICATION_LABEL +from .project import Project # Project-specific search configuration (persisted to the database) class SearchProfile(models.Model): diff --git a/cog/models/signals.py b/cog/models/signals.py index 452f4ca2d..50d4ab077 100644 --- a/cog/models/signals.py +++ b/cog/models/signals.py @@ -29,7 +29,7 @@ def account_created_receiver(sender, **kwargs): # change the username lastPartOfOpenid = userp.openid().split("/")[-1] username = createUsername(lastPartOfOpenid) - print "New user: changing the username from: %s to: %s" % (userp.user.username, username) + print("New user: changing the username from: %s to: %s" % (userp.user.username, username)) userp.user.username = username userp.user.save() @@ -57,13 +57,13 @@ def update_user_projects(user): # add new memberships for remote projects remoteGroups = [] # updated list of remote groups for (project, roles) in projTuples: - print 'Updating membership for user: %s project: %s roles: %s' % (user.profile.openid(), project.short_name, roles) + print('Updating membership for user: %s project: %s roles: %s' % (user.profile.openid(), project.short_name, roles)) for role in roles: group = project.getGroup(role) remoteGroups.append(group) if not group in ugroups: - print 'Adding group: %s to user: %s' % (group, user) + print('Adding group: %s to user: %s' % (group, user)) user.groups.add(group) # persist changes to local database @@ -76,10 +76,10 @@ def update_user_projects(user): # do not change local projects if not project.isLocal(): if not group in remoteGroups: - print 'Removing group: %s from user: %s' % (group, user) + print('Removing group: %s from user: %s' % (group, user)) user.groups.remove( group ) except ObjectDoesNotExist: - print 'WARNING: cannot retrieve project for group=%s, removing obsolete group' % group + print('WARNING: cannot retrieve project for group=%s, removing obsolete group' % group) user.groups.remove( group ) # persist changes to local database @@ -92,7 +92,7 @@ def update_user_tags(user): openid = user.profile.openid() url = "http://%s/share/user/?openid=%s" % (user.profile.site.domain, user.profile.openid()) - print 'Updating user tags: querying URL=%s' % url + print('Updating user tags: querying URL=%s' % url) jobj = getJson(url) if jobj is not None and openid in jobj['users'] and 'project_tags' in jobj['users'][openid]: @@ -111,7 +111,7 @@ def update_user_tags(user): userProfile.tags = tags userProfile.save() transaction.commit() - print 'User: %s updated for tags: %s' % (user, tags) + print('User: %s updated for tags: %s' % (user, tags)) # NOTE: connecting the login signal is not needed because every time the user logs in, # the session is refreshed and updating of projects is triggered already by the CoG session middleware diff --git a/cog/models/topic.py b/cog/models/topic.py index 4f1d7b04e..13c32826f 100644 --- a/cog/models/topic.py +++ b/cog/models/topic.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL # A category to group Posts class Topic(models.Model): diff --git a/cog/models/user_profile.py b/cog/models/user_profile.py index 3cc6f2a22..e0ef876f0 100644 --- a/cog/models/user_profile.py +++ b/cog/models/user_profile.py @@ -1,6 +1,6 @@ from django.db import models from django.contrib.auth.models import User -from constants import APPLICATION_LABEL, RESEARCH_KEYWORDS_MAX_CHARS, RESEARCH_INTERESTS_MAX_CHARS +from .constants import APPLICATION_LABEL, RESEARCH_KEYWORDS_MAX_CHARS, RESEARCH_INTERESTS_MAX_CHARS from django.conf import settings from cog.utils import hasText @@ -146,7 +146,7 @@ def discoverSiteForUser(openid): url = "http://%s/share/user/?openid=%s" % (site.domain, openid) jobj = getJson(url) if jobj is not None: - for key, value in jobj['users'].items(): + for key, value in list(jobj['users'].items()): if str(value['home_site_domain']) == site.domain: return site # node found @@ -166,10 +166,10 @@ def getDataCartsForUser(openid): #for site in Site.objects.all(): # loop over all sites (e.g. nodes) in database. Note: includes current node for site in getPeerSites(): # loop over nodes that are federated url = "http://%s/share/user/?openid=%s" % (site.domain, openid) - print 'Querying for datacart: url=%s' % url + print('Querying for datacart: url=%s' % url) jobj = getJson(url) if jobj is not None: - for key, value in jobj['users'].items(): + for key, value in list(jobj['users'].items()): dcs[ site ] = int( value['datacart']['size'] ) return dcs diff --git a/cog/models/user_url.py b/cog/models/user_url.py index 578307b06..8d5cb281a 100644 --- a/cog/models/user_url.py +++ b/cog/models/user_url.py @@ -1,5 +1,5 @@ from django.db import models -from constants import APPLICATION_LABEL +from .constants import APPLICATION_LABEL from cog.models import UserProfile class UserUrl(models.Model): diff --git a/cog/models/utils.py b/cog/models/utils.py index 7e10a64f7..f053a4094 100644 --- a/cog/models/utils.py +++ b/cog/models/utils.py @@ -1,25 +1,25 @@ -from project import Project -from collaborator import Collaborator -from project_topic import ProjectTopic -from search_profile import SearchProfile -from communication_means import CommunicationMeans -from search_facet import SearchFacet -from search_group import SearchGroup -from post import Post -from bookmark import Bookmark -from doc import Doc -from navbar import PROJECT_PAGES, DEFAULT_TABS +from .project import Project +from .collaborator import Collaborator +from .project_topic import ProjectTopic +from .search_profile import SearchProfile +from .communication_means import CommunicationMeans +from .search_facet import SearchFacet +from .search_group import SearchGroup +from .post import Post +from .bookmark import Bookmark +from .doc import Doc +from .navbar import PROJECT_PAGES, DEFAULT_TABS from django.conf import settings from django.utils.timezone import now -from news import News +from .news import News from django.db.models import Q from django.contrib.contenttypes.models import ContentType -from folder import Folder, getTopFolder, TOP_SUB_FOLDERS +from .folder import Folder, getTopFolder, TOP_SUB_FOLDERS from cog.models.constants import DEFAULT_SEARCH_FACETS -from project_tab import ProjectTab +from .project_tab import ProjectTab import shutil import os -from urllib import quote +from urllib.parse import quote # method to retrieve all news for a given project, ordered by original publication date @@ -63,7 +63,7 @@ def site_index(project): # -) page numbers start at 1 (for Home) def init_site_index(project): - print 'Initializing project index' + print('Initializing project index') project.topics.clear() # list all top-level project pages, order by topic first, then title @@ -94,7 +94,7 @@ def create_project_search_profile(project): try: profile = project.searchprofile except SearchProfile.DoesNotExist: - print 'Configuring the project search profile' + print('Configuring the project search profile') # assign default URL, if available url = getattr(settings, "DEFAULT_SEARCH_URL", "") profile = SearchProfile(project=project, url=url) @@ -104,7 +104,7 @@ def create_project_search_profile(project): group.save() # assign default facets facets = DEFAULT_SEARCH_FACETS - for key, label in facets.items(): + for key, label in list(facets.items()): facet = SearchFacet(key=key, label=label, group=group) facet.save() project.searchprofile = profile @@ -159,7 +159,7 @@ def create_project_page(url, project): if _page[0] == 'Logistics': page.title = '%s Agenda' % project.short_name page.save() - print "Created project page: %s" % url + print("Created project page: %s" % url) return page return None @@ -194,7 +194,7 @@ def get_or_create_default_search_group(project): try: group = SearchGroup.objects.filter(profile=profile).filter(name=SearchGroup.DEFAULT_NAME)[0] except IndexError: - print 'Creating default search group for project=%s' % project.short_name + print('Creating default search group for project=%s' % project.short_name) group = SearchGroup(profile=profile, name=SearchGroup.DEFAULT_NAME, order=len(list(profile.groups.all()))) group.save() return group @@ -228,13 +228,13 @@ def get_or_create_project_tabs(project, save=True): active = False tab = ProjectTab(project=project, label=label, url=url, active=active) if save: - print "Creating tab= %s" % tab + print("Creating tab= %s" % tab) tab.save() # assign parent tab if i > 0: tab.parent = tablist[0] tab.save() - print "Assigned parent tab=%s to child tab=%s" % (tablist[0], tab) + print("Assigned parent tab=%s to child tab=%s" % (tablist[0], tab)) tablist.append(tab) tabs.append(tablist) @@ -249,7 +249,7 @@ def setActiveProjectTabs(tabs, request, save=False): # Home tab MUST always be active if tab.label.endswith("Home"): tab.active = True - elif "tab_%s" % tab.label in request.POST.keys(): + elif "tab_%s" % tab.label in list(request.POST.keys()): tab.active = True else: tab.active = False @@ -273,11 +273,11 @@ def createOrUpdateProjectSubFolders(project, request=None): topFolder = getTopFolder(project) - for name in TOP_SUB_FOLDERS.values(): + for name in list(TOP_SUB_FOLDERS.values()): folder, created = Folder.objects.get_or_create(name=name, parent=topFolder, project=project) if created: - print 'Project=%s: created top-level folder=%s' % (project.short_name, folder.name) - if request is not None and ("folder_%s" % folder.name) in request.POST.keys(): + print('Project=%s: created top-level folder=%s' % (project.short_name, folder.name)) + if request is not None and ("folder_%s" % folder.name) in list(request.POST.keys()): folder.active = True else: folder.active = False @@ -287,7 +287,7 @@ def getBookmarkFromDoc(doc): '''Returns the first Bookmark with URL matching the Document path.''' url_fragment = quote(doc.path, safe="%/:=&?~#+!$,;'@()*[]") - print 'Looking for bookmark that contains URL fragment: %s' % url_fragment + print('Looking for bookmark that contains URL fragment: %s' % url_fragment) bookmarks = Bookmark.objects.filter(url__contains=url_fragment) for bookmark in bookmarks: return bookmark @@ -298,7 +298,7 @@ def getDocFromBookmark(bookmark): if 'site_media/' in bookmark.url: _, url_fragment = bookmark.url.split('site_media/', 1) - print 'Looking for doc that contains path: %s' % url_fragment + print('Looking for doc that contains path: %s' % url_fragment) docs = Doc.objects.filter(path__contains=url_fragment) for doc in docs: return doc @@ -318,7 +318,7 @@ def delete_doc(doc): # remove possible associated resource bookmark = getBookmarkFromDoc(doc) if bookmark is not None: - print 'Deleting associated bookmark: %s' % bookmark.url + print('Deleting associated bookmark: %s' % bookmark.url) bookmark.delete() # obtain document full path (before deleting object from database) @@ -331,7 +331,7 @@ def delete_doc(doc): # delete document from file system for fp in [fullpath, fullpath2]: if os.path.exists(fp): - print 'Deleting document=%s' % fp + print('Deleting document=%s' % fp) os.remove(fp) # also delete possible thumbnail files (created by File Browser) @@ -349,47 +349,47 @@ def deleteProject(project, dryrun=True, rmdir=False): Utility method to delete a project and associated objects, media. """ - print "Deleting project=%s" % project.short_name + print("Deleting project=%s" % project.short_name) # delete project User group, permissions ug = project.getUserGroup() for p in ug.permissions.all(): - print '\tDeleting permission: %s' % p + print('\tDeleting permission: %s' % p) if not dryrun: p.delete() - print '\tDeleting group: %s' % ug + print('\tDeleting group: %s' % ug) if not dryrun: ug.delete() # delete project Admin group, permissions ag = project.getAdminGroup() for p in ag.permissions.all(): - print '\tDeleting permission: %s' % p + print('\tDeleting permission: %s' % p) if not dryrun: p.delete() - print '\tDeleting group: %s' % ag + print('\tDeleting group: %s' % ag) if not dryrun: ag.delete() # delete project Contributor group, permissions cg = project.getContributorGroup() for p in cg.permissions.all(): - print '\tDeleting permission: %s' % p + print('\tDeleting permission: %s' % p) if not dryrun: p.delete() - print '\tDeleting group: %s' % cg + print('\tDeleting group: %s' % cg) if not dryrun: cg.delete() if rmdir: media_dir = os.path.join(settings.MEDIA_ROOT, settings.FILEBROWSER_DIRECTORY, project.short_name.lower()) - print "\tRemoving directory tree: %s" % media_dir + print("\tRemoving directory tree: %s" % media_dir) if not dryrun: try: shutil.rmtree(media_dir) except OSError as e: - print e + print(e) - print '\tDeleting project: %s' % project + print('\tDeleting project: %s' % project) if not dryrun: project.delete() \ No newline at end of file diff --git a/cog/notification.py b/cog/notification.py index 057e1060d..4ccb4e140 100644 --- a/cog/notification.py +++ b/cog/notification.py @@ -24,16 +24,16 @@ def __init__(self): self.username = siteManager.get('EMAIL_USERNAME', section=SECTION_EMAIL) self.password = siteManager.get('EMAIL_PASSWORD', section=SECTION_EMAIL) self.security = siteManager.get('EMAIL_SECURITY', section=SECTION_EMAIL) - print 'Using email server=%s' % self.server - print 'Using email port=%s' % self.port - print 'Using email sender=%s' % self.sender - print 'Using email username=%s' % self.username + print('Using email server=%s' % self.server) + print('Using email port=%s' % self.port) + print('Using email sender=%s' % self.sender) + print('Using email username=%s' % self.username) #print 'Using email password=%s' % self.password - print 'Using email security=%s' % self.security + print('Using email security=%s' % self.security) self.init = True if not self.init: - print "Email configuration not found, email notification disabled" + print("Email configuration not found, email notification disabled") # module scope email configuration @@ -74,10 +74,10 @@ def __init__ (self, toAddress, subject, message, fromAddress=None, mime_type='pl def run(self): #print "From: %s" % self.fromAddress - print "To: %s" % self.toAddress - print "Subject: %s" % self.subject - print "Message: %s" % self.message - print "Mime Type: %s" % self.mime_type + print("To: %s" % self.toAddress) + print("Subject: %s" % self.subject) + print("Message: %s" % self.message) + print("Mime Type: %s" % self.mime_type) # use local mail server #toUser.email_user(subject, message, from_email=fromAddress) @@ -100,4 +100,4 @@ def run(self): s.login(emailConfig.username, emailConfig.password ) s.sendmail(emailConfig.sender, [self.toAddress], msg.as_string()) s.quit() - print 'Email sent.' + print('Email sent.') diff --git a/cog/plugins/esgf/registry.py b/cog/plugins/esgf/registry.py index 133314d2f..04ac6fd5c 100644 --- a/cog/plugins/esgf/registry.py +++ b/cog/plugins/esgf/registry.py @@ -20,18 +20,14 @@ #import certifi NS = "http://www.esgf.org/whitelist" -class WhiteList(object): - - __metaclass__ = abc.ABCMeta +class WhiteList(object, metaclass=abc.ABCMeta): @abc.abstractmethod def trust(self, openid): '''Returns true if an openid can be trusted, false otherwise.''' pass -class KnownProvidersDict(object): - - __metaclass__ = abc.ABCMeta +class KnownProvidersDict(object, metaclass=abc.ABCMeta): @abc.abstractmethod def idpDict(self): @@ -84,7 +80,7 @@ def _reload(self, force=False): if force or modtime > self.modtime: - print 'Loading known IdPs from file: %s, last modified: %s' % (self.filepath, modtime) + print('Loading known IdPs from file: %s, last modified: %s' % (self.filepath, modtime)) self.modtime = modtime idps = {} @@ -104,7 +100,7 @@ def _reload(self, force=False): if name is not None and len(name.strip()) > 0: url = idp.find('URL').text idps[name] = url - print 'Using known IdP: name=%s url=%s' % (name, url) + print('Using known IdP: name=%s url=%s' % (name, url)) # switch the dictionary of knwon providers self.idps = idps @@ -136,7 +132,7 @@ def __init__(self, filepath_string): try: self._reload(filepath, force=True) except ParseError as e: - print e # print error from parsing single white-list files and continue + print(e) # print error from parsing single white-list files and continue def _reload(self, filepath, force=False): @@ -146,7 +142,7 @@ def _reload(self, filepath, force=False): if force or modtime > self.modtimes[filepath]: - print 'Loading IdP white list: %s, last modified: %s' % (filepath, modtime) + print('Loading IdP white list: %s, last modified: %s' % (filepath, modtime)) self.modtimes[filepath] = modtime idps = [] @@ -162,7 +158,7 @@ def _reload(self, filepath, force=False): if match: idp = match.group(1) idps.append(idp.lower()) - print 'Using trusted IdP: %s' % idp + print('Using trusted IdP: %s' % idp) # switch the list for this file path self.idps[filepath] = idps @@ -202,7 +198,7 @@ def reload(self, delete=False): if self.filepath is not None and os.path.exists(self.filepath): - print('Updating list of CoG sites from: %s (delete: %s)' % (self.filepath, delete) ) + print(('Updating list of CoG sites from: %s (delete: %s)' % (self.filepath, delete) )) # current site - must not be updated from file list current_site = Site.objects.get_current() @@ -221,7 +217,7 @@ def reload(self, delete=False): name = site.attrib['name'] domain = site.attrib['domain'] domains.append(domain) - print 'Updating site domain: %s name: %s' % (domain, name) + print('Updating site domain: %s name: %s' % (domain, name)) # update Site objects try: @@ -230,25 +226,25 @@ def reload(self, delete=False): # update site _site.name = name _site.save() - print('Updated site: %s' % _site) + print(('Updated site: %s' % _site)) except ObjectDoesNotExist: _site = Site.objects.create(name=name, domain=domain) - print 'Created site: %s' % _site + print('Created site: %s' % _site) # update PeerSite objects try: peersite = PeerSite.objects.get(site=_site) except ObjectDoesNotExist: peersite = PeerSite.objects.create(site=_site, enabled=False) - print '\tPeer site: %s' % peersite + print('\tPeer site: %s' % peersite) # clean up stale sites if delete: for peer in PeerSite.objects.all(): if peer.site.domain not in domains: if peer.site != current_site: - print 'Stale peer site found at domain: %s' % peer.site.domain + ", deleting it..." + print('Stale peer site found at domain: %s' % peer.site.domain + ", deleting it...") peer.site.delete() # will also delete the PeerSite object on cascade else: - print 'WARNING: File %s does not exist, skipping update of ESGF peer nodes' % self.filepath \ No newline at end of file + print('WARNING: File %s does not exist, skipping update of ESGF peer nodes' % self.filepath) \ No newline at end of file diff --git a/cog/plugins/esgf/security.py b/cog/plugins/esgf/security.py index 7f4f7f969..2f92bf997 100644 --- a/cog/plugins/esgf/security.py +++ b/cog/plugins/esgf/security.py @@ -69,7 +69,7 @@ def createGroup(self, name, description='', visible=True, automatic_approval=Fal session = self.Session() group = session.query(ESGFGroup).filter( func.lower(ESGFGroup.name) == func.lower(name) ).one() - print "Group with name=%s already exists" % group.name + print("Group with name=%s already exists" % group.name) created = False return group @@ -125,13 +125,13 @@ def insertEsgfUser(self, userProfile): session.add(esgfUser) session.commit() - print 'Inserted user with openid=%s into ESGF database' % openid + print('Inserted user with openid=%s into ESGF database' % openid) finally: session.close() else: - print 'User with openid: %s already existing in ESGF database, no action taken' % esgfUser.openid + print('User with openid: %s already existing in ESGF database, no action taken' % esgfUser.openid) pass def getUserByOpenid(self, openid): @@ -184,7 +184,7 @@ def listUsers(self): for user in users: parts = user.openid.split('/') new_username = parts[-1] - print 'Updating user: openid=%s new username=%s' % (user.openid, new_username) + print('Updating user: openid=%s new username=%s' % (user.openid, new_username)) user.username = new_username session.commit() @@ -228,7 +228,7 @@ def updatePassword(self, user, clearTextPwd): session = self.Session() encPasword = md5_crypt.encrypt(clearTextPwd) esgfUser.password = encPasword - print 'Updated ESGF password for user with openid: %s' % openid + print('Updated ESGF password for user with openid: %s' % openid) session.add(esgfUser) session.commit() session.close() @@ -251,7 +251,7 @@ def updateUser(self, user_profile): esgfUser.city = user_profile.city esgfUser.state = user_profile.state esgfUser.country = user_profile.country - print 'Updated ESGF data for user with openid: %s' % openid + print('Updated ESGF data for user with openid: %s' % openid) session.add(esgfUser) session.commit() session.close() @@ -265,7 +265,7 @@ def deleteUser(self, user): esgfUser = self.getUserByOpenid(openid) if esgfUser is not None: - print 'Deleting ESGF user with openid=%s' % openid + print('Deleting ESGF user with openid=%s' % openid) session = self.Session() # delete user permissions permissions = session.query(ESGFPermission).filter(ESGFPermission.user_id==esgfUser.id) diff --git a/cog/plugins/globus/directory_transfer.py b/cog/plugins/globus/directory_transfer.py index 20eafedf9..178f827fc 100644 --- a/cog/plugins/globus/directory_transfer.py +++ b/cog/plugins/globus/directory_transfer.py @@ -42,20 +42,20 @@ def arguments(argv): dlocation = args.download_location if '#' not in uendpoint: - print "Please supply a valid Globus endpoint (local host)" + print("Please supply a valid Globus endpoint (local host)") sys.exit() if '#' not in gendpoint: - print "Please supply a valid Globus endpoint (remote host)" + print("Please supply a valid Globus endpoint (remote host)") if '/' in uendpoint: - print "Do not include the download path in the local endpoint name, please use the -p option" + print("Do not include the download path in the local endpoint name, please use the -p option") sys.exit() if '/' in gendpoint: - print "Do not include the download path in the remote endpoint name, please use the -p option" + print("Do not include the download path in the remote endpoint name, please use the -p option") if '#' in path: - print "The '#' character is invalid in your path, please re-enter" + print("The '#' character is invalid in your path, please re-enter") sys.exit() if '#' in dlocation: - print "The '#' character is invalid in your download location, please re-enter" + print("The '#' character is invalid in your download location, please re-enter") if path[-1] != '/': path = path + '/' if path[0] != '/': diff --git a/cog/plugins/globus/download.py b/cog/plugins/globus/download.py index d6b6f94c4..4383f9a3f 100644 --- a/cog/plugins/globus/download.py +++ b/cog/plugins/globus/download.py @@ -9,10 +9,10 @@ def listEndpoints(gendpointDict): - endNames = gendpointDict.keys() - print "Endpoints involved:" + endNames = list(gendpointDict.keys()) + print("Endpoints involved:") for thisEndName in endNames: - print thisEndName + print(thisEndName) def arguments(argv): @@ -42,10 +42,10 @@ def arguments(argv): listonly = args.list_endpoints if '/' in uendpoint: - print "Do not include the download path in the endpoint name, please use the -p option" + print("Do not include the download path in the endpoint name, please use the -p option") sys.exit() if '#' in upath: - print "The '#' character is invalid in your path, please re-enter" + print("The '#' character is invalid in your path, please re-enter") sys.exit() if upath[0] != '/' and upath != '/~/': upath = '/' + upath @@ -56,7 +56,7 @@ def getFiles(gendpointDict, uendpoint, username, upath): label = str(uuid4()) - endNames = gendpointDict.keys() + endNames = list(gendpointDict.keys()) for thisEndName in endNames: diff --git a/cog/plugins/globus/transfer.py b/cog/plugins/globus/transfer.py index 7c55b9c41..fff96e57e 100644 --- a/cog/plugins/globus/transfer.py +++ b/cog/plugins/globus/transfer.py @@ -9,14 +9,14 @@ if siteManager.isGlobusEnabled(): from globus_sdk.transfer import TransferData import os -import urlparse +import urllib.parse DOWNLOAD_SCRIPT = "download.py" def generateGlobusDownloadScript(download_map): - print "Generating script for downloading files: " - print download_map + print("Generating script for downloading files: ") + print(download_map) # read script 'download.py' located in same directory as this module scriptFile = os.path.join(os.path.dirname(__file__), DOWNLOAD_SCRIPT) @@ -32,12 +32,12 @@ def activateEndpoint(transfer_client, endpoint, openid=None, password=None): if not openid or not password: # Try to autoactivate the endpoint code, reason, result = transfer_client.endpoint_autoactivate(endpoint, if_expires_in=2880) - print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) + print("Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"])) if result["code"] == "AutoActivationFailed": return (False, "") return (True, "") - openid_parsed = urlparse.urlparse(openid) + openid_parsed = urllib.parse.urlparse(openid) hostname = openid_parsed.hostname username = os.path.basename(openid_parsed.path) code, reason, reqs = transfer_client.endpoint_get_activation_requirements(endpoint) @@ -67,13 +67,13 @@ def activateEndpoint(transfer_client, endpoint, openid=None, password=None): try: code, reason, result = transfer_client.endpoint_activate(endpoint, reqs) except Exception as e: - print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) + print("Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e))) return (False, str(e)) if code != 200: - print "Could not aactivate the endpoint: %s. Error: %s - %s" % (endpoint, result["code"], result["message"]) + print("Could not aactivate the endpoint: %s. Error: %s - %s" % (endpoint, result["code"], result["message"])) return (False, result["message"]) - print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) + print("Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"])) return (True, "") @@ -93,7 +93,7 @@ def submitTransfer(transfer_client, source_endpoint, source_files, target_endpoi # create a transfer request transfer_task = Transfer(transfer_client, source_endpoint, target_endpoint, deadline=deadline) - print "Obtained transfer submission id: %s" % transfer_task["submission_id"] + print("Obtained transfer submission id: %s" % transfer_task["submission_id"]) for source_file in source_files: source_directory, filename = os.path.split(source_file) target_file = os.path.join(target_directory, filename) @@ -103,9 +103,9 @@ def submitTransfer(transfer_client, source_endpoint, source_files, target_endpoi try: code, reason, data = transfer_client.submit_transfer(transfer_task) task_id = data["task_id"] - print "Submitted transfer task with id: %s" % task_id + print("Submitted transfer task with id: %s" % task_id) except Exception as e: - print "Could not submit the transfer. Error: %s" % str(e) + print("Could not submit the transfer. Error: %s" % str(e)) task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue." return task_id diff --git a/cog/project_manager.py b/cog/project_manager.py index 3a21e1e8e..77a53a7dc 100644 --- a/cog/project_manager.py +++ b/cog/project_manager.py @@ -58,7 +58,7 @@ def _associateProjects(self, objList, apDictList): site__domain=site_domain) objList.add(aproject) except Project.DoesNotExist: # correct short name, wrong site ? - print 'Associated project does not exist in local database: short_name=%s site_domain=%s, will ignore' % (apdict['short_name'], apdict['site_domain']) + print('Associated project does not exist in local database: short_name=%s site_domain=%s, will ignore' % (apdict['short_name'], apdict['site_domain'])) pass @@ -79,14 +79,14 @@ def _harvest(self, jobj): remote_site, created = Site.objects.get_or_create(domain=sdict['domain']) if created: - print 'Created remote site: %s' % remote_site + print('Created remote site: %s' % remote_site) else: - print 'Remote site %s already existing' % remote_site + print('Remote site %s already existing' % remote_site) #remote_site.name = sdict["name"] # don't change the site 'name', keep value from esgf_cogs.xml instead #remote_site.save() # first loop to create ALL projects first - for key, pdict in jobj["projects"].items(): + for key, pdict in list(jobj["projects"].items()): short_name = pdict['short_name'] long_name = pdict['long_name'] @@ -97,17 +97,17 @@ def _harvest(self, jobj): if not Project.objects.filter(short_name__iexact=short_name).exists(): # avoid conflicts with existing projects, from ANY site # create new project - print 'Creating project=%s (%s) for site=%s in local database' % (short_name, long_name, remote_site) + print('Creating project=%s (%s) for site=%s in local database' % (short_name, long_name, remote_site)) try: Project.objects.create(short_name=short_name, long_name=long_name, site=remote_site, active=True) - print 'Created project=%s for site=%s in local database' % (short_name, remote_site) + print('Created project=%s for site=%s in local database' % (short_name, remote_site)) except Exception as e: - print e # ignore errors while creating any one project from remote site, continue iteration + print(e) # ignore errors while creating any one project from remote site, continue iteration else: - print 'Project with name:%s already exists (local or remote)' % short_name + print('Project with name:%s already exists (local or remote)' % short_name) # second loop to update project attributes and associations - for key, pdict in jobj["projects"].items(): + for key, pdict in list(jobj["projects"].items()): short_name = pdict['short_name'] long_name = pdict['long_name'] @@ -121,7 +121,7 @@ def _harvest(self, jobj): try: # load existing project from remote site project = Project.objects.get(short_name=short_name, site=remote_site) - print 'Loaded project: %s from site: %s' % (short_name, site_domain) + print('Loaded project: %s from site: %s' % (short_name, site_domain)) # update project attributes project.long_name = long_name @@ -140,9 +140,9 @@ def _harvest(self, jobj): # update project associations self._associateProjects(project.peers, pdict['peers']) - print 'Updated project peers=%s' % project.peers.all() + print('Updated project peers=%s' % project.peers.all()) self._associateProjects(project.parents, pdict['parents']) - print 'Updated project parents=%s' % project.parents.all() + print('Updated project parents=%s' % project.parents.all()) project.save() diff --git a/cog/services/SolrSerializer.py b/cog/services/SolrSerializer.py index 8c9e6637c..f59960e46 100644 --- a/cog/services/SolrSerializer.py +++ b/cog/services/SolrSerializer.py @@ -19,7 +19,7 @@ def serialize(input, output): strEl = SubElement(docEl, "str", attrib={ "name":"id" } ) strEl.text = record.id - for key, values in record.fields.items(): + for key, values in list(record.fields.items()): # Dataset # single-valued fields (as from Solr schema) if key == 'url' or key=='type': @@ -41,9 +41,9 @@ def serialize(input, output): facetFieldsEl = SubElement(facetCountsEl, "lst", attrib={ "name": "facet_fields" }) # loop over facets - for key, facet in output.facets.items(): + for key, facet in list(output.facets.items()): facetEl = SubElement(facetFieldsEl, "lst", attrib={ "name": key }) - for value, counts in facet.getValues().items(): + for value, counts in list(facet.getValues().items()): facetSubEl = SubElement(facetEl, "int", attrib={ "name": value }) facetSubEl.text = str(counts) @@ -115,7 +115,7 @@ def deserialize(xml, facetProfile): except (ValueError, LookupError) as err: - print "Error: %s" % err + print("Error: %s" % err) return output def _encode_field_value(record, name, value): diff --git a/cog/services/membership.py b/cog/services/membership.py index 975c23010..2fe92a808 100644 --- a/cog/services/membership.py +++ b/cog/services/membership.py @@ -35,7 +35,7 @@ def cancelMembershipRequest(user, group): mrlist = MembershipRequest.objects.filter(group=group).filter(user=user) if len(mrlist)>0: for mr in mrlist: - print 'Cancelling membership request for user=%s group=%s' % (user.username, group.name) + print('Cancelling membership request for user=%s group=%s' % (user.username, group.name)) mr.delete() # Method to cancel ALL membership requests for a given user, project @@ -51,7 +51,7 @@ def addMembership(user, group, admin=None): if not group in user.groups.all(): user.groups.add(group) - print "Enrolled user=%s in group=%s" % (user.username, group.name) + print("Enrolled user=%s in group=%s" % (user.username, group.name)) cancelMembershipRequests(user, project) # log event if admin is not None: @@ -62,7 +62,7 @@ def addMembership(user, group, admin=None): return RESULT_SUCCESS else: - print "User=%s is already enrolled in group=%s" % (user.username, group.name) + print("User=%s is already enrolled in group=%s" % (user.username, group.name)) cancelMembershipRequests(user, project) return RESULT_DUPLICATE @@ -78,7 +78,7 @@ def cancelMembership(user, group, admin=None): # first remove user from that group user.groups.remove(group) - print "Removed user=%s from group=%s" % (user.username, group.name) + print("Removed user=%s from group=%s" % (user.username, group.name)) # if user is not part of the project any more, remove from all project management bodies project = getProjectForGroup(group) @@ -86,17 +86,17 @@ def cancelMembership(user, group, admin=None): # Management Bodies objs = ManagementBodyMember.objects.filter(user=user).filter(managementBody__project=project) for obj in objs: - print 'Deleting ManagementBodyMember for project=%s user=%s managementBody=%s' % (project, user, obj.managementBody.title) + print('Deleting ManagementBodyMember for project=%s user=%s managementBody=%s' % (project, user, obj.managementBody.title)) obj.delete() # Organization Roles objs = OrganizationalRoleMember.objects.filter(user=user).filter(organizationalRole__project=project) for obj in objs: - print 'Deleting OrganizationalRoleMember for project=%s user=%s organizationalRole=%s' % (project, user, obj.organizationalRole.title) + print('Deleting OrganizationalRoleMember for project=%s user=%s organizationalRole=%s' % (project, user, obj.organizationalRole.title)) obj.delete() # Communication Means objs = CommunicationMeansMember.objects.filter(user=user).filter(communicationMeans__project=project) for obj in objs: - print 'Deleting CommunicationMeansMember for project=%s user=%s communicationMeans=%s' % (project, user, obj.communicationMeans.title) + print('Deleting CommunicationMeansMember for project=%s user=%s communicationMeans=%s' % (project, user, obj.communicationMeans.title)) obj.delete() # log event @@ -110,5 +110,5 @@ def cancelMembership(user, group, admin=None): return RESULT_SUCCESS else: - print "User=%s not found in group=%s" % (user.username, group.name) + print("User=%s not found in group=%s" % (user.username, group.name)) return RESULT_NOT_FOUND \ No newline at end of file diff --git a/cog/services/registration/__init__.py b/cog/services/registration/__init__.py index e60dffc6b..ca8b642f7 100644 --- a/cog/services/registration/__init__.py +++ b/cog/services/registration/__init__.py @@ -1,2 +1,2 @@ -from registration import RegistrationService -from registration_impl import ESGFRegistrationServiceImpl, esgfRegistrationServiceImpl \ No newline at end of file +from .registration import RegistrationService +from .registration_impl import ESGFRegistrationServiceImpl, esgfRegistrationServiceImpl \ No newline at end of file diff --git a/cog/services/registration/registration.py b/cog/services/registration/registration.py index 3f211f977..f57a096d9 100644 --- a/cog/services/registration/registration.py +++ b/cog/services/registration/registration.py @@ -6,10 +6,8 @@ import abc -class RegistrationService(object): +class RegistrationService(object, metaclass=abc.ABCMeta): - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def createGroup(self, name, description='', visible=True, automatic_approval=False): ''' diff --git a/cog/services/search.py b/cog/services/search.py index 18cf6a4ba..c71564084 100644 --- a/cog/services/search.py +++ b/cog/services/search.py @@ -1,4 +1,4 @@ -import urllib, urllib2 +import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse from string import join # search service that queries a Solr server through the ESGF RESTful search API @@ -30,7 +30,7 @@ def search(self, searchInput, allFacets=False): if searchInput.min_version: params.append( ('min_version', searchInput.min_version.strip()) ) - for key, values in searchInput.constraints.items(): + for key, values in list(searchInput.constraints.items()): for value in values: params.append( (key, value) ) @@ -42,9 +42,9 @@ def search(self, searchInput, allFacets=False): facetlist = join([facet.key for facet in self.facets], ',') params.append( ("facets", facetlist) ) - url = self.url+"?"+urllib.urlencode(params) - print 'ESGF search URL=%s' % url - fh = urllib2.urlopen( url ) + url = self.url+"?"+urllib.parse.urlencode(params) + print('ESGF search URL=%s' % url) + fh = urllib.request.urlopen( url ) xml = fh.read().decode("UTF-8") # return search URL and corresponding results as XML diff --git a/cog/site_manager.py b/cog/site_manager.py index 557381675..120293a0e 100644 --- a/cog/site_manager.py +++ b/cog/site_manager.py @@ -1,6 +1,6 @@ import os import time -import ConfigParser +import configparser import logging from cog.constants import (SECTION_ESGF, SECTION_GLOBUS, SECTION_PID) @@ -22,16 +22,16 @@ def __init__(self): logging.info("Waiting to read configuration file: %s" % SiteManager.CONFIGFILEPATH ) time.sleep(1) - self.config = ConfigParser.ConfigParser(allow_no_value=True) + self.config = configparser.ConfigParser(allow_no_value=True) # location of site specific settings configuration file self.cog_config_dir = SiteManager.COG_CONFIG_DIR try: config = self.config.read( SiteManager.CONFIGFILEPATH ) logging.info("Site manager: using CoG settings from file(s): %s" % config) - print 'Initialized CoG settings from file: %s' % SiteManager.CONFIGFILEPATH + print('Initialized CoG settings from file: %s' % SiteManager.CONFIGFILEPATH) except Exception as e: - print "Error reading site settings configuration file: %s" % SiteManager.CONFIGFILEPATH + print("Error reading site settings configuration file: %s" % SiteManager.CONFIGFILEPATH) def get(self, name, section='DEFAULT', default=None): '''Method that retrieves a settings value from a specified section of the configuration file.''' diff --git a/cog/templatetags/cog_utils.py b/cog/templatetags/cog_utils.py index a9c2d6180..3c33bdaac 100644 --- a/cog/templatetags/cog_utils.py +++ b/cog/templatetags/cog_utils.py @@ -18,7 +18,7 @@ from cog.util.thumbnails import getThumbnailPath from django.contrib.auth.models import AnonymousUser from django.core.exceptions import ObjectDoesNotExist -import urlparse +import urllib.parse import string import bleach from cog.views.utils import getKnownIdentityProviders, getQueryDict, paginate @@ -37,7 +37,7 @@ def wps_arguments(the_dict): @register.filter def knownIdentityProviders(request): - return getKnownIdentityProviders().items() + return list(getKnownIdentityProviders().items()) @register.filter @@ -48,7 +48,7 @@ def concat(astring, bstring): @register.filter def sortdict(the_dict): tuples = [] - for key, value in the_dict.iteritems(): + for key, value in the_dict.items(): tuples.append((key, value)) return sorted(tuples, key=lambda tuple: tuple[0]) @@ -178,7 +178,7 @@ def _folder_tree(folder, user, esc, expanded=False, icon='folder'): # its child folders in reverse order to keep the most recent folder on top. # closing upper-level child folders by default (e.g. folders under Presentations) - years = reversed(range(2000, 2025)) # create list of reversed integers representing years + years = reversed(list(range(2000, 2025))) # create list of reversed integers representing years years_str = [str(_year) for _year in years] # convert that list to a list of strings if folder.name == "Bookmarks": @@ -776,7 +776,7 @@ def get_domain(url): Returns the domain part of a URL """ - return urlparse.urlparse(url)[1] + return urllib.parse.urlparse(url)[1] @register.filter def get_target_url_with_next_url(request, target_url_name): @@ -884,7 +884,7 @@ def getDisplayStatus(flag1, flag2): @register.filter def startsWith(text, starts): - if isinstance(text, basestring): + if isinstance(text, str): return text.starts_with(starts) return False diff --git a/cog/templatetags/search_utils.py b/cog/templatetags/search_utils.py index 8453dd13b..ab138cb40 100644 --- a/cog/templatetags/search_utils.py +++ b/cog/templatetags/search_utils.py @@ -162,7 +162,7 @@ def qcflags(record): qcflags[qcflag_name][int(qcflag_order)] = qcflag_value # for each qcflag, sort dictionary of values by their key (i.e. by the QC flag value order) for key in qcflags: - qcflags[key] = OrderedDict(sorted(qcflags[key].items(), key=lambda t: t[0])) + qcflags[key] = OrderedDict(sorted(list(qcflags[key].items()), key=lambda t: t[0])) # note: to enable easy access in html template, return the sorted set of (key, value) pairs # where the value is itself an ordered dictionary diff --git a/cog/tests.py b/cog/tests.py index 2247054b3..c7c4668e1 100644 --- a/cog/tests.py +++ b/cog/tests.py @@ -12,7 +12,7 @@ def test_basic_addition(self): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = {"doctest": """ Another way to test that 1 + 1 is equal to 2. diff --git a/cog/tests/test_openid.py b/cog/tests/test_openid.py index 1a7862363..20b51aad9 100644 --- a/cog/tests/test_openid.py +++ b/cog/tests/test_openid.py @@ -6,7 +6,7 @@ import os import sys import unittest -import urllib2 +import urllib.request, urllib.error, urllib.parse # list of openids from different ESGF IdPs OPENIDS = ['https://pcmdi9.llnl.gov/esgf-idp/openid/lucacinquini', @@ -55,10 +55,10 @@ def testValidOpenids(self): for openid in OPENIDS: try: - response = urllib2.urlopen(openid) + response = urllib.request.urlopen(openid) self.assertEqual(response.getcode(), 200, 'Invalid openid: %s' % openid) except Exception as e: - print 'Invalid openid: %s' % openid + print('Invalid openid: %s' % openid) raise e diff --git a/cog/tests/test_registration.py b/cog/tests/test_registration.py index 30d25576d..0521039a0 100644 --- a/cog/tests/test_registration.py +++ b/cog/tests/test_registration.py @@ -35,7 +35,7 @@ def tearDown(self): def testGroups(self): groups = self.registrationService.listGroups() - g = {'visible': True, 'automatic_approval': False, 'name': u'TestGroup', 'description': u'Description'} + g = {'visible': True, 'automatic_approval': False, 'name': 'TestGroup', 'description': 'Description'} self.assertIn(g, groups) def _testUserRegistration(self): diff --git a/cog/util/thumbnails.py b/cog/util/thumbnails.py index 42c24ddcb..061c41e6b 100644 --- a/cog/util/thumbnails.py +++ b/cog/util/thumbnails.py @@ -53,8 +53,8 @@ def generateThumbnail(filePath, thumbnail_size): im.thumbnail(thumbnail_size) im.save(thumbnailPath, "PNG") except IOError as error: - print "Cannot create thumbnail for %s, using full image instead " % filePath - print error + print("Cannot create thumbnail for %s, using full image instead " % filePath) + print(error) shutil.copy(filePath, thumbnailPath) def deleteThumbnail(filePath): @@ -63,8 +63,8 @@ def deleteThumbnail(filePath): try: os.remove(thumbnailPath) except IOError as error: - print "Cannot delete thumbnail for", filePath - print error + print("Cannot delete thumbnail for", filePath) + print(error) def deleteImageAndThumbnail(obj): '''Method to delete an object image and thumbnail simultaneously.''' diff --git a/cog/utils.py b/cog/utils.py index 90d7474d0..717580169 100644 --- a/cog/utils.py +++ b/cog/utils.py @@ -2,7 +2,7 @@ from django import forms import os import datetime -import urllib2 +import urllib.request, urllib.error, urllib.parse import json # timeout for JSON HTTP requests @@ -35,7 +35,7 @@ def clean_field(form, field, invalid_characters): # raise forms.ValidationError("The field %s contains invalid characters" % field) for c in data: if re.match(invalid_characters, c): - print 'Invalid character: %s' % c + print('Invalid character: %s' % c) raise forms.ValidationError("The character '%s' is invalid." % c) return data @@ -65,15 +65,15 @@ def getJson(url): '''Retrieves and parses a JSON document at some URL.''' try: - opener = urllib2.build_opener() - request = urllib2.Request(url) + opener = urllib.request.build_opener() + request = urllib.request.Request(url) response = opener.open(request, timeout=TIMEOUT) jdoc = response.read() return json.loads(jdoc) except Exception as e: - print e - print 'Error retrieving URL=%s' % url + print(e) + print('Error retrieving URL=%s' % url) return None def check_filepath(file_full_path, expected_file_names): diff --git a/cog/views/__init__.py b/cog/views/__init__.py index 0df89efd3..0ade7a3f0 100644 --- a/cog/views/__init__.py +++ b/cog/views/__init__.py @@ -1,20 +1,20 @@ from cog.views.constants import * -from views_templated import * -from views_bookmarks import * -from views_account import * -from views_datacart import * -from views_doc import * -from views_external_urls import * -from views_account import * -from views_governance import * -from views_admin import * -from views_membership import * -from views_news import * -from views_post import * -from views_project import * -from views_signal import * -from views_aboutus import * -from views_search import * +from .views_templated import * +from .views_bookmarks import * +from .views_account import * +from .views_datacart import * +from .views_doc import * +from .views_external_urls import * +from .views_account import * +from .views_governance import * +from .views_admin import * +from .views_membership import * +from .views_news import * +from .views_post import * +from .views_project import * +from .views_signal import * +from .views_aboutus import * +from .views_search import * from cog.views.views_share import * from cog.views.views_access_control import * from cog.views.views_globus import * diff --git a/cog/views/utils.py b/cog/views/utils.py index 05fbc5bc1..fa871ca5a 100644 --- a/cog/views/utils.py +++ b/cog/views/utils.py @@ -8,7 +8,7 @@ from cog.models.peer_site import getPeerSites from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist -import urllib +import urllib.request, urllib.parse, urllib.error from collections import OrderedDict from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger @@ -68,7 +68,7 @@ def getProjectNotVisibleRedirect(request, project): def set_openid_cookie(response, openid): """Utility method to consistently set the openid cookie.""" - print 'SETTING openid cookie to: %s' % openid + print('SETTING openid cookie to: %s' % openid) response.set_cookie('openid', openid, expires=(datetime.datetime.now() + datetime.timedelta(days=3650)), # expires in 10 years @@ -151,7 +151,7 @@ def get_all_shared_user_info(user, includeCurrentSite=True): if user.profile.openid() is not None: openid = user.profile.openid() - print 'Retrieving projects, groups for user with openid=%s' % openid + print('Retrieving projects, groups for user with openid=%s' % openid) # loop over remote (enabled) nodes, possibly add current node sites = list(getPeerSites()) @@ -161,12 +161,12 @@ def get_all_shared_user_info(user, includeCurrentSite=True): for site in sites: url = "http://%s/share/user/?openid=%s" % (site.domain, openid) - print 'Retrieving user projects and groups from URL=%s' % url + print('Retrieving user projects and groups from URL=%s' % url) jobj = getJson(url) if jobj is not None and openid in jobj['users']: userDict[site] = jobj['users'][openid] else: - print 'Openid=%s not found at site=%s' % (openid, site) + print('Openid=%s not found at site=%s' % (openid, site)) except UserProfile.DoesNotExist: pass # user profile not yet created @@ -174,18 +174,18 @@ def get_all_shared_user_info(user, includeCurrentSite=True): # restructure information as list of (project object, user roles) and (group name, group roles) tuples projects = [] groups = [] - for usite, udict in userDict.items(): + for usite, udict in list(userDict.items()): if udict.get('projects', None): - for pname, proles in udict["projects"].items(): + for pname, proles in list(udict["projects"].items()): try: proj = Project.objects.get(short_name__iexact=pname) projects.append((proj, proles)) except ObjectDoesNotExist: pass if udict.get('groups', None): - for gname, gdict in udict["groups"].items(): + for gname, gdict in list(udict["groups"].items()): groles = [] - for grole, approved in gdict.items(): + for grole, approved in list(gdict.items()): if approved: groles.append(grole) groups.append((gname,groles)) @@ -199,9 +199,9 @@ def add_get_parameter(url, key, value): """ if '?' in url: - return url + "&%s" % urllib.urlencode([(key, value)]) + return url + "&%s" % urllib.parse.urlencode([(key, value)]) else: - return url + "?%s" % urllib.urlencode([(key, value)]) + return url + "?%s" % urllib.parse.urlencode([(key, value)]) def getQueryDict(request): '''Utiity method to return the query dictionary for a GET or POST request.''' diff --git a/cog/views/views_aboutus.py b/cog/views/views_aboutus.py index 65e32716d..81c29cf02 100644 --- a/cog/views/views_aboutus.py +++ b/cog/views/views_aboutus.py @@ -20,7 +20,7 @@ from cog.util.thumbnails import * from cog.models.auth import userHasAdminPermission -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE def aboutus_display(request, project_short_name, tab): @@ -95,7 +95,7 @@ def aboutus_update(request, project_short_name, tab): else: # re-display form view if not form.is_valid(): - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_aboutus_form(request, project, tab, form) @@ -159,7 +159,7 @@ def impacts_update(request, project_short_name, tab): return HttpResponseRedirect(redirect) else: - print formset.errors + print(formset.errors) return render_impacts_form(request, project, formset, tab) @@ -276,11 +276,11 @@ def _imageformset_update(request, project, tab, deleteImageAndThumbnail(form.instance) except ValueError as error: - print error + print(error) # delete object if form.cleaned_data.get('DELETE', False): - print 'Deleting instance=%s' % form.instance + print('Deleting instance=%s' % form.instance) form.instance.delete() # persist formset data @@ -310,14 +310,14 @@ def _imageformset_update(request, project, tab, try: generateThumbnail(instance.image.path, thumbnail_size) except ValueError as e: - print e + print(e) pass # no image supplied # redirect to people display (GET-POST-REDIRECT) return HttpResponseRedirect(reverse('aboutus_display', args=[project.short_name.lower(), tab])) else: - print 'Formset is invalid %s' % formset.errors + print('Formset is invalid %s' % formset.errors) return render_formset(form_template, request, project, tab, formset) diff --git a/cog/views/views_access_control.py b/cog/views/views_access_control.py index 87d79f91d..e02071366 100644 --- a/cog/views/views_access_control.py +++ b/cog/views/views_access_control.py @@ -24,7 +24,7 @@ from cog.plugins.esgf.objects import ROLE_USER, ROLE_PUBLISHER, ROLE_SUPERUSER, ROLE_ADMIN from cog.services.registration import esgfRegistrationServiceImpl as registrationService from cog.utils import getJson -from constants import PERMISSION_DENIED_MESSAGE, SAVED, GROUP_NOT_FOUND_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE, SAVED, GROUP_NOT_FOUND_MESSAGE from cog.plugins.esgf.security import esgfDatabaseManager @@ -55,7 +55,7 @@ def ac_subscribe(request, group_name): status = registrationService.status(request.user.profile.openid(), group_name, ROLE_USER) except ObjectDoesNotExist: # user does not exist in ESGF database - print 'Inserting user into ESGF security database' + print('Inserting user into ESGF security database') esgfDatabaseManager.insertEsgfUser(request.user.profile) status = None @@ -114,7 +114,7 @@ def ac_process(request, group_name, user_id): # set initial status of check boxes from database initial = {} permissions = registrationService.list(openid, group_name) - for role, approved in permissions.items(): + for role, approved in list(permissions.items()): initial['%sPermissionCheckbox' % role] = approved form = PermissionForm(initial=initial) @@ -150,7 +150,7 @@ def ac_process(request, group_name, user_id): + "?message=%s" % SAVED) else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render(request, template, {'group_name': group_name, 'title': title, 'user': user, 'form': form}) @@ -173,7 +173,7 @@ def ac_list(request): site_domain = jobj['site']['domain'] # loop over groups for this node - for group_name, group_dict in jobj['groups'].items(): + for group_name, group_dict in list(jobj['groups'].items()): # augment group dictionary group_dict['site_name'] = site_name group_dict['site_domain'] = site_domain @@ -227,7 +227,7 @@ def notifyUser(group_name, user, permissions): subject = "'%s' Data Access Notification" % group_name message = "Your permissions in group '%s' have been updated" % group_name - for (role, status) in permissions.items(): + for (role, status) in list(permissions.items()): message += "\nRole: %s status=%s" % (role, status) notify(user, subject, message) diff --git a/cog/views/views_account.py b/cog/views/views_account.py index 3ab5e00c2..239634e99 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -1,5 +1,5 @@ -import urllib -from urlparse import urlparse +import urllib.request, urllib.parse, urllib.error +from urllib.parse import urlparse from django.contrib.auth import logout from django.contrib.auth.decorators import login_required, user_passes_test @@ -66,7 +66,7 @@ def custom_login_complete(request, **kwargs): except ObjectDoesNotExist: - print 'Discovering site for user with openid=%s' % openid + print('Discovering site for user with openid=%s' % openid) # retrieve user home node site = discoverSiteForUser(openid) @@ -74,7 +74,7 @@ def custom_login_complete(request, **kwargs): # set user home node to current node site = Site.objects.get_current() - print 'User site=%s... creating user profile...' % site + print('User site=%s... creating user profile...' % site) # create new ESGF/OpenID login, type=2: ESGF UserProfile.objects.create(user=request.user, institution='', city='', country='', type=2, site=site) @@ -96,7 +96,7 @@ def _custom_login(request, response): # missing information if isUserLocal(request.user) and not isUserValid(request.user): - print 'User is local but some information is missing, redirecting to user update page' + print('User is local but some information is missing, redirecting to user update page') return HttpResponseRedirect(reverse('user_update', kwargs={'user_id': request.user.id}) + "?message=incomplete_profile") @@ -180,7 +180,7 @@ def _sendSubsriptionEmail(user, action): # body message = '' - print 'Sending subscription email: To=%s Subject=%s' % (toAddress, subject) + print('Sending subscription email: To=%s Subject=%s' % (toAddress, subject)) sendEmail(toAddress, subject, message, fromAddress=user.email) @@ -194,8 +194,8 @@ def user_add(request): if redirectToIdp(): redirect_url = settings.IDP_REDIRECT + request.path if _next is not None: - redirect_url += ("?next=%s" % urllib.quote_plus(_next)) - print 'Redirecting account creation to: %s' % redirect_url + redirect_url += ("?next=%s" % urllib.parse.quote_plus(_next)) + print('Redirecting account creation to: %s' % redirect_url) return HttpResponseRedirect(redirect_url) # create URLs formset @@ -226,14 +226,14 @@ def user_add(request): # save user to database user.save() - print 'Created user=%s' % user.username + print('Created user=%s' % user.username) # create openid if settings.ESGF_CONFIG: openid = form.cleaned_data['openid'] - print 'Creating openid=%s' % openid + print('Creating openid=%s' % openid) userOpenID = UserOpenID.objects.create(user=user, claimed_id=openid, display_id=openid) - print 'Added openid=%s for user=%s into COG database' % (openid, user.username) + print('Added openid=%s for user=%s into COG database' % (openid, user.username)) # use additional form fields to create user profile userp = UserProfile(user=user, @@ -287,29 +287,29 @@ def user_add(request): # redirect to login page with special message login_url = reverse('login')+"?message=user_add" if _next is not None and len(_next.strip()) > 0: - login_url += ("&next=%s" % urllib.quote_plus(_next)) + login_url += ("&next=%s" % urllib.parse.quote_plus(_next)) # redirect to absolute URL (possibly at an another node) if 'http' in _next: url = urlparse(_next) login_url = '%s://%s%s' % (url.scheme, url.netloc, login_url) # append openid to initial login_url if userp.openid() is not None: - login_url += "&openid=%s" % urllib.quote_plus(userp.openid()) - login_url += "&username=%s" % urllib.quote_plus(userp.user.username) + login_url += "&openid=%s" % urllib.parse.quote_plus(userp.openid()) + login_url += "&username=%s" % urllib.parse.quote_plus(userp.user.username) response = HttpResponseRedirect(login_url) # set openid cookie on this host set_openid_cookie(response, userp.openid()) - print 'New user account created: redirecting to login url: %s' % login_url + print('New user account created: redirecting to login url: %s' % login_url) return response else: if not form.is_valid(): - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) elif not formset.is_valid(): - print "URL formset is invalid: %s" % formset.errors + print("URL formset is invalid: %s" % formset.errors) return render_user_form(request, form, formset, title='Create User Profile') @@ -327,12 +327,12 @@ def user_detail(request, user_id): except: user_profile = UserProfile(user=user) user_profile.save() - print "Created empty profile for user=%s" % user + print("Created empty profile for user=%s" % user) # retrieve map of (project, roles) for this user (projTuples, groupTuples) = get_all_shared_user_info(user) - print "\nprojTuples=" - print projTuples + print("\nprojTuples=") + print(projTuples) # sort projects, groups alphabetically projects = sorted(projTuples, key=lambda x: x[0].short_name) @@ -486,7 +486,7 @@ def user_update(request, user_id): # delete UserUrls if found urls = UserUrl.objects.filter(profile=profile) for url in urls: - print 'Deleting user URL: %s' % url.url + print('Deleting user URL: %s' % url.url) url.delete() # update user @@ -515,7 +515,7 @@ def user_update(request, user_id): if not is_password_usable(user.password): user.set_password(form.cleaned_data['password']) user.save() - print 'Reset password for user=%s' % user + print('Reset password for user=%s' % user) # image management _generateThumbnail = False @@ -571,9 +571,9 @@ def user_update(request, user_id): else: if not form.is_valid(): - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) elif not formset.is_valid(): - print "URL formset is invalid: %s" % formset.errors + print("URL formset is invalid: %s" % formset.errors) return render_user_form(request, form, formset, title='Update User Profile') @@ -639,7 +639,7 @@ def password_update(request, user_id): kwargs={'user_id': user.id})+"?message=password_updated_by_admin") else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render_password_change_form(request, form, user.username) @@ -684,7 +684,7 @@ def user_reminder(request): return render_user_reminder_form(request, form, "This email address cannot be found.") else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render_user_reminder_form(request, form) @@ -708,7 +708,7 @@ def password_reset(request): # check form is valid first if not form.is_valid(): - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render_password_reset_form(request, form) openid = form.cleaned_data.get('openid') diff --git a/cog/views/views_admin.py b/cog/views/views_admin.py index af29d0f44..a1e7cd77e 100644 --- a/cog/views/views_admin.py +++ b/cog/views/views_admin.py @@ -97,6 +97,6 @@ def admin_peers(request): return HttpResponseRedirect(reverse('admin_peers')+"?status=success") else: - print formset.errors + print(formset.errors) return render(request, 'cog/admin/admin_peers.html', {'formset': formset}) \ No newline at end of file diff --git a/cog/views/views_bookmarks.py b/cog/views/views_bookmarks.py index 0e740df57..64fe65f7e 100644 --- a/cog/views/views_bookmarks.py +++ b/cog/views/views_bookmarks.py @@ -7,9 +7,9 @@ from django.http import HttpResponseRedirect, HttpResponseForbidden import json from django.http import HttpResponse -from constants import PERMISSION_DENIED_MESSAGE, BAD_REQUEST -from utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect -from views_post import post_add +from .constants import PERMISSION_DENIED_MESSAGE, BAD_REQUEST +from .utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect +from .views_post import post_add from cog.models.auth import userHasContributorPermission @@ -102,7 +102,7 @@ def bookmark_add(request, project_short_name): return HttpResponseRedirect(reverse('bookmark_list', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) return render_bookmark_form(request, project, form) @@ -132,9 +132,9 @@ def bookmark_add2(request, project_short_name): response_data['message'] = 'Your bookmark was saved.' else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) # encode errors in response - although not used - for key, value in form.errors.items(): + for key, value in list(form.errors.items()): response_data['errors'][key] = value response_data['result'] = 'Error' response_data['message'] = 'Sorry, the form data is invalid: %s' % form.errors @@ -198,7 +198,7 @@ def bookmark_update(request, project_short_name, bookmark_id): # update associated Doc, if any doc = getDocFromBookmark(bookmark) if doc is not None: - print 'Updating associated doc: %s' % doc + print('Updating associated doc: %s' % doc) doc.title = bookmark.name doc.description = bookmark.description doc.save() @@ -207,7 +207,7 @@ def bookmark_update(request, project_short_name, bookmark_id): return HttpResponseRedirect(reverse('bookmark_list', args=[project.short_name.lower()])) else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) # return to view return render_bookmark_form(request, project, form) @@ -261,7 +261,7 @@ def folder_add(request, project_short_name): else: # return to view - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render_folder_form(request, project, form) @@ -295,7 +295,7 @@ def folder_update(request, project_short_name, folder_id): else: # return to view - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) return render_folder_form(request, folder.project, form) @@ -331,11 +331,11 @@ def delete_folder(folder): # delete folder bookmarks for bookmark in folder.bookmark_set.all(): - print 'Deleting bookmark=%s' % bookmark + print('Deleting bookmark=%s' % bookmark) bookmark.delete() # delete this folder - print 'Deleting folder=%s' % folder + print('Deleting folder=%s' % folder) folder.delete() diff --git a/cog/views/views_datacart.py b/cog/views/views_datacart.py index 359902a66..67df18783 100644 --- a/cog/views/views_datacart.py +++ b/cog/views/views_datacart.py @@ -44,8 +44,8 @@ def datacart_display(request, site_id, user_id): _dcs = getDataCartsForUser(openid) if len(_dcs) > 0: dcs[openid] = {} - for site, size in _dcs.items(): - print site, size + for site, size in list(_dcs.items()): + print(site, size) dcs[openid][site] = size return render(request, @@ -135,15 +135,15 @@ def datacart_add_all(request, site_id, user_id): searchOutput = data[SEARCH_OUTPUT] # loop over record - print "Adding %s items" % len(searchOutput.results) + print("Adding %s items" % len(searchOutput.results)) for record in searchOutput.results: # check item is not in cart already if DataCartItem.objects.filter(cart=datacart, identifier=record.id).exists(): - print 'Item %s already in Data Cart' % record.id + print('Item %s already in Data Cart' % record.id) else: item = DataCartItem.fromRecord(datacart, record) - print 'Added item: %s' % record.id + print('Added item: %s' % record.id) # redirect to search results back = request.GET.get('back', '/') @@ -173,12 +173,12 @@ def datacart_delete_all(request, site_id, user_id): searchOutput = data[SEARCH_OUTPUT] # loop over record - print "Deleting %s items" % len(searchOutput.results) + print("Deleting %s items" % len(searchOutput.results)) for record in searchOutput.results: try: item = DataCartItem.objects.get(cart=datacart, identifier=record.id) - print 'Deleting item: %s' % item.id + print('Deleting item: %s' % item.id) item.delete() except ObjectDoesNotExist: pass @@ -366,6 +366,6 @@ def datacart_pid(request, site_id, user_id): pid = connector.create_data_cart_pid(dataset_ids) connector.finish_messaging_thread() - print 'Generated data cart PID for %d datasets: %s' % (len(ids), pid) + print('Generated data cart PID for %d datasets: %s' % (len(ids), pid)) return HttpResponse(json.dumps(pid), content_type="application/json") diff --git a/cog/views/views_doc.py b/cog/views/views_doc.py index 67554356d..9cb610629 100644 --- a/cog/views/views_doc.py +++ b/cog/views/views_doc.py @@ -5,7 +5,7 @@ from django.core.urlresolvers import reverse from cog.models import * from cog.forms import * -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE from os.path import basename from django.views.decorators.csrf import csrf_exempt from cog.forms import UploadImageForm @@ -46,7 +46,7 @@ def doc_upload(request, project_short_name): error = '' else: - print 'Form errors:%s' % form.errors + print('Form errors:%s' % form.errors) error = 'The file uploaded is not an image. Valid files include PNG, JPG, and PDF.' url = "%s%s" % (settings.STATIC_URL, 'cog/img/error.jpeg') @@ -155,7 +155,7 @@ def data_download(request, path): project_short_name = path.split("/")[0] project = get_object_or_404(Project, short_name__iexact=project_short_name) - print 'Data for project=%s' % project + print('Data for project=%s' % project) # TODO: check if data is public before forcing login return secure_data_download(request, path, project) @@ -235,7 +235,7 @@ def doc_update(request, doc_id): bookmark.name = doc.title bookmark.description = doc.description bookmark.save() - print 'Updated associated bookmark: %s' % bookmark + print('Updated associated bookmark: %s' % bookmark) # redirect to document detail (GET-POST-REDIRECT) return HttpResponseRedirect(reverse('doc_detail', kwargs={'doc_id': doc.id})) diff --git a/cog/views/views_external_urls.py b/cog/views/views_external_urls.py index e388b7718..a4bf54423 100644 --- a/cog/views/views_external_urls.py +++ b/cog/views/views_external_urls.py @@ -8,9 +8,9 @@ from django.core.urlresolvers import reverse from django.http import HttpResponse, HttpResponseForbidden from django.forms.models import modelformset_factory -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE from cog.models.constants import * -from utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect +from .utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect from cog.models.navbar import TABS from cog.models.external_url_conf import externalUrlManager from cog.models.auth import userHasContributorPermission @@ -107,7 +107,7 @@ def external_urls_update(request, project_short_name, suburl): # GET if request.method == 'GET': - print type + print(type) # create formset instance backed by current saved instances # must provide the initial data to all the extra instances, @@ -148,7 +148,7 @@ def external_urls_update(request, project_short_name, suburl): return HttpResponseRedirect(redirect) else: - print formset.errors + print(formset.errors) return render_external_urls_form(request, project, formset, externalUrlConf, redirect) diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index 52d00c461..66e5dd202 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -4,13 +4,13 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError from django.shortcuts import render from django.template import RequestContext -import urllib +import urllib.request, urllib.parse, urllib.error from cog.utils import getJson -from urlparse import urlparse +from urllib.parse import urlparse from cog.constants import SECTION_GLOBUS from cog.site_manager import siteManager import datetime -from constants import GLOBUS_NOT_ENABLED_MESSAGE +from .constants import GLOBUS_NOT_ENABLED_MESSAGE from functools import wraps import os import re @@ -113,8 +113,8 @@ def download(request): params.append(("distrib", "false")) - url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) - print 'Searching for files at URL: %s' % url + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) + print('Searching for files at URL: %s' % url) jobj = getJson(url) # parse response for GridFTP URls @@ -138,13 +138,13 @@ def download(request): download_map[gendpoint_name] = [] # insert empty list of paths download_map[gendpoint_name].append(path) else: - print 'The file is not accessible through Globus' + print('The file is not accessible through Globus') else: return HttpResponseServerError("Error querying for files URL") # store map in session request.session[GLOBUS_DOWNLOAD_MAP] = download_map - print 'Stored Globus Download Map=%s at session scope' % download_map + print('Stored Globus Download Map=%s at session scope' % download_map) # redirect after post (to display page) return HttpResponseRedirect( reverse('globus_transfer') ) @@ -171,8 +171,8 @@ def transfer(request): # ('lock', 'ep'), # do NOT allow the user to change their endpoint ('action', request.build_absolute_uri(reverse("globus_oauth")) ), # redirect to CoG Oauth URL ] - globus_url = GLOBUS_SELECT_DESTINATION_URL + "?" + urllib.urlencode(params) - print "Redirecting to: %s" % globus_url + globus_url = GLOBUS_SELECT_DESTINATION_URL + "?" + urllib.parse.urlencode(params) + print("Redirecting to: %s" % globus_url) return HttpResponseRedirect(globus_url) # replacement URL for localhost development #return HttpResponseRedirect( request.build_absolute_uri(reverse("globus_oauth")) ) # FIXME @@ -185,14 +185,14 @@ def oauth(request): # retrieve destination parameters from Globus redirect # example URL with added parameters from Globus redirect: # /globus/oauth/?label=&verify_checksum=on&submitForm=&folder[0]=tmp&endpoint=cinquiniluca#mymac&path=/~/&ep=GC&lock=ep&method=get&folderlimit=1&action=http://localhost:8000/globus/oauth/ - print request.GET + print(request.GET) request.session[TARGET_ENDPOINT] = getQueryDict(request).get('endpoint','#') request.session[TARGET_FOLDER] = getQueryDict(request).get('path','/~/') + getQueryDict(request).get('folder[0]', '') # default value: user home directory - print 'User selected destionation endpoint:%s, path:%s, folder:%s' % (request.session[TARGET_ENDPOINT], getQueryDict(request).get('path','/~/'), getQueryDict(request).get('folder[0]', '')) + print('User selected destionation endpoint:%s, path:%s, folder:%s' % (request.session[TARGET_ENDPOINT], getQueryDict(request).get('path','/~/'), getQueryDict(request).get('folder[0]', ''))) # Redirect the user to Globus OAuth server to get an authorization code if the user approves the access request. globus_authorize_url = establishFlow(request).step1_get_authorize_url() - print "Redirecting to: %s" % globus_authorize_url + print("Redirecting to: %s" % globus_authorize_url) return HttpResponseRedirect(globus_authorize_url) @@ -239,15 +239,15 @@ def submit(request): target_endpoint = request.session[TARGET_ENDPOINT] target_folder = request.session[TARGET_FOLDER] - print 'Downloading files=%s' % download_map.items() - print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder) + print('Downloading files=%s' % list(download_map.items())) + print('User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder)) authorizer = AccessTokenAuthorizer(access_token) transfer_client = TransferClient(authorizer) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(transfer_client, target_endpoint) - for source_endpoint, source_files in download_map.items(): + for source_endpoint, source_files in list(download_map.items()): status, message = activateEndpoint(transfer_client, source_endpoint, openid, password) if not status: return render(request, @@ -256,7 +256,7 @@ def submit(request): # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids - for source_endpoint, source_files in download_map.items(): + for source_endpoint, source_files in list(download_map.items()): # submit transfer request task_id = submitTransfer(transfer_client, source_endpoint, source_files, target_endpoint, target_folder) diff --git a/cog/views/views_governance.py b/cog/views/views_governance.py index 3eebe14f7..251cd1b42 100644 --- a/cog/views/views_governance.py +++ b/cog/views/views_governance.py @@ -3,14 +3,14 @@ from cog.models.constants import LEAD_ORGANIZATIONAL_ROLES_DICT, \ ROLE_CATEGORY_LEAD, ROLE_CATEGORY_MEMBER, MANAGEMENT_BODY_CATEGORY_STRATEGIC, \ MANAGEMENT_BODY_CATEGORY_OPERATIONAL -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE from django.contrib.auth.decorators import login_required from django.forms.models import BaseInlineFormSet, inlineformset_factory from django.http import HttpResponseRedirect, HttpResponse, HttpResponseForbidden from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.utils.functional import curry -from utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect +from .utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect from cog.models.navbar import TABS, TAB_LABELS from cog.views.views_templated import templated_page_display from cog.models.auth import userHasAdminPermission @@ -120,7 +120,7 @@ def governance_overview_update(request, project_short_name): # return to form else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_governance_overview_form(request, form, project) @@ -195,7 +195,7 @@ def governance_object_update(request, project_short_name, tab, objectType, objec return redirect else: - print 'Formset is invalid %s' % formset.errors + print('Formset is invalid %s' % formset.errors) return render_governance_object_form(request, project, formset, title, template) @@ -291,7 +291,7 @@ def members_update(request, tab, objectId, objectType, objectMemberType, objectM return HttpResponseRedirect(redirect) else: - print 'Formset is invalid: %s' % formset.errors + print('Formset is invalid: %s' % formset.errors) # redirect to form view return render_members_form(request, obj, formset, redirect) @@ -343,7 +343,7 @@ def processes_update(request, project_short_name): # return to form else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_governance_processes_form(request, form, project) def render_governance_processes_form(request, form, project): @@ -400,7 +400,7 @@ def organizational_role_update(request, project_short_name): return HttpResponseRedirect(reverse('governance_display', args=[project.short_name.lower(), tab])) else: - print 'Organizational Role formset is invalid: %s' % organizational_role_formset.errors + print('Organizational Role formset is invalid: %s' % organizational_role_formset.errors) # redorect to form return render_organizational_role_form(request, project, organizational_role_formset) diff --git a/cog/views/views_membership.py b/cog/views/views_membership.py index 49c998585..982db0b64 100644 --- a/cog/views/views_membership.py +++ b/cog/views/views_membership.py @@ -12,7 +12,7 @@ from cog.views.utils import paginate from cog.notification import notify -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE # HTTP parameters NEW_MEMBERSHIP = "new_membership" @@ -216,7 +216,7 @@ def membership_process(request, project_short_name): queryDict = getQueryDict(request) - for (name, value) in queryDict.items(): + for (name, value) in list(queryDict.items()): if name.startswith(NEW_MEMBERSHIP) or name.startswith(OLD_MEMBERSHIP) or name.startswith(NO_MEMBERSHIP): (prefix, group_name, user_id) = name.split(":") diff --git a/cog/views/views_news.py b/cog/views/views_news.py index 10fb0fde9..4ea9cd7b1 100644 --- a/cog/views/views_news.py +++ b/cog/views/views_news.py @@ -7,7 +7,7 @@ from cog.models import * from cog.models.auth import userHasContributorPermission from cog.forms import * -from constants import PERMISSION_DENIED_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE from cog.views.utils import paginate @@ -75,7 +75,7 @@ def news_update(request, news_id): # invalid data else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) news = form.instance return render_news_form(request, request.POST, form, news.project) @@ -130,7 +130,7 @@ def news_add(request, project_short_name): # invalid data else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) news = form.instance return render_news_form(request, request.POST, form, news.project) diff --git a/cog/views/views_post.py b/cog/views/views_post.py index 94e2126d2..3d6975a07 100644 --- a/cog/views/views_post.py +++ b/cog/views/views_post.py @@ -10,11 +10,11 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from string import Template -from urllib import quote, unquote +from urllib.parse import quote, unquote import copy -from constants import PERMISSION_DENIED_MESSAGE, LOCAL_PROJECTS_ONLY_MESSAGE +from .constants import PERMISSION_DENIED_MESSAGE, LOCAL_PROJECTS_ONLY_MESSAGE from cog.models.constants import SIGNAL_OBJECT_CREATED, SIGNAL_OBJECT_UPDATED, SIGNAL_OBJECT_DELETED -from utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect +from .utils import getProjectNotActiveRedirect, getProjectNotVisibleRedirect from django.utils.timezone import now from cog.models.utils import delete_doc from django.conf import settings @@ -292,7 +292,7 @@ def post_add(request, project_short_name, owner=None): # invalid data else: - print form.errors + print(form.errors) return render_post_form(request, form, project, postType) @@ -382,7 +382,7 @@ def post_update(request, post_id): # check versions queryDict = getQueryDict(request) if post.version != int(queryDict.get('version', -1)): - print 'database version=%s form version=%s' % (post.version, queryDict.get('version', -1)) + print('database version=%s form version=%s' % (post.version, queryDict.get('version', -1))) return getLostLockRedirect(request, post.project, post, lock) #print "1 PAGE URL=%s" % form.data['url'] @@ -433,7 +433,7 @@ def post_update(request, post_id): return HttpResponseRedirect(reverse('project_home', args=[post.project.short_name.lower()])) else: - print form.errors + print(form.errors) return render_post_form(request, form, post.project, post.type, lock=lock) diff --git a/cog/views/views_project.py b/cog/views/views_project.py index 90f92637e..9fc3bcad0 100644 --- a/cog/views/views_project.py +++ b/cog/views/views_project.py @@ -96,7 +96,7 @@ def project_add(request): # invalid data else: - print "Form is invalid: %s" % form.errors + print("Form is invalid: %s" % form.errors) project = form.instance # create and set state of project tabs, do not persist @@ -160,7 +160,7 @@ def project_index(request, project_short_name): topicvalue = request.POST[topickey] ii.order = topicvalue # validate topic order - if topicvalue in topicMap.values(): + if topicvalue in list(topicMap.values()): valid = False errors[topickey] = "Duplicate topic number: %d" % int(topicvalue) else: @@ -173,7 +173,7 @@ def project_index(request, project_short_name): pagevalue = request.POST[pagekey] page.order = pagevalue # validate page order - if pagevalue in pageMap.values(): + if pagevalue in list(pageMap.values()): valid = False errors[pagekey] = "Duplicate page number: %d" % int(pagevalue) else: @@ -271,7 +271,7 @@ def project_update(request, project_short_name): return HttpResponseRedirect(reverse('project_home', args=[project.short_name.lower()])) else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) # update project tabs, but do not persist state since form had errors tabs = get_or_create_project_tabs(project, save=False) @@ -371,7 +371,7 @@ def contactus_update(request, project_short_name): else: # re-display form view if not form.is_valid(): - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_contactus_form(request, project, form) @@ -419,7 +419,7 @@ def notifyAuthorOfProjectApproval(project, request): def initProject(project): - print "Initializing project: %s" % project.short_name + print("Initializing project: %s" % project.short_name) # create project home page create_project_home(project, project.author) @@ -540,7 +540,7 @@ def tags_update(request, project_short_name): return HttpResponseRedirect(reverse('project_home', args=[project.short_name.lower()])) else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_tags_form(request, project, form) @@ -613,22 +613,22 @@ def save_user_tag(request): queryDict = getQueryDict(request) tagName = queryDict['tag'] redirect = queryDict['redirect'] - print 'Saving user tag: %s' % tagName - print 'Eventually redirecting to: %s' % redirect + print('Saving user tag: %s' % tagName) + print('Eventually redirecting to: %s' % redirect) if isUserLocal(request.user): try: tag = ProjectTag.objects.get(name__iexact=tagName) except ObjectDoesNotExist: tag = ProjectTag.objects.create(name=tagName) - print 'Created new tag: %s' % tag + print('Created new tag: %s' % tag) # add this tag to the user preferences utags = request.user.profile.tags if tag not in utags.all(): utags.add(tag) request.user.profile.save() - print 'Tag: %s added to user: %s' % (tagName, request.user) + print('Tag: %s added to user: %s' % (tagName, request.user)) # set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 @@ -638,7 +638,7 @@ def save_user_tag(request): else: url = "http://%s%s?tag=%s&redirect=%s" % (request.user.profile.site.domain, reverse('save_user_tag'), tagName, redirect) - print 'Redirecting save request to URL=%s' % url + print('Redirecting save request to URL=%s' % url) # set session flag to eventually force reloading of user tags request.session['LAST_ACCESSED'] = 0 # also set session flag to preselect a tab @@ -656,8 +656,8 @@ def delete_user_tag(request): queryDict = getQueryDict(request) tagName = queryDict['tag'] redirect = queryDict['redirect'] - print 'Deleting user tag: %s' % tagName - print 'Eventually redirecting to: %s' % redirect + print('Deleting user tag: %s' % tagName) + print('Eventually redirecting to: %s' % redirect) if isUserLocal(request.user): try: @@ -668,7 +668,7 @@ def delete_user_tag(request): request.user.profile.save() except ObjectDoesNotExist: - print "Invalid project tag." + print("Invalid project tag.") # set session flag to preselect a tab request.session['PROJECT_BROWSER_TAB'] = 3 @@ -678,7 +678,7 @@ def delete_user_tag(request): else: url = "http://%s%s?tag=%s&redirect=%s" % (request.user.profile.site.domain, reverse('delete_user_tag'), tagName, redirect) - print 'Redirecting delete request to URL=%s' % url + print('Redirecting delete request to URL=%s' % url) # set session flag to eventually force reloading of user tags request.session['LAST_ACCESSED'] = 0 # also set session flag to preselect a tab @@ -704,7 +704,7 @@ def render_project_list(project, tab, tag_name, user, widget_name, widget_id, di if tag_name is not None: try: tag = ProjectTag.objects.get(name__iexact=tag_name) - print "tag in render_project_list = ", tag + print("tag in render_project_list = ", tag) except ObjectDoesNotExist: # store error associated with non-existing tag tag_error = "Tag does not exist." @@ -867,7 +867,7 @@ def development_update(request, project_short_name): # return to form else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render_development_form(request, project, form) @@ -933,7 +933,7 @@ def _project_page_update(request, project_short_name, # return to form else: - print 'Form is invalid %s' % form.errors + print('Form is invalid %s' % form.errors) return render(request, form_template, {'title': form_template_title, 'project': project, 'form': form}) @@ -952,9 +952,9 @@ def _getUnsavedProjectSubFolders(project, request): """ folders = [] - for key, value in TOP_SUB_FOLDERS.items(): + for key, value in list(TOP_SUB_FOLDERS.items()): folder = Folder(name=value, project=project, active=False) - if request is not None and ("folder_%s" % value) in getQueryDict(request).keys(): + if request is not None and ("folder_%s" % value) in list(getQueryDict(request).keys()): folder.active = True folders.append(folder) return folders diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 44395b33e..0f69c2e46 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -1,7 +1,7 @@ from copy import copy, deepcopy import json -import urllib, urllib2 -from urllib2 import HTTPError +import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse +from urllib.error import HTTPError from cog.config.search import SearchConfigParser from cog.forms.forms_search import * @@ -93,7 +93,7 @@ def _addConfigConstraints(searchInput, searchConfig): _searchInput = deepcopy(searchInput) # add fixed constraints - but do NOT override previous values - for key, values in searchConfig.fixedConstraints.items(): + for key, values in list(searchConfig.fixedConstraints.items()): if not _searchInput.hasConstraint(key): _searchInput.setConstraint(key, values) return _searchInput @@ -168,11 +168,11 @@ def search_config(request, searchConfig, extra={}, fromRedirectFlag=False): # GET/POST switch queryDict = getQueryDict(request) - print "Search() view: HTTP Request method=%s fromRedirectFlag flag=%s HTTP parameters=%s" % (request.method, fromRedirectFlag, queryDict) + print("Search() view: HTTP Request method=%s fromRedirectFlag flag=%s HTTP parameters=%s" % (request.method, fromRedirectFlag, queryDict)) if request.method == 'GET': # GET pre-seeded search URL -> invoke POST immediately - if len(queryDict.keys()) > 0 and not fromRedirectFlag: + if len(list(queryDict.keys())) > 0 and not fromRedirectFlag: return search_post(request, searchInput, searchConfig, extra) else: return search_get(request, searchInput, searchConfig, extra, fromRedirectFlag) @@ -196,7 +196,7 @@ def search_get(request, searchInput, searchConfig, extra={}, fromRedirectFlag=Fa # GET request after POST redirection if fromRedirectFlag: - print "Retrieving search data from session" + print("Retrieving search data from session") data = request.session.get(SEARCH_DATA) # direct GET request: must query for all facet values with project-specific constraints @@ -206,7 +206,7 @@ def search_get(request, searchInput, searchConfig, extra={}, fromRedirectFlag=Fa request.session[SEARCH_PATH] = [] # add project fixed constraints - print 'Search GET: adding fixed project constraints' + print('Search GET: adding fixed project constraints') _searchInput = _addConfigConstraints(searchInput, searchConfig) _searchInput.printme() @@ -230,7 +230,7 @@ def search_get(request, searchInput, searchConfig, extra={}, fromRedirectFlag=Fa request.session[SEARCH_DATA] = data except HTTPError: - print "HTTP Request Error" + print("HTTP Request Error") # data = request.session[SEARCH_DATA] data[SEARCH_INPUT] = searchInput @@ -302,7 +302,7 @@ def search_post(request, searchInput, searchConfig, extra={}): if valid: # add project fixed constraints - print 'Search POST: adding fixed project constraints' + print('Search POST: adding fixed project constraints') _searchInput = _addConfigConstraints(searchInput, searchConfig) _searchInput.printme() @@ -324,7 +324,7 @@ def search_post(request, searchInput, searchConfig, extra={}): # data[FACET_PROFILE] = sorted( facetProfile.getKeys() ) # sort facets by key except HTTPError: - print "HTTP Request Error" + print("HTTP Request Error") data = request.session[SEARCH_DATA] data[SEARCH_INPUT] = searchInput @@ -332,7 +332,7 @@ def search_post(request, searchInput, searchConfig, extra={}): "Administrator." # invalid user input else: - print "Invalid Search Input" + print("Invalid Search Input") # re-use previous data (output, profile and any extra argument) from session data = request.session[SEARCH_DATA] # override search input from request @@ -348,10 +348,10 @@ def search_post(request, searchInput, searchConfig, extra={}): # for key, values in searchInput.constraints.items(): # note: request parameters do NOT include the project fixed constraints req_constraints = [] # latest constraints from request - for key, value in queryDict.items(): + for key, value in list(queryDict.items()): if not key in SEARCH_PATH_EXCLUDE and value != 'on': # value from 'checkbox_...' if value is not None and len(value) > 0: # disregard empty facet - print 'key=%s value=%s' % (key, value) + print('key=%s value=%s' % (key, value)) constraint = (key, value) req_constraints.append(constraint) if not constraint in sp: @@ -392,9 +392,9 @@ def metadata_display(request, project_short_name): if type == 'File': params.append(('dataset_id', dataset_id)) - url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) - print 'Metadata Solr search URL=%s' % url - fh = urllib2.urlopen(url) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) + print('Metadata Solr search URL=%s' % url) + fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") # parse JSON response (containing only one matching 'doc) @@ -405,9 +405,9 @@ def metadata_display(request, project_short_name): parentMetadata = {} if type == 'File': params = [('type', 'Dataset'), ('id', dataset_id), ("format", "application/solr+json"), ("distrib", "false")] - url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) # print 'Solr search URL=%s' % url - fh = urllib2.urlopen(url) + fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") jsondoc = json.loads(response) parentMetadata = _processDoc(jsondoc["response"]["docs"][0]) @@ -538,7 +538,7 @@ def search_profile_export(request, project_short_name): scp.write() message = 'search_config_exported' except Exception as e: - print "ERROR: %s" % e + print("ERROR: %s" % e) message = e return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])+"?message=%s" % message) @@ -559,7 +559,7 @@ def search_profile_import(request, project_short_name): scp.read() message = 'search_config_imported' except Exception as e: - print "ERROR: %s" % e + print("ERROR: %s" % e) message = 'search_config_not_found' return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])+"?message=%s" % message) @@ -607,7 +607,7 @@ def search_profile_config(request, project_short_name): return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form + print('Form is invalid: %s' % form) return render_search_profile_form(request, project, form, search_groups) @@ -663,7 +663,7 @@ def search_facet_add(request, project_short_name): return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) # must retrieve facets again facets = _queryFacets(request, project) @@ -698,7 +698,7 @@ def search_group_add(request, project_short_name): return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) return render_search_group_form(request, project, form) @@ -727,7 +727,7 @@ def search_group_update(request, group_id): return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) return render_search_group_form(request, project, form) @@ -759,7 +759,7 @@ def search_facet_update(request, facet_id): return HttpResponseRedirect(reverse('search_profile_config', args=[project.short_name.lower()])) else: - print 'Form is invalid: %s' % form.errors + print('Form is invalid: %s' % form.errors) return render_search_facet_form(request, project, form, facets) @@ -850,9 +850,9 @@ def search_files(request, dataset_id, index_node): else: params.append(("distrib", "false")) - url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) - print 'Searching for files: URL=%s' % url - fh = urllib2.urlopen(url) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) + print('Searching for files: URL=%s' % url) + fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") return HttpResponse(response, content_type="application/json") @@ -863,7 +863,7 @@ def search_reload(request): including constraints and results.""" if request.session.get(LAST_SEARCH_URL, None): - print 'Reloading search page: %s' % request.session[LAST_SEARCH_URL] + print('Reloading search page: %s' % request.session[LAST_SEARCH_URL]) request.session[SEARCH_REDIRECT] = True # flag to retrieve constraints, results return HttpResponseRedirect(request.session[LAST_SEARCH_URL]) # just like after the last POST @@ -920,13 +920,13 @@ def search_profile_order(request, project_short_name): valid = True # form data validation flag errors = {} # form validation errors - for group, facets in groups.items(): + for group, facets in list(groups.items()): group_key = SEARCH_GROUP_KEY + str(group.name) group_order = request.POST[group_key] group.order = int(group_order) # reassign the group orde WITHOUT saving to the database for now # validate group order - if group_order in groupOrderMap.values(): + if group_order in list(groupOrderMap.values()): valid = False errors[group_key] = "Duplicate search facet number: %d" % int(group_order) else: @@ -939,7 +939,7 @@ def search_profile_order(request, project_short_name): facet_order = request.POST[facet_key] facet.order = facet_order # validate facet order within this group - if facet_order in facetOrderMap.values(): + if facet_order in list(facetOrderMap.values()): valid = False errors[facet_key] = "Duplicate facet number: %d" % int(facet_order) else: @@ -949,7 +949,7 @@ def search_profile_order(request, project_short_name): if valid: # save new ordering for groups, facets - for group, facets in groups.items(): + for group, facets in list(groups.items()): group.save() for facet in facets: facet.save() @@ -991,21 +991,21 @@ def citation_display(request): url = request.GET.get('url', '') try: - fh = urllib2.urlopen(url) + fh = urllib.request.urlopen(url) response = fh.read() headers = fh.info().dict - except HTTPError, e: - print('HTTPError %s for %s' % (str(e.code), url)) + except HTTPError as e: + print(('HTTPError %s for %s' % (str(e.code), url))) return HttpResponseNotFound() if int(headers['x-cera-rc']) > 0: - print 'Citation not found: %s' % url + print('Citation not found: %s' % url) return HttpResponseNotFound() try: json.loads(response) - except ValueError, e: - print 'Citation not valid json: %s' % url + except ValueError as e: + print('Citation not valid json: %s' % url) return HttpResponseNotFound() return HttpResponse(response, content_type="application/json") diff --git a/cog/views/views_share.py b/cog/views/views_share.py index 9f1704d16..a4b23b02a 100644 --- a/cog/views/views_share.py +++ b/cog/views/views_share.py @@ -97,7 +97,7 @@ def share_projects(request): # list projects from this node projects = {} - print 'Listing ACTIVE projects for current site=%s' % current_site + print('Listing ACTIVE projects for current site=%s' % current_site) for project in Project.objects.filter(active=True).filter(site=current_site): projects[project.short_name] = serialize_project(project) @@ -123,7 +123,7 @@ def share_groups(request): response_data['site'] = serialize_site(current_site) # list groups from this node, index by group name - print 'Listing visible groups for current site=%s' % current_site + print('Listing visible groups for current site=%s' % current_site) groups = {} for group in registrationService.listGroups(): if group['visible'] and group['name'].lower() != 'wheel': @@ -149,7 +149,7 @@ def share_user(request): except ObjectDoesNotExist: # return empty dictionary - print 'User with openid=%s found at this site' % openid + print('User with openid=%s found at this site' % openid) users = {} response_data["users"] = users @@ -169,7 +169,7 @@ def sync_projects(request): return render(request, 'cog/admin/sync_projects.html', - {'sites':sorted(sites.iteritems(), key=lambda (siteid, sitedict): sitedict['name']), + {'sites':sorted(iter(sites.items()), key=lambda siteid_sitedict: siteid_sitedict[1]['name']), 'totalNumberOfProjects':totalNumberOfProjects, 'totalNumberOfUsers':totalNumberOfUsers }) \ No newline at end of file diff --git a/filebrowser/actions.py b/filebrowser/actions.py index fd275c066..2df6db6ff 100644 --- a/filebrowser/actions.py +++ b/filebrowser/actions.py @@ -53,30 +53,30 @@ def transpose_image(request, fileobjects, operation): def flip_horizontal(request, fileobjects): transpose_image(request, fileobjects, 0) -flip_horizontal.short_description = _(u'Flip horizontal') +flip_horizontal.short_description = _('Flip horizontal') flip_horizontal.applies_to = applies_to_all_images def flip_vertical(request, fileobjects): transpose_image(request, fileobjects, 1) -flip_vertical.short_description = _(u'Flip vertical') +flip_vertical.short_description = _('Flip vertical') flip_vertical.applies_to = applies_to_all_images def rotate_90_clockwise(request, fileobjects): transpose_image(request, fileobjects, 4) -rotate_90_clockwise.short_description = _(u'Rotate 90° CW') +rotate_90_clockwise.short_description = _('Rotate 90° CW') rotate_90_clockwise.applies_to = applies_to_all_images def rotate_90_counterclockwise(request, fileobjects): transpose_image(request, fileobjects, 2) -rotate_90_counterclockwise.short_description = _(u'Rotate 90° CCW') +rotate_90_counterclockwise.short_description = _('Rotate 90° CCW') rotate_90_counterclockwise.applies_to = applies_to_all_images def rotate_180(request, fileobjects): transpose_image(request, fileobjects, 3) -rotate_180.short_description = _(u'Rotate 180°') +rotate_180.short_description = _('Rotate 180°') rotate_180.applies_to = applies_to_all_images diff --git a/filebrowser/base.py b/filebrowser/base.py index ad05b21c8..88d03e8fa 100644 --- a/filebrowser/base.py +++ b/filebrowser/base.py @@ -132,7 +132,7 @@ def files_walk_total(self): def files_listing_filtered(self): "Returns FileObjects for filtered files in listing" if self.filter_func: - listing = filter(self.filter_func, self.files_listing_total()) + listing = list(filter(self.filter_func, self.files_listing_total())) else: listing = self.files_listing_total() self._results_listing_filtered = len(listing) @@ -141,7 +141,7 @@ def files_listing_filtered(self): def files_walk_filtered(self): "Returns FileObjects for filtered files in walk" if self.filter_func: - listing = filter(self.filter_func, self.files_walk_total()) + listing = list(filter(self.filter_func, self.files_walk_total())) else: listing = self.files_walk_total() self._results_walk_filtered = len(listing) diff --git a/filebrowser/decorators.py b/filebrowser/decorators.py index bd309f77f..6a33d49f3 100644 --- a/filebrowser/decorators.py +++ b/filebrowser/decorators.py @@ -20,7 +20,7 @@ def path_exists(site, function): def decorator(request, *args, **kwargs): if get_path('', site=site) == None: # The DIRECTORY does not exist, raise an error to prevent eternal redirecting. - raise ImproperlyConfigured, _("Error finding Upload-Folder (MEDIA_ROOT + DIRECTORY). Maybe it does not exist?") + raise ImproperlyConfigured(_("Error finding Upload-Folder (MEDIA_ROOT + DIRECTORY). Maybe it does not exist?")) if get_path(request.GET.get('dir', ''), site=site) == None: msg = _('The requested Folder does not exist.') messages.add_message(request, messages.ERROR, msg) diff --git a/filebrowser/fields.py b/filebrowser/fields.py index 7f922dbdc..a823ac6dc 100644 --- a/filebrowser/fields.py +++ b/filebrowser/fields.py @@ -62,7 +62,7 @@ def render(self, name, value, attrs=None): class FileBrowseFormField(forms.CharField): default_error_messages = { - 'extension': _(u'Extension %(ext)s is not allowed. Only %(allowed)s is allowed.'), + 'extension': _('Extension %(ext)s is not allowed. Only %(allowed)s is allowed.'), } def __init__(self, max_length=None, min_length=None, site=None, directory=None, extensions=None, format=None, *args, **kwargs): @@ -85,9 +85,8 @@ def clean(self, value): return value -class FileBrowseField(CharField): +class FileBrowseField(CharField, metaclass=models.SubfieldBase): description = "FileBrowseField" - __metaclass__ = models.SubfieldBase def __init__(self, *args, **kwargs): self.site = kwargs.pop('filebrowser_site', site) diff --git a/filebrowser/forms.py b/filebrowser/forms.py index 4cd9973b1..8eae475e9 100644 --- a/filebrowser/forms.py +++ b/filebrowser/forms.py @@ -17,12 +17,12 @@ # CHOICES TRANSPOSE_CHOICES = ( - ("", u"-----"), - ("0", _(u"Flip horizontal")), - ("1", _(u"Flip vertical")), - ("2", _(u"Rotate 90° CW")), - ("4", _(u"Rotate 90° CCW")), - ("3", _(u"Rotate 180°")), + ("", "-----"), + ("0", _("Flip horizontal")), + ("1", _("Flip vertical")), + ("2", _("Rotate 90° CW")), + ("4", _("Rotate 90° CCW")), + ("3", _("Rotate 180°")), ) @@ -36,16 +36,16 @@ def __init__(self, path, *args, **kwargs): self.site = kwargs.pop("filebrowser_site", None) super(CreateDirForm, self).__init__(*args, **kwargs) - name = forms.CharField(widget=forms.TextInput(attrs=dict({ 'class': 'vTextField' }, max_length=50, min_length=3)), label=_(u'Name'), help_text=_(u'Only letters, numbers, underscores, spaces and hyphens are allowed.'), required=True) + name = forms.CharField(widget=forms.TextInput(attrs=dict({ 'class': 'vTextField' }, max_length=50, min_length=3)), label=_('Name'), help_text=_('Only letters, numbers, underscores, spaces and hyphens are allowed.'), required=True) def clean_name(self): if self.cleaned_data['name']: # only letters, numbers, underscores, spaces and hyphens are allowed. if not alnum_name_re.search(self.cleaned_data['name']): - raise forms.ValidationError(_(u'Only letters, numbers, underscores, spaces and hyphens are allowed.')) + raise forms.ValidationError(_('Only letters, numbers, underscores, spaces and hyphens are allowed.')) # Folder must not already exist. if self.site.storage.isdir(os.path.join(self.path, convert_filename(self.cleaned_data['name']))): - raise forms.ValidationError(_(u'The Folder already exists.')) + raise forms.ValidationError(_('The Folder already exists.')) return convert_filename(self.cleaned_data['name']) @@ -54,8 +54,8 @@ class ChangeForm(forms.Form): Form for renaming a file/folder. """ - custom_action = forms.ChoiceField(label=_(u'Actions'), required=False) - name = forms.CharField(widget=forms.TextInput(attrs=dict({ 'class': 'vTextField' }, max_length=50, min_length=3)), label=_(u'Name'), help_text=_(u'Only letters, numbers, underscores, spaces and hyphens are allowed.'), required=True) + custom_action = forms.ChoiceField(label=_('Actions'), required=False) + name = forms.CharField(widget=forms.TextInput(attrs=dict({ 'class': 'vTextField' }, max_length=50, min_length=3)), label=_('Name'), help_text=_('Only letters, numbers, underscores, spaces and hyphens are allowed.'), required=True) def __init__(self, *args, **kwargs): self.path = kwargs.pop("path", None) @@ -65,7 +65,7 @@ def __init__(self, *args, **kwargs): super(ChangeForm, self).__init__(*args, **kwargs) # Initialize choices of custom actions - choices = [("",u"-----"),] + choices = [("","-----"),] for name, action in self.site.applicable_actions(self.fileobject): choices.append((name, action.short_description)) @@ -76,12 +76,12 @@ def clean_name(self): if self.cleaned_data['name']: # only letters, numbers, underscores, spaces and hyphens are allowed. if not alnum_name_re.search(self.cleaned_data['name']): - raise forms.ValidationError(_(u'Only letters, numbers, underscores, spaces and hyphens are allowed.')) + raise forms.ValidationError(_('Only letters, numbers, underscores, spaces and hyphens are allowed.')) # folder/file must not already exist. if self.site.storage.isdir(os.path.join(self.path, convert_filename(self.cleaned_data['name']))) and os.path.join(self.path, convert_filename(self.cleaned_data['name'])) != self.fileobject.path: - raise forms.ValidationError(_(u'The Folder already exists.')) + raise forms.ValidationError(_('The Folder already exists.')) elif self.site.storage.isfile(os.path.join(self.path, convert_filename(self.cleaned_data['name']))) and os.path.join(self.path, convert_filename(self.cleaned_data['name'])) != self.fileobject.path: - raise forms.ValidationError(_(u'The File already exists.')) + raise forms.ValidationError(_('The File already exists.')) return convert_filename(self.cleaned_data['name']) diff --git a/filebrowser/functions.py b/filebrowser/functions.py index 8c86f9ca2..4cc17f7e2 100644 --- a/filebrowser/functions.py +++ b/filebrowser/functions.py @@ -125,9 +125,9 @@ def sort_by_attr(seq, attr): # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not # only to provide stable sorting, but mainly to eliminate comparison of objects # (which can be expensive or prohibited) in case of equal attribute values. - intermed = map(None, map(getattr, seq, (attr,)*len(seq)), xrange(len(seq)), seq) + intermed = map(None, list(map(getattr, seq, (attr,)*len(seq))), range(len(seq)), seq) intermed.sort() - return map(operator.getitem, intermed, (-1,) * len(intermed)) + return list(map(operator.getitem, intermed, (-1,) * len(intermed))) def url_join(*args): @@ -179,7 +179,7 @@ def get_file_type(filename): file_extension = os.path.splitext(filename)[1].lower() file_type = '' - for k,v in EXTENSIONS.iteritems(): + for k,v in EXTENSIONS.items(): for extension in v: if file_extension == extension.lower(): file_type = k @@ -261,7 +261,7 @@ def handle_file_upload(path, file, site): try: file_path = os.path.join(path, file.name) uploadedfile = site.storage.save(file_path, file) - except Exception, inst: + except Exception as inst: raise inst return uploadedfile @@ -273,7 +273,7 @@ def is_selectable(filename, selecttype): file_extension = os.path.splitext(filename)[1].lower() select_types = [] - for k,v in SELECT_FORMATS.iteritems(): + for k,v in SELECT_FORMATS.items(): for extension in v: if file_extension == extension.lower(): select_types.append(k) @@ -309,7 +309,7 @@ def version_generator(value, version_prefix, force=None, site=None): version = scale_and_crop(im, VERSIONS[version_prefix]['width'], VERSIONS[version_prefix]['height'], VERSIONS[version_prefix]['opts']) if not version: version = im - if 'methods' in VERSIONS[version_prefix].keys(): + if 'methods' in list(VERSIONS[version_prefix].keys()): for m in VERSIONS[version_prefix]['methods']: if callable(m): version = m(version) @@ -379,7 +379,7 @@ def convert_filename(value): chunks = value.split(os.extsep) normalized = [] for v in chunks: - v = unicodedata.normalize('NFKD', unicode(v)).encode('ascii', 'ignore') + v = unicodedata.normalize('NFKD', str(v)).encode('ascii', 'ignore') v = re.sub('[^\w\s-]', '', v).strip() normalized.append(v) diff --git a/filebrowser/management/commands/fb_version_generate.py b/filebrowser/management/commands/fb_version_generate.py index 595965e42..4728a9e10 100644 --- a/filebrowser/management/commands/fb_version_generate.py +++ b/filebrowser/management/commands/fb_version_generate.py @@ -33,7 +33,7 @@ def handle(self, *args, **options): for version in VERSIONS: self.stdout.write(' * %s\n' % version) - version_name = raw_input('(leave blank to generate all versions): ') + version_name = input('(leave blank to generate all versions): ') if version_name == "": selected_version = None @@ -52,7 +52,7 @@ def handle(self, *args, **options): filter_re = [] for exp in EXCLUDE: filter_re.append(re.compile(exp)) - for k,v in VERSIONS.iteritems(): + for k,v in VERSIONS.items(): exp = (r'_%s(%s)') % (k, '|'.join(EXTENSION_LIST)) filter_re.append(re.compile(exp)) diff --git a/filebrowser/management/commands/fb_version_remove.py b/filebrowser/management/commands/fb_version_remove.py index cb698a419..78f91d36e 100644 --- a/filebrowser/management/commands/fb_version_remove.py +++ b/filebrowser/management/commands/fb_version_remove.py @@ -34,7 +34,7 @@ def handle(self, *args, **options): while 1: self.stdout.write('\nOlder versions of the FileBrowser used to prefix the filename with the version name.\n') self.stdout.write('Current version of the FileBrowser adds the version name as suffix.\n') - prefix_or_suffix = raw_input('"p" for prefix or "s" for suffix (leave blank for "%s"): ' % default_prefix_or_suffix) + prefix_or_suffix = input('"p" for prefix or "s" for suffix (leave blank for "%s"): ' % default_prefix_or_suffix) if default_prefix_or_suffix and prefix_or_suffix == '': prefix_or_suffix = default_prefix_or_suffix @@ -46,7 +46,7 @@ def handle(self, *args, **options): # get version name while 1: - version_name = raw_input('\nversion name as defined with VERSIONS: ') + version_name = input('\nversion name as defined with VERSIONS: ') if version_name == "": self.stderr.write('Error: You have to enter a version name.\n') @@ -82,7 +82,7 @@ def handle(self, *args, **options): # ask to make sure do_remove = "" self.stdout.write('Are Sure you want to delete these files?\n') - do_remove = raw_input('"y" for Yes or "n" for No (leave blank for "n"): ') + do_remove = input('"y" for Yes or "n" for No (leave blank for "n"): ') # if "yes" we delete. any different case we finish without removing anything if do_remove == "y": diff --git a/filebrowser/settings.py b/filebrowser/settings.py index 3069e89f5..7797b6a1c 100644 --- a/filebrowser/settings.py +++ b/filebrowser/settings.py @@ -97,7 +97,7 @@ # Exclude files matching any of the following regular expressions # Default is to exclude 'thumbnail' style naming of image-thumbnails. EXTENSION_LIST = [] -for exts in EXTENSIONS.values(): +for exts in list(EXTENSIONS.values()): EXTENSION_LIST += exts EXCLUDE = getattr(settings, 'FILEBROWSER_EXCLUDE', (r'_(%(exts)s)_.*_q\d{1,3}\.(%(exts)s)' % {'exts': ('|'.join(EXTENSION_LIST))},)) @@ -122,7 +122,7 @@ # Traverse directories when searching SEARCH_TRAVERSE = getattr(settings, "FILEBROWSER_SEARCH_TRAVERSE", False) # Default Upload and Version Permissions -DEFAULT_PERMISSIONS = getattr(settings, "FILEBROWSER_DEFAULT_PERMISSIONS", 0755) +DEFAULT_PERMISSIONS = getattr(settings, "FILEBROWSER_DEFAULT_PERMISSIONS", 0o755) # EXTRA TRANSLATION STRINGS # The following strings are not availabe within views or templates diff --git a/filebrowser/sites.py b/filebrowser/sites.py index e0a496c53..2d0e6d4cf 100644 --- a/filebrowser/sites.py +++ b/filebrowser/sites.py @@ -70,19 +70,19 @@ def get_site_dict(app_name='filebrowser'): Return a dict with all *deployed* FileBrowser sites that have a given app_name. """ - if not _sites_cache.has_key(app_name): + if app_name not in _sites_cache: return {} # Get names of all deployed filebrowser sites with a give app_name deployed = get_resolver(get_urlconf()).app_dict[app_name] # Get the deployed subset from the cache - return dict((k,v) for k, v in _sites_cache[app_name].iteritems() if k in deployed) + return dict((k,v) for k, v in _sites_cache[app_name].items() if k in deployed) def register_site(app_name, site_name, site): """ Add a site into the site dict. """ - if not _sites_cache.has_key(app_name): + if app_name not in _sites_cache: _sites_cache[app_name] = {} _sites_cache[app_name][site_name] = site @@ -196,7 +196,7 @@ def actions(self): Get all the enabled actions as a list of (name, func). The list is sorted alphabetically by actions names """ - res = self._actions.items() + res = list(self._actions.items()) res.sort(key=lambda name_func: name_func[0]) return res @@ -212,7 +212,7 @@ def browse(self, request): filter_re = [] for exp in EXCLUDE: filter_re.append(re.compile(exp)) - for k,v in VERSIONS.iteritems(): + for k,v in VERSIONS.items(): exp = (r'_%s(%s)') % (k, '|'.join(EXTENSION_LIST)) filter_re.append(re.compile(exp)) @@ -233,7 +233,7 @@ def filter_browse(item): return True query = request.GET.copy() - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) filelisting = FileListing(path, filter_func=filter_browse, @@ -283,7 +283,7 @@ def filter_browse(item): 'page': page, 'filelisting': filelisting, 'query': query, - 'title': _(u'FileBrowser'), + 'title': _('FileBrowser'), 'settings_var': get_settings_var(directory=self.directory), 'breadcrumbs': get_breadcrumbs(query, query.get('dir', '')), 'breadcrumbs_title': "", @@ -301,7 +301,7 @@ def createdir(self, request): """ from filebrowser.forms import CreateDirForm query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) if request.method == 'POST': form = CreateDirForm(path, request.POST, filebrowser_site=self) @@ -316,7 +316,8 @@ def createdir(self, request): messages.add_message(request, messages.SUCCESS, _('The Folder %s was successfully created.') % form.cleaned_data['name']) redirect_url = reverse("filebrowser:fb_browse", current_app=self.name) + query_helper(query, "ot=desc,o=date", "ot,o,filter_type,filter_date,q,p") return HttpResponseRedirect(redirect_url) - except OSError, (errno, strerror): + except OSError as xxx_todo_changeme: + (errno, strerror) = xxx_todo_changeme.args if errno == 13: form.errors['name'] = forms.util.ErrorList([_('Permission denied.')]) else: @@ -328,10 +329,10 @@ def createdir(self, request): 'filebrowser/createdir.html', {'form': form, 'query': query, - 'title': _(u'New Folder'), + 'title': _('New Folder'), 'settings_var': get_settings_var(directory=self.directory), 'breadcrumbs': get_breadcrumbs(query, query.get('dir', '')), - 'breadcrumbs_title': _(u'New Folder'), + 'breadcrumbs_title': _('New Folder'), 'filebrowser_site': self } ) #context_instance=Context(request, current_app=self.name)) @@ -341,15 +342,15 @@ def upload(self, request): Multipe File Upload. """ query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) return render(request, 'filebrowser/upload.html', { 'query': query, - 'title': _(u'Select files to upload'), + 'title': _('Select files to upload'), 'settings_var': get_settings_var(directory=self.directory), 'breadcrumbs': get_breadcrumbs(query, query.get('dir', '')), - 'breadcrumbs_title': _(u'Upload'), + 'breadcrumbs_title': _('Upload'), 'filebrowser_site': self }) #context_instance=Context(request, current_app=self.name)) @filebrowser_check() @@ -358,7 +359,7 @@ def delete_confirm(self, request): Delete existing File/Directory. """ query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) fileobject = FileObject(os.path.join(path, query.get('filename', '')), site=self) if fileobject.filetype == "Folder": filelisting = FileListing(os.path.join(path, fileobject.filename), @@ -380,10 +381,10 @@ def delete_confirm(self, request): 'filelisting': filelisting, 'additional_files': additional_files, 'query': query, - 'title': _(u'Confirm delete'), + 'title': _('Confirm delete'), 'settings_var': get_settings_var(directory=self.directory), 'breadcrumbs': get_breadcrumbs(query, query.get('dir', '')), - 'breadcrumbs_title': _(u'Confirm delete'), + 'breadcrumbs_title': _('Confirm delete'), 'filebrowser_site': self }) #, context_instance=Context(request, current_app=self.name)) @@ -397,7 +398,7 @@ def delete(self, request): Delete existing File/Directory. """ query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) fileobject = FileObject(os.path.join(path, query.get('filename', '')), site=self) if request.GET: @@ -406,7 +407,7 @@ def delete(self, request): # COG: must delete Doc objects docs = Doc.objects.filter(file=fileobject.path) for doc in docs: - print 'Deleting doc=%s' % doc + print('Deleting doc=%s' % doc) doc.delete() self.filebrowser_pre_delete.send(sender=request, path=fileobject.path, name=fileobject.filename) @@ -414,7 +415,9 @@ def delete(self, request): fileobject.delete() self.filebrowser_post_delete.send(sender=request, path=fileobject.path, name=fileobject.filename) messages.add_message(request, messages.SUCCESS, _('Successfully deleted %s') % fileobject.filename) - except OSError, (errno, strerror): + except OSError as xxx_todo_changeme2: + # TODO: define error-message + (errno, strerror) = xxx_todo_changeme2.args # TODO: define error-message pass redirect_url = reverse("filebrowser:fb_browse", current_app=self.name) + query_helper(query, "", "filename,filetype") @@ -436,7 +439,7 @@ def detail(self, request): """ from filebrowser.forms import ChangeForm query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) fileobject = FileObject(os.path.join(path, query.get('filename', '')), site=self) if request.method == 'POST': @@ -467,7 +470,8 @@ def detail(self, request): else: redirect_url = reverse("filebrowser:fb_browse", current_app=self.name) + query_helper(query, "", "filename") return HttpResponseRedirect(redirect_url) - except OSError, (errno, strerror): + except OSError as xxx_todo_changeme1: + (errno, strerror) = xxx_todo_changeme1.args form.errors['name'] = forms.util.ErrorList([_('Error.')]) else: form = ChangeForm(initial={"name": fileobject.filename}, path=path, fileobject=fileobject, filebrowser_site=self) @@ -476,10 +480,10 @@ def detail(self, request): 'form': form, 'fileobject': fileobject, 'query': query, - 'title': u'%s' % fileobject.filename, + 'title': '%s' % fileobject.filename, 'settings_var': get_settings_var(directory=self.directory), 'breadcrumbs': get_breadcrumbs(query, query.get('dir', '')), - 'breadcrumbs_title': u'%s' % fileobject.filename, + 'breadcrumbs_title': '%s' % fileobject.filename, 'filebrowser_site': self }) # context_instance=Context(request, current_app=self.name)) @@ -488,7 +492,7 @@ def version(self, request): Version detail. """ query = request.GET - path = u'%s' % os.path.join(self.directory, query.get('dir', '')) + path = '%s' % os.path.join(self.directory, query.get('dir', '')) fileobject = FileObject(os.path.join(path, query.get('filename', '')), site=self) return render(request, 'filebrowser/version.html', { @@ -518,7 +522,7 @@ def _upload_file(self, request): # TODO: This needs some attention, do we use this at all? folder = request.POST.get('folder') if len(request.FILES) == 1: - filedata = request.FILES.values()[0] + filedata = list(request.FILES.values())[0] else: raise Http404('Invalid request! Multiple files included.') # filedata.name = convert_filename(upload.name) @@ -557,7 +561,7 @@ def _upload_file(self, request): site = FileBrowserSite(name='filebrowser', storage=storage) # Default actions -from actions import * +from .actions import * site.add_action(flip_horizontal) site.add_action(flip_vertical) site.add_action(rotate_90_clockwise) diff --git a/filebrowser/templatetags/fb_csrf.py b/filebrowser/templatetags/fb_csrf.py index 93803fae4..48bdeef89 100644 --- a/filebrowser/templatetags/fb_csrf.py +++ b/filebrowser/templatetags/fb_csrf.py @@ -12,9 +12,9 @@ def render(self, context): csrf_token = context.get('csrf_token', None) if csrf_token: if csrf_token == 'NOTPROVIDED': - return mark_safe(u"") + return mark_safe("") else: - return mark_safe(u"
" % (csrf_token)) + return mark_safe("
" % (csrf_token)) else: # It's very probable that the token is missing because of # misconfiguration, so we raise a warning @@ -22,7 +22,7 @@ def render(self, context): if settings.DEBUG: import warnings warnings.warn("A {% csrf_token %} was used in a template, but the context did not provide the value. This is usually caused by not using RequestContext.") - return u'' + return '' def fb_csrf_token(parser, token): return CsrfTokenNode() diff --git a/filebrowser/templatetags/fb_pagination.py b/filebrowser/templatetags/fb_pagination.py index 13e07a479..97460bf7c 100644 --- a/filebrowser/templatetags/fb_pagination.py +++ b/filebrowser/templatetags/fb_pagination.py @@ -23,24 +23,24 @@ def pagination(context): # If there are 10 or fewer pages, display links to every page. # Otherwise, do some fancy if paginator.num_pages <= 10: - page_range = range(paginator.num_pages) + page_range = list(range(paginator.num_pages)) else: # Insert "smart" pagination links, so that there are always ON_ENDS # links at either end of the list of pages, and there are always # ON_EACH_SIDE links at either end of the "current page" link. page_range = [] if page_num > (ON_EACH_SIDE + ON_ENDS): - page_range.extend(range(0, ON_EACH_SIDE - 1)) + page_range.extend(list(range(0, ON_EACH_SIDE - 1))) page_range.append(DOT) - page_range.extend(range(page_num - ON_EACH_SIDE, page_num + 1)) + page_range.extend(list(range(page_num - ON_EACH_SIDE, page_num + 1))) else: - page_range.extend(range(0, page_num + 1)) + page_range.extend(list(range(0, page_num + 1))) if page_num < (paginator.num_pages - ON_EACH_SIDE - ON_ENDS - 1): - page_range.extend(range(page_num + 1, page_num + ON_EACH_SIDE + 1)) + page_range.extend(list(range(page_num + 1, page_num + ON_EACH_SIDE + 1))) page_range.append(DOT) - page_range.extend(range(paginator.num_pages - ON_ENDS, paginator.num_pages)) + page_range.extend(list(range(paginator.num_pages - ON_ENDS, paginator.num_pages))) else: - page_range.extend(range(page_num + 1, paginator.num_pages)) + page_range.extend(list(range(page_num + 1, paginator.num_pages))) return { 'page_range': page_range, diff --git a/filebrowser/templatetags/fb_tags.py b/filebrowser/templatetags/fb_tags.py index dd00ce3a3..2334d266b 100644 --- a/filebrowser/templatetags/fb_tags.py +++ b/filebrowser/templatetags/fb_tags.py @@ -50,16 +50,16 @@ def get_query_string(p, new_params=None, remove=None): if new_params is None: new_params = {} if remove is None: remove = [] for r in remove: - for k in p.keys(): + for k in list(p.keys()): #if k.startswith(r): if k == r: del p[k] - for k, v in new_params.items(): + for k, v in list(new_params.items()): if k in p and v is None: del p[k] elif v is not None: p[k] = v - return '?' + '&'.join([u'%s=%s' % (urlquote(k), urlquote(v)) for k, v in p.items()]) + return '?' + '&'.join(['%s=%s' % (urlquote(k), urlquote(v)) for k, v in list(p.items())]) def string_to_dict(string): @@ -132,7 +132,7 @@ def selectable(parser, token): try: tag, filetype, format = token.split_contents() except: - raise TemplateSyntaxError, "%s tag requires 2 arguments" % token.contents.split()[0] + raise TemplateSyntaxError("%s tag requires 2 arguments" % token.contents.split()[0]) return SelectableNode(filetype, format) diff --git a/filebrowser/templatetags/fb_versions.py b/filebrowser/templatetags/fb_versions.py index ac66e0269..10af1ec1f 100644 --- a/filebrowser/templatetags/fb_versions.py +++ b/filebrowser/templatetags/fb_versions.py @@ -77,9 +77,9 @@ def version(parser, token): try: tag, src, version_prefix = token.split_contents() except: - raise TemplateSyntaxError, "%s tag requires 2 arguments" % token.contents.split()[0] + raise TemplateSyntaxError("%s tag requires 2 arguments" % token.contents.split()[0]) if (version_prefix[0] == version_prefix[-1] and version_prefix[0] in ('"', "'")) and version_prefix.lower()[1:-1] not in VERSIONS: - raise TemplateSyntaxError, "%s tag received bad version_prefix %s" % (tag, version_prefix) + raise TemplateSyntaxError("%s tag received bad version_prefix %s" % (tag, version_prefix)) return VersionNode(src, version_prefix) @@ -146,13 +146,13 @@ def version_object(parser, token): #tag, src, version_prefix = token.split_contents() tag, arg = token.contents.split(None, 1) except: - raise TemplateSyntaxError, "%s tag requires arguments" % token.contents.split()[0] + raise TemplateSyntaxError("%s tag requires arguments" % token.contents.split()[0]) m = re.search(r'(.*?) (.*?) as (\w+)', arg) if not m: - raise TemplateSyntaxError, "%r tag had invalid arguments" % tag + raise TemplateSyntaxError("%r tag had invalid arguments" % tag) src, version_prefix, var_name = m.groups() if (version_prefix[0] == version_prefix[-1] and version_prefix[0] in ('"', "'")) and version_prefix.lower()[1:-1] not in VERSIONS: - raise TemplateSyntaxError, "%s tag received bad version_prefix %s" % (tag, version_prefix) + raise TemplateSyntaxError("%s tag received bad version_prefix %s" % (tag, version_prefix)) return VersionObjectNode(src, version_prefix, var_name) @@ -184,9 +184,9 @@ def version_setting(parser, token): try: tag, version_prefix = token.split_contents() except: - raise TemplateSyntaxError, "%s tag requires 1 argument" % token.contents.split()[0] + raise TemplateSyntaxError("%s tag requires 1 argument" % token.contents.split()[0]) if (version_prefix[0] == version_prefix[-1] and version_prefix[0] in ('"', "'")) and version_prefix.lower()[1:-1] not in VERSIONS: - raise TemplateSyntaxError, "%s tag received bad version_prefix %s" % (tag, version_prefix) + raise TemplateSyntaxError("%s tag received bad version_prefix %s" % (tag, version_prefix)) return VersionSettingNode(version_prefix) diff --git a/filebrowser/tests/sites.py b/filebrowser/tests/sites.py index dc8f5b9bc..ecd123fea 100644 --- a/filebrowser/tests/sites.py +++ b/filebrowser/tests/sites.py @@ -12,7 +12,7 @@ import os import sys import shutil -from urllib import urlencode +from urllib.parse import urlencode from types import MethodType # DJANGO IMPORTS @@ -136,7 +136,7 @@ def test_detail(test): # At this moment all versions should be generated. Check that. for version_suffix in VERSIONS: path = get_version_path(test.testfile.path, version_suffix, site=test.site) - print 'PATH=%s' % path + print('PATH=%s' % path) test.assertTrue(test.site.storage.exists(path)) # Attemp renaming the file @@ -218,7 +218,7 @@ def setUp(self): def tearDown(self): # Delete a left-over tmp directories, if there's any if hasattr(self, 'tmpdir') and self.tmpdir: - print "Removing left-over tmp dir:", self.tmpdir.path + print("Removing left-over tmp dir:", self.tmpdir.path) self.site.storage.rmtree(self.tmpdir.path) def runTest(self): @@ -243,7 +243,7 @@ def runTest(self): ## Create a test class for each deployed filebrowser site for site in all_sites: - print 'Creating Test for the FileBrowser site:', site + print('Creating Test for the FileBrowser site:', site) # Create a subclass of TestCase testcase_class = type('TestSite_' + site, (TestCase,), {'site_name': site, 'c': Client(), 'tmpdirs': None}) # Add setUp, tearDown, and runTest methods diff --git a/filebrowser/widgets.py b/filebrowser/widgets.py index 2b32120a1..ece1bea5f 100644 --- a/filebrowser/widgets.py +++ b/filebrowser/widgets.py @@ -19,7 +19,7 @@ class FileInput(DjangoClearableFileInput): initial_text = ugettext_lazy('Currently') input_text = ugettext_lazy('Change') clear_checkbox_label = ugettext_lazy('Clear') - template_with_initial = u'%(input)s %(preview)s' + template_with_initial = '%(input)s %(preview)s' def render(self, name, value, attrs=None): substitutions = { @@ -29,7 +29,7 @@ def render(self, name, value, attrs=None): 'preview': '', 'clear_checkbox_label': self.clear_checkbox_label, } - template = u'%(input)s' + template = '%(input)s' substitutions['input'] = super(DjangoClearableFileInput, self).render(name, value, attrs) if value and hasattr(value, "url"): @@ -53,8 +53,8 @@ class ClearableFileInput(DjangoClearableFileInput): input_text = ugettext_lazy('Change') clear_checkbox_label = ugettext_lazy('Clear') - template_with_initial = u'

%(initial_text)s: %(initial)s%(clear_template)s
%(input_text)s: %(input)s %(preview)s

' - template_with_clear = u'%(clear)s ' + template_with_initial = '

%(initial_text)s: %(initial)s%(clear_template)s
%(input_text)s: %(input)s %(preview)s

' + template_with_clear = '%(clear)s ' # template_with_initial = u'%(initial_text)s: %(initial)s %(clear_template)s
%(input_text)s: %(input)s' # template_with_clear = u'%(clear)s ' @@ -70,12 +70,12 @@ def render(self, name, value, attrs=None): 'preview': '', 'clear_checkbox_label': self.clear_checkbox_label, } - template = u'%(input)s' + template = '%(input)s' substitutions['input'] = super(DjangoClearableFileInput, self).render(name, value, attrs) if value and hasattr(value, "url"): template = self.template_with_initial - substitutions['initial'] = (u'%s' % (value.url, value)) + substitutions['initial'] = ('%s' % (value.url, value)) if not self.is_required: checkbox_name = self.clear_checkbox_name(name) checkbox_id = self.clear_checkbox_id(checkbox_name) diff --git a/resources/scripts/configure_search.py b/resources/scripts/configure_search.py index cf43083f3..93271a64c 100644 --- a/resources/scripts/configure_search.py +++ b/resources/scripts/configure_search.py @@ -1,5 +1,5 @@ # Python script to configure off-band a project advanced search -import sys, os, ConfigParser +import sys, os, configparser sys.path.append( os.path.abspath(os.path.dirname('.')) ) os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' @@ -7,7 +7,7 @@ from cog.config.search import read_config #from django.conf import settings -print 'Upgrading COG' +print('Upgrading COG') # read search configurations configs = { 'StandardDistribution': 'cog/config/search/standard_distribution.cfg', diff --git a/resources/scripts/fix_broken_links.py b/resources/scripts/fix_broken_links.py index c0a825ec2..406800add 100644 --- a/resources/scripts/fix_broken_links.py +++ b/resources/scripts/fix_broken_links.py @@ -19,7 +19,7 @@ for post in Post.objects.all(): if BAD_LINK in post.body: - print "Found bad link at URL: %s" % post.url + print("Found bad link at URL: %s" % post.url) post.body = post.body.replace(BAD_LINK, GOOD_LINK) post.save() diff --git a/resources/scripts/insert_esgf_user.py b/resources/scripts/insert_esgf_user.py index 6daf1f098..3e4f11e20 100644 --- a/resources/scripts/insert_esgf_user.py +++ b/resources/scripts/insert_esgf_user.py @@ -28,11 +28,11 @@ openid = '' for userOpenID in user.useropenid_set.all(): openid = userOpenID.claimed_id - print 'User openid=%s' % openid + print('User openid=%s' % openid) # verify user was inserted esgfUser2 = esgfDatabaseManager.getUserByOpenid( openid ) -print "Retrieved user with openid=%s" % esgfUser2.openid +print("Retrieved user with openid=%s" % esgfUser2.openid) # cleanup tgis execution user.delete() \ No newline at end of file diff --git a/resources/scripts/migrate_roles.py b/resources/scripts/migrate_roles.py index 66343912d..7610e8463 100644 --- a/resources/scripts/migrate_roles.py +++ b/resources/scripts/migrate_roles.py @@ -14,7 +14,7 @@ # loop over users for user in User.objects.all(): - print 'Processing user=%s' % user + print('Processing user=%s' % user) # FIXME #if user.last_name=='Cinquini': @@ -28,7 +28,7 @@ user.groups.remove( ugroup ) # add 'contributor' group cgroup = project.getContributorGroup() - print '\tUser %s: changing group %s to %s' % (user, ugroup, cgroup) + print('\tUser %s: changing group %s to %s' % (user, ugroup, cgroup)) user.groups.add( cgroup ) user.save() except Project.DoesNotExist: @@ -37,7 +37,7 @@ # delete obsolete Permission objects for permission in user.user_permissions.all(): if 'Admin Permission' in permission.name or 'User Permission' in permission.name: - print '\tUser: %s deleting permission: %s' % (user, permission) + print('\tUser: %s deleting permission: %s' % (user, permission)) user.user_permissions.remove(permission) user.save() permission.delete() \ No newline at end of file diff --git a/resources/scripts/migrate_users.py b/resources/scripts/migrate_users.py index 03015ae89..cbe0af392 100644 --- a/resources/scripts/migrate_users.py +++ b/resources/scripts/migrate_users.py @@ -64,7 +64,7 @@ if settings.ESGF_HOSTNAME in esgfUser.openid and user.username in esgfUser.openid: if not UserOpenID.objects.filter(claimed_id=esgfUser.openid).exists(): openid = UserOpenID.objects.create(user=user, claimed_id=esgfUser.openid, display_id=esgfUser.openid) - print 'Assigned ESGF openid=%s to CoG user=%s' % (openid.claimed_id, user) + print('Assigned ESGF openid=%s to CoG user=%s' % (openid.claimed_id, user)) found = True if not found: esgfDatabaseManager.insertUser(userp) diff --git a/resources/scripts/send_email.py b/resources/scripts/send_email.py index 044aee449..c37e93fa0 100644 --- a/resources/scripts/send_email.py +++ b/resources/scripts/send_email.py @@ -36,11 +36,11 @@ def __init__ (self, toAddress, subject, message, fromAddress=EMAIL_SENDER, mime_ def run(self): #print "From: %s" % self.fromAddress - print "To: %s" % self.toAddress - print 'From: %s' % self.fromAddress - print "Subject: %s" % self.subject - print "Message: %s" % self.message - print "Mime Type: %s" % self.mime_type + print("To: %s" % self.toAddress) + print('From: %s' % self.fromAddress) + print("Subject: %s" % self.subject) + print("Message: %s" % self.message) + print("Mime Type: %s" % self.mime_type) # use local mail server #toUser.email_user(subject, message, from_email=fromAddress) @@ -59,7 +59,7 @@ def run(self): s.login(EMAIL_USERNAME, EMAIL_PASSWORD) s.sendmail(self.fromAddress, [self.toAddress], msg.as_string()) s.quit() - print 'Email sent.' + print('Email sent.') if __name__ == '__main__': diff --git a/settings.py b/settings.py index 0e6d74925..58b5e8b03 100644 --- a/settings.py +++ b/settings.py @@ -1,7 +1,7 @@ import os import logging import re -from cog.utils import str2bool +from .cog.utils import str2bool rel = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x) @@ -12,8 +12,8 @@ Each parameter has a default value. ''' -from cog.site_manager import siteManager -from cog.constants import SECTION_ESGF, SECTION_PID +from .cog.site_manager import siteManager +from .cog.constants import SECTION_ESGF, SECTION_PID SITE_NAME = siteManager.get('SITE_NAME', default='Local CoG') SITE_DOMAIN = siteManager.get('SITE_DOMAIN', default='localhost:8000') @@ -41,23 +41,23 @@ else: DEBUG = False ALLOWED_HOSTS = siteManager.get('ALLOWED_HOSTS', default=SITE_DOMAIN).split(",") -print 'Using DEBUG=%s ALLOWED_HOSTS=%s' % (DEBUG, ALLOWED_HOSTS) +print('Using DEBUG=%s ALLOWED_HOSTS=%s' % (DEBUG, ALLOWED_HOSTS)) IDP_WHITELIST = siteManager.get('IDP_WHITELIST', default=None) -print 'Using IdP whitelist(s): %s' % IDP_WHITELIST +print('Using IdP whitelist(s): %s' % IDP_WHITELIST) KNOWN_PROVIDERS = siteManager.get('KNOWN_PROVIDERS', default=None) -print 'Using list of known Identity Providers: %s' % KNOWN_PROVIDERS +print('Using list of known Identity Providers: %s' % KNOWN_PROVIDERS) PEER_NODES = siteManager.get('PEER_NODES', default=None) USE_CAPTCHA = str2bool(siteManager.get('USE_CAPTCHA', default='True')) -print 'Using list of ESGF/CoG peer nodes from: %s' % PEER_NODES +print('Using list of ESGF/CoG peer nodes from: %s' % PEER_NODES) # DEVELOPMENT/PRODUCTION server switch PRODUCTION_SERVER = str2bool(siteManager.get('PRODUCTION_SERVER', default='False')) -print 'Production server flag=%s' % PRODUCTION_SERVER +print('Production server flag=%s' % PRODUCTION_SERVER) WPS_ENDPOINT = siteManager.get('WPS_ENDPOINT', default=None); # Fields that will be added to the query string WPS_FIELDS = siteManager.get('WPS_FIELDS', default='index_node').split(','); WPS_DATACART = str2bool(siteManager.get('WPS_DATACART', default='False')) -print 'WPS endpoint: %s, datacart enabled: %s, fields: %s' % (WPS_ENDPOINT, WPS_DATACART, ','.join(WPS_FIELDS)) +print('WPS endpoint: %s, datacart enabled: %s, fields: %s' % (WPS_ENDPOINT, WPS_DATACART, ','.join(WPS_FIELDS))) # FIXME # ESGF specific settings @@ -169,7 +169,7 @@ # must be writable by web server PROJECT_CONFIG_DIR = os.path.join(MEDIA_ROOT, 'config') -print 'Loading custom templates from directories: %s, %s' % (MYTEMPLATES, MYMEDIA) +print('Loading custom templates from directories: %s, %s' % (MYTEMPLATES, MYMEDIA)) # Make this unique, and don't share it with anybody. #SECRET_KEY = 'yb@$-bub$i_mrxqe5it)v%p=^(f-h&x3%uy040x))19g^iha&#' diff --git a/setup.py b/setup.py index eca7d56a4..128b911c4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ import os from setuptools import setup, find_packages -from cog.installation.setup import CogSetupCommand +from .cog.installation.setup import CogSetupCommand # Utility function to read the README file. def read(fname): diff --git a/urls.py b/urls.py index abde4229a..d8139f2df 100644 --- a/urls.py +++ b/urls.py @@ -2,10 +2,10 @@ from django.conf.urls import url, include from django.contrib import admin import django.views.static, django.views.generic -import cog.views +from . import cog.views from django.http.response import HttpResponseNotFound -from filebrowser.sites import site +from .filebrowser.sites import site admin.autodiscover() From 0d9354ddaf58366c69e5b372dbb326310c05b30e Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 14:44:28 -0700 Subject: [PATCH 09/65] Remove periods from imports in the installation Python scripts --- settings.py | 6 +++--- setup.py | 2 +- urls.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/settings.py b/settings.py index 58b5e8b03..21d3319a2 100644 --- a/settings.py +++ b/settings.py @@ -1,7 +1,7 @@ import os import logging import re -from .cog.utils import str2bool +from cog.utils import str2bool rel = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x) @@ -12,8 +12,8 @@ Each parameter has a default value. ''' -from .cog.site_manager import siteManager -from .cog.constants import SECTION_ESGF, SECTION_PID +from cog.site_manager import siteManager +from cog.constants import SECTION_ESGF, SECTION_PID SITE_NAME = siteManager.get('SITE_NAME', default='Local CoG') SITE_DOMAIN = siteManager.get('SITE_DOMAIN', default='localhost:8000') diff --git a/setup.py b/setup.py index 128b911c4..eca7d56a4 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ import os from setuptools import setup, find_packages -from .cog.installation.setup import CogSetupCommand +from cog.installation.setup import CogSetupCommand # Utility function to read the README file. def read(fname): diff --git a/urls.py b/urls.py index d8139f2df..6c487c598 100644 --- a/urls.py +++ b/urls.py @@ -5,7 +5,7 @@ from . import cog.views from django.http.response import HttpResponseNotFound -from .filebrowser.sites import site +from filebrowser.sites import site admin.autodiscover() From 50a079bb8b9606660997b6879c9c10e03fd45a6c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 14:46:47 -0700 Subject: [PATCH 10/65] Turn values into strings for ConfigParser --- cog/installation/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cog/installation/config.py b/cog/installation/config.py index 506eefd8b..d88289357 100644 --- a/cog/installation/config.py +++ b/cog/installation/config.py @@ -160,7 +160,7 @@ def _setupConfig(self): # default search service URL, before any project customization self._safeSet('DEFAULT_SEARCH_URL','http://%s/esg-search/search/' % hostName) # interval between updates of user's projects, during user session - self._safeSet('MY_PROJECTS_REFRESH_SECONDS', 3600) + self._safeSet('MY_PROJECTS_REFRESH_SECONDS', '3600') # optional number of days after which password expire self._safeSet('PWD_EXPIRATION_DAYS','0') # optional top-level URL to redirect user registration (no trailing '/') @@ -180,13 +180,13 @@ def _setupConfig(self): # PEER_NODES = /esg/config/esgf_cogs.xml self._safeSet('PEER_NODES', PEER_NODES) # option to send SESSION and CSRF cookies via SSL only - requires full SSL-encrypted site - self._safeSet('PRODUCTION_SERVER', True) + self._safeSet('PRODUCTION_SERVER', 'True') # ESGF software stack version esgfVersion = self._safeGet("version", default=None) if esgfVersion: self._safeSet('ESGF_VERSION', esgfVersion, override=True) # option to disable CAPTCHA for creating account in automatic testing - self._safeSet('USE_CAPTCHA', True) + self._safeSet('USE_CAPTCHA', 'True') #[ESGF] From 5bdde5089cba7daa98612b375f5b31574efa4d6b Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 14:48:03 -0700 Subject: [PATCH 11/65] Fix indent --- cog/templatetags/cog_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cog/templatetags/cog_utils.py b/cog/templatetags/cog_utils.py index 3c33bdaac..259153ac8 100644 --- a/cog/templatetags/cog_utils.py +++ b/cog/templatetags/cog_utils.py @@ -31,7 +31,7 @@ def wps_arguments(the_dict): for f in settings.WPS_FIELDS: if f in the_dict: - args[f] = the_dict[f] + args[f] = the_dict[f] return args From f5568f86b70ff8e1176df17bce2fc524d4114360 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 10 May 2019 16:01:59 -0700 Subject: [PATCH 12/65] Remove importing 'join' and 'replace' from module 'string' --- cog/services/search.py | 3 +-- cog/templatetags/search_utils.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cog/services/search.py b/cog/services/search.py index c71564084..d1a29f330 100644 --- a/cog/services/search.py +++ b/cog/services/search.py @@ -1,5 +1,4 @@ import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse -from string import join # search service that queries a Solr server through the ESGF RESTful search API class SolrSearchService: @@ -39,7 +38,7 @@ def search(self, searchInput, allFacets=False): params.append( ("facets", "*") ) else: if len(self.facets)>0: - facetlist = join([facet.key for facet in self.facets], ',') + facetlist = ','.join([facet.key for facet in self.facets]) params.append( ("facets", facetlist) ) url = self.url+"?"+urllib.parse.urlencode(params) diff --git a/cog/templatetags/search_utils.py b/cog/templatetags/search_utils.py index ab138cb40..56434213b 100644 --- a/cog/templatetags/search_utils.py +++ b/cog/templatetags/search_utils.py @@ -1,7 +1,6 @@ from django import template from cog.models.search import searchMappings from cog.site_manager import siteManager -from string import replace import json from collections import OrderedDict from django.conf import settings From c26fd56be0a5f9393540393bea7a410dbe56c2cf Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 24 May 2019 16:10:34 -0700 Subject: [PATCH 13/65] Replace StringIO.StringIO with io.StringIO --- cog/installation/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cog/installation/config.py b/cog/installation/config.py index 903c8146f..bbbd88ad9 100644 --- a/cog/installation/config.py +++ b/cog/installation/config.py @@ -103,7 +103,7 @@ def _readEsgfConfig(self): with open(ESGF_PROPERTIES_FILE, 'r') as f: # transform Java properties file into python configuration file: must prepend a section config_string = '[%s]\n' % SECTION_DEFAULT + f.read() - config_file = StringIO.StringIO(config_string) + config_file = io.StringIO(config_string) self.esgfConfig.readfp(config_file) logging.info("Read ESGF configuration parameters from file: %s" % ESGF_PROPERTIES_FILE) From 7ca57742aeadce14a1b1e27d6002447754d7cd28 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 24 May 2019 16:22:32 -0700 Subject: [PATCH 14/65] Changed ConfigParser.ConfigParser to configparser.ConfigParser --- cog/installation/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cog/installation/config.py b/cog/installation/config.py index bbbd88ad9..489a4e2da 100644 --- a/cog/installation/config.py +++ b/cog/installation/config.py @@ -23,7 +23,7 @@ ''' -import ConfigParser +import configparser import io import collections import logging @@ -59,7 +59,7 @@ def _readCogConfig(self): '''Method that reads an existing COG configuration file, or create a new one if not existing.''' # initialize COG configuration file - self.cogConfig = ConfigParser.ConfigParser(allow_no_value=True, + self.cogConfig = configparser.ConfigParser(allow_no_value=True, dict_type=collections.OrderedDict) # must set following line explicitly to preserve the case of configuration keys self.cogConfig.optionxform = str @@ -88,7 +88,7 @@ def _readEsgfConfig(self): '''Method that reads local parameters from ESGF configuration file esgf.properties.''' # read ESGF configuration file ($esg_config_dir/esgf.properties), if available - self.esgfConfig = ConfigParser.ConfigParser() + self.esgfConfig = configparser.ConfigParser() try: self.esgfConfig.read(ESGF_PROPERTIES_FILE) except IOError: From e22777295c9b19986bb5c52532ff45ead863bb3e Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 24 May 2019 16:32:05 -0700 Subject: [PATCH 15/65] Add "from __future__ import unicode_literals" back to files --- cog/db_migrations/django_openid_auth/0001_initial.py | 1 + cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py | 1 + cog/migrations/0001_initial.py | 1 + cog/migrations/0002_auto_20150706_1045.py | 1 + cog/migrations/0003_project_nodeswidgetenabled.py | 1 + cog/migrations/0004_auto_20160106_0812.py | 1 + cog/migrations/0005_project_shared.py | 1 + cog/migrations/0006_auto_20160303_1043.py | 1 + cog/migrations/0007_auto_20160303_1524.py | 1 + cog/migrations/0008_auto_20160609_1504.py | 1 + cog/migrations/0009_auto_20160826_0339.py | 1 + 11 files changed, 11 insertions(+) diff --git a/cog/db_migrations/django_openid_auth/0001_initial.py b/cog/db_migrations/django_openid_auth/0001_initial.py index 83d3a6a65..0059caf34 100644 --- a/cog/db_migrations/django_openid_auth/0001_initial.py +++ b/cog/db_migrations/django_openid_auth/0001_initial.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals from django.db import models, migrations diff --git a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py index 5b1d561fe..a6962cf16 100644 --- a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py +++ b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-06 08:12 +from __future__ import unicode_literals from django.db import migrations diff --git a/cog/migrations/0001_initial.py b/cog/migrations/0001_initial.py index 19a353034..85e19c1b2 100644 --- a/cog/migrations/0001_initial.py +++ b/cog/migrations/0001_initial.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals from django.db import models, migrations diff --git a/cog/migrations/0002_auto_20150706_1045.py b/cog/migrations/0002_auto_20150706_1045.py index 9bc18ac83..7b9824831 100644 --- a/cog/migrations/0002_auto_20150706_1045.py +++ b/cog/migrations/0002_auto_20150706_1045.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals from django.db import models, migrations diff --git a/cog/migrations/0003_project_nodeswidgetenabled.py b/cog/migrations/0003_project_nodeswidgetenabled.py index 89c039ce1..36a64dc69 100644 --- a/cog/migrations/0003_project_nodeswidgetenabled.py +++ b/cog/migrations/0003_project_nodeswidgetenabled.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals from django.db import models, migrations diff --git a/cog/migrations/0004_auto_20160106_0812.py b/cog/migrations/0004_auto_20160106_0812.py index 39abc04c1..0bb8b5070 100644 --- a/cog/migrations/0004_auto_20160106_0812.py +++ b/cog/migrations/0004_auto_20160106_0812.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-06 08:12 +from __future__ import unicode_literals from django.db import migrations, models diff --git a/cog/migrations/0005_project_shared.py b/cog/migrations/0005_project_shared.py index 50a3145aa..463ee2fe8 100644 --- a/cog/migrations/0005_project_shared.py +++ b/cog/migrations/0005_project_shared.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-01-22 01:52 +from __future__ import unicode_literals from django.db import migrations, models diff --git a/cog/migrations/0006_auto_20160303_1043.py b/cog/migrations/0006_auto_20160303_1043.py index d4d7a4ec1..b8f19e6d9 100644 --- a/cog/migrations/0006_auto_20160303_1043.py +++ b/cog/migrations/0006_auto_20160303_1043.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-03-03 10:43 +from __future__ import unicode_literals from django.db import migrations, models diff --git a/cog/migrations/0007_auto_20160303_1524.py b/cog/migrations/0007_auto_20160303_1524.py index be721d1cd..07b46dfca 100644 --- a/cog/migrations/0007_auto_20160303_1524.py +++ b/cog/migrations/0007_auto_20160303_1524.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.1 on 2016-03-03 15:24 +from __future__ import unicode_literals from django.db import migrations, models diff --git a/cog/migrations/0008_auto_20160609_1504.py b/cog/migrations/0008_auto_20160609_1504.py index 27258b99a..a7a4879b3 100644 --- a/cog/migrations/0008_auto_20160609_1504.py +++ b/cog/migrations/0008_auto_20160609_1504.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.4 on 2016-06-09 15:04 +from __future__ import unicode_literals from django.db import migrations, models diff --git a/cog/migrations/0009_auto_20160826_0339.py b/cog/migrations/0009_auto_20160826_0339.py index 2f37a86f2..23f362c96 100644 --- a/cog/migrations/0009_auto_20160826_0339.py +++ b/cog/migrations/0009_auto_20160826_0339.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.4 on 2016-08-26 03:39 +from __future__ import unicode_literals from django.db import migrations From 76498ad8ca089b3a03ba144fee0d5c1d3588475d Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 24 May 2019 16:35:47 -0700 Subject: [PATCH 16/65] Remove extra line --- cog/db_migrations/django_openid_auth/0001_initial.py | 1 - cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py | 1 - cog/migrations/0001_initial.py | 1 - cog/migrations/0002_auto_20150706_1045.py | 1 - cog/migrations/0003_project_nodeswidgetenabled.py | 1 - cog/migrations/0004_auto_20160106_0812.py | 1 - cog/migrations/0005_project_shared.py | 1 - cog/migrations/0006_auto_20160303_1043.py | 1 - cog/migrations/0007_auto_20160303_1524.py | 1 - cog/migrations/0008_auto_20160609_1504.py | 1 - cog/migrations/0009_auto_20160826_0339.py | 1 - 11 files changed, 11 deletions(-) diff --git a/cog/db_migrations/django_openid_auth/0001_initial.py b/cog/db_migrations/django_openid_auth/0001_initial.py index 0059caf34..8a4aba821 100644 --- a/cog/db_migrations/django_openid_auth/0001_initial.py +++ b/cog/db_migrations/django_openid_auth/0001_initial.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - from django.db import models, migrations from django.conf import settings diff --git a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py index a6962cf16..f2dcf4692 100644 --- a/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py +++ b/cog/db_migrations/django_openid_auth/0002_auto_20160106_0812.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.1 on 2016-01-06 08:12 from __future__ import unicode_literals - from django.db import migrations diff --git a/cog/migrations/0001_initial.py b/cog/migrations/0001_initial.py index 85e19c1b2..038e8dc77 100644 --- a/cog/migrations/0001_initial.py +++ b/cog/migrations/0001_initial.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - from django.db import models, migrations import django.db.models.deletion from django.conf import settings diff --git a/cog/migrations/0002_auto_20150706_1045.py b/cog/migrations/0002_auto_20150706_1045.py index 7b9824831..2b5dc67a3 100644 --- a/cog/migrations/0002_auto_20150706_1045.py +++ b/cog/migrations/0002_auto_20150706_1045.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - from django.db import models, migrations import cog.models.dbutils diff --git a/cog/migrations/0003_project_nodeswidgetenabled.py b/cog/migrations/0003_project_nodeswidgetenabled.py index 36a64dc69..591b92c52 100644 --- a/cog/migrations/0003_project_nodeswidgetenabled.py +++ b/cog/migrations/0003_project_nodeswidgetenabled.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals - from django.db import models, migrations diff --git a/cog/migrations/0004_auto_20160106_0812.py b/cog/migrations/0004_auto_20160106_0812.py index 0bb8b5070..a708531fb 100644 --- a/cog/migrations/0004_auto_20160106_0812.py +++ b/cog/migrations/0004_auto_20160106_0812.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.1 on 2016-01-06 08:12 from __future__ import unicode_literals - from django.db import migrations, models diff --git a/cog/migrations/0005_project_shared.py b/cog/migrations/0005_project_shared.py index 463ee2fe8..0bd618ff4 100644 --- a/cog/migrations/0005_project_shared.py +++ b/cog/migrations/0005_project_shared.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.1 on 2016-01-22 01:52 from __future__ import unicode_literals - from django.db import migrations, models diff --git a/cog/migrations/0006_auto_20160303_1043.py b/cog/migrations/0006_auto_20160303_1043.py index b8f19e6d9..5990789e1 100644 --- a/cog/migrations/0006_auto_20160303_1043.py +++ b/cog/migrations/0006_auto_20160303_1043.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.1 on 2016-03-03 10:43 from __future__ import unicode_literals - from django.db import migrations, models diff --git a/cog/migrations/0007_auto_20160303_1524.py b/cog/migrations/0007_auto_20160303_1524.py index 07b46dfca..c943da26d 100644 --- a/cog/migrations/0007_auto_20160303_1524.py +++ b/cog/migrations/0007_auto_20160303_1524.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.1 on 2016-03-03 15:24 from __future__ import unicode_literals - from django.db import migrations, models diff --git a/cog/migrations/0008_auto_20160609_1504.py b/cog/migrations/0008_auto_20160609_1504.py index a7a4879b3..4a7276989 100644 --- a/cog/migrations/0008_auto_20160609_1504.py +++ b/cog/migrations/0008_auto_20160609_1504.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.4 on 2016-06-09 15:04 from __future__ import unicode_literals - from django.db import migrations, models diff --git a/cog/migrations/0009_auto_20160826_0339.py b/cog/migrations/0009_auto_20160826_0339.py index 23f362c96..274ac56dd 100644 --- a/cog/migrations/0009_auto_20160826_0339.py +++ b/cog/migrations/0009_auto_20160826_0339.py @@ -2,7 +2,6 @@ # Generated by Django 1.9.4 on 2016-08-26 03:39 from __future__ import unicode_literals - from django.db import migrations From c7473e8dedb54b383f4a026a29d1ebee24a53c53 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 29 May 2019 15:53:29 -0700 Subject: [PATCH 17/65] Fix exception and print for Python 3. --- cog/management/commands/sync_sites.py | 5 +++-- cog/middleware/init_middleware.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cog/management/commands/sync_sites.py b/cog/management/commands/sync_sites.py index d21b1a336..7aa8d3649 100644 --- a/cog/management/commands/sync_sites.py +++ b/cog/management/commands/sync_sites.py @@ -36,5 +36,6 @@ def handle(self, *args, **options): try: pnl = PeerNodesList(FILEPATH) pnl.reload(delete=options['delete']) - except Exception, error: - print "Could not update peer nodes from xml file", error + except Exception as error: + print("Could not update peer nodes from xml file") + print(error) diff --git a/cog/middleware/init_middleware.py b/cog/middleware/init_middleware.py index b42f51128..2573fc63d 100644 --- a/cog/middleware/init_middleware.py +++ b/cog/middleware/init_middleware.py @@ -24,8 +24,9 @@ def __init__(self): try: pnl = PeerNodesList(filepath) pnl.reload() # delete=False - except Exception, error: - print "Could not update peer nodes from xml file", error + except Exception as error: + print("Could not update peer nodes from xml file") + print(error) # read IdP whitelist From 9ee5e32f0205c0e03b7415f24d7a84e4657b19cd Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 3 Jun 2019 16:57:05 -0700 Subject: [PATCH 18/65] Change 'from . import cog.views' to 'import cog.views' --- urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urls.py b/urls.py index 6c487c598..abde4229a 100644 --- a/urls.py +++ b/urls.py @@ -2,7 +2,7 @@ from django.conf.urls import url, include from django.contrib import admin import django.views.static, django.views.generic -from . import cog.views +import cog.views from django.http.response import HttpResponseNotFound from filebrowser.sites import site From 63487638b18c9631200a5b44d07ded975c9c9132 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 3 Jun 2019 17:17:31 -0700 Subject: [PATCH 19/65] Change str and unicode to bytes and text --- filebrowser/base.py | 6 +++--- filebrowser/fields.py | 1 - filebrowser/functions.py | 4 ++-- filebrowser/sites.py | 8 ++++---- filebrowser/templatetags/fb_versions.py | 6 +++--- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/filebrowser/base.py b/filebrowser/base.py index 88d03e8fa..c47a5a260 100644 --- a/filebrowser/base.py +++ b/filebrowser/base.py @@ -4,7 +4,7 @@ import mimetypes import os, shutil, re, datetime, time -from django.utils.encoding import smart_str, smart_unicode +from django.utils.encoding import smart_bytes, smart_text from django.utils.translation import ugettext as _ from filebrowser.functions import (get_file_type, url_join, get_version_path, get_original_path, sort_by_attr, version_generator, path_strip, url_strip, validate_path) @@ -199,10 +199,10 @@ def __init__(self, path, site=None): def __str__(self): - return smart_str(self.path) + return smart_bytes(self.path) def __unicode__(self): - return smart_unicode(self.path) + return smart_text(self.path) @property def name(self): diff --git a/filebrowser/fields.py b/filebrowser/fields.py index a823ac6dc..dc6fd304a 100644 --- a/filebrowser/fields.py +++ b/filebrowser/fields.py @@ -8,7 +8,6 @@ from django import forms from django.forms.widgets import Input from django.db.models.fields import Field, CharField -from django.utils.encoding import force_unicode from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ from django.core import urlresolvers diff --git a/filebrowser/functions.py b/filebrowser/functions.py index 4cc17f7e2..399e040b5 100644 --- a/filebrowser/functions.py +++ b/filebrowser/functions.py @@ -8,7 +8,7 @@ # django imports from django.utils.translation import ugettext as _ from django.core.files import File -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text # filebrowser imports from filebrowser.settings import * @@ -166,7 +166,7 @@ def get_file(path, filename, site=None): """ Get file (or folder). """ - converted_path = smart_unicode(os.path.join(site.directory, path, filename)) + converted_path = smart_text(os.path.join(site.directory, path, filename)) if not site.storage.isfile(converted_path) and not site.storage.isdir(converted_path): return None return filename diff --git a/filebrowser/sites.py b/filebrowser/sites.py index 2d0e6d4cf..5025657e5 100644 --- a/filebrowser/sites.py +++ b/filebrowser/sites.py @@ -28,7 +28,7 @@ from django.core.urlresolvers import reverse, get_urlconf, get_resolver from django.dispatch import Signal from django.core.paginator import Paginator, InvalidPage, EmptyPage -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from django.contrib import messages from django.views.decorators.csrf import csrf_exempt, csrf_protect from django.core.files.base import ContentFile @@ -544,11 +544,11 @@ def _upload_file(self, request): uploadedfile = handle_file_upload(path, filedata, site=self) if file_already_exists: - old_file = smart_unicode(file_name) - new_file = smart_unicode(uploadedfile) + old_file = smart_text(file_name) + new_file = smart_text(uploadedfile) self.storage.move(new_file, old_file, allow_overwrite=True) - self.filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_unicode(file_name), site=self)) + self.filebrowser_post_upload.send(sender=request, path=request.POST.get('folder'), file=FileObject(smart_text(file_name), site=self)) # let Ajax Upload know whether we saved it or not ret_json = {'success': True, 'filename': filedata.name} diff --git a/filebrowser/templatetags/fb_versions.py b/filebrowser/templatetags/fb_versions.py index 10af1ec1f..d7f979a28 100644 --- a/filebrowser/templatetags/fb_versions.py +++ b/filebrowser/templatetags/fb_versions.py @@ -7,7 +7,7 @@ # DJANGO IMPORTS from django.template import Library, Node, Variable, VariableDoesNotExist, TemplateSyntaxError from django.conf import settings -from django.utils.encoding import force_unicode, smart_str +from django.utils.encoding import force_text from django.core.files import File @@ -48,7 +48,7 @@ def render(self, context): source = source.path if isinstance(source, File): source = source.name - source = force_unicode(source) + source = force_text(source) if FORCE_PLACEHOLDER: source = PLACEHOLDER elif SHOW_PLACEHOLDER and not site.storage.isfile(source): @@ -113,7 +113,7 @@ def render(self, context): source = source.path if isinstance(source, File): source = source.name - source = force_unicode(source) + source = force_text(source) if FORCE_PLACEHOLDER: source = PLACEHOLDER elif SHOW_PLACEHOLDER and not site.storage.isfile(source): From 6944ad17abdd1cde72ec64eb6baff97fafdc1a08 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 7 Jun 2019 15:43:36 -0700 Subject: [PATCH 20/65] Use psycopg2 >=2.7 for Python 3.7 support due to use of 'async' as a keyword. Replace python-openid with python3-openid for Python 3 support. --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f13680a45..937c6b320 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,8 @@ django-grappelli==2.8.1 #django-openid-auth==0.7', # must be installed independently from fork sqlalchemy>=1.0.11,<1.3 south==1.0.2 -psycopg2>=2.5.2,<2.7 -python-openid==2.2.5 +psycopg2>=2.7,<2.8 +python3-openid==3.1.0 passlib==1.6.5 django-contrib-comments==1.6.2 oauth2client==2.0.1 From 7ea2827b34552309f4665e78181f7129501f8ede Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 7 Jun 2019 16:49:35 -0700 Subject: [PATCH 21/65] Updated packages in requirements.txt; updated Django to 2.2 --- requirements.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 937c6b320..a2703e716 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,18 @@ # CoG application dependencies -django==1.10.4 -django-grappelli==2.8.1 +django>=2.2,<2.3 +django-grappelli==2.12.3 #django-openid-auth==0.7', # must be installed independently from fork sqlalchemy>=1.0.11,<1.3 south==1.0.2 psycopg2>=2.7,<2.8 python3-openid==3.1.0 -passlib==1.6.5 -django-contrib-comments==1.6.2 -oauth2client==2.0.1 -globus-sdk==1.7.1 +passlib==1.7.1 +django-contrib-comments==1.9.1 +oauth2client==4.1.3 +globus-sdk==1.7.1 #'pillow==3.1.0', # pre-requisite: must be installed with --use-wheel on MAC-OSX -django-simple-captcha==0.5.1 -html5lib==1.0b8 -bleach==1.4.2 -python-magic==0.4.12 -esgfpid==0.7.10 +django-simple-captcha==0.5.11 +html5lib==1.0.1 +bleach==3.1.0 +python-magic==0.4.15 +esgfpid==0.7.14 From 6c7253b35e1ce0838d0506b8e59c88e4d628259c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 7 Jun 2019 16:57:06 -0700 Subject: [PATCH 22/65] change django.core.urlresolvers to django.urls --- cog/middleware/login_middleware.py | 2 +- cog/middleware/password_middleware.py | 2 +- cog/models/logged_event.py | 2 +- cog/models/project.py | 2 +- cog/models/project_tab.py | 2 +- cog/models/user_profile.py | 2 +- cog/services/membership.py | 2 +- cog/templatetags/cog_utils.py | 3 +-- cog/views/views_aboutus.py | 2 +- cog/views/views_access_control.py | 3 +-- cog/views/views_account.py | 2 +- cog/views/views_admin.py | 2 +- cog/views/views_datacart.py | 2 +- cog/views/views_doc.py | 2 +- cog/views/views_external_urls.py | 2 +- cog/views/views_globus.py | 2 +- cog/views/views_membership.py | 2 +- cog/views/views_news.py | 2 +- cog/views/views_post.py | 2 +- cog/views/views_search.py | 2 +- filebrowser/decorators.py | 2 +- filebrowser/sites.py | 4 ++-- filebrowser/sites.py-original | 4 ++-- filebrowser/tests/sites.py | 2 +- 24 files changed, 26 insertions(+), 28 deletions(-) diff --git a/cog/middleware/login_middleware.py b/cog/middleware/login_middleware.py index db3335e92..23fb04e0f 100644 --- a/cog/middleware/login_middleware.py +++ b/cog/middleware/login_middleware.py @@ -6,7 +6,7 @@ ''' from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse +from django.urls import reverse from django.conf import settings from cog.plugins.esgf.registry import LocalWhiteList diff --git a/cog/middleware/password_middleware.py b/cog/middleware/password_middleware.py index 7d7dd4387..e509cd6dd 100644 --- a/cog/middleware/password_middleware.py +++ b/cog/middleware/password_middleware.py @@ -3,7 +3,7 @@ ''' from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse +from django.urls import reverse from django.core.exceptions import ObjectDoesNotExist EXEMPT_URLS = ['/password/update/', 'site_media', 'logout'] diff --git a/cog/models/logged_event.py b/cog/models/logged_event.py index 9e5eddcc4..f177157b5 100644 --- a/cog/models/logged_event.py +++ b/cog/models/logged_event.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.db.models.signals import post_save from django.core.signals import request_finished -from django.core.urlresolvers import reverse +from django.urls import reverse from django.dispatch import receiver from .post import Post, post_signal from .doc import Doc diff --git a/cog/models/project.py b/cog/models/project.py index f569c3a60..1d5030aa6 100644 --- a/cog/models/project.py +++ b/cog/models/project.py @@ -17,7 +17,7 @@ import sys import re from django.contrib.sites.models import Site -from django.core.urlresolvers import reverse +from django.urls import reverse from collections import OrderedDict from django.core.exceptions import ObjectDoesNotExist from cog.models.auth import (getAdminGroupName, getContributorGroupName, getUserGroupName, diff --git a/cog/models/project_tab.py b/cog/models/project_tab.py index 0d1b7ecab..429dc8396 100644 --- a/cog/models/project_tab.py +++ b/cog/models/project_tab.py @@ -2,7 +2,7 @@ from .constants import APPLICATION_LABEL from .navbar import PROJECT_PAGES, DEFAULT_TABS from .project import Project -from django.core.urlresolvers import reverse +from django.urls import reverse from cog.models.dbutils import UnsavedForeignKey diff --git a/cog/models/user_profile.py b/cog/models/user_profile.py index e0ef876f0..657b6dda0 100644 --- a/cog/models/user_profile.py +++ b/cog/models/user_profile.py @@ -5,7 +5,7 @@ from cog.utils import hasText from django.contrib.sites.models import Site -from django.core.urlresolvers import reverse +from django.urls import reverse from cog.utils import getJson from cog.models.peer_site import getPeerSites from cog.models.project_tag import ProjectTag diff --git a/cog/services/membership.py b/cog/services/membership.py index 2fe92a808..3d7418314 100644 --- a/cog/services/membership.py +++ b/cog/services/membership.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User, Group, Permission from cog.models import (MembershipRequest, ManagementBodyMember, OrganizationalRoleMember, getProjectForGroup, CommunicationMeansMember, LoggedEvent) -from django.core.urlresolvers import reverse +from django.urls import reverse import django.dispatch # return codes diff --git a/cog/templatetags/cog_utils.py b/cog/templatetags/cog_utils.py index 259153ac8..5fed2dcb1 100644 --- a/cog/templatetags/cog_utils.py +++ b/cog/templatetags/cog_utils.py @@ -4,10 +4,9 @@ from cog.views import encodeMembershipPar, NEW_MEMBERSHIP, OLD_MEMBERSHIP, NO_MEMBERSHIP from cog.views import userCanPost, userCanView from django import template -from django.core.urlresolvers import reverse from django.utils.html import conditional_escape from django.utils.safestring import mark_safe -from django.core.urlresolvers import reverse +from django.urls import reverse import re from cog.utils import smart_truncate, INVALID_CHARS from cog.models.utils import get_project_communication_means diff --git a/cog/views/views_aboutus.py b/cog/views/views_aboutus.py index 81c29cf02..a7fca44ce 100644 --- a/cog/views/views_aboutus.py +++ b/cog/views/views_aboutus.py @@ -1,7 +1,7 @@ from django.shortcuts import get_object_or_404, render, redirect from django.template import RequestContext from django.http import HttpResponseRedirect, HttpResponseForbidden -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.decorators import login_required, user_passes_test, permission_required from django.forms.models import modelformset_factory, inlineformset_factory import string diff --git a/cog/views/views_access_control.py b/cog/views/views_access_control.py index e02071366..16cf5020c 100644 --- a/cog/views/views_access_control.py +++ b/cog/views/views_access_control.py @@ -7,8 +7,7 @@ from django.contrib.auth.decorators import login_required from django.contrib.sites.models import Site -from django.core.urlresolvers import reverse -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpRequest, HttpResponseForbidden, HttpResponseRedirect from django.shortcuts import get_object_or_404, render from django.template import RequestContext diff --git a/cog/views/views_account.py b/cog/views/views_account.py index 239634e99..e780e6753 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -7,7 +7,7 @@ from django.contrib.auth.views import login from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse +from django.urls import reverse from django.forms.models import modelformset_factory from django.http import HttpResponseRedirect, HttpResponseNotAllowed, HttpResponseServerError from django.shortcuts import get_object_or_404, render diff --git a/cog/views/views_admin.py b/cog/views/views_admin.py index a1e7cd77e..d5819e86c 100644 --- a/cog/views/views_admin.py +++ b/cog/views/views_admin.py @@ -1,7 +1,7 @@ from cog.forms import * from cog.models import * from django.contrib.auth.decorators import login_required, user_passes_test -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, render from django.template import RequestContext diff --git a/cog/views/views_datacart.py b/cog/views/views_datacart.py index 67df18783..7c05bbb9b 100644 --- a/cog/views/views_datacart.py +++ b/cog/views/views_datacart.py @@ -1,7 +1,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.http import HttpResponseRedirect, HttpResponse, HttpResponseForbidden, HttpResponseNotAllowed -from django.core.urlresolvers import reverse +from django.urls import reverse from cog.models import * from django.contrib.auth.decorators import login_required import re diff --git a/cog/views/views_doc.py b/cog/views/views_doc.py index 9cb610629..0768064a9 100644 --- a/cog/views/views_doc.py +++ b/cog/views/views_doc.py @@ -2,7 +2,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.http import HttpResponseRedirect, HttpResponseForbidden -from django.core.urlresolvers import reverse +from django.urls import reverse from cog.models import * from cog.forms import * from .constants import PERMISSION_DENIED_MESSAGE diff --git a/cog/views/views_external_urls.py b/cog/views/views_external_urls.py index a4bf54423..298d1445e 100644 --- a/cog/views/views_external_urls.py +++ b/cog/views/views_external_urls.py @@ -5,7 +5,7 @@ from django.contrib.auth.decorators import login_required from cog.forms import * from django.http import HttpResponseRedirect -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponse, HttpResponseForbidden from django.forms.models import modelformset_factory from .constants import PERMISSION_DENIED_MESSAGE diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index 7f1bc610c..81c40ffea 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -1,5 +1,5 @@ -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.decorators import login_required from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError from django.shortcuts import render diff --git a/cog/views/views_membership.py b/cog/views/views_membership.py index 982db0b64..f26a01bff 100644 --- a/cog/views/views_membership.py +++ b/cog/views/views_membership.py @@ -4,7 +4,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.http import HttpRequest, HttpResponseRedirect, HttpResponseForbidden -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.decorators import login_required, user_passes_test, permission_required from cog.views.utils import getUsersThatMatch, getQueryDict from django.contrib.sites.models import Site diff --git a/cog/views/views_news.py b/cog/views/views_news.py index 4ea9cd7b1..01136a650 100644 --- a/cog/views/views_news.py +++ b/cog/views/views_news.py @@ -2,7 +2,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.http import HttpResponseRedirect, HttpResponseForbidden -from django.core.urlresolvers import reverse +from django.urls import reverse from cog.models import * from cog.models.auth import userHasContributorPermission diff --git a/cog/views/views_post.py b/cog/views/views_post.py index 3d6975a07..089fd9c71 100644 --- a/cog/views/views_post.py +++ b/cog/views/views_post.py @@ -5,7 +5,7 @@ from datetime import datetime from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponseRedirect, Http404, HttpResponseForbidden from django.shortcuts import get_object_or_404, render from django.template import RequestContext diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 0f69c2e46..335e077b2 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -17,7 +17,7 @@ from cog.views.utils import getQueryDict from django.contrib.auth.decorators import login_required, user_passes_test from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse +from django.urls import reverse from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseBadRequest, HttpResponseNotFound from django.http.response import HttpResponseServerError from django.shortcuts import get_object_or_404, render diff --git a/filebrowser/decorators.py b/filebrowser/decorators.py index 6a33d49f3..0c4cadb1f 100644 --- a/filebrowser/decorators.py +++ b/filebrowser/decorators.py @@ -3,7 +3,7 @@ # DJANGO IMPORTS from django.http import HttpResponseRedirect from django.utils.translation import ugettext as _ -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib import messages from django.core.exceptions import ImproperlyConfigured diff --git a/filebrowser/sites.py b/filebrowser/sites.py index 5025657e5..8403f498b 100644 --- a/filebrowser/sites.py +++ b/filebrowser/sites.py @@ -25,7 +25,7 @@ from django.views.decorators.cache import never_cache from django.utils.translation import ugettext as _ from django import forms -from django.core.urlresolvers import reverse, get_urlconf, get_resolver +from django.urls import reverse, get_urlconf, get_resolver from django.dispatch import Signal from django.core.paginator import Paginator, InvalidPage, EmptyPage from django.utils.encoding import smart_text @@ -96,7 +96,7 @@ def get_default_site(app_name='filebrowser'): resolver = get_resolver(get_urlconf()) name = 'filebrowser' - # Django's default name resolution method (see django.core.urlresolvers.reverse()) + # Django's default name resolution method (see django.urls.reverse()) app_list = resolver.app_dict[app_name] if not name in app_list: name = app_list[0] diff --git a/filebrowser/sites.py-original b/filebrowser/sites.py-original index a45f67491..4b5134ada 100644 --- a/filebrowser/sites.py-original +++ b/filebrowser/sites.py-original @@ -12,7 +12,7 @@ from django.contrib.admin.views.decorators import staff_member_required from django.views.decorators.cache import never_cache from django.utils.translation import ugettext as _ from django import forms -from django.core.urlresolvers import reverse, get_urlconf, get_resolver +from django.urls import reverse, get_urlconf, get_resolver from django.dispatch import Signal from django.core.paginator import Paginator, InvalidPage, EmptyPage from django.utils.encoding import smart_unicode @@ -84,7 +84,7 @@ def get_default_site(app_name='filebrowser'): resolver = get_resolver(get_urlconf()) name = 'filebrowser' - # Django's default name resolution method (see django.core.urlresolvers.reverse()) + # Django's default name resolution method (see django.urls.reverse()) app_list = resolver.app_dict[app_name] if not name in app_list: name = app_list[0] diff --git a/filebrowser/tests/sites.py b/filebrowser/tests/sites.py index ecd123fea..445d802fd 100644 --- a/filebrowser/tests/sites.py +++ b/filebrowser/tests/sites.py @@ -18,7 +18,7 @@ # DJANGO IMPORTS from django.test import TestCase from django.test.client import Client -from django.core.urlresolvers import get_resolver, get_urlconf, resolve, reverse +from django.urls import get_resolver, get_urlconf, resolve, reverse # FILEBROWSER IMPORTS from filebrowser.settings import * From 15f8e2a612db148c77c2f09d7b5bb9a4db895c17 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 7 Jun 2019 17:18:45 -0700 Subject: [PATCH 23/65] change MIDDLEWARE_CLASSES to MIDDLEWARE in settings.py --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 21d3319a2..b7c57ae0e 100644 --- a/settings.py +++ b/settings.py @@ -201,7 +201,7 @@ }, ] -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.middleware.common.CommonMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', From 78be6c06d8fb571197516815659296623f3e3831 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 7 Jun 2019 17:26:01 -0700 Subject: [PATCH 24/65] Remove parentheses from User.is_authenticated and User.is_anonymous --- cog/middleware/login_middleware.py | 4 ++-- cog/middleware/password_middleware.py | 2 +- cog/middleware/session_middleware.py | 2 +- cog/models/signals.py | 2 +- cog/views/views_account.py | 4 ++-- cog/views/views_doc.py | 2 +- cog/views/views_post.py | 2 +- cog/views/views_project.py | 8 ++++---- cog/views/views_search.py | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cog/middleware/login_middleware.py b/cog/middleware/login_middleware.py index 23fb04e0f..177dd41c2 100644 --- a/cog/middleware/login_middleware.py +++ b/cog/middleware/login_middleware.py @@ -78,7 +78,7 @@ def process_response(self, request, response): # process errors from openid authentication if request.path == self.url2: - if request.method=='POST' and not request.user.is_authenticated(): + if request.method=='POST' and not request.user.is_authenticated: if response.status_code == 500: print('Authentication Error') print(response) @@ -88,7 +88,7 @@ def process_response(self, request, response): # process errors from standard authentication elif request.path == self.url1: - if request.method=='POST' and not request.user.is_authenticated(): + if request.method=='POST' and not request.user.is_authenticated: return HttpResponseRedirect(reverse('login')+"?message=login_failed&next=%s&username=%s" % (next, username) ) return response diff --git a/cog/middleware/password_middleware.py b/cog/middleware/password_middleware.py index e509cd6dd..0fbfa909f 100644 --- a/cog/middleware/password_middleware.py +++ b/cog/middleware/password_middleware.py @@ -18,7 +18,7 @@ def process_request(self, request): if not any(url in request.path for url in EXEMPT_URLS): try: - if request.user.is_authenticated() and request.user.profile.type==1 and request.user.profile.hasPasswordExpired(): + if request.user.is_authenticated and request.user.profile.type==1 and request.user.profile.hasPasswordExpired(): print('Password for user %s has expired, forcing mandatory change.' % request.user) return HttpResponseRedirect(reverse('password_update', kwargs={'user_id':request.user.id})+"?message=password_expired&next=%s" % request.path) diff --git a/cog/middleware/session_middleware.py b/cog/middleware/session_middleware.py index 0f24a9701..a24544308 100644 --- a/cog/middleware/session_middleware.py +++ b/cog/middleware/session_middleware.py @@ -19,7 +19,7 @@ def process_request(self, request): # must not use shared anonymous session try: - if request.user.is_authenticated() and request.user.profile.openid() is not None: + if request.user.is_authenticated and request.user.profile.openid() is not None: s = request.session last_accessed_seconds = s.get('LAST_ACCESSED', 0) # defaults to Unix Epoch diff --git a/cog/models/signals.py b/cog/models/signals.py index 50d4ab077..e2ceead40 100644 --- a/cog/models/signals.py +++ b/cog/models/signals.py @@ -46,7 +46,7 @@ def update_user_projects(user): and save the updated information in the local database. ''' - if user.is_authenticated(): + if user.is_authenticated: # current user groups in local database ugroups = user.groups.all() diff --git a/cog/views/views_account.py b/cog/views/views_account.py index e780e6753..e1ec344bd 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -59,7 +59,7 @@ def custom_login_complete(request, **kwargs): response = login_complete(request, **kwargs) # create a stub profile with blank mandatory fields - if not request.user.is_anonymous(): + if not request.user.is_anonymous: openid = request.GET.get('openid.claimed_id', None) try: request.user.profile @@ -92,7 +92,7 @@ def custom_login_complete(request, **kwargs): def _custom_login(request, response): # successful login - if not request.user.is_anonymous(): + if not request.user.is_anonymous: # missing information if isUserLocal(request.user) and not isUserValid(request.user): diff --git a/cog/views/views_doc.py b/cog/views/views_doc.py index 0768064a9..107a90d58 100644 --- a/cog/views/views_doc.py +++ b/cog/views/views_doc.py @@ -253,7 +253,7 @@ def doc_list(request, project_short_name): #list_title = 'All Documents' # do not list private documents unless user is a project member - if request.user.is_anonymous() or not userHasUserPermission(request.user, project): + if request.user.is_anonymous or not userHasUserPermission(request.user, project): qset = qset & Q(is_private=False) # optional query parameters diff --git a/cog/views/views_post.py b/cog/views/views_post.py index 089fd9c71..927093867 100644 --- a/cog/views/views_post.py +++ b/cog/views/views_post.py @@ -533,7 +533,7 @@ def getNotAuthorizedRedirect(request, post): # user is NOT authorized if not userCanView(request.user, post): - if not request.user.is_authenticated(): + if not request.user.is_authenticated: return HttpResponseRedirect(reverse('login')+"?next=%s" % request.path) else: messages = ['This page is only viewable to members of %s.' % post.project.short_name, diff --git a/cog/views/views_project.py b/cog/views/views_project.py index 9fc3bcad0..4c721dc77 100644 --- a/cog/views/views_project.py +++ b/cog/views/views_project.py @@ -577,13 +577,13 @@ def project_browser(request, project_short_name, tab): elif tab == 'all': html += render_project_list(project, tab, tag, request.user, None, 'all_projects', None) elif tab == 'my': - if not request.user.is_anonymous(): + if not request.user.is_anonymous: html += render_project_list(project, tab, tag, request.user, None, 'my_projects', None) else: html += '
' \ 'Please login to display your projects.
' elif tab == 'tags': - if not request.user.is_anonymous(): + if not request.user.is_anonymous: display = DisplayStatus(True) # open all sub-widgets by default # loop over user tags (sorted by name) utags = request.user.profile.tags.all() @@ -745,7 +745,7 @@ def render_project_list(project, tab, tag_name, user, widget_name, widget_id, di html += ''+tag_error+'' else: # special case: cannot retrieve list of projects for guest user - if (tab == 'my' or tab == 'tags') and not user.is_authenticated(): + if (tab == 'my' or tab == 'tags') and not user.is_authenticated: html += 'Please login to display your projects.' else: html += 'No projects found.' @@ -788,7 +788,7 @@ def listBrowsableProjects(project, tab, tag, user, widgetName): # retrieve all active projects for this user projects = getProjectsForUser(user, False) # includePending==False elif tab == 'tags': - if not user.is_authenticated(): + if not user.is_authenticated: projects = Project.objects.none() else: # widgetName==user tag name diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 335e077b2..31ae692e2 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -64,7 +64,7 @@ def search(request, project_short_name): # check permission if project.private: - if request.user.is_anonymous(): + if request.user.is_anonymous: return HttpResponseRedirect(reverse('login')+"?next=%s" % request.path) else: if not userHasUserPermission(request.user, project): From 9c002d1516380caf11b7744b655cc0c49c5fc320 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 1 Jul 2019 10:25:02 -0700 Subject: [PATCH 25/65] Add on_delete=models.CASCADE to ForeignKey calls --- .../django_openid_auth/0001_initial.py | 2 +- cog/migrations/0001_initial.py | 88 +++++++++---------- cog/migrations/0002_auto_20150706_1045.py | 4 +- cog/models/bookmark.py | 2 +- cog/models/collaborator.py | 2 +- cog/models/communication_means.py | 2 +- cog/models/communication_means_member.py | 4 +- cog/models/datacart.py | 8 +- cog/models/doc.py | 2 +- cog/models/external_url.py | 2 +- cog/models/folder.py | 4 +- cog/models/funding_source.py | 2 +- cog/models/lock.py | 2 +- cog/models/logged_event.py | 4 +- cog/models/management_body.py | 2 +- cog/models/management_body_member.py | 4 +- cog/models/membership.py | 4 +- cog/models/news.py | 2 +- cog/models/organization.py | 2 +- cog/models/organizational_role.py | 2 +- cog/models/organizational_role_member.py | 4 +- cog/models/peer_site.py | 2 +- cog/models/post.py | 2 +- cog/models/project.py | 2 +- cog/models/project_impact.py | 2 +- cog/models/project_tab.py | 4 +- cog/models/project_topic.py | 4 +- cog/models/search_facet.py | 2 +- cog/models/search_group.py | 2 +- cog/models/search_profile.py | 2 +- cog/models/user_profile.py | 4 +- cog/models/user_url.py | 2 +- 32 files changed, 88 insertions(+), 88 deletions(-) diff --git a/cog/db_migrations/django_openid_auth/0001_initial.py b/cog/db_migrations/django_openid_auth/0001_initial.py index 8a4aba821..24f3d5068 100644 --- a/cog/db_migrations/django_openid_auth/0001_initial.py +++ b/cog/db_migrations/django_openid_auth/0001_initial.py @@ -39,7 +39,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('claimed_id', models.TextField(unique=True, max_length=2047)), ('display_id', models.TextField(max_length=2047)), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), ] diff --git a/cog/migrations/0001_initial.py b/cog/migrations/0001_initial.py index 038e8dc77..5604a2765 100644 --- a/cog/migrations/0001_initial.py +++ b/cog/migrations/0001_initial.py @@ -54,15 +54,15 @@ class Migration(migrations.Migration): name='CommunicationMeansMember', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('communicationMeans', models.ForeignKey(to='cog.CommunicationMeans')), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('communicationMeans', models.ForeignKey(to='cog.CommunicationMeans', on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( name='DataCart', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('user', models.OneToOneField(related_name='datacart', to=settings.AUTH_USER_MODEL)), + ('user', models.OneToOneField(related_name='datacart', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -71,7 +71,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('identifier', models.CharField(max_length=200)), ('date', models.DateTimeField(auto_now_add=True, verbose_name=b'Date Time')), - ('cart', models.ForeignKey(related_name='items', to='cog.DataCart')), + ('cart', models.ForeignKey(related_name='items', to='cog.DataCart', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -79,7 +79,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('key', models.CharField(max_length=200)), - ('item', models.ForeignKey(related_name='keys', to='cog.DataCartItem')), + ('item', models.ForeignKey(related_name='keys', to='cog.DataCartItem', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -87,7 +87,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('value', models.CharField(max_length=1000, null=True, blank=True)), - ('key', models.ForeignKey(related_name='values', to='cog.DataCartItemMetadataKey')), + ('key', models.ForeignKey(related_name='values', to='cog.DataCartItemMetadataKey', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -121,7 +121,7 @@ class Migration(migrations.Migration): ('name', models.CharField(max_length=200)), ('active', models.BooleanField(default=True)), ('order', models.IntegerField(default=0, blank=True)), - ('parent', models.ForeignKey(related_name='parent_folder', blank=True, to='cog.Folder', null=True)), + ('parent', models.ForeignKey(related_name='parent_folder', blank=True, to='cog.Folder', null=True, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -151,7 +151,7 @@ class Migration(migrations.Migration): ('is_private', models.BooleanField(default=False, verbose_name=b'Private?')), ('order', models.IntegerField(default=0, blank=True)), ('author', models.ForeignKey(related_name='forum_topics', on_delete=django.db.models.deletion.SET_NULL, verbose_name=b'Author', to=settings.AUTH_USER_MODEL, null=True)), - ('forum', models.ForeignKey(related_name='topics', to='cog.Forum')), + ('forum', models.ForeignKey(related_name='topics', to='cog.Forum', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -170,7 +170,7 @@ class Migration(migrations.Migration): ('object_type', models.CharField(max_length=100, verbose_name=b'Object Type')), ('object_id', models.IntegerField(verbose_name=b'Object Identifier')), ('timestamp', models.DateTimeField(auto_now=True, verbose_name=b'Last Update Date')), - ('owner', models.ForeignKey(verbose_name=b'Owner', to=settings.AUTH_USER_MODEL)), + ('owner', models.ForeignKey(verbose_name=b'Owner', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -199,8 +199,8 @@ class Migration(migrations.Migration): name='ManagementBodyMember', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('managementBody', models.ForeignKey(to='cog.ManagementBody')), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('managementBody', models.ForeignKey(to='cog.ManagementBody', on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -217,8 +217,8 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('date', models.DateTimeField(auto_now=True, verbose_name=b'Request Date')), - ('group', models.ForeignKey(to='auth.Group')), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('group', models.ForeignKey(to='auth.Group', on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -255,8 +255,8 @@ class Migration(migrations.Migration): name='OrganizationalRoleMember', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('organizationalRole', models.ForeignKey(to='cog.OrganizationalRole')), - ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL)), + ('organizationalRole', models.ForeignKey(to='cog.OrganizationalRole', on_delete=models.CASCADE)), + ('user', models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -264,7 +264,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('enabled', models.BooleanField(default=False)), - ('site', models.OneToOneField(related_name='peersite', to='sites.Site')), + ('site', models.OneToOneField(related_name='peersite', to='sites.Site', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -327,7 +327,7 @@ class Migration(migrations.Migration): ('author', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, default=None, blank=True, to=settings.AUTH_USER_MODEL, null=True)), ('parents', models.ManyToManyField(related_name='parent_projects', to='cog.Project', blank=True)), ('peers', models.ManyToManyField(related_name='peer_projects', to='cog.Project', blank=True)), - ('site', models.ForeignKey(default=1, to='sites.Site')), + ('site', models.ForeignKey(default=1, to='sites.Site', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -337,7 +337,7 @@ class Migration(migrations.Migration): ('title', models.CharField(default=b'', help_text=b'Title for this impact.', max_length=200)), ('description', models.TextField(help_text=b'Describe a major impact of this project in its field.', verbose_name=b'Project Impact')), ('order', models.IntegerField(blank=True)), - ('project', models.ForeignKey(related_name='impacts', to='cog.Project')), + ('project', models.ForeignKey(related_name='impacts', to='cog.Project', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -347,8 +347,8 @@ class Migration(migrations.Migration): ('url', models.CharField(default=b'', unique=True, max_length=200, verbose_name=b'URL', blank=True)), ('label', models.CharField(max_length=40)), ('active', models.BooleanField(default=True)), - ('parent', models.ForeignKey(blank=True, to='cog.ProjectTab', null=True)), - ('project', models.ForeignKey(related_name='tabs', to='cog.Project')), + ('parent', models.ForeignKey(blank=True, to='cog.ProjectTab', null=True, on_delete=models.CASCADE)), + ('project', models.ForeignKey(related_name='tabs', to='cog.Project', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -363,7 +363,7 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('order', models.IntegerField(default=0)), - ('project', models.ForeignKey(to='cog.Project')), + ('project', models.ForeignKey(to='cog.Project', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -394,7 +394,7 @@ class Migration(migrations.Migration): ('latestSearchFlag', models.BooleanField(default=False)), ('localSearchFlag', models.BooleanField(default=False)), ('description', models.TextField(help_text=b'Optional description of this project search capabilities.', null=True, verbose_name=b'Search Help', blank=True)), - ('project', models.OneToOneField(to='cog.Project')), + ('project', models.OneToOneField(to='cog.Project', on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -421,9 +421,9 @@ class Migration(migrations.Migration): ('researchKeywords', models.CharField(default=b'', max_length=60, null=True, blank=True)), ('type', models.IntegerField(default=1)), ('last_password_update', models.DateTimeField(null=True, verbose_name=b'Date and Time when Password was Last Updated', blank=True)), - ('site', models.ForeignKey(default=1, to='sites.Site')), + ('site', models.ForeignKey(default=1, to='sites.Site', on_delete=models.CASCADE)), ('tags', models.ManyToManyField(related_name='users', to='cog.ProjectTag', blank=True)), - ('user', models.OneToOneField(related_name='profile', to=settings.AUTH_USER_MODEL)), + ('user', models.OneToOneField(related_name='profile', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -432,23 +432,23 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('url', models.URLField(max_length=1000, verbose_name=b'URL')), ('name', models.CharField(max_length=200)), - ('profile', models.ForeignKey(to='cog.UserProfile')), + ('profile', models.ForeignKey(to='cog.UserProfile', on_delete=models.CASCADE)), ], ), migrations.AddField( model_name='searchgroup', name='profile', - field=models.ForeignKey(related_name='groups', to='cog.SearchProfile'), + field=models.ForeignKey(related_name='groups', to='cog.SearchProfile', on_delete=models.CASCADE), ), migrations.AddField( model_name='searchfacet', name='group', - field=models.ForeignKey(related_name='facets', to='cog.SearchGroup', null=True), + field=models.ForeignKey(related_name='facets', to='cog.SearchGroup', null=True, on_delete=models.CASCADE), ), migrations.AddField( model_name='projecttopic', name='topic', - field=models.ForeignKey(to='cog.Topic'), + field=models.ForeignKey(to='cog.Topic', on_delete=models.CASCADE), ), migrations.AddField( model_name='project', @@ -463,7 +463,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='post', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='post', @@ -473,12 +473,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='organizationalrole', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='organization', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='news', @@ -488,12 +488,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='news', name='project', - field=models.ForeignKey(verbose_name=b'About Project', to='cog.Project'), + field=models.ForeignKey(verbose_name=b'About Project', to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='managementbody', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='managementbody', @@ -503,57 +503,57 @@ class Migration(migrations.Migration): migrations.AddField( model_name='loggedevent', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='loggedevent', name='user', - field=models.ForeignKey(to=settings.AUTH_USER_MODEL), + field=models.ForeignKey(to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE), ), migrations.AddField( model_name='fundingsource', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='forumthread', name='topic', - field=models.ForeignKey(related_name='threads', to='cog.ForumTopic'), + field=models.ForeignKey(related_name='threads', to='cog.ForumTopic', on_delete=models.CASCADE), ), migrations.AddField( model_name='forum', name='project', - field=models.OneToOneField(related_name='forum', to='cog.Project'), + field=models.OneToOneField(related_name='forum', to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='folder', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='externalurl', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='doc', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='communicationmeans', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='collaborator', name='project', - field=models.ForeignKey(to='cog.Project'), + field=models.ForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AddField( model_name='bookmark', name='folder', - field=models.ForeignKey(to='cog.Folder'), + field=models.ForeignKey(to='cog.Folder', on_delete=models.CASCADE), ), migrations.AddField( model_name='bookmark', diff --git a/cog/migrations/0002_auto_20150706_1045.py b/cog/migrations/0002_auto_20150706_1045.py index 2b5dc67a3..5a2732500 100644 --- a/cog/migrations/0002_auto_20150706_1045.py +++ b/cog/migrations/0002_auto_20150706_1045.py @@ -15,11 +15,11 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='folder', name='project', - field=cog.models.dbutils.UnsavedForeignKey(to='cog.Project'), + field=cog.models.dbutils.UnsavedForeignKey(to='cog.Project', on_delete=models.CASCADE), ), migrations.AlterField( model_name='projecttab', name='project', - field=cog.models.dbutils.UnsavedForeignKey(related_name='tabs', to='cog.Project'), + field=cog.models.dbutils.UnsavedForeignKey(related_name='tabs', to='cog.Project', on_delete=models.CASCADE), ), ] diff --git a/cog/models/bookmark.py b/cog/models/bookmark.py index d6c5abe6c..edd2b42c7 100644 --- a/cog/models/bookmark.py +++ b/cog/models/bookmark.py @@ -7,7 +7,7 @@ class Bookmark(models.Model): url = models.URLField('URL', blank=False, max_length=1000) name = models.CharField(max_length=1000, blank=False) - folder = models.ForeignKey('Folder', blank=False, null=False) + folder = models.ForeignKey('Folder', blank=False, null=False, on_delete=models.CASCADE) description = models.TextField(max_length=1000, blank=True, null=True) order = models.IntegerField(blank=True, default=0) # note: do not delete bookmark if notes is deleted diff --git a/cog/models/collaborator.py b/cog/models/collaborator.py index 7c041b145..8ec3a38af 100644 --- a/cog/models/collaborator.py +++ b/cog/models/collaborator.py @@ -8,7 +8,7 @@ class Collaborator(models.Model): last_name = models.CharField(max_length=100, blank=False, default='') institution = models.CharField(max_length=100, blank=False, default='') researchKeywords = models.CharField(max_length=RESEARCH_KEYWORDS_MAX_CHARS, blank=True, null=True, default='') - project = models.ForeignKey(Project, blank=False) + project = models.ForeignKey(Project, blank=False, on_delete=models.CASCADE) # optional picture image = models.ImageField(upload_to='photos/', blank=True, null=True) diff --git a/cog/models/communication_means.py b/cog/models/communication_means.py index 184189b30..8f7ccaea1 100644 --- a/cog/models/communication_means.py +++ b/cog/models/communication_means.py @@ -23,7 +23,7 @@ class CommunicationMeans(models.Model): participationDetails = models.TextField(blank=True, null=True, verbose_name='Participation Details', help_text='Information about how a person would participate: phone number, ' 'pass code, meeting venue, etc.') - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) # opt-out privacy option internal = models.BooleanField(default=True, null=False) diff --git a/cog/models/communication_means_member.py b/cog/models/communication_means_member.py index 1989f1b82..d9dbdd0ea 100644 --- a/cog/models/communication_means_member.py +++ b/cog/models/communication_means_member.py @@ -5,8 +5,8 @@ class CommunicationMeansMember(models.Model): - user = models.ForeignKey(User, blank=False, null=False) - communicationMeans = models.ForeignKey(CommunicationMeans, blank=False, null=False) + user = models.ForeignKey(User, blank=False, null=False, on_delete=models.CASCADE) + communicationMeans = models.ForeignKey(CommunicationMeans, blank=False, null=False, on_delete=models.CASCADE) class Meta: unique_together = (("user", "communicationMeans"),) diff --git a/cog/models/datacart.py b/cog/models/datacart.py index 394d78856..9cf148a64 100644 --- a/cog/models/datacart.py +++ b/cog/models/datacart.py @@ -8,7 +8,7 @@ class DataCart(models.Model): - user = models.OneToOneField(User, related_name='datacart') + user = models.OneToOneField(User, related_name='datacart', on_delete=models.CASCADE) def contains(self, item_identifier): '''Checks whether the data cart contains an item with the given identifier.''' @@ -20,7 +20,7 @@ class Meta: class DataCartItem(models.Model): - cart = models.ForeignKey(DataCart, related_name="items", blank=False, null=False) + cart = models.ForeignKey(DataCart, related_name="items", blank=False, null=False, on_delete=models.CASCADE) # used to enforce uniqueness within a single user datacart identifier = models.CharField(max_length=200, blank=False, null=False) @@ -109,7 +109,7 @@ def getValue(self, key): class DataCartItemMetadataKey(models.Model): - item = models.ForeignKey(DataCartItem, related_name='keys', blank=False, null=False) + item = models.ForeignKey(DataCartItem, related_name='keys', blank=False, null=False, on_delete=models.CASCADE) key = models.CharField(max_length=200, blank=False, null=False) class Meta: @@ -117,7 +117,7 @@ class Meta: class DataCartItemMetadataValue(models.Model): - key = models.ForeignKey(DataCartItemMetadataKey, related_name='values', blank=False, null=False) + key = models.ForeignKey(DataCartItemMetadataKey, related_name='values', blank=False, null=False, on_delete=models.CASCADE) value = models.CharField(max_length=1000, blank=True, null=True) class Meta: diff --git a/cog/models/doc.py b/cog/models/doc.py index 52ba48ffb..a3f248e24 100644 --- a/cog/models/doc.py +++ b/cog/models/doc.py @@ -47,7 +47,7 @@ class Doc(models.Model): file = models.FileField(upload_to=get_upload_path, storage=ofs, max_length=400) publication_date = models.DateTimeField('Date Published', auto_now_add=True) update_date = models.DateTimeField('Date Updated', auto_now=True) - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) # public/private flag is_private = models.BooleanField(verbose_name='Private?', default=False, null=False) diff --git a/cog/models/external_url.py b/cog/models/external_url.py index 67d42ae17..0b7a9c647 100644 --- a/cog/models/external_url.py +++ b/cog/models/external_url.py @@ -12,7 +12,7 @@ class ExternalUrl(models.Model): url = models.URLField('URL', blank=False, max_length=1000) type = models.CharField(max_length=20, verbose_name='URL Type', blank=False, choices=externalUrlManager.external_url_choices()) - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) def __unicode__(self): return "URL Type=%s Title='%s' url='%s' description='%s'" % (self.type, self.title, self.url, self.description) diff --git a/cog/models/folder.py b/cog/models/folder.py index 3f7d88aa1..1efb29b91 100644 --- a/cog/models/folder.py +++ b/cog/models/folder.py @@ -20,10 +20,10 @@ class Folder(models.Model): - project = UnsavedForeignKey(Project, blank=False) + project = UnsavedForeignKey(Project, blank=False, on_delete=models.CASCADE) name = models.CharField(max_length=200, blank=False) - parent = models.ForeignKey('self', blank=True, null=True, related_name='parent_folder') + parent = models.ForeignKey('self', blank=True, null=True, related_name='parent_folder', on_delete=models.CASCADE) active = models.BooleanField(default=True, blank=False, null=False) order = models.IntegerField(blank=True, default=0) diff --git a/cog/models/funding_source.py b/cog/models/funding_source.py index 780bda767..2de14683f 100644 --- a/cog/models/funding_source.py +++ b/cog/models/funding_source.py @@ -7,7 +7,7 @@ class FundingSource(models.Model): name = models.CharField(max_length=200, blank=False, help_text='Organization or agency that financially supports the project') url = models.URLField(max_length=200, blank=True, null=True, help_text='Funding Source URL') image = models.ImageField(upload_to=UPLOAD_DIR_LOGOS, blank=True, null=True) - project = models.ForeignKey(Project, blank=False) + project = models.ForeignKey(Project, blank=False, on_delete=models.CASCADE) def __unicode__(self): return "Project=%s Funding Source=%s" % (self.project, self.name) diff --git a/cog/models/lock.py b/cog/models/lock.py index 9c74f8345..d3f8311d9 100644 --- a/cog/models/lock.py +++ b/cog/models/lock.py @@ -13,7 +13,7 @@ class Lock(models.Model): object_type = models.CharField(max_length=100, verbose_name='Object Type', blank=False) object_id = models.IntegerField(verbose_name='Object Identifier', blank=False) timestamp = models.DateTimeField('Last Update Date', auto_now=True) - owner = models.ForeignKey(User, verbose_name='Owner', blank=False) + owner = models.ForeignKey(User, verbose_name='Owner', blank=False, on_delete=models.CASCADE) def __unicode__(self): return "Object type=%s id=%s expiration=%s owner=%s" % (self.object_type, self.object_id, self.get_expiration(), self.owner.get_full_name()) diff --git a/cog/models/logged_event.py b/cog/models/logged_event.py index f177157b5..496194b8f 100644 --- a/cog/models/logged_event.py +++ b/cog/models/logged_event.py @@ -13,8 +13,8 @@ class LoggedEvent(models.Model): '''Class that represents an important event that is logged to the database.''' - user = models.ForeignKey(User, blank=False) - project = models.ForeignKey('Project', blank=False) + user = models.ForeignKey(User, blank=False, on_delete=models.CASCADE) + project = models.ForeignKey('Project', blank=False, on_delete=models.CASCADE) title = models.CharField(max_length=200, blank=False) description = models.CharField(max_length=200, blank=False) url = models.URLField(blank=True) diff --git a/cog/models/management_body.py b/cog/models/management_body.py index 27ea5e76a..3a75ada0a 100644 --- a/cog/models/management_body.py +++ b/cog/models/management_body.py @@ -39,7 +39,7 @@ class ManagementBody(models.Model): category = models.CharField(max_length=50, blank=True, null=False, default=MANAGEMENT_BODY_CATEGORY_STRATEGIC, choices=MANAGEMENT_BODY_CATEGORIES_CV, verbose_name='Category', help_text='Strategic or Operational management body purpose.') - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) # sort list of members my last name and ignore case def members(self): diff --git a/cog/models/management_body_member.py b/cog/models/management_body_member.py index be1decf53..5ae4c4d74 100644 --- a/cog/models/management_body_member.py +++ b/cog/models/management_body_member.py @@ -5,8 +5,8 @@ class ManagementBodyMember(models.Model): - user = models.ForeignKey(User, blank=False, null=False) - managementBody = models.ForeignKey(ManagementBody, blank=False, null=False) + user = models.ForeignKey(User, blank=False, null=False, on_delete=models.CASCADE) + managementBody = models.ForeignKey(ManagementBody, blank=False, null=False, on_delete=models.CASCADE) class Meta: unique_together = (("user", "managementBody"),) diff --git a/cog/models/membership.py b/cog/models/membership.py index 99d1e2d6b..06832058b 100644 --- a/cog/models/membership.py +++ b/cog/models/membership.py @@ -6,8 +6,8 @@ # Object holding user requests to join a group class MembershipRequest(models.Model): - user = models.ForeignKey(User) - group = models.ForeignKey(Group) + user = models.ForeignKey(User, on_delete=models.CASCADE) + group = models.ForeignKey(Group, on_delete=models.CASCADE) date = models.DateTimeField('Request Date', auto_now=True) class Meta: diff --git a/cog/models/news.py b/cog/models/news.py index df78f3d72..f55a99f50 100644 --- a/cog/models/news.py +++ b/cog/models/news.py @@ -11,7 +11,7 @@ class News(models.Model): author = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) publication_date = models.DateTimeField('Date Published', auto_now_add=True) update_date = models.DateTimeField('Date Updated', auto_now=True) - project = models.ForeignKey(Project, verbose_name='About Project') + project = models.ForeignKey(Project, verbose_name='About Project', on_delete=models.CASCADE) other_projects = models.ManyToManyField(Project, verbose_name='Projects Notified', related_name='other_news', blank=True) def __unicode__(self): diff --git a/cog/models/organization.py b/cog/models/organization.py index b82a46468..346fbf570 100644 --- a/cog/models/organization.py +++ b/cog/models/organization.py @@ -8,7 +8,7 @@ class Organization(models.Model): name = models.CharField(max_length=200, blank=False, help_text='Project or organization that collaborates on this project') url = models.URLField(max_length=200, blank=True, null=True, help_text='Organization URL') image = models.ImageField(upload_to=UPLOAD_DIR_LOGOS, blank=True, null=True) - project = models.ForeignKey(Project, blank=False) + project = models.ForeignKey(Project, blank=False, on_delete=models.CASCADE) def __unicode__(self): return "Project=%s Organization=%s" % (self.project, self.name) diff --git a/cog/models/organizational_role.py b/cog/models/organizational_role.py index f85647aa1..ef3ea19d9 100644 --- a/cog/models/organizational_role.py +++ b/cog/models/organizational_role.py @@ -11,7 +11,7 @@ class OrganizationalRole(models.Model): title = models.CharField(max_length=200, blank=True, null=True, verbose_name='Title', help_text='Optional string used to succinctly describe an organizational role.') description = models.TextField(blank=True, null=True, verbose_name='Description', help_text='Long description providing extra information about an organizational role.') category = models.CharField(max_length=50, blank=True, null=False, default='Member', choices=ORGANIZATIONAL_ROLE_CATEGORIES_CV, verbose_name='Category', help_text='Lead or Member role.') - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) def set_category(self, dict={}): """Method to select the object category from the other fields. """ diff --git a/cog/models/organizational_role_member.py b/cog/models/organizational_role_member.py index c04a560c8..d71eea141 100644 --- a/cog/models/organizational_role_member.py +++ b/cog/models/organizational_role_member.py @@ -5,8 +5,8 @@ class OrganizationalRoleMember(models.Model): - user = models.ForeignKey(User, blank=False, null=False) - organizationalRole = models.ForeignKey(OrganizationalRole, blank=False, null=False) + user = models.ForeignKey(User, blank=False, null=False, on_delete=models.CASCADE) + organizationalRole = models.ForeignKey(OrganizationalRole, blank=False, null=False, on_delete=models.CASCADE) class Meta: unique_together = (("user", "organizationalRole"),) diff --git a/cog/models/peer_site.py b/cog/models/peer_site.py index 7bf500c07..f0bbe46c0 100644 --- a/cog/models/peer_site.py +++ b/cog/models/peer_site.py @@ -10,7 +10,7 @@ class PeerSite(models.Model): - site = models.OneToOneField(Site, blank=False, null=False, related_name='peersite') + site = models.OneToOneField(Site, blank=False, null=False, related_name='peersite', on_delete=models.CASCADE) enabled = models.BooleanField(default=False, null=False) class Meta: diff --git a/cog/models/post.py b/cog/models/post.py index 6f6c99ef1..45a9c4e4a 100644 --- a/cog/models/post.py +++ b/cog/models/post.py @@ -27,7 +27,7 @@ class Post(models.Model): publication_date = models.DateTimeField('Date Published', auto_now_add=True) update_date = models.DateTimeField('Date Updated') # project context - project = models.ForeignKey(Project) + project = models.ForeignKey(Project, on_delete=models.CASCADE) # order of post within project index order = models.IntegerField(blank=True, null=False, default=0) # optional topic diff --git a/cog/models/project.py b/cog/models/project.py index 1d5030aa6..68702a7a2 100644 --- a/cog/models/project.py +++ b/cog/models/project.py @@ -35,7 +35,7 @@ class Project(models.Model): long_name = models.CharField(max_length=120, unique=True, help_text='Fully spelled project name.') description = models.TextField(blank=False, null=True, help_text='A short paragraph that describes the project.') - site = models.ForeignKey(Site, default=settings.SITE_ID) + site = models.ForeignKey(Site, default=settings.SITE_ID, on_delete=models.CASCADE) # optional attributes mission = models.TextField(blank=True, help_text='Succinctly describes why the project exists and what it does.') diff --git a/cog/models/project_impact.py b/cog/models/project_impact.py index ba2484324..5e6cd59e1 100644 --- a/cog/models/project_impact.py +++ b/cog/models/project_impact.py @@ -4,7 +4,7 @@ class ProjectImpact(models.Model): - project = models.ForeignKey(Project, blank=False, related_name='impacts') + project = models.ForeignKey(Project, blank=False, related_name='impacts', on_delete=models.CASCADE) title = models.CharField(max_length=200, help_text='Title for this impact.', blank=False, null=False, default='') description = models.TextField(blank=False, null=False, verbose_name='Project Impact', help_text='Describe a major impact of this project in its field.') # IMPORTANT: NEVER USE A DEFAULT WITH A FORMSET, OTHERWISE has_changed=True for empty forms! diff --git a/cog/models/project_tab.py b/cog/models/project_tab.py index 429dc8396..6fd9f9b3b 100644 --- a/cog/models/project_tab.py +++ b/cog/models/project_tab.py @@ -9,7 +9,7 @@ # Tab displayed in project top navigation menu class ProjectTab(models.Model): - project = UnsavedForeignKey(Project, blank=False, null=False, related_name="tabs") + project = UnsavedForeignKey(Project, blank=False, null=False, related_name="tabs", on_delete=models.CASCADE) # the URL of a corresponding project page url = models.CharField(max_length=200, verbose_name='URL', blank=True, unique=True, default='') @@ -18,7 +18,7 @@ class ProjectTab(models.Model): # whether or not the tab will be displayed active = models.BooleanField(default=True, null=False, blank=False) # optional parent tab (null for top-level tabs) - parent = models.ForeignKey('self', blank=True, null=True) + parent = models.ForeignKey('self', blank=True, null=True, on_delete=models.CASCADE) def __unicode__(self): return "Project Tab label='%s', url='%s', active=%s" % (self.label, self.url, self.active) diff --git a/cog/models/project_topic.py b/cog/models/project_topic.py index e0c245ce7..483e3b2e3 100644 --- a/cog/models/project_topic.py +++ b/cog/models/project_topic.py @@ -5,8 +5,8 @@ # intermediate model for Project-Topic association class ProjectTopic(models.Model): - topic = models.ForeignKey(Topic) - project = models.ForeignKey(Project) + topic = models.ForeignKey(Topic, on_delete=models.CASCADE) + project = models.ForeignKey(Project, on_delete=models.CASCADE) # topic order within project index order = models.IntegerField(blank=False, null=False, default=0) diff --git a/cog/models/search_facet.py b/cog/models/search_facet.py index d226cf7ce..5e5a81a51 100644 --- a/cog/models/search_facet.py +++ b/cog/models/search_facet.py @@ -5,7 +5,7 @@ # Search facet displayed in user interface class SearchFacet(models.Model): - group = models.ForeignKey(SearchGroup, related_name="facets", blank=False, null=True) + group = models.ForeignKey(SearchGroup, related_name="facets", blank=False, null=True, on_delete=models.CASCADE) key = models.CharField(max_length=40, blank=False) label = models.CharField(max_length=40, blank=False) diff --git a/cog/models/search_group.py b/cog/models/search_group.py index 0a0c4d1e4..9e759f7d0 100644 --- a/cog/models/search_group.py +++ b/cog/models/search_group.py @@ -7,7 +7,7 @@ class SearchGroup(models.Model): DEFAULT_NAME = 'default' - profile = models.ForeignKey(SearchProfile, related_name="groups", blank=False, null=False) + profile = models.ForeignKey(SearchProfile, related_name="groups", blank=False, null=False, on_delete=models.CASCADE) name = models.CharField(max_length=40, null=False, blank=False, default=DEFAULT_NAME) order = models.IntegerField(blank=True, default=0) diff --git a/cog/models/search_profile.py b/cog/models/search_profile.py index 2fa20149e..c78387020 100644 --- a/cog/models/search_profile.py +++ b/cog/models/search_profile.py @@ -5,7 +5,7 @@ # Project-specific search configuration (persisted to the database) class SearchProfile(models.Model): - project = models.OneToOneField(Project, blank=False, null=False) + project = models.OneToOneField(Project, blank=False, null=False, on_delete=models.CASCADE) # name that identifies this configuration #name = models.CharField(max_length=50, blank=False, unique=True, default='') diff --git a/cog/models/user_profile.py b/cog/models/user_profile.py index 657b6dda0..36d50f4f7 100644 --- a/cog/models/user_profile.py +++ b/cog/models/user_profile.py @@ -14,10 +14,10 @@ class UserProfile(models.Model): # user - user = models.OneToOneField(User, related_name='profile') + user = models.OneToOneField(User, related_name='profile', on_delete=models.CASCADE) # node (using the django site object) - site = models.ForeignKey(Site, default=1) + site = models.ForeignKey(Site, default=1, on_delete=models.CASCADE) # additional mandatory fields institution = models.CharField(max_length=100, blank=False, default='') diff --git a/cog/models/user_url.py b/cog/models/user_url.py index 8d5cb281a..c545c4d10 100644 --- a/cog/models/user_url.py +++ b/cog/models/user_url.py @@ -7,7 +7,7 @@ class UserUrl(models.Model): url = models.URLField('URL', blank=False, null=False, max_length=1000) name = models.CharField(max_length=200, blank=False, null=False) - profile = models.ForeignKey(UserProfile, blank=False, null=False) + profile = models.ForeignKey(UserProfile, blank=False, null=False, on_delete=models.CASCADE) class Meta: app_label = APPLICATION_LABEL From a426c0079c1b4c74841b820bff127ebe725d17ae Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 26 Jul 2019 18:02:12 -0700 Subject: [PATCH 26/65] Remove include function for admin URLs --- urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urls.py b/urls.py index abde4229a..70902ef9d 100644 --- a/urls.py +++ b/urls.py @@ -32,11 +32,11 @@ # Filebrowser Admin pages #(r'^filebrowser/', include('filebrowser.urls')), - url(r'^admin/filebrowser/', include(site.urls)), + url(r'^admin/filebrowser/', site.urls), # Administrator application #(r'^admin/doc/', include('django.contrib.admindocs.urls')), - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), # django-simple-captcha url(r'^captcha/', include('captcha.urls')), From 85ba1f7caf79c781a17a828fb527a755d56c64c9 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 26 Jul 2019 18:03:37 -0700 Subject: [PATCH 27/65] Get login and logout from django.contrib.auth --- cog/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cog/urls.py b/cog/urls.py index 36faefca4..5fbb19094 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -38,7 +38,7 @@ # authentication options # a) django (username/password) login #url(r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'cog/account/login.html'}, name='login'), - url(r'^login2/$', django.contrib.auth.views.login, {'template_name': 'cog/account/login.html'}, name='login'), + url(r'^login2/$', django.contrib.auth.login, {'template_name': 'cog/account/login.html'}, name='login'), # b) combined django + openid login #url(r'^login/$', custom_login, {'template_name': 'cog/openid/login2.html'}, name='login'), @@ -57,7 +57,7 @@ # force redirection to login page after logout #url(r'^logout/$', 'django.contrib.auth.views.logout_then_login', name='logout'), # use next=... to redirect to previous page after logout - url(r'^logout/$', django.contrib.auth.views.logout, name='logout'), + url(r'^logout/$', django.contrib.auth.logout, name='logout'), # user management url(r'^user/add/$', cog.views.user_add, name='user_add' ), From 70983aff4049f51e73ad90c5feca0a024c8df835 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 26 Jul 2019 18:10:03 -0700 Subject: [PATCH 28/65] Middleware now accepts get_response argument --- cog/middleware/init_middleware.py | 7 ++++++- cog/middleware/login_middleware.py | 9 +++++++-- cog/middleware/session_middleware.py | 7 +++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cog/middleware/init_middleware.py b/cog/middleware/init_middleware.py index 2573fc63d..c4ec2240e 100644 --- a/cog/middleware/init_middleware.py +++ b/cog/middleware/init_middleware.py @@ -8,7 +8,8 @@ class InitMiddleware(object): - def __init__(self): + def __init__(self, get_response): + self.get_response = get_response print('Executing CoG initialization tasks') @@ -33,6 +34,10 @@ def __init__(self): # remove this class from the middleware that is invoked for every request raise MiddlewareNotUsed('Do not invoke ever again') + def __call__(self, request): + response = self.get_response(request) + return response + def process_request(self, request): print('This line should never be printed...') return None diff --git a/cog/middleware/login_middleware.py b/cog/middleware/login_middleware.py index 177dd41c2..0fe2d105c 100644 --- a/cog/middleware/login_middleware.py +++ b/cog/middleware/login_middleware.py @@ -12,10 +12,11 @@ class LoginMiddleware(object): - def __init__(self): + def __init__(self, get_response): - try: + self.get_response = get_response + # initialize the white list service self.whitelist = LocalWhiteList(settings.IDP_WHITELIST) @@ -32,6 +33,10 @@ def __init__(self): # OSError: [Errno 2] No such file or directory: '/esg/config/esgf_idp_static.xml' self.init = False + def __call__(self, request): + response = self.get_response(request) + return response + def process_request(self, request): ''' Method called before processing of the view. diff --git a/cog/middleware/session_middleware.py b/cog/middleware/session_middleware.py index a24544308..a16d99258 100644 --- a/cog/middleware/session_middleware.py +++ b/cog/middleware/session_middleware.py @@ -11,6 +11,13 @@ class SessionMiddleware(object): + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + return response + def process_request(self, request): ''' Method called before processing of the view. From 81c58634b67303d1d918dc0073e4a8d6d91a5ddf Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 31 Jul 2019 15:46:29 -0700 Subject: [PATCH 29/65] Replace login and logout with LoginView and LogoutView --- cog/urls.py | 4 ++-- cog/views/views_account.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cog/urls.py b/cog/urls.py index 5fbb19094..9687249ed 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -38,7 +38,7 @@ # authentication options # a) django (username/password) login #url(r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'cog/account/login.html'}, name='login'), - url(r'^login2/$', django.contrib.auth.login, {'template_name': 'cog/account/login.html'}, name='login'), + url(r'^login2/$', django.contrib.auth.views.LoginView.as_view(template_name='cog/account/login.html'), name='login'), # b) combined django + openid login #url(r'^login/$', custom_login, {'template_name': 'cog/openid/login2.html'}, name='login'), @@ -57,7 +57,7 @@ # force redirection to login page after logout #url(r'^logout/$', 'django.contrib.auth.views.logout_then_login', name='logout'), # use next=... to redirect to previous page after logout - url(r'^logout/$', django.contrib.auth.logout, name='logout'), + url(r'^logout/$', django.contrib.auth.views.LogoutView.as_view(), name='logout'), # user management url(r'^user/add/$', cog.views.user_add, name='user_add' ), diff --git a/cog/views/views_account.py b/cog/views/views_account.py index e1ec344bd..1fa479d29 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -4,7 +4,7 @@ from django.contrib.auth import logout from django.contrib.auth.decorators import login_required, user_passes_test from django.contrib.auth.hashers import is_password_usable -from django.contrib.auth.views import login +from django.contrib.auth.views import LoginView from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse @@ -43,7 +43,7 @@ def custom_login(request, **kwargs): """ # authenticate user via standard login - response = login(request, **kwargs) + response = LoginView.as_view(**kwargs)(request) # check if user is valid return _custom_login(request, response) From abcc80546b94521d462d949c1f6909e1d1d86236 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 1 Aug 2019 12:13:12 -0700 Subject: [PATCH 30/65] Put ignore-case expression (?i) at the end of the regex pattern --- urls.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/urls.py b/urls.py index 70902ef9d..89bc4acde 100644 --- a/urls.py +++ b/urls.py @@ -12,15 +12,15 @@ urlpatterns = [ # forbidden extensions (case-insensitive) - url(r'(?i).*\.asp\/?$', HttpResponseNotFound), - url(r'(?i).*\.aspx\/?$', HttpResponseNotFound), - url(r'(?i).*\.cfm\/?$', HttpResponseNotFound), - url(r'.*\.cgi\/?$', HttpResponseNotFound), - url(r'(?i).*\.jsp\/?$', HttpResponseNotFound), - url(r'(?i).*\.php\/?$', HttpResponseNotFound), - url(r'(?i).*\.php3\/?$', HttpResponseNotFound), - url(r'(?i).*\.pl\/?$', HttpResponseNotFound), - url(r'(?i).*\.shtml\/?$', HttpResponseNotFound), + url(r'.*\.asp\/?$(?i)', HttpResponseNotFound), + url(r'.*\.aspx\/?$(?i)', HttpResponseNotFound), + url(r'.*\.cfm\/?$(?i)', HttpResponseNotFound), + url(r'.*\.cgi\/?$(?i)', HttpResponseNotFound), + url(r'.*\.jsp\/?$(?i)', HttpResponseNotFound), + url(r'.*\.php\/?$(?i)', HttpResponseNotFound), + url(r'.*\.php3\/?$(?i)', HttpResponseNotFound), + url(r'.*\.pl\/?$(?i)', HttpResponseNotFound), + url(r'.*\.shtml\/?$(?i)', HttpResponseNotFound), url(r'^robots\.txt$', django.views.generic.TemplateView.as_view(template_name='robots.txt', content_type='text/plain'), name='robots.txt'), From 32a9bbc077e9786d0cb41be182600a7f3c95f56c Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 1 Aug 2019 13:46:17 -0700 Subject: [PATCH 31/65] Replace name of field with queryset=Project.objects.all() --- cog/forms/forms_project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cog/forms/forms_project.py b/cog/forms/forms_project.py index 3d3e3bb2a..c9a9dd234 100644 --- a/cog/forms/forms_project.py +++ b/cog/forms/forms_project.py @@ -19,10 +19,10 @@ class ProjectForm(ModelForm): # define the widget for parent/peer selection so we can set the styling. The class is set to .selectfilter and its # styles are controlled in cogstyle.css - parents = forms.ModelMultipleChoiceField("parents", required=False, + parents = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False, widget=forms.SelectMultiple(attrs={'size': '20', 'class': 'selectprojects'})) - peers = forms.ModelMultipleChoiceField("peers", required=False, + peers = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False, widget=forms.SelectMultiple(attrs={'size': '20', 'class': 'selectprojects'})) # filtering of what is see in the form is done down below. @@ -174,7 +174,7 @@ class ProjectTagForm(ModelForm): # form constructor in views_project.py # field['tags'] is the list of preexisting tags - tags = forms.ModelMultipleChoiceField("tags", required=False, + tags = forms.ModelMultipleChoiceField(queryset=Project.objects.all(), required=False, widget=forms.SelectMultiple(attrs={'size': '7'})) # override __init__ method to change the queryset for 'tags' From c1c8896e67272dbb02013296dfdffc3abc159f88 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 1 Aug 2019 15:03:48 -0700 Subject: [PATCH 32/65] Replace string decode --- cog/forms/forms_account.py | 2 +- cog/forms/forms_project.py | 2 +- cog/management/commands/sync_users.py | 2 +- cog/services/search.py | 2 +- cog/views/views_search.py | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cog/forms/forms_account.py b/cog/forms/forms_account.py index 0a93499e2..458fa239d 100644 --- a/cog/forms/forms_account.py +++ b/cog/forms/forms_account.py @@ -304,6 +304,6 @@ def validate_field(form, field_name, field_value): def validate_ascii(form, field_name, field_value): if field_value: try: - field_value.decode('ascii') + field_value.encode('ascii') except (UnicodeDecodeError, UnicodeEncodeError): form._errors[field_name] = form.error_class(["'%s' contains invalid characters." % field_name]) \ No newline at end of file diff --git a/cog/forms/forms_project.py b/cog/forms/forms_project.py index c9a9dd234..1c5b1b4d0 100644 --- a/cog/forms/forms_project.py +++ b/cog/forms/forms_project.py @@ -99,7 +99,7 @@ def clean_long_name(self): # check for non-ascii characters try: - long_name.decode('ascii') + long_name.encode('ascii') except (UnicodeDecodeError, UnicodeEncodeError): raise forms.ValidationError("Project long name contains invalid non-ASCII characters") return long_name diff --git a/cog/management/commands/sync_users.py b/cog/management/commands/sync_users.py index 6a532863b..7886dec54 100644 --- a/cog/management/commands/sync_users.py +++ b/cog/management/commands/sync_users.py @@ -24,7 +24,7 @@ def handle(self, *ags, **options): # test user existence by accessing the profile page userProfileUrl = user.profile.getAbsoluteUrl() - print("\nChecking user at URL: %s" % userProfileUrl.decode("utf-8")) + print("\nChecking user at URL: %s" % userProfileUrl) try: response = urllib.request.urlopen( userProfileUrl ) diff --git a/cog/services/search.py b/cog/services/search.py index d1a29f330..7ba36b9d8 100644 --- a/cog/services/search.py +++ b/cog/services/search.py @@ -44,7 +44,7 @@ def search(self, searchInput, allFacets=False): url = self.url+"?"+urllib.parse.urlencode(params) print('ESGF search URL=%s' % url) fh = urllib.request.urlopen( url ) - xml = fh.read().decode("UTF-8") + xml = fh.read() # return search URL and corresponding results as XML return (url, xml) \ No newline at end of file diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 31ae692e2..5d928898f 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -395,7 +395,7 @@ def metadata_display(request, project_short_name): url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) print('Metadata Solr search URL=%s' % url) fh = urllib.request.urlopen(url) - response = fh.read().decode("UTF-8") + response = fh.read() # parse JSON response (containing only one matching 'doc) jsondoc = json.loads(response) @@ -408,7 +408,7 @@ def metadata_display(request, project_short_name): url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) # print 'Solr search URL=%s' % url fh = urllib.request.urlopen(url) - response = fh.read().decode("UTF-8") + response = fh.read() jsondoc = json.loads(response) parentMetadata = _processDoc(jsondoc["response"]["docs"][0]) @@ -853,7 +853,7 @@ def search_files(request, dataset_id, index_node): url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) print('Searching for files: URL=%s' % url) fh = urllib.request.urlopen(url) - response = fh.read().decode("UTF-8") + response = fh.read() return HttpResponse(response, content_type="application/json") From 7a2cfba38b0f7326f2b797ba215c9765550c0102 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 6 Aug 2019 15:53:44 -0700 Subject: [PATCH 33/65] Use floor division when dividing offset by limit when calculating the current page number in search --- cog/views/views_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 5d928898f..8ad179de1 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -241,7 +241,7 @@ def search_get(request, searchInput, searchConfig, extra={}, fromRedirectFlag=Fa offset = data[SEARCH_INPUT].offset limit = data[SEARCH_INPUT].limit if limit > 0 and data.get(SEARCH_OUTPUT, None): - currentPage = offset/limit + 1 + currentPage = offset//limit + 1 numResults = len(data[SEARCH_OUTPUT].results) totResults = data[SEARCH_OUTPUT].counts data[SEARCH_PAGES] = [] From 18a4296a3be74240b714abdecc5acc80e0182b97 Mon Sep 17 00:00:00 2001 From: watucker Date: Mon, 12 Aug 2019 17:03:11 +0100 Subject: [PATCH 34/65] Added view to generate OAuth2 wget scripts for datasets --- cog/templates/cog/download/wget_script.sh | 7 +++ cog/urls.py | 3 + cog/views/__init__.py | 1 + cog/views/views_download.py | 76 +++++++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 cog/templates/cog/download/wget_script.sh create mode 100644 cog/views/views_download.py diff --git a/cog/templates/cog/download/wget_script.sh b/cog/templates/cog/download/wget_script.sh new file mode 100644 index 000000000..3e046e8d0 --- /dev/null +++ b/cog/templates/cog/download/wget_script.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +{% for url in download_urls %}wget --certificate $0 {{ url }} +{% endfor %} + +creds=" +{{ certificate }}" diff --git a/cog/urls.py b/cog/urls.py index 7a75d34b3..cac3d3f15 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -35,6 +35,9 @@ url(r'^search_reload/$', cog.views.search_reload, name='search_reload' ), url(r'^citation_display/$', cog.views.citation_display, name='citation_display'), + # WGET Script + url(r'^wget/script/$', cog.views.wget_script, name='wget_script'), + # authentication options # a) django (username/password) login #url(r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'cog/account/login.html'}, name='login'), diff --git a/cog/views/__init__.py b/cog/views/__init__.py index 0df89efd3..849008acb 100644 --- a/cog/views/__init__.py +++ b/cog/views/__init__.py @@ -3,6 +3,7 @@ from views_bookmarks import * from views_account import * from views_datacart import * +from views_download import * from views_doc import * from views_external_urls import * from views_account import * diff --git a/cog/views/views_download.py b/cog/views/views_download.py new file mode 100644 index 000000000..4c4a97c4b --- /dev/null +++ b/cog/views/views_download.py @@ -0,0 +1,76 @@ +import os +import urllib +import urllib2 +import json + +from django.template.loader import render_to_string +from django.conf import settings +from django.http import HttpResponse, HttpResponseBadRequest +from social_django.utils import load_strategy + +from cog.models.search import INVALID_CHARACTERS, ERROR_MESSAGE_INVALID_TEXT + + +def wget_script(request): + + certificate = get_certificate(request) + + dataset_id = request.GET.get('dataset_id') + index_node = request.GET.get('index_node') + shard = request.GET.get('shard') + + # maximum number of files to query for + limit = request.GET.get('limit', 1000) + + params = [('type', "File"), ('dataset_id', dataset_id), + ("format", "application/solr+json"), ('offset', '0'), ('limit', limit)] + + # optional query filter + query = request.GET.get('query', None) + if query is not None and len(query.strip()) > 0: + # validate query value + for c in INVALID_CHARACTERS: + if c in query: + return HttpResponseBadRequest(ERROR_MESSAGE_INVALID_TEXT, content_type="text/plain") + params.append(('query', query)) + + # optional shard + shard = request.GET.get('shard', '') + if shard is not None and len(shard.strip()) > 0: + params.append(('shards', shard+"/solr")) + else: + params.append(("distrib", "false")) + + url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) + fh = urllib2.urlopen(url) + response = fh.read().decode("UTF-8") + data = json.loads(response) + + download_urls = [] + try: + for doc in data['response']['docs']: + download_urls.append(doc['url'][0].split('|')[0]) + except (KeyError, IndexError) as e: + raise e + context = { + 'download_urls': download_urls, + 'certificate': certificate, + } + + script = render_to_string('cog/download/wget_script.sh', context) + + response = HttpResponse(script, content_type='application/x-sh') + response['Content-Disposition'] = 'attachment; filename="get_script.sh"' + + return response + + +def get_certificate(request): + + social = request.user.social_auth.get(provider='esgf') + access_token = social.extra_data['access_token'] + strategy = load_strategy() + backend = social.get_backend_instance(strategy) + + key, cert, _ = backend.get_certificate(access_token) + return key + cert From f8332cadbb8bb4b120263eeca2d75cb03165e265 Mon Sep 17 00:00:00 2001 From: watucker Date: Mon, 12 Aug 2019 17:09:54 +0100 Subject: [PATCH 35/65] Added button for new wget scripts to record displays Replaced old esg-search wget link --- cog/templates/cog/search/_record_display.html | 7 ------- cog/templatetags/search_utils.py | 15 +++++++++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cog/templates/cog/search/_record_display.html b/cog/templates/cog/search/_record_display.html index 8ea9906f1..02aa9cc23 100644 --- a/cog/templates/cog/search/_record_display.html +++ b/cog/templates/cog/search/_record_display.html @@ -79,13 +79,6 @@ [ LAS Visualization ] - - {% elif url.1 == 'application/wget' %} -   -
- [ WGET Script ] -
- {% elif url.1 == 'application/gridftp' %}   diff --git a/cog/templatetags/search_utils.py b/cog/templatetags/search_utils.py index 8453dd13b..a49f96042 100644 --- a/cog/templatetags/search_utils.py +++ b/cog/templatetags/search_utils.py @@ -1,4 +1,6 @@ from django import template +from django.urls import reverse_lazy +from urllib import urlencode from cog.models.search import searchMappings from cog.site_manager import siteManager from string import replace @@ -115,12 +117,17 @@ def recordUrls(record): if 'url' in record.fields: for value in record.fields['url']: urls.append(value) - + # add special WGET endpoint - urls.append( ("javascript:wgetScript('%s','%s','%s')" % (record.fields['index_node'][0], record.fields.get('shard', [''])[0], record.id) , - "application/wget", - "WGET Script") ) + query = { + 'dataset_id': record.id, + 'index_node': record.fields['index_node'][0], + 'shard': record.fields.get('shard', [''])[0], + } + script_url = '{url}?{query}'.format(url=reverse_lazy('wget_script'), query=urlencode(query)) + urls.append((script_url, "", "WGET Script")) + # add Globus endpoints if siteManager.isGlobusEnabled(): # only if this node has been registered with Globus if 'access' in record.fields and 'index_node' in record.fields and 'data_node' in record.fields: From aa45bb1f3160f7a8a6c1a3efbd9aa99ad729ec13 Mon Sep 17 00:00:00 2001 From: watucker Date: Tue, 20 Aug 2019 13:48:30 +0100 Subject: [PATCH 36/65] Re-added old wget script for non OAuth users --- cog/templatetags/search_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cog/templatetags/search_utils.py b/cog/templatetags/search_utils.py index a49f96042..7ff098879 100644 --- a/cog/templatetags/search_utils.py +++ b/cog/templatetags/search_utils.py @@ -118,7 +118,11 @@ def recordUrls(record): for value in record.fields['url']: urls.append(value) - # add special WGET endpoint + # add special WGET endpoints + urls.append( ("javascript:wgetScript('%s','%s','%s')" % (record.fields['index_node'][0], record.fields.get('shard', [''])[0], record.id) , + "application/wget", + "WGET Script") ) + query = { 'dataset_id': record.id, 'index_node': record.fields['index_node'][0], @@ -126,7 +130,7 @@ def recordUrls(record): } script_url = '{url}?{query}'.format(url=reverse_lazy('wget_script'), query=urlencode(query)) - urls.append((script_url, "", "WGET Script")) + urls.append((script_url, "application/simple-wget", "Simplified WGET Script")) # add Globus endpoints if siteManager.isGlobusEnabled(): # only if this node has been registered with Globus From 7ed94292bc024e5d7487214f5fd079e3a70d4a11 Mon Sep 17 00:00:00 2001 From: watucker Date: Tue, 20 Aug 2019 13:50:50 +0100 Subject: [PATCH 37/65] Refactored wget script to allow multi-dataset downloads --- cog/templates/cog/datacart/_wget_js.html | 10 ++ cog/templates/cog/search/_record_display.html | 9 +- cog/urls.py | 2 +- cog/views/views_download.py | 167 ++++++++++-------- 4 files changed, 115 insertions(+), 73 deletions(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index a21ee2443..a3fae20e9 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -135,6 +135,16 @@ html += " (shard="+shard+")"; } html += ""; + + // Generate simplified wget script links + {% if request.user.is_authenticated %} + var query_string = "index_node=" + index_node + "&shard=" + shard; + for (var id_index in ids) { + dataset_id = jsonResponse[wget_key][id_index]; + query_string += "&dataset_id=" + dataset_id; + } + html += " ( Simple WGET Script )"; + {% endif %} } } html += ""; diff --git a/cog/templates/cog/search/_record_display.html b/cog/templates/cog/search/_record_display.html index 02aa9cc23..02a2f6524 100644 --- a/cog/templates/cog/search/_record_display.html +++ b/cog/templates/cog/search/_record_display.html @@ -78,7 +78,14 @@ - + + + {% elif url.1 == 'application/wget' %} +   +
+ [ WGET Script ] +
+ {% elif url.1 == 'application/gridftp' %}   diff --git a/cog/urls.py b/cog/urls.py index cac3d3f15..dfe30d52a 100644 --- a/cog/urls.py +++ b/cog/urls.py @@ -36,7 +36,7 @@ url(r'^citation_display/$', cog.views.citation_display, name='citation_display'), # WGET Script - url(r'^wget/script/$', cog.views.wget_script, name='wget_script'), + url(r'^download/wget/$', cog.views.WgetScriptView.as_view(), name='wget_script'), # authentication options # a) django (username/password) login diff --git a/cog/views/views_download.py b/cog/views/views_download.py index 4c4a97c4b..504f4afe9 100644 --- a/cog/views/views_download.py +++ b/cog/views/views_download.py @@ -1,76 +1,101 @@ -import os -import urllib -import urllib2 import json +import requests +from django.utils import timezone +from django.http import HttpResponse from django.template.loader import render_to_string -from django.conf import settings -from django.http import HttpResponse, HttpResponseBadRequest +from django.views.generic import View from social_django.utils import load_strategy -from cog.models.search import INVALID_CHARACTERS, ERROR_MESSAGE_INVALID_TEXT - - -def wget_script(request): - - certificate = get_certificate(request) - - dataset_id = request.GET.get('dataset_id') - index_node = request.GET.get('index_node') - shard = request.GET.get('shard') - - # maximum number of files to query for - limit = request.GET.get('limit', 1000) - - params = [('type', "File"), ('dataset_id', dataset_id), - ("format", "application/solr+json"), ('offset', '0'), ('limit', limit)] - - # optional query filter - query = request.GET.get('query', None) - if query is not None and len(query.strip()) > 0: - # validate query value - for c in INVALID_CHARACTERS: - if c in query: - return HttpResponseBadRequest(ERROR_MESSAGE_INVALID_TEXT, content_type="text/plain") - params.append(('query', query)) - - # optional shard - shard = request.GET.get('shard', '') - if shard is not None and len(shard.strip()) > 0: - params.append(('shards', shard+"/solr")) - else: - params.append(("distrib", "false")) - - url = "http://"+index_node+"/esg-search/search?"+urllib.urlencode(params) - fh = urllib2.urlopen(url) - response = fh.read().decode("UTF-8") - data = json.loads(response) - - download_urls = [] - try: - for doc in data['response']['docs']: - download_urls.append(doc['url'][0].split('|')[0]) - except (KeyError, IndexError) as e: - raise e - context = { - 'download_urls': download_urls, - 'certificate': certificate, - } - - script = render_to_string('cog/download/wget_script.sh', context) - - response = HttpResponse(script, content_type='application/x-sh') - response['Content-Disposition'] = 'attachment; filename="get_script.sh"' - - return response - - -def get_certificate(request): - - social = request.user.social_auth.get(provider='esgf') - access_token = social.extra_data['access_token'] - strategy = load_strategy() - backend = social.get_backend_instance(strategy) - - key, cert, _ = backend.get_certificate(access_token) - return key + cert + +class WgetScriptView(View): + """ + View for generating a wget shell script for downloading files from one or + more datasets. The script includes a short-lived certificate retrieved from + a social auth provider. + """ + + script_template = 'cog/download/wget_script.sh' + search_limit = 10000 + + def get(self, request, *args, **kwargs): + """ + Get method for the view. Query parameters specify dataset IDs and which + index to search on. + """ + + index_node = request.GET.get('index_node') + index_node_url = 'http://{}/esg-search/search'.format(index_node) + + # Construct params for search request + params = { + 'type': 'File', + 'format': 'application/solr+json', + 'offset': '0', + 'limit': self.search_limit + } + + # Optional shard + shard = request.GET.get('shard', '') + if shard is not None and len(shard.strip()) > 0: + params['shards'] = shard + '/solr' + else: + params['distrib'] = 'false' + + # Query index nodes for all file download urls for each dataset + dataset_ids = request.GET.getlist('dataset_id') + download_urls = [] + for dataset_id in dataset_ids: + params['dataset_id'] = dataset_id + response = requests.get(url=index_node_url, params=params) + download_urls += self._parse_download_urls(response.json()) + + # Request a short-lived certificate to be placed in the script + certificate = self._get_certificate(request) + + script = render_to_string(self.script_template, + { 'download_urls': download_urls, 'certificate': certificate } + ) + response = HttpResponse(script, content_type='application/x-sh') + + # Specify downloaded file name + script_timestamp = timezone.now().strftime('%Y%m%d%f') + response['Content-Disposition'] = \ + 'attachment; filename="wget-{}.sh"'.format(script_timestamp) + + return response + + @staticmethod + def _parse_download_urls(data): + """ + Parses the output of an index search request. + + Returns a list of urls + """ + + download_urls = [] + try: + for doc in data['response']['docs']: + download_urls.append(doc['url'][0].split('|')[0]) + except (KeyError, IndexError) as e: + raise e + + return download_urls + + @staticmethod + def _get_certificate(request): + """ + Generates a short-lived certificate from the user's session. + The user must have been authenticated with the 'esgf' social auth + provider. + + Returns a pem-style certificate + """ + + social = request.user.social_auth.get(provider='esgf') + access_token = social.extra_data['access_token'] + strategy = load_strategy() + backend = social.get_backend_instance(strategy) + + key, cert, _ = backend.get_certificate(access_token) + return key + cert From 8c18694de536204f815a9b6890dd90d2f08726d4 Mon Sep 17 00:00:00 2001 From: watucker Date: Thu, 22 Aug 2019 16:11:12 +0100 Subject: [PATCH 38/65] Fixed server error from missing session --- cog/views/views_download.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cog/views/views_download.py b/cog/views/views_download.py index 504f4afe9..b517787a5 100644 --- a/cog/views/views_download.py +++ b/cog/views/views_download.py @@ -6,6 +6,7 @@ from django.template.loader import render_to_string from django.views.generic import View from social_django.utils import load_strategy +from social_django.models import UserSocialAuth class WgetScriptView(View): @@ -16,6 +17,8 @@ class WgetScriptView(View): """ script_template = 'cog/download/wget_script.sh' + + # High arbitrary file search limit search_limit = 10000 def get(self, request, *args, **kwargs): @@ -24,6 +27,20 @@ def get(self, request, *args, **kwargs): index to search on. """ + social_session = None + if hasattr(request.user, "social_auth"): + + try: + social_session = request.user.social_auth.get(provider='esgf') + except UserSocialAuth.DoesNotExist: + pass + + if not social_session: + return HttpResponse('Unauthorized', status=401) + + # Request a short-lived certificate to be placed in the script + certificate = self._get_certificate(social_session) + index_node = request.GET.get('index_node') index_node_url = 'http://{}/esg-search/search'.format(index_node) @@ -50,9 +67,6 @@ def get(self, request, *args, **kwargs): response = requests.get(url=index_node_url, params=params) download_urls += self._parse_download_urls(response.json()) - # Request a short-lived certificate to be placed in the script - certificate = self._get_certificate(request) - script = render_to_string(self.script_template, { 'download_urls': download_urls, 'certificate': certificate } ) @@ -83,7 +97,7 @@ def _parse_download_urls(data): return download_urls @staticmethod - def _get_certificate(request): + def _get_certificate(social_session): """ Generates a short-lived certificate from the user's session. The user must have been authenticated with the 'esgf' social auth @@ -92,10 +106,9 @@ def _get_certificate(request): Returns a pem-style certificate """ - social = request.user.social_auth.get(provider='esgf') - access_token = social.extra_data['access_token'] + access_token = social_session.extra_data['access_token'] strategy = load_strategy() - backend = social.get_backend_instance(strategy) + backend = social_session.get_backend_instance(strategy) key, cert, _ = backend.get_certificate(access_token) return key + cert From d3ca8c461a228497a1c7d66295c409a9a76b068b Mon Sep 17 00:00:00 2001 From: watucker Date: Thu, 22 Aug 2019 16:12:00 +0100 Subject: [PATCH 39/65] Changed presentation of new WGET link --- cog/templates/cog/datacart/_wget_js.html | 4 ++-- cog/templates/cog/search/_record_display.html | 11 ++++++++++- cog/templatetags/search_utils.py | 11 ----------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index a3fae20e9..3a6a4bb78 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -137,13 +137,13 @@ html += ""; // Generate simplified wget script links - {% if request.user.is_authenticated %} + {% if request.user.is_authenticated and backends.associated %} var query_string = "index_node=" + index_node + "&shard=" + shard; for (var id_index in ids) { dataset_id = jsonResponse[wget_key][id_index]; query_string += "&dataset_id=" + dataset_id; } - html += " ( Simple WGET Script )"; + html += " ( NEW Simplified Script )"; {% endif %} } } diff --git a/cog/templates/cog/search/_record_display.html b/cog/templates/cog/search/_record_display.html index 02a2f6524..6e80cd591 100644 --- a/cog/templates/cog/search/_record_display.html +++ b/cog/templates/cog/search/_record_display.html @@ -83,7 +83,16 @@ {% elif url.1 == 'application/wget' %}  
- [ WGET Script ] + [ + + WGET Script + + {% if request.user.is_authenticated and backends.associated %} + ( + NEW Simplified Script + ) + {% endif %} + ]
diff --git a/cog/templatetags/search_utils.py b/cog/templatetags/search_utils.py index 7ff098879..61dbee465 100644 --- a/cog/templatetags/search_utils.py +++ b/cog/templatetags/search_utils.py @@ -1,6 +1,4 @@ from django import template -from django.urls import reverse_lazy -from urllib import urlencode from cog.models.search import searchMappings from cog.site_manager import siteManager from string import replace @@ -123,15 +121,6 @@ def recordUrls(record): "application/wget", "WGET Script") ) - query = { - 'dataset_id': record.id, - 'index_node': record.fields['index_node'][0], - 'shard': record.fields.get('shard', [''])[0], - } - script_url = '{url}?{query}'.format(url=reverse_lazy('wget_script'), query=urlencode(query)) - - urls.append((script_url, "application/simple-wget", "Simplified WGET Script")) - # add Globus endpoints if siteManager.isGlobusEnabled(): # only if this node has been registered with Globus if 'access' in record.fields and 'index_node' in record.fields and 'data_node' in record.fields: From 5928a0262004f02a9f673ca252ea4b94c3ac6639 Mon Sep 17 00:00:00 2001 From: watucker Date: Thu, 22 Aug 2019 17:00:56 +0100 Subject: [PATCH 40/65] Added hover text to WGET links --- cog/templates/cog/datacart/_wget_js.html | 6 +++++- cog/templates/cog/search/_record_display.html | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index 3a6a4bb78..b49bd6a64 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -143,7 +143,11 @@ dataset_id = jsonResponse[wget_key][id_index]; query_string += "&dataset_id=" + dataset_id; } - html += " ( NEW Simplified Script )"; + var link_parts = [ + " | NEW Simplified Script"] + html += link_parts.join(""); {% endif %} } } diff --git a/cog/templates/cog/search/_record_display.html b/cog/templates/cog/search/_record_display.html index 6e80cd591..29ddafb67 100644 --- a/cog/templates/cog/search/_record_display.html +++ b/cog/templates/cog/search/_record_display.html @@ -88,9 +88,10 @@ WGET Script {% if request.user.is_authenticated and backends.associated %} - ( - NEW Simplified Script - ) + | + NEW Simplified Script + {% endif %} ] From 80d908bde0a6b812a23a8a66cdb580004d7bf48a Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 12 Nov 2019 17:43:08 -0800 Subject: [PATCH 41/65] Update Globus downloads to use Globus SDK --- cog/plugins/globus/transfer.py | 75 ++++++++++++++-------------------- cog/views/views_globus.py | 12 +++--- requirements.txt | 2 +- 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/cog/plugins/globus/transfer.py b/cog/plugins/globus/transfer.py index ce6e1bdd8..ea2e890f9 100644 --- a/cog/plugins/globus/transfer.py +++ b/cog/plugins/globus/transfer.py @@ -7,10 +7,7 @@ from datetime import datetime, timedelta from cog.site_manager import siteManager if siteManager.isGlobusEnabled(): - from globusonline.transfer.api_client import Transfer - from globusonline.transfer.api_client import TransferAPIClient - from globusonline.transfer.api_client import TransferAPIError - from globusonline.transfer.api_client import x509_proxy + from globus_sdk import TransferData import os import urlparse @@ -30,66 +27,56 @@ def generateGlobusDownloadScript(download_map): return script -def activateEndpoint(api_client, endpoint, myproxy_server=None, username=None, password=None, cert=None, key=None): - if (not myproxy_server or not password) and (not myproxy_server or not cert): +def activateEndpoint(transfer_client, endpoint, myproxy_server=None, username=None, password=None): + if not myproxy_server or not password: # Try to autoactivate the endpoint - code, reason, result = api_client.endpoint_autoactivate(endpoint, if_expires_in=2880) - print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) + result = transfer_client.endpoint_autoactivate(endpoint, if_expires_in=2880) + print("Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"])) if result["code"] == "AutoActivationFailed": return (False, "") return (True, "") - code, reason, reqs = api_client.endpoint_activation_requirements(endpoint) - - # Activate the endpoint using an X.509 user credential stored by esgf-idp in /tmp/x509up__ - if cert and key: - cred_file = "/tmp/x509up_%s_%s" % (myproxy_server, username) - with open(cred_file, 'w') as cred: - cred.write(cert) - cred.write(key) - public_key = reqs.get_requirement_value("delegate_proxy", "public_key") - try: - proxy = x509_proxy.create_proxy_from_file(cred_file, public_key, lifetime_hours=72) - except Exception as e: - print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) - return False - reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) - else: - # Activate the endpoint using MyProxy server method - reqs.set_requirement_value("myproxy", "hostname", myproxy_server) - reqs.set_requirement_value("myproxy", "username", username) - reqs.set_requirement_value("myproxy", "passphrase", password) - reqs.set_requirement_value("myproxy", "lifetime_in_hours", "168") + requirements = transfer_client.endpoint_get_activation_requirements(endpoint) + requirements_json = requirements.data + + # Activate the endpoint using MyProxy server method + for i, d in enumerate(requirements_json["DATA"]): + if d["type"] == "myproxy": + if d["name"] == "hostname": + requirements_json["DATA"][i]["value"] = myproxy_server + elif d["name"] == "username": + requirements_json["DATA"][i]["value"] = username + elif d["name"] == "passphrase": + requirements_json["DATA"][i]["value"] = password + elif d["name"] == "lifetime_in_hours": + requirements_json["DATA"][i]["value"] = "168" try: - code, reason, result = api_client.endpoint_activate(endpoint, reqs) + result = transfer_client.endpoint_activate(endpoint, requirements_json) except Exception as e: - print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) + print("Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e))) return (False, str(e)) - if code != 200: - print "Could not aactivate the endpoint: %s. Error: %s - %s" % (endpoint, result["code"], result["message"]) + if result["code"] != "Activated.MyProxyCredential": + print("Could not aactivate the endpoint: %s. Error: %s - %s" % (endpoint, result["code"], result["message"])) return (False, result["message"]) - print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) + print("Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"])) return (True, "") -def submitTransfer(api_client, source_endpoint, source_files, target_endpoint, target_directory): +def submitTransfer(transfer_client, source_endpoint, source_files, target_endpoint, target_directory): ''' Method to submit a data transfer request to Globus. ''' - # obtain a submission id from Globus - code, message, data = api_client.transfer_submission_id() - submission_id = data["value"] - print "Obtained transfer submission id: %s" % submission_id - # maximum time for completing the transfer deadline = datetime.utcnow() + timedelta(days=10) # create a transfer request - transfer_task = Transfer(submission_id, source_endpoint, target_endpoint, deadline) + transfer_task = TransferData(transfer_client, source_endpoint, target_endpoint, deadline=deadline) + print("Obtained transfer submission id: %s" % transfer_task["submission_id"]) + for source_file in source_files: source_directory, filename = os.path.split(source_file) target_file = os.path.join(target_directory, filename) @@ -97,11 +84,11 @@ def submitTransfer(api_client, source_endpoint, source_files, target_endpoint, t # submit the transfer request try: - code, reason, data = api_client.transfer(transfer_task) + data = transfer_client.submit_transfer(transfer_task) task_id = data["task_id"] - print "Submitted transfer task with id: %s" % task_id + print("Submitted transfer task with id: %s" % task_id) except Exception as e: - print "Could not submit the transfer. Error: %s" % str(e) + print("Could not submit the transfer. Error: %s" % str(e)) task_id = "Could not submit the transfer. Please contact the ESGF node admin to investigate the issue." return task_id diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index 73180fbd7..a79fd566b 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -41,7 +41,8 @@ from base64 import urlsafe_b64encode from oauth2client import client as oauth_client from cog.plugins.globus.transfer import activateEndpoint, submitTransfer, generateGlobusDownloadScript - from globusonline.transfer.api_client import TransferAPIClient + from globus_sdk.transfer import TransferClient + from globus_sdk import AccessTokenAuthorizer client_id = siteManager.get('OAUTH_CLIENT_ID', section=SECTION_GLOBUS) client_secret = siteManager.get('OAUTH_CLIENT_SECRET', section=SECTION_GLOBUS) @@ -262,13 +263,14 @@ def submit(request): #print 'Downloading files=%s' % download_map.items() print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder) - api_client = TransferAPIClient(username, goauth=access_token) + token_authorizer = AccessTokenAuthorizer(access_token) + transfer_client = TransferClient(token_authorizer) # loop over source endpoints and autoactivate them # if the autoactivation fails, redirect to a form asking for a password - activateEndpoint(api_client, target_endpoint) + activateEndpoint(transfer_client, target_endpoint) for source_endpoint, source_files in download_map.items(): status, message = activateEndpoint( - api_client, source_endpoint, + transfer_client, source_endpoint, myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) if not status: print hostname @@ -282,7 +284,7 @@ def submit(request): # submit transfer request task_id = submitTransfer( - api_client, source_endpoint, source_files, + transfer_client, source_endpoint, source_files, target_endpoint, target_folder) task_ids.append(task_id) diff --git a/requirements.txt b/requirements.txt index ffc1f6a61..1a31112d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ passlib==1.6.5 pysqlite==2.8.1 django-contrib-comments==1.6.2 oauth2client==2.0.1 -globusonline-transfer-api-client==0.10.18 +globus-sdk==1.8.0 #'pillow==3.1.0', # pre-requisite: must be installed with --use-wheel on MAC-OSX django-simple-captcha==0.5.1 html5lib==1.0b8 From 2840e5908148f8528f72657fff8b4032883f6858 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 25 Nov 2019 14:16:32 -0800 Subject: [PATCH 42/65] Add parentheses to print statements --- cog/plugins/globus/transfer.py | 6 +++--- cog/views/views_globus.py | 6 +++--- cog/views/views_search.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cog/plugins/globus/transfer.py b/cog/plugins/globus/transfer.py index c8043392e..9078c2820 100644 --- a/cog/plugins/globus/transfer.py +++ b/cog/plugins/globus/transfer.py @@ -34,7 +34,7 @@ def activateEndpoint(api_client, endpoint, myproxy_server=None, username=None, p if (not myproxy_server or not password) and (not myproxy_server or not cert): # Try to autoactivate the endpoint code, reason, result = api_client.endpoint_autoactivate(endpoint, if_expires_in=2880) - print "Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"]) + print("Endpoint Activation: %s. %s: %s" % (endpoint, result["code"], result["message"])) if result["code"] == "AutoActivationFailed": return (False, "") return (True, "") @@ -51,7 +51,7 @@ def activateEndpoint(api_client, endpoint, myproxy_server=None, username=None, p try: proxy = x509_proxy.create_proxy_from_file(cred_file, public_key, lifetime_hours=72) except Exception as e: - print "Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e)) + print("Could not activate the endpoint: %s. Error: %s" % (endpoint, str(e))) return False reqs.set_requirement_value("delegate_proxy", "proxy_chain", proxy) else: @@ -83,7 +83,7 @@ def submitTransfer(api_client, source_endpoint, source_files, target_endpoint, t # obtain a submission id from Globus code, message, data = api_client.transfer_submission_id() submission_id = data["value"] - print "Obtained transfer submission id: %s" % submission_id + print("Obtained transfer submission id: %s" % submission_id) # maximum time for completing the transfer deadline = datetime.utcnow() + timedelta(days=10) diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index 6db796965..80f257920 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -127,7 +127,7 @@ def download(request): url = settings.DEFAULT_SEARCH_URL+"?"+urllib.urlencode(params) - print 'Searching for files at URL: %s' % url + print('Searching for files at URL: %s' % url) jobj = getJson(url) # parse response for GridFTP URls @@ -260,7 +260,7 @@ def submit(request): target_folder = request.session.get(TARGET_FOLDER) #print 'Downloading files=%s' % download_map.items() - print 'User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder) + print('User selected destionation endpoint:%s, folder: %s' % (target_endpoint, target_folder)) api_client = TransferAPIClient(username, goauth=access_token) # loop over source endpoints and autoactivate them @@ -271,7 +271,7 @@ def submit(request): api_client, source_endpoint, myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) if not status: - print hostname + print(hostname) return render(request, 'cog/globus/password.html', { 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message }) diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 5a9429c12..8ec5c156b 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -394,7 +394,7 @@ def metadata_display(request, project_short_name): params.append(('dataset_id', dataset_id)) url = project.searchprofile.url+"?"+urllib.urlencode(params) - print 'Metadata Solr search URL=%s' % url + print('Metadata Solr search URL=%s' % url) fh = urllib2.urlopen(url) response = fh.read().decode("UTF-8") @@ -849,7 +849,7 @@ def search_files(request, dataset_id, index_node): params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' url = settings.DEFAULT_SEARCH_URL+"?"+urllib.urlencode(params) - print 'Searching for files: URL=%s' % url + print('Searching for files: URL=%s' % url) fh = urllib2.urlopen(url) response = fh.read().decode("UTF-8") From 0d211450f45120789936fe7d1439b251369a5df7 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Mon, 25 Nov 2019 14:48:26 -0800 Subject: [PATCH 43/65] Update requirements.txt package versions --- requirements.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6a328360f..7ef0ee723 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,17 @@ # CoG application dependencies django>=2.2,<2.3 -django-grappelli==2.12.3 +django-grappelli==2.13.2 #django-openid-auth==0.7', # must be installed independently from fork -sqlalchemy>=1.0.11,<1.3 +sqlalchemy==1.3.11 south==1.0.2 -psycopg2>=2.5.2,<2.7 -python-openid==2.2.5 -passlib==1.6.5 -pysqlite==2.8.1 -django-contrib-comments==1.6.2 -oauth2client==2.0.1 +psycopg2==2.8.4 +python3-openid==3.1.0 +passlib==1.7.2 +django-contrib-comments==1.9.1 +oauth2client==4.1.3 globusonline-transfer-api-client==0.10.18 #'pillow==3.1.0', # pre-requisite: must be installed with --use-wheel on MAC-OSX -django-simple-captcha==0.5.11 +django-simple-captcha==0.5.12 html5lib==1.0.1 bleach==3.1.0 python-magic==0.4.15 From 467a28eeb23715c7e4453a254c2d899434f40f27 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 10 Dec 2019 18:20:40 -0800 Subject: [PATCH 44/65] Apply changes for urllib in Python 3. Change django.core.urlresolvers to django.urls --- cog/views/views_datacart.py | 2 +- cog/views/views_search.py | 10 +++++----- cog/views/views_status.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cog/views/views_datacart.py b/cog/views/views_datacart.py index 28e435f07..2c8ac3328 100644 --- a/cog/views/views_datacart.py +++ b/cog/views/views_datacart.py @@ -1,7 +1,7 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext from django.conf import settings -from urlparse import urlparse +from urllib.parse import urlparse from django.http import HttpResponseRedirect, HttpResponse, HttpResponseForbidden, HttpResponseNotAllowed from django.urls import reverse from cog.models import * diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 8ec5c156b..9e768ce6e 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -393,9 +393,9 @@ def metadata_display(request, project_short_name): if type == 'File': params.append(('dataset_id', dataset_id)) - url = project.searchprofile.url+"?"+urllib.urlencode(params) + url = project.searchprofile.url+"?"+urllib.parse.urlencode(params) print('Metadata Solr search URL=%s' % url) - fh = urllib2.urlopen(url) + fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") # parse JSON response (containing only one matching 'doc) @@ -406,7 +406,7 @@ def metadata_display(request, project_short_name): parentMetadata = {} if type == 'File': params = [('type', 'Dataset'), ('id', dataset_id), ("format", "application/solr+json")] - url = project.searchprofile.url+"?"+urllib.urlencode(params) + url = project.searchprofile.url+"?"+urllib.parse.urlencode(params) # print 'Solr search URL=%s' % url fh = urllib.request.urlopen(url) response = fh.read() @@ -848,9 +848,9 @@ def search_files(request, dataset_id, index_node): if shard is not None and len(shard.strip()) > 0: params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' - url = settings.DEFAULT_SEARCH_URL+"?"+urllib.urlencode(params) + url = settings.DEFAULT_SEARCH_URL+"?"+urllib.parse.urlencode(params) print('Searching for files: URL=%s' % url) - fh = urllib2.urlopen(url) + fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") return HttpResponse(response, content_type="application/json") diff --git a/cog/views/views_status.py b/cog/views/views_status.py index a961935cc..ddc4560c7 100644 --- a/cog/views/views_status.py +++ b/cog/views/views_status.py @@ -1,7 +1,7 @@ from django.shortcuts import get_object_or_404, render, redirect from django.template import RequestContext from django.http import HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.decorators import login_required, user_passes_test, permission_required from django.forms.models import modelformset_factory, inlineformset_factory import string From 8950f7da6acd0abab86247a875cdfd04bd31f85e Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 10 Dec 2019 18:28:15 -0800 Subject: [PATCH 45/65] Fix import of views_status in __init__.py --- cog/views/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cog/views/__init__.py b/cog/views/__init__.py index 2aa57c8cd..1d8de486c 100644 --- a/cog/views/__init__.py +++ b/cog/views/__init__.py @@ -19,4 +19,4 @@ from cog.views.views_access_control import * from cog.views.views_globus import * from cog.views.utils import * -from views_status import * +from cog.views.views_status import * From 95c268af653c5f3f38ad7c178c707af5c3b56458 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Wed, 11 Dec 2019 16:48:40 -0800 Subject: [PATCH 46/65] Update reading of 'x-cera-rc' header value when displaying citations. --- cog/views/views_search.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 9e768ce6e..5c6173319 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -991,12 +991,11 @@ def citation_display(request): try: fh = urllib.request.urlopen(url) response = fh.read() - headers = fh.info().dict except HTTPError as e: print(('HTTPError %s for %s' % (str(e.code), url))) return HttpResponseNotFound() - if int(headers['x-cera-rc']) > 0: + if int(fh.getheader('x-cera-rc')) > 0: print('Citation not found: %s' % url) return HttpResponseNotFound() From 3d1270450319bd3be1e8cf2ef44cdfdef477b128 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 13 Dec 2019 14:24:08 -0800 Subject: [PATCH 47/65] Update views_globus --- cog/views/views_globus.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index 88f837e62..c4194df19 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.core.urlresolvers import reverse +from django.urls import reverse from django.contrib.auth.decorators import login_required from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseServerError from django.shortcuts import render @@ -61,8 +61,8 @@ def discover_myproxy(openid): def establishFlow(request): - basic_auth_str = urlsafe_b64encode("{}:{}".format(client_id, client_secret)) - auth_header = "Basic " + basic_auth_str + basic_auth_str = urlsafe_b64encode('{}:{}'.format(client_id, client_secret).encode()) + auth_header = b'Basic ' + basic_auth_str return oauth_client.OAuth2WebServerFlow( client_id = client_id, authorization_header = auth_header, @@ -127,7 +127,7 @@ def download(request): params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' - url = settings.DEFAULT_SEARCH_URL+"?"+urllib.urlencode(params) + url = settings.DEFAULT_SEARCH_URL+"?"+urllib.parse.urlencode(params) print('Searching for files at URL: %s' % url) jobj = getJson(url) @@ -269,14 +269,10 @@ def submit(request): # if the autoactivation fails, redirect to a form asking for a password activateEndpoint(transfer_client, target_endpoint) for source_endpoint, source_files in download_map.items(): - status, message = activateEndpoint( - transfer_client, source_endpoint, - myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) + status, message = activateEndpoint(transfer_client, source_endpoint, myproxy_server=myproxy_server, username=esgf_username, password=esgf_password) if not status: - print(hostname) - return render(request, - 'cog/globus/password.html', - { 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message }) + print(hostname) + return render(request, 'cog/globus/password.html', { 'openid': openid, 'username': hostname=='ceda.ac.uk', 'message': message }) # loop over source endpoints, submit one transfer for each source endpoint task_ids = [] # list of submitted task ids From fdb6567b53236ff120947d433ac6a8215b81a269 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Fri, 13 Dec 2019 14:30:12 -0800 Subject: [PATCH 48/65] Fix wget script URL so that it has a '?' even if there are no shards to add to it. --- cog/templates/cog/datacart/_wget_js.html | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index 77c0acb55..9c6ac9b3c 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -66,21 +66,24 @@ ids = ids.split(","); // FIXME: relative wget URL is hard-wired to ESGF web service endpoint - var url = "http://"+index_node+"/esg-search/wget/"; + var url = "http://"+index_node+"/esg-search/wget/?"; + + // add dataset ids + for (i=0; i0) { + url += "&"; + } + url += "dataset_id="+encodeURIComponent(ids[i]); + } // add optional 'local' shard if (shard.length>0) { - url += "?shards=" + encodeURI(shard+"/solr"); // "?shards=localhost:8982/solr" + url += "&shards=" + encodeURI(shard+"/solr"); // "?shards=localhost:8982/solr" // or target shared "slave" shard only } // add optional query filter url += getQueryFilter('GET'); - - // add dataset ids - for (i=0; i Date: Fri, 20 Dec 2019 14:00:34 -0800 Subject: [PATCH 49/65] Comment out esgfpid in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9ca0f3a23..84f37bc5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ django-simple-captcha==0.5.12 html5lib==1.0.1 bleach==3.1.0 python-magic==0.4.15 -esgfpid==0.7.14 +#esgfpid==0.7.14 # Used for datacart PIDs. Currently not used in production. From 3ff3e5f0cb4b6cd8ccdfcd4c849979a035a2e4dc Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 14 Jan 2020 18:07:01 -0800 Subject: [PATCH 50/65] When adding pages or links, make label default to the title if the label field is left blank. --- cog/forms/forms_post.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cog/forms/forms_post.py b/cog/forms/forms_post.py index 94b8a50c7..e2a5906b1 100644 --- a/cog/forms/forms_post.py +++ b/cog/forms/forms_post.py @@ -152,6 +152,13 @@ def clean(self): cleaned_data["topic"] = topic + # check if the 'label' field is set for hyperlink and page posts + # if 'label' is blank, then fill the field with the first 28 characters of 'title' + if type == Post.TYPE_PAGE or type == Post.TYPE_HYPERLINK: + label = cleaned_data["label"] + if label is None: + cleaned_data["label"] = cleaned_data["title"][:28] + # prevent XSS on fields 'title', 'label', 'newtopic' for key in ["title", "label", "newtopic"]: if key in cleaned_data: From f42922d9380ed679941c5d1ac93ffe625e1fffdf Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Tue, 28 Jan 2020 17:57:55 -0800 Subject: [PATCH 51/65] If the constraints field is blank when configuring search, then store an emptry string for constraints --- cog/forms/forms_search.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cog/forms/forms_search.py b/cog/forms/forms_search.py index d3df7082e..80265a7c2 100644 --- a/cog/forms/forms_search.py +++ b/cog/forms/forms_search.py @@ -15,7 +15,12 @@ def clean_url(self): return clean_url_field(self, 'url') def clean_constraints(self): - return clean_url_field(self, 'constraints') + # if no constraints are found, then return an empty string + data = self.cleaned_data['constraints'] + if data is None: + return '' + else: + return clean_url_field(self, 'constraints') class SearchFacetForm(ModelForm): From a7b03e0214510f52eb53d7e701d0e5bd90ea49de Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 12 Mar 2020 14:26:24 -0700 Subject: [PATCH 52/65] Revert "Use default search path for searching and displaying files" This reverts commit d7c9e5be0d7fe44433048355543c2d6ac17a393e. --- cog/templates/cog/datacart/_datacart_js.html | 2 +- cog/views/views_search.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cog/templates/cog/datacart/_datacart_js.html b/cog/templates/cog/datacart/_datacart_js.html index 431919415..d8f3ed5c9 100644 --- a/cog/templates/cog/datacart/_datacart_js.html +++ b/cog/templates/cog/datacart/_datacart_js.html @@ -96,7 +96,7 @@ // relative path to COG application // note: cannot execute HTTP requests to other hosts because of javascript security restrictions // note: do not use django URL tag because it is interpreted before the 'dataset_id', 'index_node' values are known - var url = "/search_files/"+dataset_id+"/"+index_node+"/?limit="+limit + var url = "/search_files/"+dataset_id+"/"+index_node+"/?limit="+limit+"&project="+project // add optional query filter + getQueryFilter('GET'); // add optional 'local' shard diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 5c6173319..58ebf6cb8 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -1,6 +1,6 @@ from copy import copy, deepcopy import json -import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse +import urllib.request, urllib.parse, urllib.error from urllib.error import HTTPError from cog.config.search import SearchConfigParser @@ -15,7 +15,6 @@ from cog.templatetags.search_utils import displayMetadataKey, formatMetadataKey from cog.views.constants import PERMISSION_DENIED_MESSAGE, TEMPLATE_NOT_FOUND_MESSAGE from cog.views.utils import getQueryDict -from django.conf import settings from django.contrib.auth.decorators import login_required, user_passes_test from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse @@ -830,6 +829,12 @@ def search_files(request, dataset_id, index_node): # maximum number of files to query for limit = request.GET.get('limit', 20) + project_short_name = request.GET.get('project', None) + if project_short_name is not None: + project = get_object_or_404(Project, short_name__iexact=project_short_name) + endpoint = project.searchprofile.url + else: + endpoint = 'http://'+index_node+'/esg-search/search/' params = [('type', "File"), ('dataset_id', dataset_id), ("format", "application/solr+json"), ('offset', '0'), ('limit', limit)] @@ -848,7 +853,7 @@ def search_files(request, dataset_id, index_node): if shard is not None and len(shard.strip()) > 0: params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' - url = settings.DEFAULT_SEARCH_URL+"?"+urllib.parse.urlencode(params) + url = endpoint+"?"+urllib.parse.urlencode(params) print('Searching for files: URL=%s' % url) fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") From 740f8b7f61b1d6b2ed6e364caa0b18c10c2628bb Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 12 Mar 2020 14:31:54 -0700 Subject: [PATCH 53/65] Revert "Use project specific search path for metadata displays" This reverts commit ec8d691af25548c907a69688251966eb8b9c438e. --- cog/views/views_search.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cog/views/views_search.py b/cog/views/views_search.py index 58ebf6cb8..f3d0efa8d 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -388,11 +388,11 @@ def metadata_display(request, project_short_name): config = _getSearchConfig(request, project) # retrieve result metadata - params = [('type', type), ('id', id), ("format", "application/solr+json")] + params = [('type', type), ('id', id), ("format", "application/solr+json"), ("distrib", "false")] if type == 'File': params.append(('dataset_id', dataset_id)) - url = project.searchprofile.url+"?"+urllib.parse.urlencode(params) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) print('Metadata Solr search URL=%s' % url) fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") @@ -404,8 +404,8 @@ def metadata_display(request, project_short_name): # retrieve parent metadata parentMetadata = {} if type == 'File': - params = [('type', 'Dataset'), ('id', dataset_id), ("format", "application/solr+json")] - url = project.searchprofile.url+"?"+urllib.parse.urlencode(params) + params = [('type', 'Dataset'), ('id', dataset_id), ("format", "application/solr+json"), ("distrib", "false")] + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) # print 'Solr search URL=%s' % url fh = urllib.request.urlopen(url) response = fh.read() From b3f5a0ebba0ea5012dc5ec48d594b44aa688d782 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 12 Mar 2020 14:34:08 -0700 Subject: [PATCH 54/65] Revert "Use the default search url for globus file searches" This reverts commit 3bc58cdecb1031a20aebc86290ab633b3643a191. --- cog/views/views_globus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cog/views/views_globus.py b/cog/views/views_globus.py index c4194df19..928804348 100644 --- a/cog/views/views_globus.py +++ b/cog/views/views_globus.py @@ -125,9 +125,11 @@ def download(request): shard = request.GET.get('shard', '') if shard is not None and len(shard.strip()) > 0: params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' + else: + params.append(("distrib", "false")) - url = settings.DEFAULT_SEARCH_URL+"?"+urllib.parse.urlencode(params) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) print('Searching for files at URL: %s' % url) jobj = getJson(url) From 9a02f72136d128bfac6d08d44c6b26d3c09b9cf4 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 12 Mar 2020 14:35:58 -0700 Subject: [PATCH 55/65] Revert "User the default search location as the index node for wget scripts" This reverts commit 887cd45a81f1b5a0e0d3c2e5f9e3415ab4d416be. --- cog/templates/cog/datacart/_wget_js.html | 2 ++ cog/views/views_datacart.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index 9c6ac9b3c..11dbd985a 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -80,6 +80,8 @@ if (shard.length>0) { url += "&shards=" + encodeURI(shard+"/solr"); // "?shards=localhost:8982/solr" // or target shared "slave" shard only + } else { + url += "?distrib=false"; } // add optional query filter diff --git a/cog/views/views_datacart.py b/cog/views/views_datacart.py index 2c8ac3328..6b3d41d4d 100644 --- a/cog/views/views_datacart.py +++ b/cog/views/views_datacart.py @@ -214,7 +214,7 @@ def datacart_wget(request, site_id, user_id): if item.identifier in ids: # group selected dataset by index_node - index_node = urlparse(settings.DEFAULT_SEARCH_URL).netloc + index_node = item.getValue('index_node') wget_key = index_node shard = item.getValue('shard') if shard is not None and len(shard.strip())>0: From b946a56cb18db5e08929724a5b27f3d61fa3dd70 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 12 Mar 2020 14:38:17 -0700 Subject: [PATCH 56/65] Revert "Ensure the project URL parameter is available to determine correct search endpoint" This reverts commit 94ff176b424ec319df062667d36549dd66aa41ff. --- cog/templates/cog/datacart/_datacart_js.html | 2 +- cog/views/views_search.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cog/templates/cog/datacart/_datacart_js.html b/cog/templates/cog/datacart/_datacart_js.html index d8f3ed5c9..431919415 100644 --- a/cog/templates/cog/datacart/_datacart_js.html +++ b/cog/templates/cog/datacart/_datacart_js.html @@ -96,7 +96,7 @@ // relative path to COG application // note: cannot execute HTTP requests to other hosts because of javascript security restrictions // note: do not use django URL tag because it is interpreted before the 'dataset_id', 'index_node' values are known - var url = "/search_files/"+dataset_id+"/"+index_node+"/?limit="+limit+"&project="+project + var url = "/search_files/"+dataset_id+"/"+index_node+"/?limit="+limit // add optional query filter + getQueryFilter('GET'); // add optional 'local' shard diff --git a/cog/views/views_search.py b/cog/views/views_search.py index f3d0efa8d..d493b4909 100644 --- a/cog/views/views_search.py +++ b/cog/views/views_search.py @@ -829,12 +829,7 @@ def search_files(request, dataset_id, index_node): # maximum number of files to query for limit = request.GET.get('limit', 20) - project_short_name = request.GET.get('project', None) - if project_short_name is not None: - project = get_object_or_404(Project, short_name__iexact=project_short_name) - endpoint = project.searchprofile.url - else: - endpoint = 'http://'+index_node+'/esg-search/search/' + params = [('type', "File"), ('dataset_id', dataset_id), ("format", "application/solr+json"), ('offset', '0'), ('limit', limit)] @@ -852,8 +847,10 @@ def search_files(request, dataset_id, index_node): shard = request.GET.get('shard', '') if shard is not None and len(shard.strip()) > 0: params.append(('shards', shard+"/solr")) # '&shards=localhost:8982/solr' + else: + params.append(("distrib", "false")) - url = endpoint+"?"+urllib.parse.urlencode(params) + url = "http://"+index_node+"/esg-search/search?"+urllib.parse.urlencode(params) print('Searching for files: URL=%s' % url) fh = urllib.request.urlopen(url) response = fh.read().decode("UTF-8") From 367786ec543ea9c440496ca6c087acc423b84d5a Mon Sep 17 00:00:00 2001 From: watucker Date: Fri, 13 Mar 2020 13:06:24 +0000 Subject: [PATCH 57/65] Refactored ESGFOAuth2 backend to ensure thread-safety Replaces the usual social auth endpoint class attributes with properties to prevent instances potentially interfering with one another. --- cog/backends/esgf.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/cog/backends/esgf.py b/cog/backends/esgf.py index 9a182251c..db93fd0c3 100644 --- a/cog/backends/esgf.py +++ b/cog/backends/esgf.py @@ -13,10 +13,6 @@ class ESGFOAuth2(BaseOAuth2): name = 'esgf' - AUTHORIZATION_URL = 'https://slcs.ceda.ac.uk/oauth/authorize' - ACCESS_TOKEN_URL = 'https://slcs.ceda.ac.uk/oauth/access_token' - CERTIFICATE_URL = 'https://slcs.ceda.ac.uk/oauth/certificate/' - DEFAULT_SCOPE = ['https://slcs.ceda.ac.uk/oauth/certificate/'] REDIRECT_STATE = True ACCESS_TOKEN_METHOD = 'POST' EXTRA_DATA = [ @@ -26,8 +22,30 @@ class ESGFOAuth2(BaseOAuth2): ('openid', 'openid', True) ] + @property + def AUTHORIZATION_URL(self): + return self._authorization_url + + @property + def ACCESS_TOKEN_URL(self): + return self._access_token_url + + @property + def CERTIFICATE_URL(self): + return self._certificate_url + + @property + def DEFAULT_SCOPE(self): + return self._default_scope + def __init__(self, strategy=None, redirect_uri=None): + + self._authorization_url = None + self._access_token_url = None + self._certificate_url = None + self._default_scope = None + super(ESGFOAuth2, self).__init__(strategy, redirect_uri) # Get openid_identifier added to the session by PSA. Requires @@ -46,12 +64,12 @@ def set_urls(self, openid): return for e in endpoints: if e.matchTypes(['urn:esg:security:oauth:endpoint:authorize']): - self.AUTHORIZATION_URL = e.uri + self._authorization_url = e.uri elif e.matchTypes(['urn:esg:security:oauth:endpoint:access']): - self.ACCESS_TOKEN_URL = e.uri + self._access_token_url = e.uri elif e.matchTypes(['urn:esg:security:oauth:endpoint:resource']): - self.CERTIFICATE_URL = e.uri - self.DEFAULT_SCOPE = [e.uri] + self._certificate_url = e.uri + self._default_scope = [e.uri] From 69bdbea338aa05771d1e7e29b10411a7567cb240 Mon Sep 17 00:00:00 2001 From: watucker Date: Thu, 19 Mar 2020 17:30:32 +0000 Subject: [PATCH 58/65] Python 3 fixes for OAuth2 backend --- cog/backends/esgf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cog/backends/esgf.py b/cog/backends/esgf.py index db93fd0c3..1714e25a8 100644 --- a/cog/backends/esgf.py +++ b/cog/backends/esgf.py @@ -4,9 +4,10 @@ from six.moves.urllib_parse import urlencode, unquote import os +import logging from base64 import b64encode from OpenSSL import crypto -from urlparse import urlparse +from urllib.parse import urlparse from openid.yadis.services import getServiceEndpoints from openid.yadis.discover import DiscoveryFailure @@ -94,8 +95,8 @@ def get_certificate(self, access_token): try: response = self.request(url, method='POST', **request_args) response.raise_for_status() - except Exception, err: - print Exception, err + except Exception as e: + logging.error(e) self.cert = response.text cert = crypto.load_certificate(crypto.FILETYPE_PEM, response.text) self.subject = cert.get_subject() @@ -159,7 +160,6 @@ def openid_url(self): def get_user_details(self, response): - print response.identity_url username = os.path.basename(urlparse(response.identity_url).path) return {'openid': response.identity_url, 'username': username, From 514d5826cb1a3cfa6e3fb51585b84bc1c70ad348 Mon Sep 17 00:00:00 2001 From: watucker Date: Thu, 19 Mar 2020 17:31:20 +0000 Subject: [PATCH 59/65] Additional fixes for Python 3 --- cog/views/__init__.py | 34 +++++++++++++++++----------------- cog/views/views_account.py | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cog/views/__init__.py b/cog/views/__init__.py index 5e78ecae8..d72f607f4 100644 --- a/cog/views/__init__.py +++ b/cog/views/__init__.py @@ -1,21 +1,21 @@ from cog.views.constants import * -from views_templated import * -from views_bookmarks import * -from views_account import * -from views_datacart import * -from views_download import * -from views_doc import * -from views_external_urls import * -from views_account import * -from views_governance import * -from views_admin import * -from views_membership import * -from views_news import * -from views_post import * -from views_project import * -from views_signal import * -from views_aboutus import * -from views_search import * +from .views_templated import * +from .views_bookmarks import * +from .views_account import * +from .views_datacart import * +from .views_download import * +from .views_doc import * +from .views_external_urls import * +from .views_account import * +from .views_governance import * +from .views_admin import * +from .views_membership import * +from .views_news import * +from .views_post import * +from .views_project import * +from .views_signal import * +from .views_aboutus import * +from .views_search import * from cog.views.views_share import * from cog.views.views_access_control import * from cog.views.views_globus import * diff --git a/cog/views/views_account.py b/cog/views/views_account.py index a2156f84b..a7283d993 100644 --- a/cog/views/views_account.py +++ b/cog/views/views_account.py @@ -381,7 +381,7 @@ def user_detail(request, user_id): user_profile = UserProfile(user=user) user_profile.save() print("Created empty profile for user=%s" % user) - if request.user.is_authenticated(): + if request.user.is_authenticated: if request.user.social_auth.filter(provider='esgf'): social = request.user.social_auth.get(provider='esgf') setattr(user_profile, 'openids', [social.uid]) From b024cb7dcbd7428df04e88d1be431cff294889f4 Mon Sep 17 00:00:00 2001 From: mauzey1 Date: Thu, 19 Mar 2020 13:44:29 -0700 Subject: [PATCH 60/65] Revert "Fix wget script URL so that it has a '?' even if there are no shards to add to it." This reverts commit fdb6567b53236ff120947d433ac6a8215b81a269. --- cog/templates/cog/datacart/_wget_js.html | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/cog/templates/cog/datacart/_wget_js.html b/cog/templates/cog/datacart/_wget_js.html index 11dbd985a..a21ee2443 100644 --- a/cog/templates/cog/datacart/_wget_js.html +++ b/cog/templates/cog/datacart/_wget_js.html @@ -66,19 +66,11 @@ ids = ids.split(","); // FIXME: relative wget URL is hard-wired to ESGF web service endpoint - var url = "http://"+index_node+"/esg-search/wget/?"; - - // add dataset ids - for (i=0; i0) { - url += "&"; - } - url += "dataset_id="+encodeURIComponent(ids[i]); - } + var url = "http://"+index_node+"/esg-search/wget/"; // add optional 'local' shard if (shard.length>0) { - url += "&shards=" + encodeURI(shard+"/solr"); // "?shards=localhost:8982/solr" + url += "?shards=" + encodeURI(shard+"/solr"); // "?shards=localhost:8982/solr" // or target shared "slave" shard only } else { url += "?distrib=false"; @@ -86,6 +78,11 @@ // add optional query filter url += getQueryFilter('GET'); + + // add dataset ids + for (i=0; i Date: Fri, 20 Mar 2020 13:18:17 +0000 Subject: [PATCH 61/65] Added requirements for OAuth2 support --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 84f37bc5f..a336606cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,5 @@ html5lib==1.0.1 bleach==3.1.0 python-magic==0.4.15 #esgfpid==0.7.14 # Used for datacart PIDs. Currently not used in production. +social-auth-app-django==3.1.0 +pyOpenSSL==19.1.0 From 053d6becdbc917dcbae122fb2470283df591d1f0 Mon Sep 17 00:00:00 2001 From: watucker Date: Tue, 24 Mar 2020 13:43:37 +0000 Subject: [PATCH 62/65] Merge branch 'python_3' of https://github.com/mauzey1/COG into devel From 59e98e70dde9f58c733bf74cd05ac0ddb457b010 Mon Sep 17 00:00:00 2001 From: watucker Date: Wed, 25 Mar 2020 14:48:59 +0000 Subject: [PATCH 63/65] Adding missing migrations --- .../0003_auto_20200324_1100.py | 18 + cog/migrations/0010_auto_20200324_1100.py | 565 ++++++++++++++++++ 2 files changed, 583 insertions(+) create mode 100644 cog/db_migrations/django_openid_auth/0003_auto_20200324_1100.py create mode 100644 cog/migrations/0010_auto_20200324_1100.py diff --git a/cog/db_migrations/django_openid_auth/0003_auto_20200324_1100.py b/cog/db_migrations/django_openid_auth/0003_auto_20200324_1100.py new file mode 100644 index 000000000..c24aa8a9a --- /dev/null +++ b/cog/db_migrations/django_openid_auth/0003_auto_20200324_1100.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.11 on 2020-03-24 11:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_openid_auth', '0002_auto_20160106_0812'), + ] + + operations = [ + migrations.AlterField( + model_name='useropenid', + name='claimed_id', + field=models.TextField(max_length=2047), + ), + ] diff --git a/cog/migrations/0010_auto_20200324_1100.py b/cog/migrations/0010_auto_20200324_1100.py new file mode 100644 index 000000000..c4a20c99a --- /dev/null +++ b/cog/migrations/0010_auto_20200324_1100.py @@ -0,0 +1,565 @@ +# Generated by Django 2.2.11 on 2020-03-24 11:00 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cog', '0009_auto_20160826_0339'), + ] + + operations = [ + migrations.AlterField( + model_name='bookmark', + name='url', + field=models.URLField(max_length=1000, verbose_name='URL'), + ), + migrations.AlterField( + model_name='collaborator', + name='first_name', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='collaborator', + name='image', + field=models.ImageField(blank=True, null=True, upload_to='photos/'), + ), + migrations.AlterField( + model_name='collaborator', + name='institution', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='collaborator', + name='last_name', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='collaborator', + name='researchKeywords', + field=models.CharField(blank=True, default='', max_length=60, null=True), + ), + migrations.AlterField( + model_name='communicationmeans', + name='frequency', + field=models.CharField(blank=True, help_text='String used to describe the frequency of the meeting (e.g. Weekly on Wednesdays, Monthly, Yearly, ...).', max_length=50, null='True', verbose_name='Frequency'), + ), + migrations.AlterField( + model_name='communicationmeans', + name='membership', + field=models.CharField(blank=True, choices=[('Open', 'Open'), ('Closed', 'Closed'), ('By Invitation', 'By Invitation')], help_text='A field that indicates whether the means is open or closed to non-members.', max_length=50, null=True, verbose_name='Membership'), + ), + migrations.AlterField( + model_name='communicationmeans', + name='participationDetails', + field=models.TextField(blank=True, help_text='Information about how a person would participate: phone number, pass code, meeting venue, etc.', null=True, verbose_name='Participation Details'), + ), + migrations.AlterField( + model_name='communicationmeans', + name='purpose', + field=models.CharField(blank=True, choices=[('Overall Project Coordination', 'Overall Project Coordination'), ('Steering Committee', 'Steering Committee'), ('Design', 'Design'), ('Design and Implementation Review', 'Design and Implementation Review'), ('Task Prioritization', 'Task Prioritization'), ('Requirements Identification', 'Requirements Identification'), ('Strategic Direction', 'Strategic Direction'), ('External Review', 'External Review'), ('Implementation', 'Implementation'), ('Meeting Planning', 'Meeting Planning'), ('Testing', 'Testing'), ('Knowledge Transfer', 'Knowledge Transfer'), ('Grant Writing', 'Grant Writing'), ('Other', 'Other')], help_text='Purpose of meeting (choose from controlled vocabulary).', max_length=50, null=True, verbose_name='Purpose'), + ), + migrations.AlterField( + model_name='communicationmeans', + name='title', + field=models.CharField(help_text='String used to describe the name of the meeting.', max_length=200, verbose_name='Title'), + ), + migrations.AlterField( + model_name='communicationmeans', + name='type', + field=models.CharField(choices=[('Telco', 'Telco'), ('Face-to-face', 'Face-to-face'), ('Webinar', 'Webinar'), ('Video Conference', 'Video Conference'), ('Internet Chat', 'Internet Chat'), ('Wiki', 'Wiki'), ('Mailing List', 'Mailing List')], help_text='String used to describe the type of meeting (choose from controlled vocabulary).', max_length=50, verbose_name='Type'), + ), + migrations.AlterField( + model_name='datacartitem', + name='date', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date Time'), + ), + migrations.AlterField( + model_name='doc', + name='author', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='documents', to=settings.AUTH_USER_MODEL, verbose_name='Publisher'), + ), + migrations.AlterField( + model_name='doc', + name='is_private', + field=models.BooleanField(default=False, verbose_name='Private?'), + ), + migrations.AlterField( + model_name='doc', + name='publication_date', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date Published'), + ), + migrations.AlterField( + model_name='doc', + name='update_date', + field=models.DateTimeField(auto_now=True, verbose_name='Date Updated'), + ), + migrations.AlterField( + model_name='externalurl', + name='title', + field=models.CharField(max_length=200, verbose_name='Title'), + ), + migrations.AlterField( + model_name='externalurl', + name='type', + field=models.CharField(choices=[('blog', 'Blog'), ('repository', 'Repositories'), ('homepage', 'Home Page'), ('reference', 'Reference'), ('tracker', 'Trackers'), ('usecase', 'Use Cases'), ('policy', ' Policies'), ('roadmaps', 'Roadmaps'), ('download', 'Download / Releases'), ('admin_docs', 'Install / Admin Docs'), ('user_docs', 'User Docs'), ('faq', 'FAQ'), ('code_example', 'Code Examples'), ('dev_docs', 'Dev Docs'), ('testing', 'Testing'), ('checklist', 'Checklists'), ('prioritization', 'Prioritization'), ('metric', 'Metrics'), ('release_schedule', 'Release Schedules')], max_length=20, verbose_name='URL Type'), + ), + migrations.AlterField( + model_name='externalurl', + name='url', + field=models.URLField(max_length=1000, verbose_name='URL'), + ), + migrations.AlterField( + model_name='fundingsource', + name='image', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + migrations.AlterField( + model_name='fundingsource', + name='name', + field=models.CharField(help_text='Organization or agency that financially supports the project', max_length=200), + ), + migrations.AlterField( + model_name='fundingsource', + name='url', + field=models.URLField(blank=True, help_text='Funding Source URL', null=True), + ), + migrations.AlterField( + model_name='lock', + name='object_id', + field=models.IntegerField(verbose_name='Object Identifier'), + ), + migrations.AlterField( + model_name='lock', + name='object_type', + field=models.CharField(max_length=100, verbose_name='Object Type'), + ), + migrations.AlterField( + model_name='lock', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='Owner'), + ), + migrations.AlterField( + model_name='lock', + name='timestamp', + field=models.DateTimeField(auto_now=True, verbose_name='Last Update Date'), + ), + migrations.AlterField( + model_name='loggedevent', + name='update_date', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date Time'), + ), + migrations.AlterField( + model_name='managementbody', + name='category', + field=models.CharField(blank=True, choices=[('Strategic', 'Strategic'), ('Operational', 'Operational')], default='Strategic', help_text='Strategic or Operational management body purpose.', max_length=50, verbose_name='Category'), + ), + migrations.AlterField( + model_name='managementbody', + name='description', + field=models.TextField(help_text='A short description providing extra information about organizational body.', verbose_name='Description'), + ), + migrations.AlterField( + model_name='managementbody', + name='other', + field=models.CharField(blank=True, help_text='Specify any other purpose(s) not included in the controlled vocabulary.', max_length=200, null=True, verbose_name='Other Purpose'), + ), + migrations.AlterField( + model_name='managementbody', + name='termsOfReference', + field=models.TextField(blank=True, help_text='A description of the duties and responsibilities of the organizational body.', verbose_name='Terms of Reference'), + ), + migrations.AlterField( + model_name='managementbody', + name='title', + field=models.CharField(help_text='String used to succinctly describe an organizational body.', max_length=200, verbose_name='Title'), + ), + migrations.AlterField( + model_name='managementbodypurpose', + name='category', + field=models.CharField(choices=[('Strategic', 'Strategic'), ('Operational', 'Operational')], default='Strategic', help_text='Strategic or Operational management body purpose.', max_length=50, verbose_name='Category'), + ), + migrations.AlterField( + model_name='managementbodypurpose', + name='purpose', + field=models.CharField(choices=[('Strategic Direction', ' Strategic Direction (Strategic)'), ('Advice or Guidance', ' Advice or Guidance (Strategic)'), ('Program Direction', ' Program Direction (Strategic)'), ('Review', ' Review (Strategic)'), ('Research', ' Research (Operational)'), ('Development', ' Development (Operational)'), ('Requirements Identification', ' Requirements Identification (Operational)'), ('Task Prioritization', ' Task Prioritization (Operational)'), ('Testing', ' Testing (Operational)'), ('Review', ' Review (Operational)'), ('Meeting and Event Planning', ' Meeting and Event Planning (Operational)'), ('Administration', ' Administration (Operational)')], default='Other', help_text='Type of organizational body (choose from controlled vocabulary).', max_length=50, verbose_name='Purpose/Role'), + ), + migrations.AlterField( + model_name='membershiprequest', + name='date', + field=models.DateTimeField(auto_now=True, verbose_name='Request Date'), + ), + migrations.AlterField( + model_name='news', + name='other_projects', + field=models.ManyToManyField(blank=True, related_name='other_news', to='cog.Project', verbose_name='Projects Notified'), + ), + migrations.AlterField( + model_name='news', + name='project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='cog.Project', verbose_name='About Project'), + ), + migrations.AlterField( + model_name='news', + name='publication_date', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date Published'), + ), + migrations.AlterField( + model_name='news', + name='update_date', + field=models.DateTimeField(auto_now=True, verbose_name='Date Updated'), + ), + migrations.AlterField( + model_name='organization', + name='image', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + migrations.AlterField( + model_name='organization', + name='name', + field=models.CharField(help_text='Project or organization that collaborates on this project', max_length=200), + ), + migrations.AlterField( + model_name='organization', + name='url', + field=models.URLField(blank=True, help_text='Organization URL', null=True), + ), + migrations.AlterField( + model_name='organizationalrole', + name='category', + field=models.CharField(blank=True, choices=[('Lead', 'Lead'), ('Member', 'Member')], default='Member', help_text='Lead or Member role.', max_length=50, verbose_name='Category'), + ), + migrations.AlterField( + model_name='organizationalrole', + name='description', + field=models.TextField(blank=True, help_text='Long description providing extra information about an organizational role.', null=True, verbose_name='Description'), + ), + migrations.AlterField( + model_name='organizationalrole', + name='title', + field=models.CharField(blank=True, help_text='Optional string used to succinctly describe an organizational role.', max_length=200, null=True, verbose_name='Title'), + ), + migrations.AlterField( + model_name='organizationalrole', + name='type', + field=models.CharField(choices=[('Principal Investigator', ' Principal Investigator (Lead Role)'), ('Co-Investigator', ' Co-Investigator (Lead Role)'), ('Program Manager', ' Program Manager (Lead Role)'), ('Project Manager', ' Project Manager (Lead Role)'), ('Software Architect', ' Software Architect (Lead Role)'), ('Lead', ' Lead (Lead Role)'), ('Other Lead', ' Other Lead (Lead Role)'), ('', '--------------'), ('Administrative Assistant', ' Administrative Assistant (Member Role)'), ('Data Manager', ' Data Manager (Member Role)'), ('Outreach Coordinator', ' Outreach Coordinator (Member Role)'), ('Researcher', ' Researcher (Member Role)'), ('Software Developer', ' Software Developer (Member Role)'), ('Webmaster', ' Webmaster (Member Role)'), ('Other Member', ' Other Member (Member Role)')], default='', help_text='Type of organizational role (choose from controlled vocabulary).', max_length=50, verbose_name='Type'), + ), + migrations.AlterField( + model_name='post', + name='author', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='posts', to=settings.AUTH_USER_MODEL, verbose_name='Author'), + ), + migrations.AlterField( + model_name='post', + name='body', + field=models.TextField(blank=True, default='', verbose_name='Content'), + ), + migrations.AlterField( + model_name='post', + name='docs', + field=models.ManyToManyField(blank=True, to='cog.Doc', verbose_name='Attachments'), + ), + migrations.AlterField( + model_name='post', + name='is_home', + field=models.BooleanField(default=False, verbose_name='Is Home Page?'), + ), + migrations.AlterField( + model_name='post', + name='is_private', + field=models.BooleanField(default=False, verbose_name='Private?'), + ), + migrations.AlterField( + model_name='post', + name='is_restricted', + field=models.BooleanField(default=False, verbose_name='Restricted?'), + ), + migrations.AlterField( + model_name='post', + name='label', + field=models.CharField(blank=True, help_text='Short index label', max_length=28, null=True, verbose_name='Label'), + ), + migrations.AlterField( + model_name='post', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='cog.Post', verbose_name='Parent Post'), + ), + migrations.AlterField( + model_name='post', + name='publication_date', + field=models.DateTimeField(auto_now_add=True, verbose_name='Date Published'), + ), + migrations.AlterField( + model_name='post', + name='template', + field=models.CharField(blank=True, max_length=200, verbose_name='Template'), + ), + migrations.AlterField( + model_name='post', + name='title', + field=models.CharField(max_length=200, verbose_name='Title'), + ), + migrations.AlterField( + model_name='post', + name='type', + field=models.CharField(choices=[('blog', 'Blog'), ('page', 'Page'), ('hyperlink', 'Hyperlink'), ('notes', 'Notes')], max_length=10, verbose_name='Type'), + ), + migrations.AlterField( + model_name='post', + name='update_date', + field=models.DateTimeField(verbose_name='Date Updated'), + ), + migrations.AlterField( + model_name='post', + name='url', + field=models.CharField(blank=True, default='', max_length=200, unique=True, verbose_name='URL'), + ), + migrations.AlterField( + model_name='project', + name='bindingLanguage', + field=models.TextField(blank=True, help_text='The binding language(s) of the software code.', null=True, verbose_name='Binding Language'), + ), + migrations.AlterField( + model_name='project', + name='dataSearchEnabled', + field=models.BooleanField(default=False, help_text='Enable data search?'), + ), + migrations.AlterField( + model_name='project', + name='description', + field=models.TextField(help_text='A short paragraph that describes the project.', null=True), + ), + migrations.AlterField( + model_name='project', + name='developmentOverview', + field=models.TextField(blank=True, help_text='One or more paragraphs providing a general overview of the development processes for the project.', null=True, verbose_name='Development Overview'), + ), + migrations.AlterField( + model_name='project', + name='externalDependencies', + field=models.TextField(blank=True, help_text='The major libraries and packages the software depends on.', null=True, verbose_name='External Dependencies'), + ), + migrations.AlterField( + model_name='project', + name='external_homepage', + field=models.URLField(blank=True, help_text='External Home Page', null=True), + ), + migrations.AlterField( + model_name='project', + name='getInvolved', + field=models.TextField(blank=True, help_text='Describe how to participate in the project.', null=True, verbose_name='Get Involved'), + ), + migrations.AlterField( + model_name='project', + name='getting_started', + field=models.TextField(blank=True, help_text='Describe how users can get started with this project.', null=True, verbose_name='Getting Started'), + ), + migrations.AlterField( + model_name='project', + name='governanceOverview', + field=models.TextField(blank=True, help_text='One or more paragraphs providing a general overview of the governance structure for the project.', null=True, verbose_name='Governance Overview'), + ), + migrations.AlterField( + model_name='project', + name='history', + field=models.TextField(blank=True, help_text='A narrative describing the origination and evolution of the project.'), + ), + migrations.AlterField( + model_name='project', + name='implementationLanguage', + field=models.TextField(blank=True, help_text='The implementation language(s) of the software code.', null=True, verbose_name='Implementation Language'), + ), + migrations.AlterField( + model_name='project', + name='license', + field=models.TextField(blank=True, help_text='Name of license used for the software, if any.', null=True, verbose_name='License'), + ), + migrations.AlterField( + model_name='project', + name='logo', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + migrations.AlterField( + model_name='project', + name='logo_url', + field=models.CharField(blank=True, help_text='Optional logo hyperlink URL.', max_length=200, null=True), + ), + migrations.AlterField( + model_name='project', + name='long_name', + field=models.CharField(help_text='Fully spelled project name.', max_length=120, unique=True), + ), + migrations.AlterField( + model_name='project', + name='maxUploadSize', + field=models.IntegerField(blank=True, default=52428800, help_text='Maximum upload size in bytes'), + ), + migrations.AlterField( + model_name='project', + name='meetingSupport', + field=models.TextField(blank=True, help_text='Describe how to setup meetings.', null=True, verbose_name='Meeting Support'), + ), + migrations.AlterField( + model_name='project', + name='mission', + field=models.TextField(blank=True, help_text='Succinctly describes why the project exists and what it does.'), + ), + migrations.AlterField( + model_name='project', + name='nodesWidgetEnabled', + field=models.BooleanField(default=False, help_text='Enable federated nodes widget ?'), + ), + migrations.AlterField( + model_name='project', + name='projectContacts', + field=models.TextField(blank=True, default='', help_text='Describe how to contact the project.', null=True, verbose_name='Project Contacts'), + ), + migrations.AlterField( + model_name='project', + name='requirementsIdentificationProcess', + field=models.TextField(blank=True, help_text='A paragraph describing how requirements are identified. This description may include who participates, what system is used to track requirements, and whether the results are public.', null=True, verbose_name='Requirements Identification Process'), + ), + migrations.AlterField( + model_name='project', + name='short_name', + field=models.CharField(help_text="Short project acronym, 20 characters maximum, use only letters, numbers and '_', '-', no spaces.", max_length=20, unique=True), + ), + migrations.AlterField( + model_name='project', + name='software_features', + field=models.TextField(blank=True, help_text=None, null=True, verbose_name='Software Features'), + ), + migrations.AlterField( + model_name='project', + name='supportedPlatforms', + field=models.TextField(blank=True, help_text='The computing platforms that the software can run on.', null=True, verbose_name='Supported Platforms'), + ), + migrations.AlterField( + model_name='project', + name='system_requirements', + field=models.TextField(blank=True, help_text=None, null=True, verbose_name='Software System Requirements'), + ), + migrations.AlterField( + model_name='project', + name='taskPrioritizationStrategy', + field=models.TextField(blank=True, help_text='A paragraph describing how tasks are prioritized. This description may include who participates, how often they meet, how they meet, and whether the results are public.', null=True, verbose_name='Task Prioritization Strategy.'), + ), + migrations.AlterField( + model_name='project', + name='technicalSupport', + field=models.TextField(blank=True, help_text='Email address for technical questions.', null=True, verbose_name='Technical Support'), + ), + migrations.AlterField( + model_name='project', + name='values', + field=models.TextField(blank=True, help_text='Beliefs that are shared among the members of a project. Values influence culture and priorities and provide a framework for informing decisions.'), + ), + migrations.AlterField( + model_name='project', + name='vision', + field=models.TextField(blank=True, help_text='Outlines what a project wants to be, or how it wants the world in which it operates to be. It is a long-term view.'), + ), + migrations.AlterField( + model_name='projectimpact', + name='description', + field=models.TextField(help_text='Describe a major impact of this project in its field.', verbose_name='Project Impact'), + ), + migrations.AlterField( + model_name='projectimpact', + name='title', + field=models.CharField(default='', help_text='Title for this impact.', max_length=200), + ), + migrations.AlterField( + model_name='projecttab', + name='url', + field=models.CharField(blank=True, default='', max_length=200, unique=True, verbose_name='URL'), + ), + migrations.AlterField( + model_name='searchgroup', + name='name', + field=models.CharField(default='default', max_length=40), + ), + migrations.AlterField( + model_name='searchprofile', + name='constraints', + field=models.CharField(blank=True, default='', max_length=500, null=True), + ), + migrations.AlterField( + model_name='searchprofile', + name='description', + field=models.TextField(blank=True, help_text='Optional description of this project search capabilities.', null=True, verbose_name='Search Help'), + ), + migrations.AlterField( + model_name='topic', + name='description', + field=models.TextField(blank=True, verbose_name='Description'), + ), + migrations.AlterField( + model_name='topic', + name='name', + field=models.CharField(max_length=200, verbose_name='Name'), + ), + migrations.AlterField( + model_name='userprofile', + name='city', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='userprofile', + name='country', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='userprofile', + name='department', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='image', + field=models.ImageField(blank=True, null=True, upload_to='photos/'), + ), + migrations.AlterField( + model_name='userprofile', + name='institution', + field=models.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='userprofile', + name='last_password_update', + field=models.DateTimeField(blank=True, null=True, verbose_name='Date and Time when Password was Last Updated'), + ), + migrations.AlterField( + model_name='userprofile', + name='private', + field=models.BooleanField(default=False, verbose_name='Do NOT list me among project members'), + ), + migrations.AlterField( + model_name='userprofile', + name='researchInterests', + field=models.CharField(blank=True, default='', max_length=1000, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='researchKeywords', + field=models.CharField(blank=True, default='', max_length=60, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='state', + field=models.CharField(blank=True, default='', max_length=100, null=True), + ), + migrations.AlterField( + model_name='userprofile', + name='subscribed', + field=models.BooleanField(default=True, verbose_name='Subscribe to COG mailing list?'), + ), + migrations.AlterField( + model_name='userurl', + name='url', + field=models.URLField(max_length=1000, verbose_name='URL'), + ), + ] From 35babdd436b312c4cca6fb42f1e1447a989f78b3 Mon Sep 17 00:00:00 2001 From: watucker Date: Wed, 25 Mar 2020 14:52:48 +0000 Subject: [PATCH 64/65] Removed unneeded social auth setting It appears to have been setting the field to None instead of storing it. --- settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 1b3706824..836997ab5 100644 --- a/settings.py +++ b/settings.py @@ -264,7 +264,9 @@ 'social_core.pipeline.user.user_details' ) -SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] +# Appears not to work correctly + not be needed in Django 2.2 +#SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] + SOCIAL_AUTH_SANITIZE_REDIRECTS = False SOCIAL_AUTH_ESGF_AUTH_EXTRA_ARGUMENTS = { 'access_type': 'offline', From f15febafefa117753141f09c7e92f0e46c16c7ef Mon Sep 17 00:00:00 2001 From: watucker Date: Wed, 8 Jul 2020 12:30:06 +0100 Subject: [PATCH 65/65] Added setting to force social auth redirects to use HTTPS --- settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.py b/settings.py index 836997ab5..b20ddec3d 100644 --- a/settings.py +++ b/settings.py @@ -267,6 +267,7 @@ # Appears not to work correctly + not be needed in Django 2.2 #SOCIAL_AUTH_FIELDS_STORED_IN_SESSION = ['openid_identifier',] +SOCIAL_AUTH_REDIRECT_IS_HTTPS = True SOCIAL_AUTH_SANITIZE_REDIRECTS = False SOCIAL_AUTH_ESGF_AUTH_EXTRA_ARGUMENTS = { 'access_type': 'offline',