From 79fc62f3fcf563ddf9bb03f62e3fa6f7865734ee Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Mon, 31 Aug 2020 21:42:20 +0200 Subject: [PATCH] Handle none-responses from open_url This handles all none-responses from open_url(). I am not sure this is the best way to handle this, it is not very elegant. Raising an exception helps in one case, but would not be elegant in the other cases either. --- resources/lib/tokenresolver.py | 20 ++++++++++++++++---- resources/lib/webscraper.py | 7 ++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/resources/lib/tokenresolver.py b/resources/lib/tokenresolver.py index 608989fa..7a5b63d6 100644 --- a/resources/lib/tokenresolver.py +++ b/resources/lib/tokenresolver.py @@ -199,7 +199,10 @@ def _get_xvrttoken(self, login_json=None): ) data = dumps(payload).encode() headers = {'Content-Type': 'application/json', 'Cookie': login_cookie} - setcookie_header = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers).info().get('Set-Cookie') + response = open_url(self._TOKEN_GATEWAY_URL, data=data, headers=headers) + if response is None: + return None + setcookie_header = response.info().get('Set-Cookie') xvrttoken = TokenResolver._create_token_dictionary(setcookie_header) if xvrttoken is None: return None @@ -213,13 +216,22 @@ def _get_roaming_xvrttoken(self): return None cookie_value = 'vrtlogin-at=' + vrtlogin_at headers = {'Cookie': cookie_value} - req_info = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False).info() + response = open_url(self._ROAMING_TOKEN_GATEWAY_URL, headers=headers, follow_redirects=False) + if response is None: + return None + req_info = response.info() cookie_value += '; state=' + req_info.get('Set-Cookie').split('state=')[1].split('; ')[0] - url = open_url(req_info.get('Location'), follow_redirects=False).info().get('Location') + response = open_url(req_info.get('Location'), follow_redirects=False) + if response is None: + return None + url = response.info().get('Location') headers = {'Cookie': cookie_value} if url is None: return None - setcookie_header = open_url(url, headers=headers, follow_redirects=False).info().get('Set-Cookie') + response = open_url(url, headers=headers, follow_redirects=False) + if response is None: + return None + setcookie_header = response.info().get('Set-Cookie') return TokenResolver._create_token_dictionary(setcookie_header) def get_token(self, name, variant=None, url=None, roaming=False): diff --git a/resources/lib/webscraper.py b/resources/lib/webscraper.py index 701f8a6b..bddfa4c3 100644 --- a/resources/lib/webscraper.py +++ b/resources/lib/webscraper.py @@ -31,6 +31,8 @@ def get_categories(): if not valid_categories(categories): from bs4 import BeautifulSoup, SoupStrainer response = open_url('https://www.vrt.be/vrtnu/categorieen/') + if response is None: + return categories tiles = SoupStrainer('nui-list--content') soup = BeautifulSoup(response.read(), 'html.parser', parse_only=tiles) @@ -79,10 +81,13 @@ def get_video_attributes(vrtnu_url): # Scrape video attributes from bs4 import BeautifulSoup, SoupStrainer try: - html_page = open_url(vrtnu_url, raise_errors='all').read() + response = open_url(vrtnu_url, raise_errors='all') except HTTPError as exc: log_error('Web scraping video attributes failed: {error}', error=exc) return None + if response is None: + return None + html_page = response.read() strainer = SoupStrainer(['section', 'div'], {'class': ['video-player', 'livestream__inner']}) soup = BeautifulSoup(html_page, 'html.parser', parse_only=strainer) item = None