From 5e08ff5103b1775335780c1a5d3230f926e10873 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 15:38:19 +0100 Subject: [PATCH 1/2] Fix redirect by swapping in requests for downloads --- .../mvnfeed/cli/transfer/transfer.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index ec1e301..9fcbcc5 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -2,18 +2,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - +import functools import logging import os.path import requests import shutil import xml.etree.ElementTree as ET -try: - # Python 3 - from urllib.request import Request, urlopen -except ImportError: - # Python 2 - from urllib2 import Request, urlopen from .configuration import get_repository, get_stagedir, get_repository_shortname from mvnfeed.cli.common.config import AUTHORIZATION, URL, load_config @@ -244,7 +238,7 @@ def _findNodeValue(node, name): return foundNode.text -def _download_file(from_repository, path, filename, length=16*1024): +def _download_file(from_repository, path, filename): """ Stores the path into the given filename. """ @@ -256,20 +250,20 @@ def _download_file(from_repository, path, filename, length=16*1024): url = _append_url(from_repository[URL], path) logging.debug('downloading from %s', url) - try: - request = Request(url) - if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: - logging.debug('authorization header added') - request.add_header('Authorization', from_repository[AUTHORIZATION]) - else: - logging.debug('no authorization configured') + if AUTHORIZATION in from_repository and from_repository[AUTHORIZATION]: + logging.debug('authorization header added') + headers = {'Authorization': from_repository[AUTHORIZATION]} + else: + logging.debug('no authorization configured') + headers = {} - response = urlopen(request) - with open(filename, 'wb') as file: - shutil.copyfileobj(response, file, length) + try: + with requests.get(url, headers=headers, stream=True) as response: + response.raw.read = functools.partial(response.raw.read, decode_content=True) + with open(filename, 'wb') as file: + shutil.copyfileobj(response.raw, file) except Exception as ex: logging.debug('exception while downloading (expected): %s', ex) - None def _already_uploaded(to_repository, path): From 9a2432fd6d1471fc38e233cefc33064579355b83 Mon Sep 17 00:00:00 2001 From: James Annesley Date: Thu, 16 May 2024 17:19:36 +0100 Subject: [PATCH 2/2] Handle 40x errors --- .../mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py index 9fcbcc5..c85fe77 100644 --- a/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py +++ b/src/mvnfeed_modules/mvnfeed-cli-transfer/mvnfeed/cli/transfer/transfer.py @@ -259,11 +259,14 @@ def _download_file(from_repository, path, filename): try: with requests.get(url, headers=headers, stream=True) as response: + if not response.ok: + logging.debug('exception while downloading (expected): %s', response.text) + return response.raw.read = functools.partial(response.raw.read, decode_content=True) with open(filename, 'wb') as file: shutil.copyfileobj(response.raw, file) except Exception as ex: - logging.debug('exception while downloading (expected): %s', ex) + logging.error('exception while downloading: %s', ex) def _already_uploaded(to_repository, path):