From 693220e08971673eb379e3f614ccafa0bc0a112f Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 16 Jun 2025 16:16:46 -0400 Subject: [PATCH 1/2] fixup upload as zip feature --- .../file_manager/file_upload/file_upload.py | 20 ++++++++------ .../file_upload/test_file_upload.py | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 4f9b28ca..37160e03 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -194,19 +194,24 @@ def simple_upload( # noqa: C901 # if the input request zip folder then process the path as single file # otherwise read throught the folder to get path underneath if os.path.isdir(input_path): - job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER - if job_type == UploadType.AS_FILE: - upload_file_path = [input_path.rstrip('/').lstrip() + '.zip'] - compress_folder_to_zip(input_path) + if compress_zip: + input_path = compress_folder_to_zip(input_path) + upload_file_path = [input_path] + job_type = UploadType.AS_FILE + # since now this is a file upload, we need to strip the folder name + current_folder_node = os.path.dirname(current_folder_node) + else: + job_type = UploadType.AS_FOLDER + upload_file_path = get_file_in_folder(input_path) + # currently not support tag and attribute for a folder upload - elif tags or attribute: + if tags or attribute: SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True) # currently not support n-to-n relationship in lineage, meaning # can ONLY specify one source id for a folder upload elif len(source_id) > 1 and job_type == UploadType.AS_FOLDER: SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_SOURCE_MANIFEST, True) - else: - upload_file_path = get_file_in_folder(input_path) + else: upload_file_path = [input_path] job_type = UploadType.AS_FILE @@ -255,7 +260,6 @@ def simple_upload( # noqa: C901 upload_client.output_manifest( pre_upload_infos, non_duplicate_file_objects[len(pre_upload_infos) + 1 :], output_path ) - # now loop over each file under the folder and start # the chunk upload pool = ThreadPool(num_of_thread) diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index a8f5f887..5f5f800e 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -349,6 +349,33 @@ def test_folder_merge_skip_with_all_duplication(mocker, mock_upload_client, capf raise AssertionError('SystemExit not raised') +def test_upload_folder_as_zip(mocker, mock_upload_client): + test_folder = 'test' + upload_event = { + 'file': test_folder, + 'project_code': 'test_project', + 'zone': 'greenroom', + 'create_folder_flag': False, + 'compress_zip': True, + } + + mocker.patch('os.path.isdir', return_value=True) + mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1)) + compress_mock = mocker.patch( + 'app.services.file_manager.file_upload.file_upload.compress_folder_to_zip', return_value='test.zip' + ) + + non_dup_list = [FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')] + mocker.patch( + 'app.services.file_manager.file_upload.file_upload.UploadClient.check_upload_duplication', + return_value=(non_dup_list, []), + ) + item_ids = simple_upload(upload_event) + assert len(item_ids) == 1 + assert item_ids[0] == non_dup_list[0].item_id + assert compress_mock.call_count == 1 + + def test_resume_upload(mocker): mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1)) test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id') From 6e47a5669049c0ce0b9064970dba5479c9a26414 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Tue, 17 Jun 2025 13:38:07 -0400 Subject: [PATCH 2/2] Update app/services/file_manager/file_upload/file_upload.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- app/services/file_manager/file_upload/file_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 37160e03..5f4f510d 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -195,8 +195,8 @@ def simple_upload( # noqa: C901 # otherwise read throught the folder to get path underneath if os.path.isdir(input_path): if compress_zip: - input_path = compress_folder_to_zip(input_path) - upload_file_path = [input_path] + zip_file_path = compress_folder_to_zip(input_path) + upload_file_path = [zip_file_path] job_type = UploadType.AS_FILE # since now this is a file upload, we need to strip the folder name current_folder_node = os.path.dirname(current_folder_node)