From e2d1fffdb24dcb2f381627242d05b8a49117e0e3 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 4 Jul 2022 23:08:47 +0200 Subject: [PATCH 01/40] fix import error --- uplink_python/project.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uplink_python/project.py b/uplink_python/project.py index 1e6b267..fa32ceb 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -385,11 +385,17 @@ def close(self): # # close Storj project by calling the exported golang function error = self.uplink.m_libuplink.uplink_close_project(self.project) + error = self.uplink.m_libuplink.uplink_close_project(self.project) # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + errorCode = error.contents.code + errorMsg = error.contents.message.decode("utf-8") + + #self.uplink.m_libuplink.uplink_free_error.argtypes = [ctypes.POINTER(_Error)] + + #self.uplink.m_libuplink.uplink_free_error(error) + raise _storj_exception(errorCode, errorMsg) def upload_object(self, bucket_name: str, storj_path: str, upload_options: UploadOptions = None): From cbf3877e96aaf513cb98abda208ce5769675e668 Mon Sep 17 00:00:00 2001 From: Fadila Date: Wed, 6 Jul 2022 12:31:37 +0200 Subject: [PATCH 02/40] free errors --- uplink_python/access.py | 5 ++--- uplink_python/download.py | 3 +-- uplink_python/uplink.py | 12 +++++++++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 51e9f19..6876bab 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -81,7 +81,7 @@ def derive_encryption_key(self, passphrase: str, salt: str): # if error occurred if bool(encryption_key_result.error): raise _storj_exception(encryption_key_result.error.contents.code, - encryption_key_result.error.contents.message.decode("utf-8")) +- encryption_key_result.error.contents.message.decode("utf-8")) return encryption_key_result.encryption_key def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): @@ -115,8 +115,7 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) # # if error occurred if bool(error_result): - raise _storj_exception(error_result.contents.code, - error_result.contents.message.decode("utf-8")) + self.free_error_and_raise_exception(error_result) def open_project(self): """ diff --git a/uplink_python/download.py b/uplink_python/download.py index 7e12650..a2bbf45 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -170,8 +170,7 @@ def close(self): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.free_error_and_raise_exception(error) def info(self): """ diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 7836090..59a8ff8 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct +from uplink_python.module_def import _AccessResult, _ConfigStruct, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -217,3 +217,13 @@ def parse_access(self, serialized_access: str): raise _storj_exception(access_result.error.contents.code, access_result.error.contents.message.decode("utf-8")) return Access(access_result.access, self) + + @classmethod + def free_error_and_raise_exception(cls,err : _Error): + errorCode = err.error.contents.code + errorMsg = err.error.contents.message.decode("utf-8") + + cls.uplink.m_libuplink.uplink_free_error.argtypes = [_Error] + cls.uplink.m_libuplink.uplink_free_error(err) + + raise _storj_exception(errorCode,errorMsg) From fac90ec123ee797db79536583bc9e3e313721db7 Mon Sep 17 00:00:00 2001 From: Fadila Date: Wed, 6 Jul 2022 14:07:25 +0200 Subject: [PATCH 03/40] access: deallocate unused c struct --- uplink_python/access.py | 87 +++++++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 6876bab..ee141d7 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -79,10 +79,19 @@ def derive_encryption_key(self, passphrase: str, salt: str): length_ptr) # # if error occurred - if bool(encryption_key_result.error): - raise _storj_exception(encryption_key_result.error.contents.code, -- encryption_key_result.error.contents.message.decode("utf-8")) - return encryption_key_result.encryption_key + error = bool(encryption_key_result.error) + if error: + errorCode = encryption_key_result.error.contents.code + errorMsg = encryption_key_result.error.contents.message.decode("utf-8") + else: + encryption_key = encryption_key_result.encryption_key + + self.uplink.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] + self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) + if error: + raise _storj_exception(errorCode, errorMsg) + + return encryption_key def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): """ @@ -135,10 +144,19 @@ def open_project(self): project_result = self.uplink.m_libuplink.uplink_open_project(self.access) # # if error occurred - if bool(project_result.error): - raise _storj_exception(project_result.error.contents.code, - project_result.error.contents.message.decode("utf-8")) - return Project(project_result.project, self.uplink) + error = bool(project_result.error) + if error: + errorCode = project_result.error.contents.code + errorMsg = project_result.error.contents.message.decode("utf-8") + else: + project = Project(project_result.project, self.uplink) + + self.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] + self.m_libuplink.uplink_free_project_result(project_result) + if error: + raise _storj_exception(errorCode,errorMsg) + + return project def config_open_project(self, config: Config): """ @@ -169,10 +187,21 @@ def config_open_project(self, config: Config): project_result = self.uplink.m_libuplink.uplink_config_open_project(config_obj, self.access) # # if error occurred - if bool(project_result.error): - raise _storj_exception(project_result.error.contents.code, - project_result.error.contents.message.decode("utf-8")) - return Project(project_result.project, self.uplink) + error = bool(project_result.error) + if error: + errorCode = project_result.error.contents.code + errorMsg = project_result.error.contents.message.decode("utf-8") + else: + project = Project(project_result.project, self.uplink) + + self.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] + self.m_libuplink.uplink_free_project_result(project_result) + + if error: + raise _storj_exception(errorCode,errorMsg) + + return project + def serialize(self): """ @@ -193,10 +222,20 @@ def serialize(self): string_result = self.uplink.m_libuplink.uplink_access_serialize(self.access) # # if error occurred - if bool(string_result.error): - raise _storj_exception(string_result.error.contents.code, - string_result.error.contents.message.decode("utf-8")) - return string_result.string.decode("utf-8") + error = bool(string_result.error) + if error: + errorCode = string_result.error.contents.code + errorMsg = string_result.error.contents.message.decode("utf-8") + else: + to_return = string_result.string.decode("utf-8") + + self.m_libuplink.uplink_free_string_result.argstypes = [_StringResult] + self.m_libuplink.uplink_free_string_result(string_result) + + if error: + raise _storj_exception(errorCode,errorMsg) + + return to_return def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = None): """ @@ -252,7 +291,15 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No shared_prefix_obj, array_size) # # if error occurred - if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) - return Access(access_result.access, self.uplink) + error = bool(access_result.error) + if error: + errorCode = access_result.error.contents.code + errorMsg = access_result.error.contents.message.decode("utf-8") + else: + access = Access(access_result.access, self.uplink) + self.m_libuplink.uplink_free_access_result.argstypes = [_AccessResult] + self.m_libuplink.uplink_free_access_result(access_result) + if error: + raise _storj_exception(errorCode,errorMsg) + + return access From 09d173d761c322e48f56204ee28139c9cd3dc7f6 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 7 Jul 2022 20:08:20 +0200 Subject: [PATCH 04/40] free unused structs --- uplink_python/access.py | 73 ++++++++++++----------- uplink_python/download.py | 49 +++++++++++----- uplink_python/hello_storj.py | 6 +- uplink_python/project.py | 111 ++++++++++++++++++++++++++--------- uplink_python/uplink.py | 30 ++++++++-- uplink_python/upload.py | 38 ++++++++---- 6 files changed, 207 insertions(+), 100 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index ee141d7..7f7a818 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -65,6 +65,7 @@ def derive_encryption_key(self, passphrase: str, salt: str): ctypes.c_void_p, ctypes.c_size_t] self.uplink.m_libuplink.uplink_derive_encryption_key.restype = _EncryptionKeyResult + self.uplink.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] # # prepare the input for the function passphrase_ptr = ctypes.c_char_p(passphrase.encode('utf-8')) @@ -79,18 +80,18 @@ def derive_encryption_key(self, passphrase: str, salt: str): length_ptr) # # if error occurred - error = bool(encryption_key_result.error) - if error: + if bool(encryption_key_result.error): errorCode = encryption_key_result.error.contents.code errorMsg = encryption_key_result.error.contents.message.decode("utf-8") - else: - encryption_key = encryption_key_result.encryption_key - self.uplink.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] - self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) - if error: + self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) + raise _storj_exception(errorCode, errorMsg) + encryption_key = encryption_key_result.encryption_key + + self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) + return encryption_key def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): @@ -144,19 +145,16 @@ def open_project(self): project_result = self.uplink.m_libuplink.uplink_open_project(self.access) # # if error occurred - error = bool(project_result.error) - if error: + if bool(project_result.error): errorCode = project_result.error.contents.code errorMsg = project_result.error.contents.message.decode("utf-8") - else: - project = Project(project_result.project, self.uplink) - self.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] - self.m_libuplink.uplink_free_project_result(project_result) - if error: + self.uplink.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] + self.uplink.m_libuplink.uplink_free_project_result(project_result) + raise _storj_exception(errorCode,errorMsg) - return project + return Project(project_result.project, self.uplink) def config_open_project(self, config: Config): """ @@ -176,6 +174,7 @@ def config_open_project(self, config: Config): self.uplink.m_libuplink.uplink_config_open_project.argtypes =\ [_ConfigStruct, ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_config_open_project.restype = _ProjectResult + self.uplink.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] # # prepare the input for the function if config is None: @@ -187,19 +186,18 @@ def config_open_project(self, config: Config): project_result = self.uplink.m_libuplink.uplink_config_open_project(config_obj, self.access) # # if error occurred - error = bool(project_result.error) - if error: + if bool(project_result.error): errorCode = project_result.error.contents.code errorMsg = project_result.error.contents.message.decode("utf-8") - else: - project = Project(project_result.project, self.uplink) - self.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] - self.m_libuplink.uplink_free_project_result(project_result) + self.uplink.m_libuplink.uplink_free_project_result(project_result) - if error: raise _storj_exception(errorCode,errorMsg) + project = Project(project_result.project, self.uplink) + + self.uplink.m_libuplink.uplink_free_project_result(project_result) + return project @@ -217,24 +215,23 @@ def serialize(self): # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_access_serialize.argtypes = [ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_access_serialize.restype = _StringResult + self.uplink.m_libuplink.uplink_free_string_result.argstypes = [_StringResult] # # get serialized access by calling the exported golang function string_result = self.uplink.m_libuplink.uplink_access_serialize(self.access) # # if error occurred - error = bool(string_result.error) - if error: + if bool(string_result.error): errorCode = string_result.error.contents.code errorMsg = string_result.error.contents.message.decode("utf-8") - else: - to_return = string_result.string.decode("utf-8") - self.m_libuplink.uplink_free_string_result.argstypes = [_StringResult] - self.m_libuplink.uplink_free_string_result(string_result) - - if error: + self.uplink.m_libuplink.uplink_free_string_result(string_result) raise _storj_exception(errorCode,errorMsg) + to_return = string_result.string.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_string_result(string_result) + return to_return def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = None): @@ -265,6 +262,7 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No ctypes.POINTER(_SharePrefixStruct), ctypes.c_size_t] self.uplink.m_libuplink.uplink_access_share.restype = _AccessResult + self.uplink.m_libuplink.uplink_free_access_result.argstypes = [_AccessResult] # # prepare the input for the function # check and create valid _PermissionStruct parameter @@ -291,15 +289,16 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No shared_prefix_obj, array_size) # # if error occurred - error = bool(access_result.error) - if error: + if bool(access_result.error): errorCode = access_result.error.contents.code errorMsg = access_result.error.contents.message.decode("utf-8") - else: - access = Access(access_result.access, self.uplink) - self.m_libuplink.uplink_free_access_result.argstypes = [_AccessResult] - self.m_libuplink.uplink_free_access_result(access_result) - if error: + + self.uplink.m_libuplink.uplink_free_access_result(access_result) + raise _storj_exception(errorCode,errorMsg) + access = Access(access_result.access, self.uplink) + + self.uplink.m_libuplink.uplink_free_access_result(access_result) + return access diff --git a/uplink_python/download.py b/uplink_python/download.py index a2bbf45..aeae8b1 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -72,6 +72,7 @@ def read(self, size_to_read: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult + self.uplink.m_libuplink.uplink_free_read_result.argstypes = [_ReadResult] # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) @@ -85,18 +86,25 @@ def read(self, size_to_read: int): size_to_read) # # if error occurred - if bool(read_result.error): - raise _storj_exception(read_result.error.contents.code, - read_result.error.contents.message.decode("utf-8")) + if read_result.error: + errorCode = read_result.error.contents.code + errorMsg = read_result.error.contents.message.decode("utf-8") + self.uplink.m_libuplink.uplink_free_read_result(read_result) + raise _storj_exception(errorCode,errorMsg) + + bytes_read = int(read_result.bytes_read) data_read = bytes() - if int(read_result.bytes_read) != 0: - # - # -------------------------------------------- - # data conversion to type python readable form - # conversion of LP_c_ubyte to python readable data variable + if bytes_read != 0: + # + # -------------------------------------------- + # data conversion to type python readable form + # conversion of LP_c_ubyte to python readable data variable data_read = ctypes.string_at(data_to_write_ptr, int(read_result.bytes_read)) - return data_read, int(read_result.bytes_read) + + self.uplink.m_libuplink.uplink_free_read_result(read_result) + + return data_read, bytes_read def read_file(self, file_handle, buffer_size: int = 0): """ @@ -141,14 +149,19 @@ def file_size(self): self.uplink.m_libuplink.uplink_stat_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_stat_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # get object information by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_stat_object(self.project, self.bucket_name, self.storj_path) # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) + errorCode = object_result.error.contents.code + errorMsg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(errorCode, errorMsg) # find object size return int(object_result.object.contents.system.content_length) @@ -190,6 +203,14 @@ def info(self): # # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + errorCode = object_result.error.contents.code + errorMsg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(errorCode, errorMsg) + + object = self.uplink.object_from_result(object_result.object) + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + return object diff --git a/uplink_python/hello_storj.py b/uplink_python/hello_storj.py index c691e5c..2a7f0ed 100644 --- a/uplink_python/hello_storj.py +++ b/uplink_python/hello_storj.py @@ -3,9 +3,9 @@ from datetime import datetime -from .errors import StorjException, BucketNotEmptyError, BucketNotFoundError -from .module_classes import ListObjectsOptions, Permission, SharePrefix -from .uplink import Uplink +from uplink_python.errors import StorjException, BucketNotEmptyError, BucketNotFoundError +from uplink_python.module_classes import ListObjectsOptions, Permission, SharePrefix +from uplink_python.uplink import Uplink if __name__ == "__main__": diff --git a/uplink_python/project.py b/uplink_python/project.py index fa32ceb..2f38e93 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -118,8 +118,13 @@ def ensure_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) + errorCode = bucket_result.error.contents.code + errorMsg = bucket_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + raise _storj_exception(errorCode, errorMsg) return self.uplink.bucket_from_result(bucket_result.bucket) def stat_bucket(self, bucket_name: str): @@ -149,8 +154,14 @@ def stat_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) + errorCode = bucket_result.error.contents.code + errorMsg = bucket_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + raise _storj_exception(errorCode, errorMsg) + return self.uplink.bucket_from_result(bucket_result.bucket) def list_buckets(self, list_bucket_options: ListBucketsOptions = None): @@ -172,11 +183,15 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): [ctypes.POINTER(_ProjectStruct), ctypes.POINTER(_ListBucketsOptionsStruct)] self.uplink.m_libuplink.uplink_list_buckets.restype =\ ctypes.POINTER(_BucketIterator) + self.uplink.m_libuplink.uplink_free_bucket_iterator.argtypes=\ + [ctypes.POINTER(_BucketIterator)] # self.uplink.m_libuplink.uplink_bucket_iterator_item.argtypes =\ [ctypes.POINTER(_BucketIterator)] self.uplink.m_libuplink.uplink_bucket_iterator_item.restype =\ ctypes.POINTER(_BucketStruct) + self.uplink.m_libuplink.uplink_free_bucket.argtypes =\ + [ctypes.POINTER(_BucketStruct)] # self.uplink.m_libuplink.uplink_bucket_iterator_err.argtypes =\ [ctypes.POINTER(_BucketIterator)] @@ -200,13 +215,14 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): bucket_iterator_err = self.uplink.m_libuplink.uplink_bucket_iterator_err(bucket_iterator) if bool(bucket_iterator_err): - raise _storj_exception(bucket_iterator_err.contents.code, - bucket_iterator_err.contents.message.decode("utf-8")) - + self.free_and_raise_error(bucket_iterator_err) bucket_list = list() while self.uplink.m_libuplink.uplink_bucket_iterator_next(bucket_iterator): bucket = self.uplink.m_libuplink.uplink_bucket_iterator_item(bucket_iterator) bucket_list.append(self.uplink.bucket_from_result(bucket)) + self.uplink.m_libuplink.uplink_free_bucket(bucket) + + self.uplink.m_libuplink.uplink_free_bucket_iterator(bucket_iterator) return bucket_list @@ -230,6 +246,7 @@ def delete_bucket(self, bucket_name: str): self.uplink.m_libuplink.uplink_delete_bucket.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p] self.uplink.m_libuplink.uplink_delete_bucket.restype = _BucketResult + self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -239,9 +256,17 @@ def delete_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) - return self.uplink.bucket_from_result(bucket_result.bucket) + errorCode = bucket_result.error.contents.code + errorMsg = bucket_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + raise _storj_exception(errorCode, errorMsg) + + bucket = self.uplink.bucket_from_result(bucket_result.bucket) + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) + + return bucket def stat_object(self, bucket_name: str, storj_path: str): """ @@ -262,6 +287,7 @@ def stat_object(self, bucket_name: str, storj_path: str): self.uplink.m_libuplink.uplink_stat_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_stat_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -273,9 +299,18 @@ def stat_object(self, bucket_name: str, storj_path: str): # # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + errorCode = object_result.error.contents.code + errorMsg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(errorCode, errorMsg) + + object = self.uplink.object_from_result(object_result.object) + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + return object def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions = None): """ @@ -298,11 +333,15 @@ def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions ctypes.POINTER(_ListObjectsOptionsStruct)] self.uplink.m_libuplink.uplink_list_objects.restype =\ ctypes.POINTER(_ObjectIterator) + self.uplink.m_libuplink.uplink_free_object_iterator.argtypes =\ + [ctypes.POINTER(_ObjectIterator)] # self.uplink.m_libuplink.uplink_object_iterator_item.argtypes =\ [ctypes.POINTER(_ObjectIterator)] self.uplink.m_libuplink.uplink_object_iterator_item.restype =\ ctypes.POINTER(_ObjectStruct) + self.uplink.m_libuplink.uplink_free_object.argtypes =\ + [ctypes.POINTER(_ObjectStruct)] # self.uplink.m_libuplink.uplink_object_iterator_err.argtypes =\ [ctypes.POINTER(_ObjectIterator)] @@ -334,6 +373,9 @@ def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions while self.uplink.m_libuplink.uplink_object_iterator_next(object_iterator): object_ = self.uplink.m_libuplink.uplink_object_iterator_item(object_iterator) object_list.append(self.uplink.object_from_result(object_)) + self.uplink.m_libuplink.uplink_free_object(object_) + + self.uplink.m_libuplink.uplink_free_object_iterator(object_iterator) return object_list def delete_object(self, bucket_name: str, storj_path: str): @@ -355,6 +397,7 @@ def delete_object(self, bucket_name: str, storj_path: str): self.uplink.m_libuplink.uplink_delete_object.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p] self.uplink.m_libuplink.uplink_delete_object.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) @@ -366,9 +409,17 @@ def delete_object(self, bucket_name: str, storj_path: str): # # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + errorCode = object_result.error.contents.code + errorMsg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(errorCode, errorMsg) + + object = self.uplink.object_from_result(object_result.object) + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + return object def close(self): """ @@ -385,17 +436,10 @@ def close(self): # # close Storj project by calling the exported golang function error = self.uplink.m_libuplink.uplink_close_project(self.project) - error = self.uplink.m_libuplink.uplink_close_project(self.project) # # if error occurred if bool(error): - errorCode = error.contents.code - errorMsg = error.contents.message.decode("utf-8") - - #self.uplink.m_libuplink.uplink_free_error.argtypes = [ctypes.POINTER(_Error)] - - #self.uplink.m_libuplink.uplink_free_error(error) - raise _storj_exception(errorCode, errorMsg) + self.free_error_and_raise_exception(error) def upload_object(self, bucket_name: str, storj_path: str, upload_options: UploadOptions = None): @@ -418,6 +462,7 @@ def upload_object(self, bucket_name: str, storj_path: str, [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(_UploadOptionsStruct)] self.uplink.m_libuplink.uplink_upload_object.restype = _UploadResult + self.uplink.m_libuplink.uplink_free_upload_result.argtypes = [_UploadResult] # # prepare the input for the function if upload_options is None: @@ -435,8 +480,12 @@ def upload_object(self, bucket_name: str, storj_path: str, # # if error occurred if bool(upload_result.error): - raise _storj_exception(upload_result.error.contents.code, - upload_result.error.contents.message.decode("utf-8")) + errorCode = upload_result.error.contents.code + errorMsg = upload_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_upload_result(upload_result) + + raise _storj_exception(errorCode,errorMsg) return Upload(upload_result.upload, self.uplink) def download_object(self, bucket_name: str, storj_path: str, @@ -460,6 +509,7 @@ def download_object(self, bucket_name: str, storj_path: str, [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(_DownloadOptionsStruct)] self.uplink.m_libuplink.uplink_download_object.restype = _DownloadResult + self.uplink.m_libuplink.uplink_free_download_result.argtypes = [_DownloadResult] # # prepare the input for the function if download_options is None: @@ -478,7 +528,12 @@ def download_object(self, bucket_name: str, storj_path: str, # # if error occurred if bool(download_result.error): - raise _storj_exception(download_result.error.contents.code, - download_result.error.contents.message.decode("utf-8")) + errorCode = download_result.error.contents.code + errorMsg = download_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_download_result(download_result) + + raise _storj_exception(errorCode, errorMsg) + return Download(download_result.download, self.uplink, self.project, bucket_name_ptr, storj_path_ptr) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 59a8ff8..c4f3cf6 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -126,8 +126,14 @@ def request_access_with_passphrase(self, satellite: str, api_key: str, passphras # # if error occurred if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) + errorCode = access_result.error.contents.code + errorMsg = access_result.error.contents.message.decode("utf-8") + + self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] + self.m_libuplink.uplink_free_access_result(access_result) + + raise _storj_exception(errorCode, errorMsg) + return Access(access_result.access, self) def config_request_access_with_passphrase(self, config: Config, satellite: str, api_key: str, @@ -180,8 +186,14 @@ def config_request_access_with_passphrase(self, config: Config, satellite: str, # # if error occurred if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) + errorCode = access_result.error.contents.code + errorMsg = access_result.error.contents.message.decode("utf-8") + + self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] + self.m_libuplink.uplink_free_access_result(access_result) + + raise _storj_exception(errorCode, errorMsg) + return Access(access_result.access, self) def parse_access(self, serialized_access: str): @@ -214,8 +226,14 @@ def parse_access(self, serialized_access: str): # # if error occurred if bool(access_result.error): - raise _storj_exception(access_result.error.contents.code, - access_result.error.contents.message.decode("utf-8")) + errorCode = access_result.error.contents.code + errorMsg = access_result.error.contents.message.decode("utf-8") + + self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] + self.m_libuplink.uplink_free_access_result(access_result) + + raise _storj_exception(errorCode, errorMsg) + return Access(access_result.access, self) @classmethod diff --git a/uplink_python/upload.py b/uplink_python/upload.py index 9b0e169..d4b0305 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -65,6 +65,7 @@ def write(self, data_to_write: bytes, size_to_write: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_upload_write.restype = _WriteResult + self.uplink.m_libuplink.uplink_free_write_result.argtypes = [_WriteResult] # # prepare the inputs for the function # -------------------------------------------- @@ -83,9 +84,18 @@ def write(self, data_to_write: bytes, size_to_write: int): # # if error occurred if bool(write_result.error): - _storj_exception(write_result.error.contents.code, - write_result.error.contents.message.decode("utf-8")) - return int(write_result.bytes_written) + errorCode = write_result.error.contents.code + errorMsg = write_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_write_result(write_result) + + _storj_exception(errorCode, errorMsg) + + bytes_written = int(write_result.bytes_written) + + self.uplink.m_libuplink.uplink_free_write_result(write_result) + + return bytes_written def write_file(self, file_handle, buffer_size: int = 0): """ @@ -134,8 +144,7 @@ def commit(self): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.free_and_raise_error(error) def abort(self): """ @@ -156,8 +165,7 @@ def abort(self): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.free_and_raise_error(error) def set_custom_metadata(self, custom_metadata: CustomMetadata = None): """ @@ -188,8 +196,7 @@ def set_custom_metadata(self, custom_metadata: CustomMetadata = None): # # if error occurred if bool(error): - raise _storj_exception(error.contents.code, - error.contents.message.decode("utf-8")) + self.free_and_raise_error(error) def info(self): """ @@ -203,12 +210,19 @@ def info(self): # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_upload_info.argtypes = [ctypes.POINTER(_UploadStruct)] self.uplink.m_libuplink.uplink_upload_info.restype = _ObjectResult + self.uplink.m_libuplink.uplink_free_object_result.argtypes = [_ObjectResult] # # get last upload info by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_upload_info(self.upload) # # if error occurred if bool(object_result.error): - raise _storj_exception(object_result.error.contents.code, - object_result.error.contents.message.decode("utf-8")) - return self.uplink.object_from_result(object_result.object) + errorCode = object_result.error.contents.code + errorMsg = object_result.error.contents.message.decode("utf-8") + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + + raise _storj_exception(errorCode, errorMsg) + + info = self.uplink.object_from_result(object_result.object) + return info From c0e0c9c0f0dd5c9f1a22285c2d2d6c8c14aeebc3 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 7 Jul 2022 22:15:08 +0200 Subject: [PATCH 05/40] fix codacy issues --- uplink_python/download.py | 4 ++-- uplink_python/project.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uplink_python/download.py b/uplink_python/download.py index aeae8b1..ae883fd 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -210,7 +210,7 @@ def info(self): raise _storj_exception(errorCode, errorMsg) - object = self.uplink.object_from_result(object_result.object) + _object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) - return object + return _object diff --git a/uplink_python/project.py b/uplink_python/project.py index 2f38e93..841adf4 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -306,11 +306,11 @@ def stat_object(self, bucket_name: str, storj_path: str): raise _storj_exception(errorCode, errorMsg) - object = self.uplink.object_from_result(object_result.object) + _object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) - return object + return _object def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions = None): """ From a8da7bbcf1922cc48dbad5c441bb7ea7be7f3a8c Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 8 Jul 2022 23:20:31 +0200 Subject: [PATCH 06/40] linting --- uplink_python/access.py | 32 +++++++++++++------------- uplink_python/download.py | 23 +++++++++---------- uplink_python/project.py | 48 +++++++++++++++++++-------------------- uplink_python/uplink.py | 34 +++++++++++++-------------- uplink_python/upload.py | 18 +++++++-------- 5 files changed, 77 insertions(+), 78 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 7f7a818..b538215 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -81,12 +81,12 @@ def derive_encryption_key(self, passphrase: str, salt: str): # # if error occurred if bool(encryption_key_result.error): - errorCode = encryption_key_result.error.contents.code - errorMsg = encryption_key_result.error.contents.message.decode("utf-8") + error_code = encryption_key_result.error.contents.code + error_msg = encryption_key_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) encryption_key = encryption_key_result.encryption_key @@ -125,7 +125,7 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) # # if error occurred if bool(error_result): - self.free_error_and_raise_exception(error_result) + self.uplink.free_error_and_raise_exception(error_result) def open_project(self): """ @@ -146,13 +146,13 @@ def open_project(self): # # if error occurred if bool(project_result.error): - errorCode = project_result.error.contents.code - errorMsg = project_result.error.contents.message.decode("utf-8") + error_code = project_result.error.contents.code + error_msg = project_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] self.uplink.m_libuplink.uplink_free_project_result(project_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) return Project(project_result.project, self.uplink) @@ -187,12 +187,12 @@ def config_open_project(self, config: Config): # # if error occurred if bool(project_result.error): - errorCode = project_result.error.contents.code - errorMsg = project_result.error.contents.message.decode("utf-8") + error_code = project_result.error.contents.code + error_msg = project_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_project_result(project_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) project = Project(project_result.project, self.uplink) @@ -222,11 +222,11 @@ def serialize(self): # # if error occurred if bool(string_result.error): - errorCode = string_result.error.contents.code - errorMsg = string_result.error.contents.message.decode("utf-8") + error_code = string_result.error.contents.code + error_msg = string_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_string_result(string_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) to_return = string_result.string.decode("utf-8") @@ -290,12 +290,12 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No # # if error occurred if bool(access_result.error): - errorCode = access_result.error.contents.code - errorMsg = access_result.error.contents.message.decode("utf-8") + error_code = access_result.error.contents.code + error_msg = access_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_access_result(access_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) access = Access(access_result.access, self.uplink) diff --git a/uplink_python/download.py b/uplink_python/download.py index ae883fd..b5ca6f2 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -87,11 +87,11 @@ def read(self, size_to_read: int): # # if error occurred if read_result.error: - errorCode = read_result.error.contents.code - errorMsg = read_result.error.contents.message.decode("utf-8") + error_code = read_result.error.contents.code + error_msg = read_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_read_result(read_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) bytes_read = int(read_result.bytes_read) data_read = bytes() @@ -128,8 +128,7 @@ def read_file(self, file_handle, buffer_size: int = 0): if not buffer_size: buffer_size = COPY_BUFSIZE file_size = self.file_size() - if buffer_size > file_size: - buffer_size = file_size + buffer_size = min(buffer_size, file_size) while file_size: buf, bytes_read = self.read(buffer_size) if buf: @@ -156,12 +155,12 @@ def file_size(self): self.storj_path) # if error occurred if bool(object_result.error): - errorCode = object_result.error.contents.code - errorMsg = object_result.error.contents.message.decode("utf-8") + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_object_result(object_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) # find object size return int(object_result.object.contents.system.content_length) @@ -183,7 +182,7 @@ def close(self): # # if error occurred if bool(error): - self.free_error_and_raise_exception(error) + self.uplink.free_error_and_raise_exception(error) def info(self): """ @@ -203,12 +202,12 @@ def info(self): # # if error occurred if bool(object_result.error): - errorCode = object_result.error.contents.code - errorMsg = object_result.error.contents.message.decode("utf-8") + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_object_result(object_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) _object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) diff --git a/uplink_python/project.py b/uplink_python/project.py index 841adf4..4d5e749 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -118,13 +118,13 @@ def ensure_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - errorCode = bucket_result.error.contents.code - errorMsg = bucket_result.error.contents.message.decode("utf-8") + error_code = bucket_result.error.contents.code + error_msg = bucket_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return self.uplink.bucket_from_result(bucket_result.bucket) def stat_bucket(self, bucket_name: str): @@ -154,13 +154,13 @@ def stat_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - errorCode = bucket_result.error.contents.code - errorMsg = bucket_result.error.contents.message.decode("utf-8") + error_code = bucket_result.error.contents.code + error_msg = bucket_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return self.uplink.bucket_from_result(bucket_result.bucket) @@ -215,7 +215,7 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): bucket_iterator_err = self.uplink.m_libuplink.uplink_bucket_iterator_err(bucket_iterator) if bool(bucket_iterator_err): - self.free_and_raise_error(bucket_iterator_err) + self.uplink.free_and_raise_error(bucket_iterator_err) bucket_list = list() while self.uplink.m_libuplink.uplink_bucket_iterator_next(bucket_iterator): bucket = self.uplink.m_libuplink.uplink_bucket_iterator_item(bucket_iterator) @@ -256,12 +256,12 @@ def delete_bucket(self, bucket_name: str): # # if error occurred if bool(bucket_result.error): - errorCode = bucket_result.error.contents.code - errorMsg = bucket_result.error.contents.message.decode("utf-8") + error_code = bucket_result.error.contents.code + error_msg = bucket_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) bucket = self.uplink.bucket_from_result(bucket_result.bucket) self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) @@ -299,12 +299,12 @@ def stat_object(self, bucket_name: str, storj_path: str): # # if error occurred if bool(object_result.error): - errorCode = object_result.error.contents.code - errorMsg = object_result.error.contents.message.decode("utf-8") + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_object_result(object_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) _object = self.uplink.object_from_result(object_result.object) @@ -369,7 +369,7 @@ def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions raise _storj_exception(object_iterator_err.contents.code, object_iterator_err.contents.message.decode("utf-8")) - object_list = list() + object_list = [] while self.uplink.m_libuplink.uplink_object_iterator_next(object_iterator): object_ = self.uplink.m_libuplink.uplink_object_iterator_item(object_iterator) object_list.append(self.uplink.object_from_result(object_)) @@ -409,12 +409,12 @@ def delete_object(self, bucket_name: str, storj_path: str): # # if error occurred if bool(object_result.error): - errorCode = object_result.error.contents.code - errorMsg = object_result.error.contents.message.decode("utf-8") + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_object_result(object_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) @@ -439,7 +439,7 @@ def close(self): # # if error occurred if bool(error): - self.free_error_and_raise_exception(error) + self.uplink.free_error_and_raise_exception(error) def upload_object(self, bucket_name: str, storj_path: str, upload_options: UploadOptions = None): @@ -480,12 +480,12 @@ def upload_object(self, bucket_name: str, storj_path: str, # # if error occurred if bool(upload_result.error): - errorCode = upload_result.error.contents.code - errorMsg = upload_result.error.contents.message.decode("utf-8") + error_code = upload_result.error.contents.code + error_msg = upload_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_upload_result(upload_result) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code,error_msg) return Upload(upload_result.upload, self.uplink) def download_object(self, bucket_name: str, storj_path: str, @@ -528,12 +528,12 @@ def download_object(self, bucket_name: str, storj_path: str, # # if error occurred if bool(download_result.error): - errorCode = download_result.error.contents.code - errorMsg = download_result.error.contents.message.decode("utf-8") + error_code = download_result.error.contents.code + error_msg = download_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_download_result(download_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return Download(download_result.download, self.uplink, self.project, bucket_name_ptr, storj_path_ptr) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index c4f3cf6..a097b87 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -61,7 +61,7 @@ def object_from_result(cls, object_): content_length=object_.contents.system.content_length) array_size = object_.contents.custom.count - entries = list() + entries = [] for i in range(array_size): if bool(object_.contents.custom.entries[i]): entries_obj = object_.contents.custom.entries[i] @@ -126,13 +126,13 @@ def request_access_with_passphrase(self, satellite: str, api_key: str, passphras # # if error occurred if bool(access_result.error): - errorCode = access_result.error.contents.code - errorMsg = access_result.error.contents.message.decode("utf-8") + error_code = access_result.error.contents.code + error_msg = access_result.error.contents.message.decode("utf-8") self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] self.m_libuplink.uplink_free_access_result(access_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return Access(access_result.access, self) @@ -186,13 +186,13 @@ def config_request_access_with_passphrase(self, config: Config, satellite: str, # # if error occurred if bool(access_result.error): - errorCode = access_result.error.contents.code - errorMsg = access_result.error.contents.message.decode("utf-8") + error_code = access_result.error.contents.code + error_msg = access_result.error.contents.message.decode("utf-8") self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] self.m_libuplink.uplink_free_access_result(access_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return Access(access_result.access, self) @@ -226,22 +226,22 @@ def parse_access(self, serialized_access: str): # # if error occurred if bool(access_result.error): - errorCode = access_result.error.contents.code - errorMsg = access_result.error.contents.message.decode("utf-8") + error_code = access_result.error.contents.code + error_msg = access_result.error.contents.message.decode("utf-8") self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] self.m_libuplink.uplink_free_access_result(access_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) return Access(access_result.access, self) - @classmethod - def free_error_and_raise_exception(cls,err : _Error): - errorCode = err.error.contents.code - errorMsg = err.error.contents.message.decode("utf-8") + def free_error_and_raise_exception(self,err : _Error): + """ free libuplinkc error and raise corresponding _storj_exception """ + error_code = err.error.contents.code + error_msg = err.error.contents.message.decode("utf-8") - cls.uplink.m_libuplink.uplink_free_error.argtypes = [_Error] - cls.uplink.m_libuplink.uplink_free_error(err) + self.m_libuplink.uplink_free_error.argtypes = [_Error] + self.m_libuplink.uplink.m_libuplink.uplink_free_error(err) - raise _storj_exception(errorCode,errorMsg) + raise _storj_exception(error_code, error_msg) diff --git a/uplink_python/upload.py b/uplink_python/upload.py index d4b0305..7573eca 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -84,12 +84,12 @@ def write(self, data_to_write: bytes, size_to_write: int): # # if error occurred if bool(write_result.error): - errorCode = write_result.error.contents.code - errorMsg = write_result.error.contents.message.decode("utf-8") + error_code = write_result.error.contents.code + error_msg = write_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_write_result(write_result) - _storj_exception(errorCode, errorMsg) + _storj_exception(error_code, error_msg) bytes_written = int(write_result.bytes_written) @@ -144,7 +144,7 @@ def commit(self): # # if error occurred if bool(error): - self.free_and_raise_error(error) + self.uplink.free_and_raise_error(error) def abort(self): """ @@ -165,7 +165,7 @@ def abort(self): # # if error occurred if bool(error): - self.free_and_raise_error(error) + self.uplink.free_and_raise_error(error) def set_custom_metadata(self, custom_metadata: CustomMetadata = None): """ @@ -196,7 +196,7 @@ def set_custom_metadata(self, custom_metadata: CustomMetadata = None): # # if error occurred if bool(error): - self.free_and_raise_error(error) + self.uplink.free_and_raise_error(error) def info(self): """ @@ -217,12 +217,12 @@ def info(self): # # if error occurred if bool(object_result.error): - errorCode = object_result.error.contents.code - errorMsg = object_result.error.contents.message.decode("utf-8") + error_code = object_result.error.contents.code + error_msg = object_result.error.contents.message.decode("utf-8") self.uplink.m_libuplink.uplink_free_object_result(object_result) - raise _storj_exception(errorCode, errorMsg) + raise _storj_exception(error_code, error_msg) info = self.uplink.object_from_result(object_result.object) return info From 7f74da29b99ae8dfef4c65ee9403a4dbbca49854 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 8 Aug 2022 23:25:14 +0200 Subject: [PATCH 07/40] remove duplicate code --- test/test_data/object_test.py | 1 - uplink_python/project.py | 17 +++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/test/test_data/object_test.py b/test/test_data/object_test.py index 82abc4e..54a6b38 100644 --- a/test/test_data/object_test.py +++ b/test/test_data/object_test.py @@ -3,7 +3,6 @@ import string import unittest -from uplink_python.errors import StorjException, ERROR_OBJECT_NOT_FOUND from .helper import TestPy diff --git a/uplink_python/project.py b/uplink_python/project.py index 4d5e749..23307ab 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -115,17 +115,8 @@ def ensure_bucket(self, bucket_name: str): # open bucket if doesn't exist by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_ensure_bucket(self.project, bucket_name_ptr) - # - # if error occurred - if bool(bucket_result.error): - error_code = bucket_result.error.contents.code - error_msg = bucket_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_bucket_result.argtypes = [_BucketResult] - self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) - raise _storj_exception(error_code, error_msg) - return self.uplink.bucket_from_result(bucket_result.bucket) + return self.check_bucket_result(bucket_result) def stat_bucket(self, bucket_name: str): """ @@ -151,8 +142,10 @@ def stat_bucket(self, bucket_name: str): # get bucket information by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_stat_bucket(self.project, bucket_name_ptr) - # - # if error occurred + + return self.check_bucket_result(bucket_result) + + def check_bucket_result(self, bucket_result): if bool(bucket_result.error): error_code = bucket_result.error.contents.code error_msg = bucket_result.error.contents.message.decode("utf-8") From 80ccdc5aeb086cef0b7a20a88ae408657f8b029e Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 8 Aug 2022 23:39:07 +0200 Subject: [PATCH 08/40] correct according to codacy checks --- uplink_python/download.py | 8 ++++---- uplink_python/project.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uplink_python/download.py b/uplink_python/download.py index b5ca6f2..6157a0a 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -96,10 +96,10 @@ def read(self, size_to_read: int): bytes_read = int(read_result.bytes_read) data_read = bytes() if bytes_read != 0: - # - # -------------------------------------------- - # data conversion to type python readable form - # conversion of LP_c_ubyte to python readable data variable + # + # -------------------------------------------- + # data conversion to type python readable form + # conversion of LP_c_ubyte to python readable data variable data_read = ctypes.string_at(data_to_write_ptr, int(read_result.bytes_read)) self.uplink.m_libuplink.uplink_free_read_result(read_result) diff --git a/uplink_python/project.py b/uplink_python/project.py index 23307ab..6a02f81 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -409,10 +409,10 @@ def delete_object(self, bucket_name: str, storj_path: str): raise _storj_exception(error_code, error_msg) - object = self.uplink.object_from_result(object_result.object) + _object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) - return object + return _object def close(self): """ From a19bcaa3da3c00fc4889823ac3207f5728a36e99 Mon Sep 17 00:00:00 2001 From: Fadila Date: Tue, 6 Sep 2022 15:12:02 +0200 Subject: [PATCH 09/40] update satellite address --- test/test_data/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_data/helper.py b/test/test_data/helper.py index 0f6aac6..4a212f7 100644 --- a/test/test_data/helper.py +++ b/test/test_data/helper.py @@ -16,7 +16,7 @@ def __init__(self): self.api_key = file_handle.read() file_handle.close() - self.satellite = "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777" + self.satellite = "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us1.storj.io:7777" self.encryption_phrase = "test" self.uplink = Uplink() From dbbed15e930f2f2b8c55712ad5df14a218e3aeb2 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sat, 29 Oct 2022 23:01:22 +0200 Subject: [PATCH 10/40] argstypes replaced by argtypes --- uplink_python/access.py | 8 ++++---- uplink_python/download.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index b538215..fdfbd48 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -149,7 +149,7 @@ def open_project(self): error_code = project_result.error.contents.code error_msg = project_result.error.contents.message.decode("utf-8") - self.uplink.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] + self.uplink.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] self.uplink.m_libuplink.uplink_free_project_result(project_result) raise _storj_exception(error_code,error_msg) @@ -174,7 +174,7 @@ def config_open_project(self, config: Config): self.uplink.m_libuplink.uplink_config_open_project.argtypes =\ [_ConfigStruct, ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_config_open_project.restype = _ProjectResult - self.uplink.m_libuplink.uplink_free_project_result.argstypes = [_ProjectResult] + self.uplink.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] # # prepare the input for the function if config is None: @@ -215,7 +215,7 @@ def serialize(self): # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_access_serialize.argtypes = [ctypes.POINTER(_AccessStruct)] self.uplink.m_libuplink.uplink_access_serialize.restype = _StringResult - self.uplink.m_libuplink.uplink_free_string_result.argstypes = [_StringResult] + self.uplink.m_libuplink.uplink_free_string_result.argtypes = [_StringResult] # # get serialized access by calling the exported golang function string_result = self.uplink.m_libuplink.uplink_access_serialize(self.access) @@ -262,7 +262,7 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No ctypes.POINTER(_SharePrefixStruct), ctypes.c_size_t] self.uplink.m_libuplink.uplink_access_share.restype = _AccessResult - self.uplink.m_libuplink.uplink_free_access_result.argstypes = [_AccessResult] + self.uplink.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] # # prepare the input for the function # check and create valid _PermissionStruct parameter diff --git a/uplink_python/download.py b/uplink_python/download.py index 6157a0a..4f64c49 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -72,7 +72,7 @@ def read(self, size_to_read: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult - self.uplink.m_libuplink.uplink_free_read_result.argstypes = [_ReadResult] + self.uplink.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) From 2fd62c8641bf2a20a94c1ca7eaa3007bd11ef349 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 31 Oct 2022 00:28:27 +0100 Subject: [PATCH 11/40] do not free upload result --- uplink_python/upload.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/uplink_python/upload.py b/uplink_python/upload.py index 7573eca..c5dac7e 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -93,8 +93,6 @@ def write(self, data_to_write: bytes, size_to_write: int): bytes_written = int(write_result.bytes_written) - self.uplink.m_libuplink.uplink_free_write_result(write_result) - return bytes_written def write_file(self, file_handle, buffer_size: int = 0): From b015729650f8fffab449c2a2ed9cbb8734a530e0 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 31 Oct 2022 12:06:44 +0100 Subject: [PATCH 12/40] test --- uplink_python/access.py | 17 ++--------------- uplink_python/project.py | 30 +++--------------------------- uplink_python/uplink.py | 3 +++ 3 files changed, 8 insertions(+), 42 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index fdfbd48..88d8470 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -8,6 +8,7 @@ _EncryptionKeyStruct from uplink_python.project import Project from uplink_python.errors import _storj_exception +import uplink_python.utils as utils class Access: @@ -78,21 +79,7 @@ def derive_encryption_key(self, passphrase: str, salt: str): encryption_key_result = self.uplink.m_libuplink.uplink_derive_encryption_key(passphrase_ptr, salt_ptr, length_ptr) - # - # if error occurred - if bool(encryption_key_result.error): - error_code = encryption_key_result.error.contents.code - error_msg = encryption_key_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) - - raise _storj_exception(error_code, error_msg) - - encryption_key = encryption_key_result.encryption_key - - self.uplink.m_libuplink.uplink_free_encryption_key_result(encryption_key_result) - - return encryption_key + return utils.unwrap_encryption_key_result(encryption_key_result, self.uplink) def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): """ diff --git a/uplink_python/project.py b/uplink_python/project.py index 6a02f81..50e66d1 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -10,6 +10,7 @@ from uplink_python.upload import Upload from uplink_python.download import Download from uplink_python.errors import _storj_exception +import uplink_python.utils as utils class Project: @@ -215,8 +216,6 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): bucket_list.append(self.uplink.bucket_from_result(bucket)) self.uplink.m_libuplink.uplink_free_bucket(bucket) - self.uplink.m_libuplink.uplink_free_bucket_iterator(bucket_iterator) - return bucket_list def delete_bucket(self, bucket_name: str): @@ -257,7 +256,6 @@ def delete_bucket(self, bucket_name: str): raise _storj_exception(error_code, error_msg) bucket = self.uplink.bucket_from_result(bucket_result.bucket) - self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) return bucket @@ -301,8 +299,6 @@ def stat_object(self, bucket_name: str, storj_path: str): _object = self.uplink.object_from_result(object_result.object) - self.uplink.m_libuplink.uplink_free_object_result(object_result) - return _object def list_objects(self, bucket_name: str, list_object_options: ListObjectsOptions = None): @@ -399,20 +395,8 @@ def delete_object(self, bucket_name: str, storj_path: str): # delete object by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_delete_object(self.project, bucket_name_ptr, storj_path_ptr) - # - # if error occurred - if bool(object_result.error): - error_code = object_result.error.contents.code - error_msg = object_result.error.contents.message.decode("utf-8") - self.uplink.m_libuplink.uplink_free_object_result(object_result) - - raise _storj_exception(error_code, error_msg) - - _object = self.uplink.object_from_result(object_result.object) - self.uplink.m_libuplink.uplink_free_object_result(object_result) - - return _object + return utils.unwrap_object_result(object_result, self.uplink) def close(self): """ @@ -470,16 +454,8 @@ def upload_object(self, bucket_name: str, storj_path: str, upload_result = self.uplink.m_libuplink.uplink_upload_object(self.project, bucket_name_ptr, storj_path_ptr, upload_options_obj) - # - # if error occurred - if bool(upload_result.error): - error_code = upload_result.error.contents.code - error_msg = upload_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_upload_result(upload_result) - raise _storj_exception(error_code,error_msg) - return Upload(upload_result.upload, self.uplink) + return utils.unwrap_upload_object_result(upload_result, self.uplink) def download_object(self, bucket_name: str, storj_path: str, download_options: DownloadOptions = None): diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index a097b87..7698932 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -245,3 +245,6 @@ def free_error_and_raise_exception(self,err : _Error): self.m_libuplink.uplink.m_libuplink.uplink_free_error(err) raise _storj_exception(error_code, error_msg) + + + From e3e693496f3bbd24747791b0f8f54ff1f448a5a8 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 31 Oct 2022 12:53:25 +0100 Subject: [PATCH 13/40] test --- uplink_python/uplink.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 7698932..a097b87 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -245,6 +245,3 @@ def free_error_and_raise_exception(self,err : _Error): self.m_libuplink.uplink.m_libuplink.uplink_free_error(err) raise _storj_exception(error_code, error_msg) - - - From 0711d1e54043534788c520488cef076d9dc6a495 Mon Sep 17 00:00:00 2001 From: Fadila Date: Mon, 31 Oct 2022 13:09:55 +0100 Subject: [PATCH 14/40] test --- uplink_python/access.py | 3 +-- uplink_python/project.py | 5 ++--- uplink_python/uplink.py | 26 ++++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 88d8470..8daf37d 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -8,7 +8,6 @@ _EncryptionKeyStruct from uplink_python.project import Project from uplink_python.errors import _storj_exception -import uplink_python.utils as utils class Access: @@ -79,7 +78,7 @@ def derive_encryption_key(self, passphrase: str, salt: str): encryption_key_result = self.uplink.m_libuplink.uplink_derive_encryption_key(passphrase_ptr, salt_ptr, length_ptr) - return utils.unwrap_encryption_key_result(encryption_key_result, self.uplink) + return self.uplink.unwrap_encryption_key_result(encryption_key_result) def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key): """ diff --git a/uplink_python/project.py b/uplink_python/project.py index 50e66d1..1ef16b6 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -10,7 +10,6 @@ from uplink_python.upload import Upload from uplink_python.download import Download from uplink_python.errors import _storj_exception -import uplink_python.utils as utils class Project: @@ -396,7 +395,7 @@ def delete_object(self, bucket_name: str, storj_path: str): object_result = self.uplink.m_libuplink.uplink_delete_object(self.project, bucket_name_ptr, storj_path_ptr) - return utils.unwrap_object_result(object_result, self.uplink) + return self.uplink.unwrap_object_result(object_result) def close(self): """ @@ -455,7 +454,7 @@ def upload_object(self, bucket_name: str, storj_path: str, storj_path_ptr, upload_options_obj) - return utils.unwrap_upload_object_result(upload_result, self.uplink) + return self.uplink.unwrap_upload_object_result(upload_result) def download_object(self, bucket_name: str, storj_path: str, download_options: DownloadOptions = None): diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index a097b87..1fceca1 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -245,3 +245,29 @@ def free_error_and_raise_exception(self,err : _Error): self.m_libuplink.uplink.m_libuplink.uplink_free_error(err) raise _storj_exception(error_code, error_msg) + + def unwrap_libuplink_result(self, result_object, finalizer, attribute_name): + if bool(result_object.error): + error_code = result_object.error.contents.code + error_msg = result_object.error.contents.message.decode("utf-8") + finalizer(result_object) + raise _storj_exception(error_code, error_msg) + + result = getattr(result_object, attribute_name) + # finalizer(result_object) + return result + + def unwrap_encryption_key_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + + def unwrap_project_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_project_result, 'project') + + def unwrap_download_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_download_result, 'download') + + def unwrap_object_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_object_result, 'object') + + def unwrap_upload_object_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_upload_result, 'upload') From 6547aa2488753e991b72de73cf66f922b2c3abeb Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 3 Nov 2022 11:40:27 +0100 Subject: [PATCH 15/40] use unwrap_libuplink_result --- uplink_python/access.py | 83 +++++--------------------------------- uplink_python/download.py | 18 +-------- uplink_python/project.py | 5 ++- uplink_python/uplink.py | 84 ++++++++++++++++++++++++--------------- 4 files changed, 69 insertions(+), 121 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 8daf37d..f47d36f 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -58,15 +58,6 @@ def derive_encryption_key(self, passphrase: str, salt: str): ------- EncryptionKey """ - - # - # declare types of arguments and response of the corresponding golang function - self.uplink.m_libuplink.uplink_derive_encryption_key.argtypes = [ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t] - self.uplink.m_libuplink.uplink_derive_encryption_key.restype = _EncryptionKeyResult - self.uplink.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] - # # prepare the input for the function passphrase_ptr = ctypes.c_char_p(passphrase.encode('utf-8')) hash_value = hashlib.sha256() # Choose SHA256 and update with bytes @@ -92,14 +83,7 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) None """ - # - # declare types of arguments and response of the corresponding golang function - self.uplink.m_libuplink.uplink_access_override_encryption_key.argtypes =\ - [ctypes.POINTER(_AccessStruct), ctypes.c_char_p, ctypes.c_char_p, - ctypes.POINTER(_EncryptionKeyStruct)] - self.uplink.m_libuplink.uplink_access_override_encryption_key.restype =\ - _EncryptionKeyResult - # + # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) prefix_ptr = ctypes.c_char_p(prefix.encode('utf-8')) @@ -121,26 +105,12 @@ def open_project(self): ------- Project """ - - # - # declare types of arguments and response of the corresponding golang function - self.uplink.m_libuplink.uplink_open_project.argtypes = [ctypes.POINTER(_AccessStruct)] - self.uplink.m_libuplink.uplink_open_project.restype = _ProjectResult - # - # open project by calling the exported golang function + # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_open_project(self.access) - # - # if error occurred - if bool(project_result.error): - error_code = project_result.error.contents.code - error_msg = project_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] - self.uplink.m_libuplink.uplink_free_project_result(project_result) - raise _storj_exception(error_code,error_msg) + _unwrapped_project = self.uplink.unwrap_project_result(project_result) - return Project(project_result.project, self.uplink) + return Project(_unwrapped_project, self.uplink) def config_open_project(self, config: Config): """ @@ -170,21 +140,10 @@ def config_open_project(self, config: Config): # # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_config_open_project(config_obj, self.access) - # - # if error occurred - if bool(project_result.error): - error_code = project_result.error.contents.code - error_msg = project_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_project_result(project_result) - - raise _storj_exception(error_code,error_msg) - project = Project(project_result.project, self.uplink) + _unwrapped_project = self.uplink.unwrap_project_result(project_result) - self.uplink.m_libuplink.uplink_free_project_result(project_result) - - return project + return Project(_unwrapped_project, self.uplink) def serialize(self): @@ -205,20 +164,10 @@ def serialize(self): # # get serialized access by calling the exported golang function string_result = self.uplink.m_libuplink.uplink_access_serialize(self.access) - # - # if error occurred - if bool(string_result.error): - error_code = string_result.error.contents.code - error_msg = string_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_string_result(string_result) - raise _storj_exception(error_code,error_msg) - to_return = string_result.string.decode("utf-8") + _unwrapped_string = self.uplink.unwrap_string_result(string_result) - self.uplink.m_libuplink.uplink_free_string_result(string_result) - - return to_return + return _unwrapped_string.decode("utf-8") def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = None): """ @@ -273,18 +222,6 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No # get shareable access by calling the exported golang function access_result = self.uplink.m_libuplink.uplink_access_share(self.access, permission_obj, shared_prefix_obj, array_size) - # - # if error occurred - if bool(access_result.error): - error_code = access_result.error.contents.code - error_msg = access_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_access_result(access_result) - - raise _storj_exception(error_code,error_msg) - - access = Access(access_result.access, self.uplink) - - self.uplink.m_libuplink.uplink_free_access_result(access_result) - return access + _unwrapped_access = self.uplink.unwrap_access_result(access_result) + return Access(_unwrapped_access, self.uplink) diff --git a/uplink_python/download.py b/uplink_python/download.py index 4f64c49..fe7b20a 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -67,13 +67,6 @@ def read(self, size_to_read: int): bytes, int """ # - # declare types of arguments and response of the corresponding golang function - self.uplink.m_libuplink.uplink_download_read.argtypes = [ctypes.POINTER(_DownloadStruct), - ctypes.POINTER(ctypes.c_uint8), - ctypes.c_size_t] - self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult - self.uplink.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] - # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) data_to_write = [0] @@ -199,17 +192,10 @@ def info(self): # # get last download info by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_download_info(self.download) - # - # if error occurred - if bool(object_result.error): - error_code = object_result.error.contents.code - error_msg = object_result.error.contents.message.decode("utf-8") - self.uplink.m_libuplink.uplink_free_object_result(object_result) - - raise _storj_exception(error_code, error_msg) + _unwrapped_object = self.uplink.unwrap_object_result(object_result) + _object = self.uplink.object_from_result(_unwrapped_object) - _object = self.uplink.object_from_result(object_result.object) self.uplink.m_libuplink.uplink_free_object_result(object_result) return _object diff --git a/uplink_python/project.py b/uplink_python/project.py index 1ef16b6..e9a2894 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -454,7 +454,10 @@ def upload_object(self, bucket_name: str, storj_path: str, storj_path_ptr, upload_options_obj) - return self.uplink.unwrap_upload_object_result(upload_result) + _upload_result = self.uplink.unwrap_libuplink_result(upload_result, self.uplink.m_libuplink.uplink_free_upload_result, 'upload') + + # _upload_result will be freed when committing or aborting upload. + return Upload(_upload_result, self.uplink) def download_object(self, bucket_name: str, storj_path: str, download_options: DownloadOptions = None): diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 1fceca1..8f4afd7 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct, _Error +from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, _DownloadStruct, _EncryptionKeyResult, _EncryptionKeyStruct, _Error, _ProjectResult, _ReadResult, _UploadResult, _UploadStruct from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -48,6 +48,30 @@ def __init__(self): self.m_libuplink = ctypes.CDLL(so_path) else: raise LibUplinkSoError + + + self.m_libuplink.uplink_derive_encryption_key.argtypes = [ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t] + self.m_libuplink.uplink_derive_encryption_key.restype = _EncryptionKeyResult + self.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] + + self.m_libuplink.uplink_download_read.argtypes = [ctypes.POINTER(_DownloadStruct), + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t] + self.m_libuplink.uplink_download_read.restype = _ReadResult + self.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] + + self.m_libuplink.uplink_access_override_encryption_key.argtypes =\ + [ctypes.POINTER(_AccessStruct), ctypes.c_char_p, ctypes.c_char_p, + ctypes.POINTER(_EncryptionKeyStruct)] + self.m_libuplink.uplink_access_override_encryption_key.restype =\ + _EncryptionKeyResult + + self.m_libuplink.uplink_open_project.argtypes = [ctypes.POINTER(_AccessStruct)] + self.m_libuplink.uplink_open_project.restype = _ProjectResult + # + Uplink.__instance = self else: self.m_libuplink = Uplink.__instance.m_libuplink @@ -183,18 +207,10 @@ def config_request_access_with_passphrase(self, config: Config, satellite: str, satellite_ptr, api_key_ptr, phrase_ptr) - # - # if error occurred - if bool(access_result.error): - error_code = access_result.error.contents.code - error_msg = access_result.error.contents.message.decode("utf-8") - - self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] - self.m_libuplink.uplink_free_access_result(access_result) - raise _storj_exception(error_code, error_msg) + _unwrapped_access = self.m_libuplink.unwrap_access_result(access_result) - return Access(access_result.access, self) + return Access(_unwrapped_access, self) def parse_access(self, serialized_access: str): """ @@ -223,26 +239,19 @@ def parse_access(self, serialized_access: str): # get parsed access by calling the exported golang function access_result = self.m_libuplink.uplink_parse_access(serialized_access_ptr) - # - # if error occurred - if bool(access_result.error): - error_code = access_result.error.contents.code - error_msg = access_result.error.contents.message.decode("utf-8") - self.m_libuplink.uplink_free_access_result.argtypes = [_AccessResult] - self.m_libuplink.uplink_free_access_result(access_result) + _unwrapped_access = self.unwrap_access_result(access_result) - raise _storj_exception(error_code, error_msg) + return Access(_unwrapped_access, self) - return Access(access_result.access, self) - def free_error_and_raise_exception(self,err : _Error): + def free_error_and_raise_exception(self, err ): """ free libuplinkc error and raise corresponding _storj_exception """ - error_code = err.error.contents.code - error_msg = err.error.contents.message.decode("utf-8") + error_code = err.contents.code + error_msg = err.contents.message.decode("utf-8") - self.m_libuplink.uplink_free_error.argtypes = [_Error] - self.m_libuplink.uplink.m_libuplink.uplink_free_error(err) + self.m_libuplink.uplink_free_error.argtypes = [ctypes.POINTER(_Error)] + self.m_libuplink.uplink_free_error(err) raise _storj_exception(error_code, error_msg) @@ -254,20 +263,33 @@ def unwrap_libuplink_result(self, result_object, finalizer, attribute_name): raise _storj_exception(error_code, error_msg) result = getattr(result_object, attribute_name) - # finalizer(result_object) return result - def unwrap_encryption_key_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') - - def unwrap_project_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_project_result, 'project') + def unwrap_access_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_access_result, 'access') def unwrap_download_result(self, result_object): return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_download_result, 'download') + def unwrap_encryption_key_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + def unwrap_object_result(self, result_object): return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_object_result, 'object') + def unwrap_project_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_project_result, 'project') + + def unwrap_string_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_string_result, 'string') + def unwrap_upload_object_result(self, result_object): return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_upload_result, 'upload') + + def unwrap_upload_write_result(self, result_object): + return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') + + def free_upload_struct(self, uploadStruct): + _uploadResult = _UploadResult() + _uploadResult.upload = uploadStruct + self.m_libuplink.uplink_free_upload_result(_uploadResult) From ea0d41becaae01710179b95033d777d05407cf61 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 3 Nov 2022 12:54:40 +0100 Subject: [PATCH 16/40] code clean-up --- uplink_python/project.py | 2 +- uplink_python/uplink.py | 134 ++++++++++++++++++++++++++++++++------- uplink_python/upload.py | 37 ++++------- 3 files changed, 123 insertions(+), 50 deletions(-) diff --git a/uplink_python/project.py b/uplink_python/project.py index e9a2894..a84d332 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -454,7 +454,7 @@ def upload_object(self, bucket_name: str, storj_path: str, storj_path_ptr, upload_options_obj) - _upload_result = self.uplink.unwrap_libuplink_result(upload_result, self.uplink.m_libuplink.uplink_free_upload_result, 'upload') + _upload_result = self.uplink.unwrap_upload_object_result(upload_result) # _upload_result will be freed when committing or aborting upload. return Upload(_upload_result, self.uplink) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 8f4afd7..789131c 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,9 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, _DownloadStruct, _EncryptionKeyResult, _EncryptionKeyStruct, _Error, _ProjectResult, _ReadResult, _UploadResult, _UploadStruct +from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, \ + _DownloadStruct, _EncryptionKeyResult, _EncryptionKeyStruct, _Error, \ + _ProjectResult, _ReadResult, _UploadResult from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -255,41 +257,127 @@ def free_error_and_raise_exception(self, err ): raise _storj_exception(error_code, error_msg) - def unwrap_libuplink_result(self, result_object, finalizer, attribute_name): - if bool(result_object.error): - error_code = result_object.error.contents.code - error_msg = result_object.error.contents.message.decode("utf-8") - finalizer(result_object) + def unwrap_libuplink_result(self, result, finalizer, attribute_name): + ''' unwrap libuplink result - raise exception if error occured''' + if bool(result.error): + error_code = result.error.contents.code + error_msg = result.error.contents.message.decode("utf-8") + finalizer(result) raise _storj_exception(error_code, error_msg) - result = getattr(result_object, attribute_name) + result = getattr(result, attribute_name) return result - def unwrap_access_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_access_result, 'access') + def unwrap_access_result(self, access_result): + """ + unwrap access result + + Parameters + ---------- + access_result : _AccessResult + + Returns + ------- + ctypes.POINTER(_AccessStruct)) + """ + return self.unwrap_libuplink_result(access_result, self.m_libuplink.uplink_free_access_result, 'access') + + def unwrap_encryption_key_result(self, encryption_key_result): + """ + unwrap encryption key result + + Parameters + ---------- + encryption_key_result : _EncryptionKeyResult + + Returns + ------- + ctypes.POINTER(_EncryptionKeyStruct)) + """ + return self.unwrap_libuplink_result(encryption_key_result, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + + def unwrap_object_result(self, object_result): + """ + unwrap object result - def unwrap_download_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_download_result, 'download') + Parameters + ---------- + object_result : _ObjectResult - def unwrap_encryption_key_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + Returns + ------- + ctypes.POINTER(_ObjectStruct)) + """ + return self.unwrap_libuplink_result(object_result, self.m_libuplink.uplink_free_object_result, 'object') - def unwrap_object_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_object_result, 'object') + def unwrap_project_result(self, project_result): + """ + unwrap project result - def unwrap_project_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_project_result, 'project') + Parameters + ---------- + project_result : _ProjectResult - def unwrap_string_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_string_result, 'string') + Returns + ------- + ctypes.POINTER(_ProjectStruct)) + """ + return self.unwrap_libuplink_result(project_result, self.m_libuplink.uplink_free_project_result, 'project') - def unwrap_upload_object_result(self, result_object): - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_upload_result, 'upload') + def unwrap_string_result(self, string_result): + """ + unwrap project result + + Parameters + ---------- + project_result : _StringResult + + Returns + ------- + ctypes.c_char_p + """ + return self.unwrap_libuplink_result(string_result, self.m_libuplink.uplink_free_string_result, 'string') + + def unwrap_upload_object_result(self, upload_object_result): + """ + unwrap project result + + Parameters + ---------- + upload_object_result : _UploadResult + + Returns + ------- + ctypes.POINTER(_UploadStruct)) + """ + return self.unwrap_libuplink_result(upload_object_result, self.m_libuplink.uplink_free_upload_result, 'upload') def unwrap_upload_write_result(self, result_object): + """ + unwrap upload write result + + Parameters + ---------- + upload_write_result : _WriteResult + + Returns + ------- + ctypes.c_size_t + """ return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') - def free_upload_struct(self, uploadStruct): + def free_upload_struct(self, upload_struct): + """ + free upload object result + + Parameters + ---------- + upload_struct : _UploadStruct + + Returns + ------- + None + """ _uploadResult = _UploadResult() - _uploadResult.upload = uploadStruct + _uploadResult.upload = upload_struct self.m_libuplink.uplink_free_upload_result(_uploadResult) diff --git a/uplink_python/upload.py b/uplink_python/upload.py index c5dac7e..cbe820b 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -81,19 +81,8 @@ def write(self, data_to_write: bytes, size_to_write: int): # upload data by calling the exported golang function write_result = self.uplink.m_libuplink.uplink_upload_write(self.upload, data_to_write_ptr, size_to_write_obj) - # - # if error occurred - if bool(write_result.error): - error_code = write_result.error.contents.code - error_msg = write_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_write_result(write_result) - - _storj_exception(error_code, error_msg) - bytes_written = int(write_result.bytes_written) - - return bytes_written + return self.uplink.unwrap_upload_write_result(write_result) def write_file(self, file_handle, buffer_size: int = 0): """ @@ -139,10 +128,12 @@ def commit(self): # upload commit by calling the exported golang function error = self.uplink.m_libuplink.uplink_upload_commit(self.upload) + + self.uplink.free_upload_struct(self.upload) # # if error occurred if bool(error): - self.uplink.free_and_raise_error(error) + self.uplink.free_error_and_raise_exception(error) def abort(self): """ @@ -162,9 +153,11 @@ def abort(self): error = self.uplink.m_libuplink.uplink_upload_abort(self.upload) # # if error occurred + self.uplink.free_upload_struct(self.upload) if bool(error): self.uplink.free_and_raise_error(error) + def set_custom_metadata(self, custom_metadata: CustomMetadata = None): """ function to set custom meta information while uploading data @@ -191,10 +184,9 @@ def set_custom_metadata(self, custom_metadata: CustomMetadata = None): # # set custom metadata to upload by calling the exported golang function error = self.uplink.m_libuplink.uplink_upload_set_custom_metadata(self.upload, custom_metadata_obj) - # - # if error occurred + if bool(error): - self.uplink.free_and_raise_error(error) + self.uplink.free_error_and_raise_exception(error) def info(self): """ @@ -212,15 +204,8 @@ def info(self): # # get last upload info by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_upload_info(self.upload) - # - # if error occurred - if bool(object_result.error): - error_code = object_result.error.contents.code - error_msg = object_result.error.contents.message.decode("utf-8") - - self.uplink.m_libuplink.uplink_free_object_result(object_result) - - raise _storj_exception(error_code, error_msg) - info = self.uplink.object_from_result(object_result.object) + _unwrapped_object = self.uplink.unwrap_object_result(object_result) + info = self.uplink.object_from_result(_unwrapped_object) + self.uplink.m_libuplink.uplink_free_object(_unwrapped_object) return info From 02721a7b715795e1cac0df3dc1af7d7c8760d3d4 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 3 Nov 2022 14:03:37 +0100 Subject: [PATCH 17/40] code clean-up --- uplink_python/uplink.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 789131c..c42d3ad 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -280,7 +280,8 @@ def unwrap_access_result(self, access_result): ------- ctypes.POINTER(_AccessStruct)) """ - return self.unwrap_libuplink_result(access_result, self.m_libuplink.uplink_free_access_result, 'access') + return self.unwrap_libuplink_result( + access_result, self.m_libuplink.uplink_free_access_result, 'access') def unwrap_encryption_key_result(self, encryption_key_result): """ @@ -294,7 +295,8 @@ def unwrap_encryption_key_result(self, encryption_key_result): ------- ctypes.POINTER(_EncryptionKeyStruct)) """ - return self.unwrap_libuplink_result(encryption_key_result, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + return self.unwrap_libuplink_result( + encryption_key_result, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') def unwrap_object_result(self, object_result): """ @@ -308,7 +310,8 @@ def unwrap_object_result(self, object_result): ------- ctypes.POINTER(_ObjectStruct)) """ - return self.unwrap_libuplink_result(object_result, self.m_libuplink.uplink_free_object_result, 'object') + return self.unwrap_libuplink_result( + object_result, self.m_libuplink.uplink_free_object_result, 'object') def unwrap_project_result(self, project_result): """ @@ -322,7 +325,8 @@ def unwrap_project_result(self, project_result): ------- ctypes.POINTER(_ProjectStruct)) """ - return self.unwrap_libuplink_result(project_result, self.m_libuplink.uplink_free_project_result, 'project') + return self.unwrap_libuplink_result( + project_result, self.m_libuplink.uplink_free_project_result, 'project') def unwrap_string_result(self, string_result): """ @@ -336,7 +340,8 @@ def unwrap_string_result(self, string_result): ------- ctypes.c_char_p """ - return self.unwrap_libuplink_result(string_result, self.m_libuplink.uplink_free_string_result, 'string') + return self.unwrap_libuplink_result( + string_result, self.m_libuplink.uplink_free_string_result, 'string') def unwrap_upload_object_result(self, upload_object_result): """ @@ -350,7 +355,8 @@ def unwrap_upload_object_result(self, upload_object_result): ------- ctypes.POINTER(_UploadStruct)) """ - return self.unwrap_libuplink_result(upload_object_result, self.m_libuplink.uplink_free_upload_result, 'upload') + return self.unwrap_libuplink_result( + upload_object_result, self.m_libuplink.uplink_free_upload_result, 'upload') def unwrap_upload_write_result(self, result_object): """ @@ -364,7 +370,8 @@ def unwrap_upload_write_result(self, result_object): ------- ctypes.c_size_t """ - return self.unwrap_libuplink_result(result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') + return self.unwrap_libuplink_result( + result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') def free_upload_struct(self, upload_struct): """ @@ -378,6 +385,6 @@ def free_upload_struct(self, upload_struct): ------- None """ - _uploadResult = _UploadResult() - _uploadResult.upload = upload_struct - self.m_libuplink.uplink_free_upload_result(_uploadResult) + _upload_result = _UploadResult() + _upload_result.upload = upload_struct + self.m_libuplink.uplink_free_upload_result(_upload_result) From 493d6d6936fbec306ff98cd6ab5cb25f0c6fa722 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 3 Nov 2022 14:08:07 +0100 Subject: [PATCH 18/40] code clean-up --- uplink_python/uplink.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index c42d3ad..fa537b9 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -295,8 +295,8 @@ def unwrap_encryption_key_result(self, encryption_key_result): ------- ctypes.POINTER(_EncryptionKeyStruct)) """ - return self.unwrap_libuplink_result( - encryption_key_result, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') + return self.unwrap_libuplink_result(encryption_key_result, + self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') def unwrap_object_result(self, object_result): """ From d3c4f4855b16051a3b7d859e309477ab9af7e021 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 3 Nov 2022 14:30:22 +0100 Subject: [PATCH 19/40] code clean-up --- uplink_python/access.py | 23 +++++++++++++++++++++++ uplink_python/download.py | 8 ++++++++ uplink_python/uplink.py | 23 ----------------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index f47d36f..1b52af8 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -58,6 +58,15 @@ def derive_encryption_key(self, passphrase: str, salt: str): ------- EncryptionKey """ + + + # + # declare types of arguments and response of the corresponding golang function + self.uplink.m_libuplink.uplink_derive_encryption_key.argtypes = [ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_size_t] + self.uplink.m_libuplink.uplink_derive_encryption_key.restype = _EncryptionKeyResult + # # prepare the input for the function passphrase_ptr = ctypes.c_char_p(passphrase.encode('utf-8')) hash_value = hashlib.sha256() # Choose SHA256 and update with bytes @@ -84,6 +93,13 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) """ # + # declare types of arguments and response of the corresponding golang function + self.uplink.m_libuplink.uplink_access_override_encryption_key.argtypes =\ + [ctypes.POINTER(_AccessStruct), ctypes.c_char_p, ctypes.c_char_p, + ctypes.POINTER(_EncryptionKeyStruct)] + self.uplink.m_libuplink.uplink_access_override_encryption_key.restype =\ + _EncryptionKeyResult + # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) prefix_ptr = ctypes.c_char_p(prefix.encode('utf-8')) @@ -105,6 +121,13 @@ def open_project(self): ------- Project """ + + # + # declare types of arguments and response of the corresponding golang function + self.uplink.m_libuplink.uplink_open_project.argtypes = [ctypes.POINTER(_AccessStruct)] + self.uplink.m_libuplink.uplink_open_project.restype = _ProjectResult + # + # open project by calling the exported golang function # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_open_project(self.access) diff --git a/uplink_python/download.py b/uplink_python/download.py index fe7b20a..9bb734a 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -53,6 +53,8 @@ def __init__(self, download, uplink, project, bucket_name, storj_path): self.storj_path = storj_path self.uplink = uplink + + def read(self, size_to_read: int): """ function downloads up to len size_to_read bytes from the object's data stream. @@ -67,6 +69,12 @@ def read(self, size_to_read: int): bytes, int """ # + # declare types of arguments and response of the corresponding golang function + self.uplink.m_libuplink.uplink_download_read.argtypes = [ctypes.POINTER(_DownloadStruct), + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t] + self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult + # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) data_to_write = [0] diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index fa537b9..78c3e86 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -51,29 +51,6 @@ def __init__(self): else: raise LibUplinkSoError - - self.m_libuplink.uplink_derive_encryption_key.argtypes = [ctypes.c_char_p, - ctypes.c_void_p, - ctypes.c_size_t] - self.m_libuplink.uplink_derive_encryption_key.restype = _EncryptionKeyResult - self.m_libuplink.uplink_free_encryption_key_result.argtypes = [_EncryptionKeyResult] - - self.m_libuplink.uplink_download_read.argtypes = [ctypes.POINTER(_DownloadStruct), - ctypes.POINTER(ctypes.c_uint8), - ctypes.c_size_t] - self.m_libuplink.uplink_download_read.restype = _ReadResult - self.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] - - self.m_libuplink.uplink_access_override_encryption_key.argtypes =\ - [ctypes.POINTER(_AccessStruct), ctypes.c_char_p, ctypes.c_char_p, - ctypes.POINTER(_EncryptionKeyStruct)] - self.m_libuplink.uplink_access_override_encryption_key.restype =\ - _EncryptionKeyResult - - self.m_libuplink.uplink_open_project.argtypes = [ctypes.POINTER(_AccessStruct)] - self.m_libuplink.uplink_open_project.restype = _ProjectResult - # - Uplink.__instance = self else: self.m_libuplink = Uplink.__instance.m_libuplink From 139c51a96752f109439d96c79e589be6153c8f54 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 01:55:05 +0100 Subject: [PATCH 20/40] code cleanup --- uplink_python/access.py | 10 ++++-- uplink_python/download.py | 26 ++++++--------- uplink_python/project.py | 14 +++++---- uplink_python/uplink.py | 66 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 26 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 1b52af8..58ff6e1 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -128,7 +128,6 @@ def open_project(self): self.uplink.m_libuplink.uplink_open_project.restype = _ProjectResult # # open project by calling the exported golang function - # open project by calling the exported golang function project_result = self.uplink.m_libuplink.uplink_open_project(self.access) _unwrapped_project = self.uplink.unwrap_project_result(project_result) @@ -190,7 +189,9 @@ def serialize(self): _unwrapped_string = self.uplink.unwrap_string_result(string_result) - return _unwrapped_string.decode("utf-8") + serialized_access = _unwrapped_string.decode("utf-8") + self.uplink.m_libuplink.uplink_free_string_result(string_result) + return serialized_access def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = None): """ @@ -248,3 +249,8 @@ def share(self, permission: Permission = None, shared_prefix: [SharePrefix] = No _unwrapped_access = self.uplink.unwrap_access_result(access_result) return Access(_unwrapped_access, self.uplink) + + + def __del__(self): + """Free memory associated to this Access""" + self.uplink.free_access_struct(self.access) diff --git a/uplink_python/download.py b/uplink_python/download.py index 9bb734a..1602b60 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -74,6 +74,7 @@ def read(self, size_to_read: int): ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] self.uplink.m_libuplink.uplink_download_read.restype = _ReadResult + self.uplink.m_libuplink.uplink_free_read_result.argtypes = [_ReadResult] # # prepare the inputs for the function data_size = ctypes.c_int32(size_to_read) @@ -85,16 +86,9 @@ def read(self, size_to_read: int): # read data from Storj by calling the exported golang function read_result = self.uplink.m_libuplink.uplink_download_read(self.download, data_to_write_ptr, size_to_read) - # - # if error occurred - if read_result.error: - error_code = read_result.error.contents.code - error_msg = read_result.error.contents.message.decode("utf-8") - self.uplink.m_libuplink.uplink_free_read_result(read_result) - raise _storj_exception(error_code,error_msg) + bytes_read = self.uplink.unwrap_read_result(read_result) - bytes_read = int(read_result.bytes_read) data_read = bytes() if bytes_read != 0: # @@ -154,16 +148,13 @@ def file_size(self): # get object information by calling the exported golang function object_result = self.uplink.m_libuplink.uplink_stat_object(self.project, self.bucket_name, self.storj_path) - # if error occurred - if bool(object_result.error): - error_code = object_result.error.contents.code - error_msg = object_result.error.contents.message.decode("utf-8") - self.uplink.m_libuplink.uplink_free_object_result(object_result) + object = self.uplink.unwrap_object_result(object_result) - raise _storj_exception(error_code, error_msg) - # find object size - return int(object_result.object.contents.system.content_length) + file_size = int(object.contents.system.content_length) + + self.uplink.m_libuplink.uplink_free_object_result(object_result) + return file_size def close(self): """ @@ -207,3 +198,6 @@ def info(self): self.uplink.m_libuplink.uplink_free_object_result(object_result) return _object + + def __del__(self): + self.uplink.free_download_struct(self.download) \ No newline at end of file diff --git a/uplink_python/project.py b/uplink_python/project.py index a84d332..29e7e13 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -76,18 +76,20 @@ def create_bucket(self, bucket_name: str): self.uplink.m_libuplink.uplink_create_bucket.argtypes = [ctypes.POINTER(_ProjectStruct), ctypes.c_char_p] self.uplink.m_libuplink.uplink_create_bucket.restype = _BucketResult + self.uplink.m_libuplink.uplink_free_bucket.argtypes = [_BucketResult] # # prepare the input for the function bucket_name_ptr = ctypes.c_char_p(bucket_name.encode('utf-8')) # create bucket by calling the exported golang function bucket_result = self.uplink.m_libuplink.uplink_create_bucket(self.project, bucket_name_ptr) - # - # if error occurred - if bool(bucket_result.error): - raise _storj_exception(bucket_result.error.contents.code, - bucket_result.error.contents.message.decode("utf-8")) - return self.uplink.bucket_from_result(bucket_result.bucket) + + _unwrapped_bucket = self.uplink.unwrap_bucket_result(bucket_result) + bucket = self.uplink.bucket_from_result(_unwrapped_bucket) + + self.uplink.uplink_free_bucket_result(bucket_result) + + return bucket def ensure_bucket(self, bucket_name: str): """ diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 78c3e86..08a6c08 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, \ +from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, _DownloadResult, \ _DownloadStruct, _EncryptionKeyResult, _EncryptionKeyStruct, _Error, \ _ProjectResult, _ReadResult, _UploadResult from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ @@ -260,6 +260,21 @@ def unwrap_access_result(self, access_result): return self.unwrap_libuplink_result( access_result, self.m_libuplink.uplink_free_access_result, 'access') + def unwrap_bucket_result(self, bucket_result): + """ + unwrap bucket result + + Parameters + ---------- + bucket_result : _BucketResult + + Returns + ------- + ctypes.POINTER(_BucketStruct)) + """ + return self.unwrap_libuplink_result( + bucket_result, self.m_libuplink.uplink_free_bucket_result, 'bucket') + def unwrap_encryption_key_result(self, encryption_key_result): """ unwrap encryption key result @@ -305,13 +320,28 @@ def unwrap_project_result(self, project_result): return self.unwrap_libuplink_result( project_result, self.m_libuplink.uplink_free_project_result, 'project') + def unwrap_read_result(self, read_result): + """ + unwrap read result + + Parameters + ---------- + read_result : _ReadResult + + Returns + ------- + ctypes.c_size_t + """ + return self.unwrap_libuplink_result( + read_result, self.m_libuplink.uplink_free_read_result, 'bytes_read') + def unwrap_string_result(self, string_result): """ unwrap project result Parameters ---------- - project_result : _StringResult + string_result : _StringResult Returns ------- @@ -350,6 +380,22 @@ def unwrap_upload_write_result(self, result_object): return self.unwrap_libuplink_result( result_object, self.m_libuplink.uplink_free_write_result, 'bytes_written') + def free_access_struct(self, access_struct): + """ + free access result + + Parameters + ---------- + access_struct : _AccessStruct + + Returns + ------- + None + """ + _access_result = _AccessResult() + _access_result.upload = access_struct + self.m_libuplink.uplink_free_access_result(_access_result) + def free_upload_struct(self, upload_struct): """ free upload object result @@ -365,3 +411,19 @@ def free_upload_struct(self, upload_struct): _upload_result = _UploadResult() _upload_result.upload = upload_struct self.m_libuplink.uplink_free_upload_result(_upload_result) + + def free_download_struct(self, download_struct): + """ + free download object result + + Parameters + ---------- + download_struct : _DownloadStruct + + Returns + ------- + None + """ + _download_result = _DownloadResult() + _download_result.download = download_struct + self.m_libuplink.uplink_free_download_result(_download_result) \ No newline at end of file From 2162e461488682dc496c483fece29cc1c125b9e4 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 02:08:36 +0100 Subject: [PATCH 21/40] code cleanup --- uplink_python/project.py | 4 ++-- uplink_python/upload.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/uplink_python/project.py b/uplink_python/project.py index 29e7e13..2034417 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -87,7 +87,7 @@ def create_bucket(self, bucket_name: str): _unwrapped_bucket = self.uplink.unwrap_bucket_result(bucket_result) bucket = self.uplink.bucket_from_result(_unwrapped_bucket) - self.uplink.uplink_free_bucket_result(bucket_result) + self.uplink.m_libuplink.uplink_free_bucket_result(bucket_result) return bucket @@ -210,7 +210,7 @@ def list_buckets(self, list_bucket_options: ListBucketsOptions = None): bucket_iterator_err = self.uplink.m_libuplink.uplink_bucket_iterator_err(bucket_iterator) if bool(bucket_iterator_err): - self.uplink.free_and_raise_error(bucket_iterator_err) + self.uplink.free_error_and_raise_exception(bucket_iterator_err) bucket_list = list() while self.uplink.m_libuplink.uplink_bucket_iterator_next(bucket_iterator): bucket = self.uplink.m_libuplink.uplink_bucket_iterator_item(bucket_iterator) diff --git a/uplink_python/upload.py b/uplink_python/upload.py index cbe820b..450cb80 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -5,7 +5,6 @@ from uplink_python.module_classes import CustomMetadata from uplink_python.module_def import _UploadStruct, _WriteResult, _Error, _CustomMetadataStruct, _ObjectResult -from uplink_python.errors import _storj_exception _WINDOWS = os.name == 'nt' COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 @@ -155,7 +154,7 @@ def abort(self): # if error occurred self.uplink.free_upload_struct(self.upload) if bool(error): - self.uplink.free_and_raise_error(error) + self.uplink.free_error_and_raise_exception(error) def set_custom_metadata(self, custom_metadata: CustomMetadata = None): From 83607ab209a2809ffde36cd235881523db1b75f6 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 02:30:21 +0100 Subject: [PATCH 22/40] code cleanup --- uplink_python/access.py | 1 - uplink_python/uplink.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 58ff6e1..904b52e 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -59,7 +59,6 @@ def derive_encryption_key(self, passphrase: str, salt: str): EncryptionKey """ - # # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_derive_encryption_key.argtypes = [ctypes.c_char_p, diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 08a6c08..0cb2a56 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -8,8 +8,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, _DownloadResult, \ - _DownloadStruct, _EncryptionKeyResult, _EncryptionKeyStruct, _Error, \ - _ProjectResult, _ReadResult, _UploadResult + _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata From 12851fcdf5dd7e23817b93859847106afb8239cf Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 02:45:14 +0100 Subject: [PATCH 23/40] code cleanup --- uplink_python/uplink.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 0cb2a56..a660373 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _AccessStruct, _ConfigStruct, _DownloadResult, \ +from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, \ _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -425,4 +425,4 @@ def free_download_struct(self, download_struct): """ _download_result = _DownloadResult() _download_result.download = download_struct - self.m_libuplink.uplink_free_download_result(_download_result) \ No newline at end of file + self.m_libuplink.uplink_free_download_result(_download_result) From d0837244c6ec4693fbc71ec3a28978d33a84ed62 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 02:58:44 +0100 Subject: [PATCH 24/40] code cleanup --- uplink_python/access.py | 1 - uplink_python/download.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 904b52e..44f06ff 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -7,7 +7,6 @@ _AccessStruct, _ProjectResult, _StringResult, _AccessResult, _EncryptionKeyResult,\ _EncryptionKeyStruct from uplink_python.project import Project -from uplink_python.errors import _storj_exception class Access: diff --git a/uplink_python/download.py b/uplink_python/download.py index 1602b60..796a0de 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -5,7 +5,6 @@ from uplink_python.module_def import _DownloadStruct, _ReadResult, _ProjectStruct,\ _ObjectResult, _Error -from uplink_python.errors import _storj_exception _WINDOWS = os.name == 'nt' COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 @@ -149,9 +148,9 @@ def file_size(self): object_result = self.uplink.m_libuplink.uplink_stat_object(self.project, self.bucket_name, self.storj_path) - object = self.uplink.unwrap_object_result(object_result) + _object = self.uplink.unwrap_object_result(object_result) - file_size = int(object.contents.system.content_length) + file_size = int(_object.contents.system.content_length) self.uplink.m_libuplink.uplink_free_object_result(object_result) return file_size From 86e51abc981778a379af5ff97f1c92c2f10de875 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 4 Nov 2022 10:54:07 +0100 Subject: [PATCH 25/40] code cleanup --- uplink_python/access.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uplink_python/access.py b/uplink_python/access.py index 44f06ff..4b39f16 100644 --- a/uplink_python/access.py +++ b/uplink_python/access.py @@ -90,8 +90,8 @@ def override_encryption_key(self, bucket_name: str, prefix: str, encryption_key) None """ - # - # declare types of arguments and response of the corresponding golang function + # + # declare types of arguments and response of the corresponding golang function self.uplink.m_libuplink.uplink_access_override_encryption_key.argtypes =\ [ctypes.POINTER(_AccessStruct), ctypes.c_char_p, ctypes.c_char_p, ctypes.POINTER(_EncryptionKeyStruct)] From 4f6c64fa5b7e88fefb0d34b8e8afce9f95cbec91 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 10 Nov 2022 12:08:56 +0100 Subject: [PATCH 26/40] add __del__ to free c memory --- uplink_python/download.py | 2 +- uplink_python/project.py | 3 +++ uplink_python/uplink.py | 25 ++++++++++++++++++++++--- uplink_python/upload.py | 4 +++- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/uplink_python/download.py b/uplink_python/download.py index 796a0de..abb671e 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -199,4 +199,4 @@ def info(self): return _object def __del__(self): - self.uplink.free_download_struct(self.download) \ No newline at end of file + self.uplink.free_download_struct(self.download) diff --git a/uplink_python/project.py b/uplink_python/project.py index 2034417..0a78743 100644 --- a/uplink_python/project.py +++ b/uplink_python/project.py @@ -510,3 +510,6 @@ def download_object(self, bucket_name: str, storj_path: str, return Download(download_result.download, self.uplink, self.project, bucket_name_ptr, storj_path_ptr) + + def __del__(self): + self.uplink.free_project_struct(self.project) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index a660373..ee9a6a6 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, \ +from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, _ProjectResult, _ProjectStruct, \ _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -397,7 +397,7 @@ def free_access_struct(self, access_struct): def free_upload_struct(self, upload_struct): """ - free upload object result + free upload struct Parameters ---------- @@ -413,7 +413,7 @@ def free_upload_struct(self, upload_struct): def free_download_struct(self, download_struct): """ - free download object result + free download struct Parameters ---------- @@ -426,3 +426,22 @@ def free_download_struct(self, download_struct): _download_result = _DownloadResult() _download_result.download = download_struct self.m_libuplink.uplink_free_download_result(_download_result) + + def free_project_struct(self, project_struct): + """ + free project struct + + Parameters + ---------- + project_struct : _ProjectStruct + + Returns + ------- + None + """ + self.m_libuplink.uplink_free_project_result.argtypes = [_ProjectResult] + + _project_result = _ProjectResult() + _project_result.project = project_struct + self.m_libuplink.uplink_free_project_result(_project_result) + diff --git a/uplink_python/upload.py b/uplink_python/upload.py index 450cb80..e0a336c 100644 --- a/uplink_python/upload.py +++ b/uplink_python/upload.py @@ -128,7 +128,6 @@ def commit(self): # upload commit by calling the exported golang function error = self.uplink.m_libuplink.uplink_upload_commit(self.upload) - self.uplink.free_upload_struct(self.upload) # # if error occurred if bool(error): @@ -208,3 +207,6 @@ def info(self): info = self.uplink.object_from_result(_unwrapped_object) self.uplink.m_libuplink.uplink_free_object(_unwrapped_object) return info + + def __del__(self): + self.uplink.free_upload_struct(self.upload) From b8f98a2889679b072896a9b2bf5e6a1175e370ad Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 10 Nov 2022 17:42:45 +0100 Subject: [PATCH 27/40] add __del__ to free c memory --- uplink_python/uplink.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index ee9a6a6..2a542bd 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,7 +7,7 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, _ProjectResult, _ProjectStruct, \ +from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, _ProjectResult, \ _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata From c30c6502016cc5439300209022c81ee66daf87f1 Mon Sep 17 00:00:00 2001 From: Fadila Date: Fri, 11 Nov 2022 13:40:39 +0100 Subject: [PATCH 28/40] typo --- uplink_python/uplink.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 2a542bd..7679c5d 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -254,7 +254,7 @@ def unwrap_access_result(self, access_result): Returns ------- - ctypes.POINTER(_AccessStruct)) + ctypes.POINTER(_AccessStruct) """ return self.unwrap_libuplink_result( access_result, self.m_libuplink.uplink_free_access_result, 'access') @@ -269,7 +269,7 @@ def unwrap_bucket_result(self, bucket_result): Returns ------- - ctypes.POINTER(_BucketStruct)) + ctypes.POINTER(_BucketStruct) """ return self.unwrap_libuplink_result( bucket_result, self.m_libuplink.uplink_free_bucket_result, 'bucket') @@ -284,7 +284,7 @@ def unwrap_encryption_key_result(self, encryption_key_result): Returns ------- - ctypes.POINTER(_EncryptionKeyStruct)) + ctypes.POINTER(_EncryptionKeyStruct) """ return self.unwrap_libuplink_result(encryption_key_result, self.m_libuplink.uplink_free_encryption_key_result, 'encryption_key') @@ -299,7 +299,7 @@ def unwrap_object_result(self, object_result): Returns ------- - ctypes.POINTER(_ObjectStruct)) + ctypes.POINTER(_ObjectStruct) """ return self.unwrap_libuplink_result( object_result, self.m_libuplink.uplink_free_object_result, 'object') @@ -314,7 +314,7 @@ def unwrap_project_result(self, project_result): Returns ------- - ctypes.POINTER(_ProjectStruct)) + ctypes.POINTER(_ProjectStruct) """ return self.unwrap_libuplink_result( project_result, self.m_libuplink.uplink_free_project_result, 'project') @@ -359,7 +359,7 @@ def unwrap_upload_object_result(self, upload_object_result): Returns ------- - ctypes.POINTER(_UploadStruct)) + ctypes.POINTER(_UploadStruct) """ return self.unwrap_libuplink_result( upload_object_result, self.m_libuplink.uplink_free_upload_result, 'upload') From d4cbf8a7eb476ac78a5383aa747a3f57f745b31d Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 19:23:45 +0100 Subject: [PATCH 29/40] storj-sim for test --- cloudbuild.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f1320ec..7662a83 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -8,12 +8,20 @@ steps: - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] + - name: 'gcr.io/${PROJECT_ID}/python3' + entrypoint: 'bash' + script: | + git clone https://github.com/storj/storj.git storj + cd storj + make install-sim + storj-sim network setup + storj-sim network run - name: 'gcr.io/${PROJECT_ID}/python3' entrypoint: 'bash' args: ["-c", "git clone -b v1.2.2 https://github.com/storj/uplink-c"] - name: 'gcr.io/${PROJECT_ID}/python3' entrypoint: 'bash' - args: ["-c", "cd uplink-c && go build -o libuplinkc.so -buildmode=c-shared && cp *.so ../uplink_python/"] + args: ["-c", "cd uplink-c && go build -o libuplinkc.so -buildmode=c-shared && cp *.so ../uplink_python/"] - name: 'gcr.io/${PROJECT_ID}/python3' entrypoint: 'python3' args: ['-m', 'unittest', 'test/test_cases.py', '-v'] From 4021f41e1922c20527df9bf9eb6bc7999f968759 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 19:28:27 +0100 Subject: [PATCH 30/40] storj-sim for test --- cloudbuild.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 7662a83..dededfa 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -9,13 +9,12 @@ steps: entrypoint: 'bash' args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] - name: 'gcr.io/${PROJECT_ID}/python3' - entrypoint: 'bash' script: | - git clone https://github.com/storj/storj.git storj - cd storj - make install-sim - storj-sim network setup - storj-sim network run + git clone https://github.com/storj/storj.git storj + cd storj + make install-sim + storj-sim network setup + storj-sim network run & - name: 'gcr.io/${PROJECT_ID}/python3' entrypoint: 'bash' args: ["-c", "git clone -b v1.2.2 https://github.com/storj/uplink-c"] From 30fc6aebae698b144b8dfe7858c2193231eac051 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:01:51 +0100 Subject: [PATCH 31/40] storj-sim for test --- uplink_python/uplink.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uplink_python/uplink.py b/uplink_python/uplink.py index 7679c5d..e5b8562 100644 --- a/uplink_python/uplink.py +++ b/uplink_python/uplink.py @@ -7,8 +7,8 @@ from uplink_python.access import Access from uplink_python.errors import _storj_exception, LibUplinkSoError -from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, _ProjectResult, \ - _UploadResult, _Error +from uplink_python.module_def import _AccessResult, _ConfigStruct, _DownloadResult, \ + _ProjectResult, _UploadResult, _Error from uplink_python.module_classes import Config, Bucket, Object, SystemMetadata, \ CustomMetadataEntry, CustomMetadata @@ -444,4 +444,3 @@ def free_project_struct(self, project_struct): _project_result = _ProjectResult() _project_result.project = project_struct self.m_libuplink.uplink_free_project_result(_project_result) - From ed9b28426b71191530ad69774b35291cdbf359d0 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:09:08 +0100 Subject: [PATCH 32/40] storj-sim for test --- Dockerfile | 2 ++ cloudbuild.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac85f53..5988855 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,3 +9,5 @@ RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin RUN pip --no-cache-dir install pylint + +RUN apt install build-essential diff --git a/cloudbuild.yaml b/cloudbuild.yaml index dededfa..0a517d9 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -10,8 +10,8 @@ steps: args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] - name: 'gcr.io/${PROJECT_ID}/python3' script: | - git clone https://github.com/storj/storj.git storj - cd storj + git clone https://github.com/storj/storj.git /storj + cd /storj make install-sim storj-sim network setup storj-sim network run & From a9e605bae7bf423b0dd8d4b5f235672d1ba8bcb4 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:13:59 +0100 Subject: [PATCH 33/40] storj-sim for test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5988855..9465be8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,4 +10,4 @@ ENV PATH=$PATH:/usr/local/go/bin RUN pip --no-cache-dir install pylint -RUN apt install build-essential +RUN apt -y update && apt install -y build-essential From 69035b780cf30bcabeb392d1899d4f5a34a3a136 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:20:11 +0100 Subject: [PATCH 34/40] storj-sim for test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9465be8..a0fabe3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.8 -RUN curl -O https://dl.google.com/go/go1.14.6.linux-amd64.tar.gz +RUN curl -O https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz RUN sha256sum go1.14.6.linux-amd64.tar.gz RUN tar xvf go1.14.6.linux-amd64.tar.gz RUN chown -R root:root ./go From 9aba7aaae18bc065f9e3e926706ba24ed4624ab9 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:27:24 +0100 Subject: [PATCH 35/40] storj-sim for test --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a0fabe3..93942f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ FROM python:3.8 -RUN curl -O https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz -RUN sha256sum go1.14.6.linux-amd64.tar.gz -RUN tar xvf go1.14.6.linux-amd64.tar.gz +RUN export GOVERSION="1.16.6" +RUN curl -O "https://dl.google.com/go/go$GOVERSION.linux-amd64.tar.gz" +RUN sha256sum "go$GOVERSION.linux-amd64.tar.gz" +RUN tar xvf "go$GOVERSION.linux-amd64.tar.gz" RUN chown -R root:root ./go RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin From 2a9579e81094bb04fee6aba7515901438cdca7c6 Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:30:55 +0100 Subject: [PATCH 36/40] storj-sim for test --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93942f9..6e09933 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM python:3.8 -RUN export GOVERSION="1.16.6" -RUN curl -O "https://dl.google.com/go/go$GOVERSION.linux-amd64.tar.gz" -RUN sha256sum "go$GOVERSION.linux-amd64.tar.gz" -RUN tar xvf "go$GOVERSION.linux-amd64.tar.gz" +RUN export GO_ARCHIVE="go1.16.6.linux-amd64.tar.gz" +RUN curl -O "https://dl.google.com/go/$GO_ARCHIVE" +RUN sha256sum $GO_ARCHIVE +RUN tar xvf $GO_ARCHIVE RUN chown -R root:root ./go RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin From 09e200d457e6711d295f424c0b7379483cf91ddf Mon Sep 17 00:00:00 2001 From: Fadila Date: Sun, 20 Nov 2022 21:46:37 +0100 Subject: [PATCH 37/40] storj-sim for test --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e09933..34f68f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,9 @@ FROM python:3.8 -RUN export GO_ARCHIVE="go1.16.6.linux-amd64.tar.gz" -RUN curl -O "https://dl.google.com/go/$GO_ARCHIVE" -RUN sha256sum $GO_ARCHIVE -RUN tar xvf $GO_ARCHIVE +RUN curl -O "https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz" +RUN sha256sum go1.16.6.linux-amd64.tar.gz +RUN tar xvf go1.16.6.linux-amd64.tar.gz RUN chown -R root:root ./go RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin From 888345eca0f2ae1b6ed4dd67ff1fd50805bc9b2e Mon Sep 17 00:00:00 2001 From: Fadila Date: Tue, 22 Nov 2022 10:06:24 +0100 Subject: [PATCH 38/40] ci back to original --- cloudbuild.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 0a517d9..388a0bb 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -8,13 +8,6 @@ steps: - name: gcr.io/cloud-builders/gcloud entrypoint: 'bash' args: ["-c","gcloud secrets versions access latest --secret=StorjAPIKey >>secret.txt" ] - - name: 'gcr.io/${PROJECT_ID}/python3' - script: | - git clone https://github.com/storj/storj.git /storj - cd /storj - make install-sim - storj-sim network setup - storj-sim network run & - name: 'gcr.io/${PROJECT_ID}/python3' entrypoint: 'bash' args: ["-c", "git clone -b v1.2.2 https://github.com/storj/uplink-c"] From a58905963f46f15fa2302670666fb4686a22364b Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 24 Nov 2022 02:36:41 +0100 Subject: [PATCH 39/40] code cleanup --- uplink_python/download.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/uplink_python/download.py b/uplink_python/download.py index abb671e..752a27a 100644 --- a/uplink_python/download.py +++ b/uplink_python/download.py @@ -52,8 +52,6 @@ def __init__(self, download, uplink, project, bucket_name, storj_path): self.storj_path = storj_path self.uplink = uplink - - def read(self, size_to_read: int): """ function downloads up to len size_to_read bytes from the object's data stream. From 90970c0307bd54cab7070f64b7a32168a49d4a33 Mon Sep 17 00:00:00 2001 From: Fadila Date: Thu, 24 Nov 2022 02:36:53 +0100 Subject: [PATCH 40/40] code cleanup --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 34f68f3..59e13f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,5 +9,3 @@ RUN mv go /usr/local ENV PATH=$PATH:/usr/local/go/bin RUN pip --no-cache-dir install pylint - -RUN apt -y update && apt install -y build-essential