From 225a2b70b745fbcebc6ebc6554efb2510d8a9b06 Mon Sep 17 00:00:00 2001 From: Jonas Lindberg Frellesen Date: Mon, 20 May 2019 11:38:45 +0200 Subject: [PATCH 1/6] Added GetFileInfo identification, operation and examples. Intended to combine and replace GetFileIDs and GetChecksums. --- .../examples/data/GetFileInfosResults.xml | 45 ++++++++++++++ .../messages/GetFileInfosFinalResponse.xml | 60 +++++++++++++++++++ .../messages/GetFileInfosProgressResponse.xml | 44 ++++++++++++++ .../examples/messages/GetFileInfosRequest.xml | 51 ++++++++++++++++ .../IdentifyPillarsForGetFileInfosRequest.xml | 39 ++++++++++++ ...IdentifyPillarsForGetFileInfosResponse.xml | 47 +++++++++++++++ 6 files changed, 286 insertions(+) create mode 100644 message-xml-xsd/src/main/resources/examples/data/GetFileInfosResults.xml create mode 100644 message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml create mode 100644 message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml create mode 100644 message-xml-xsd/src/main/resources/examples/messages/GetFileInfosRequest.xml create mode 100644 message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml create mode 100644 message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml diff --git a/message-xml-xsd/src/main/resources/examples/data/GetFileInfosResults.xml b/message-xml-xsd/src/main/resources/examples/data/GetFileInfosResults.xml new file mode 100644 index 0000000..6166645 --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/data/GetFileInfosResults.xml @@ -0,0 +1,45 @@ + + + + + MySla + MyPillar + + MyFile1 + 1 + 2019-05-20T11:11:11.111+02:00 + ABCDABCD + 2019-05-20T11:11:11.111+02:00 + + + MyFile2 + 2 + 1970-01-01T01:00:00.000+00:00 + 1234567890ABCDEF + 1970-01-01T01:00:00.000+00:00 + + diff --git a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml new file mode 100644 index 0000000..59cf956 --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml @@ -0,0 +1,60 @@ + + + + + 12345 + MySla + topic://clientTopic + queue://pillarQueue + MyPillar + + OPERATION_COMPLETED + successfull completion + + MyPillar + + MyDataID1 + + + HMAC_SHA384 + 5A17 + + + + + + MyDataID1 + 1 + 2019-05-20T11:11:11.111+02:00 + ABCDABCD + 2019-05-20T11:11:11.111+02:00 + + + + + diff --git a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml new file mode 100644 index 0000000..148f767 --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml @@ -0,0 +1,44 @@ + + + + + 12345 + MySla + topic://clientTopic + queue://pillarQueue + MyPillar + + OPERATION_ACCEPTED_PROGRESS + Message request has been received and is expected to be met successfully + + MyPillar + + MyDataID1 + + http://myServer/myPlace/Myfile + diff --git a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosRequest.xml b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosRequest.xml new file mode 100644 index 0000000..c50aaf9 --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosRequest.xml @@ -0,0 +1,51 @@ + + + + + 12345 + MySla + queue://pillarQueue + topic://clientTopic + ClientPillar + User: Name, Institution: Place + MyPillar + + MyDataID1 + + + HMAC_SHA384 + 5A17 + + true + 1970-01-01T01:00:00.000+00:00 + 2019-05-20T11:37:00.000+02:00 + 1970-01-01T01:00:00.000+00:00 + 2019-05-20T11:37:00.000+02:00 + 1 + http://myServer/myPlace/Myfile + diff --git a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml new file mode 100644 index 0000000..57f1031 --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml @@ -0,0 +1,39 @@ + + + + + 12345 + MySla + topic://collectionTopic + topic://clientTopic + ClientPillar + User: Name, Institution: Place + + MyDataID1 + + diff --git a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml new file mode 100644 index 0000000..3ae76de --- /dev/null +++ b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml @@ -0,0 +1,47 @@ + + + + + 12345 + MySla + topic://collectionTopic + topic://clientTopic + MyPillar + + IDENTIFICATION_POSITIVE + Ready to deliver fileIDS. + + MyPillar + + MyDataID1 + + + MILLISECONDS + 1000 + + From c99c271bbda5e15be067f43cbb46ebcf8f8aef1c Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 6 Sep 2022 12:11:17 +0200 Subject: [PATCH 2/6] Added removed getfileinfos stuff --- .../main/resources/xsd/BitRepositoryData.xsd | 32 +++ .../resources/xsd/BitRepositoryElements.xsd | 92 +++++++- .../resources/xsd/BitRepositoryMessages.xsd | 211 ++++++++++++++++++ 3 files changed, 333 insertions(+), 2 deletions(-) diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryData.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryData.xsd index a2ad0b1..e2952d4 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryData.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryData.xsd @@ -107,6 +107,38 @@ + + + + + + + Data format for delivery of checksums as result of GetFileInfos + + + + + + A unique identifier for bit repository collection which data belongs to. + + + + + + Identifier of pillar, that created the result + + + + + + List of data for file infos + + + + + + + diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd index ae36ffe..1429a75 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd @@ -422,6 +422,68 @@ + + + Includes relevant information on data file Infos in a result + + + + + + + List of data on data identifiers in result + + + + + + + + + + + + + + Includes relevant information on a data ID in a result + + + + + + Identifier of data + + + + + + The filesize in bytes. + + + + + + The last modify timestamp of the file, or the latest time which the pillar had the whole file (especially + for off-line media). + + + + + + The value of the checksum + + + + + + The time when the checksum was last calculated (especially for ordinary checksums on off-line medias, + there can be long intervals between calculations, - in such cases this information is crucial for alarming) + + + + + + @@ -667,6 +729,27 @@ + + + results for GetFileInfos operaiton which is either given via specified address or directly + + + + + + Address (e.g. in form of an URL) for where result data has been delivered (if completed) + + + + + + List of data for file ids in result, if given in this message instead of via data transfer + + + + + + @@ -1281,8 +1364,13 @@ - - + + + + + + + diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd index bae69ee..af50e85 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd @@ -1668,6 +1668,217 @@ + + + + + + REQUEST in IdentifyPillarsForGetFileInfos primitive + + + + + + + FileIDs uniquely identifies the files in given bit repository collection which the file ids are requested for + Given in order to be able to give possible estimate of time to deliver + + + + + + + + + + + + RESPONSE in IdentifyPillarsForGetFileInfos primitive + + + + + + + Unique identifier for pillar replying + + + + + + FileIDs uniquely identifies the files in given bit repository collection which the file ids are requested for + only to repeat parameter from request. + + + + + + Estimated time before delivery of data (fileInfos) requested. + That means the time from receiving a GetFileInfosRequest message to the time + when data has been collected and returned in a GetFileInfosFinalResponse message. + + + + + + + + + + + + REQUEST in GetFileInfos primitive + + + + + + + Identifier of receiving pillar, which can be used for possible checks + (IdentifyPillarsForGetFileInfosResponse.ReplyTo should ensure that right pillar is reached) + + + + + + FileIDs uniquely identifies the files in given bit repository collection which the file ids are requested for + + + + + Checksums type and possible salt requested. + If this element is missing, then no checksum is requested + + + + + Whether or not to require the size of the file as part of the response. + Defaults to true. + + + + + + Only fileInfos for files added/updated later or at the same time as the minFileTimestamp should be returned. + + + + + Only fileInfos for files added/updated before or at the same time as the maxFileTimestamp should be returned. + + + + + Only fileInfos for checksums younger or at the same time as the minChecksumTimestamp should be returned. + + + + + Only fileInfos for checksums with checksums older or at the same time as the maxCheckusumTimestamp should be returned. + + + + + Request the contributor to only return 'up-to' MaxNumberOfResults of fileInfo results. This can be + used to break a fileInfo retrieval into small result sets (paging). + + + + + + Address (e.g. in form of an URL) for where data (fileInfos) must be delivered via file exchange + In cases where the address is not given, the result will be given in the GetFileIdsFinalResponse message + + + + + + + + + + + + PROGRESS RESPONSE in GetFileInfos primitive + + + + + + + Identifier of responding pillar, which can be used for possible checks + (otherwise it is known via CorrelationID and ReplyTo) + + + + + + FileIDs uniquely identifies the files in given bit repository collection which the file ids are requested for + (otherwise it is known via CorrelationID) + + + + + + Address (e.g. in form of an URL) for where data (file IDs) will be delivered + + + + + + + + + + + + + FINAL RESPONSE in GetFileInfos primitive + + + + + + + Identifier of responding pillar, which can be used for possible checks + (otherwise it is known via CorrelationID and ReplyTo) + + + + + + FileIDs uniquely identifies the files in given collection which the file ids are requested for. If not present + it means all the requested files are present on the pillar. + + + + + + Checksums type and possible salt requested + (Also in result, and traceable via CorrelationID) + + + + + + Result with file infos. + + + + + + Indicates whether the list of result were limited for some reason, either a MaxLimit in the request + or a limit defined in the contributor. If all results have been returned + this should be false or undefined. + + + + + + + + + From e206546a9bb136b7025fb6a95adad458270e36b3 Mon Sep 17 00:00:00 2001 From: Kim Christensen Date: Mon, 20 May 2019 16:12:06 +0200 Subject: [PATCH 3/6] Add ChecksumRequestForExistingFile to IdentifyPillarsForGetFileInfoRequest --- .../src/main/resources/xsd/BitRepositoryMessages.xsd | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd index af50e85..27d16d9 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd @@ -1684,6 +1684,14 @@ Given in order to be able to give possible estimate of time to deliver + + + + Optional checksums type and possible salt parameter requested + (only pillars that can deliver this kind of checksum should reply). + If not present pillars should use the default checksumtype specified in RepositorySettings. + + From 06ca3a3066595d84ed5a7234c75ffd2762395199 Mon Sep 17 00:00:00 2001 From: Jonas Lindberg Frellesen Date: Mon, 20 May 2019 18:08:59 +0200 Subject: [PATCH 4/6] Added ChecksumRequestForExistingFile and PillarChecksumSpec to the IdentifyPillarsForGetFileInfosResponse. --- .../IdentifyPillarsForGetFileInfosRequest.xml | 4 ++++ .../IdentifyPillarsForGetFileInfosResponse.xml | 8 ++++++++ .../main/resources/xsd/BitRepositoryMessages.xsd | 13 +++++++++++++ 3 files changed, 25 insertions(+) diff --git a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml index 57f1031..0f5e83d 100644 --- a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml +++ b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosRequest.xml @@ -36,4 +36,8 @@ xmlns="http://bitrepository.org/BitRepositoryElements.xsd"> MyDataID1 + + HMAC_SHA384 + 5A17 + diff --git a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml index 3ae76de..8b8631b 100644 --- a/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml +++ b/message-xml-xsd/src/main/resources/examples/messages/IdentifyPillarsForGetFileInfosResponse.xml @@ -37,6 +37,14 @@ Ready to deliver fileIDS. MyPillar + + HMAC_SHA384 + 5A17 + + + HMAC_SHA384 + 5A17 + MyDataID1 diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd index 27d16d9..03d684d 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd @@ -1712,6 +1712,19 @@ + + + Checksums type and possible salt requested + + + + + + Checksums type in case it is a checksum pillar replying, + only used for extra checks + + + FileIDs uniquely identifies the files in given bit repository collection which the file ids are requested for From ebb24e22436a32cd97f1141385991b6cf48fb209 Mon Sep 17 00:00:00 2001 From: Jonas Lindberg Frellesen Date: Mon, 20 May 2019 18:37:44 +0200 Subject: [PATCH 5/6] Added ChecksumRequestForExistingFile to the GetFileInfosProgressResponse --- .../examples/messages/GetFileInfosProgressResponse.xml | 4 ++++ .../src/main/resources/xsd/BitRepositoryMessages.xsd | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml index 148f767..b203e27 100644 --- a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml +++ b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosProgressResponse.xml @@ -40,5 +40,9 @@ xmlns="http://bitrepository.org/BitRepositoryElements.xsd"> MyDataID1 + + HMAC_SHA384 + 5A17 + http://myServer/myPlace/Myfile diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd index 03d684d..c682601 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryMessages.xsd @@ -1839,6 +1839,12 @@ + + + Checksums type and possible salt requested + + + Address (e.g. in form of an URL) for where data (file IDs) will be delivered From 93fb9479454d8312a59306d4feb4b6ab20e1ece6 Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 6 Sep 2022 13:07:34 +0200 Subject: [PATCH 6/6] Flatten data part of GetFileInfosFinalResponse --- .../messages/GetFileInfosFinalResponse.xml | 18 ++++------- .../resources/xsd/BitRepositoryElements.xsd | 32 +++---------------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml index 59cf956..55449d1 100644 --- a/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml +++ b/message-xml-xsd/src/main/resources/examples/messages/GetFileInfosFinalResponse.xml @@ -45,16 +45,12 @@ 5A17 - - - - MyDataID1 - 1 - 2019-05-20T11:11:11.111+02:00 - ABCDABCD - 2019-05-20T11:11:11.111+02:00 - - - + + MyDataID1 + 1 + 2019-05-20T11:11:11.111+02:00 + ABCDABCD + 2019-05-20T11:11:11.111+02:00 + diff --git a/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd b/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd index 1429a75..213d2c2 100644 --- a/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd +++ b/message-xml-xsd/src/main/resources/xsd/BitRepositoryElements.xsd @@ -422,28 +422,7 @@ - - - Includes relevant information on data file Infos in a result - - - - - - - List of data on data identifiers in result - - - - - - - - - - - - + Includes relevant information on a data ID in a result @@ -594,9 +573,8 @@ - The optional ReponseText can be used to include additional information in addition to the information - found in the - other elements. This will typically be relevant to describe details in problem situations. + The optional ResponseText can be used to include additional information in addition to the information + found in the other elements. This will typically be relevant to describe details in problem situations. @@ -624,7 +602,7 @@ - ReponseText is a textual description for logging, in a human readable form. + ResponseText is a textual description for logging, in a human readable form. @@ -741,7 +719,7 @@ - + List of data for file ids in result, if given in this message instead of via data transfer