From 3099d3d7bb9944a2baeabd5ae2bea03864713412 Mon Sep 17 00:00:00 2001 From: rsonghuster Date: Fri, 20 Oct 2023 14:11:42 +0800 Subject: [PATCH 1/7] feat: change to fc3.0 Signed-off-by: rsonghuster --- .github/workflows/registry-publish.yml | 33 -------- publish.yaml | 34 ++++---- src/readme.md | 66 ++++++++-------- src/s.yaml | 104 ++++++++++--------------- 4 files changed, 91 insertions(+), 146 deletions(-) delete mode 100644 .github/workflows/registry-publish.yml diff --git a/.github/workflows/registry-publish.yml b/.github/workflows/registry-publish.yml deleted file mode 100644 index 71099a4..0000000 --- a/.github/workflows/registry-publish.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: publish package to serverless-hub - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - uses: actions/setup-node@v1 - with: - node-version: 12 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - pip install requests - - name: Add publish file - run: | - wget https://serverless-registry.oss-cn-hangzhou.aliyuncs.com/publish-file/python3/hub-publish.py - ls - - name: Publish package - env: - publish_token: ${{ secrets.alibaba_registry_publish_token }} - run: | - ls - python hub-publish.py \ No newline at end of file diff --git a/publish.yaml b/publish.yaml index ddd4f2f..39603d3 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,10 +1,13 @@ -Type: Application -Name: start-unzip-oss -Version: 0.0.19 +Edition: 3.0.0 +Type: Project +Name: start-unzip-oss-v3 +Version: 0.0.9 Provider: - 阿里云 Description: 使用函数计算自动解压上传到OSS指定前缀目录的zip文件 HomePage: https://github.com/devsapp/start-unzip-oss +Organization: 阿里云函数计算（FC） +Effective: Public Tags: - OSS上zip文件自动解压 - 函数计算 OSS 触发器 @@ -13,7 +16,7 @@ Service: 函数计算: Authorities: - AliyunFCFullAccess - OSS: + 对象存储: Authorities: - AliyunOSSFullAccess Parameters: @@ -21,7 +24,6 @@ Parameters: additionalProperties: false # 不允许增加其他属性 required: # 必填项 - region - - serviceName - functionName - roleArn - bucketName @@ -50,38 +52,29 @@ Parameters: - ap-southeast-3 - ap-southeast-5 - ap-northeast-1 + - ap-northeast-2 - eu-central-1 - eu-west-1 - us-west-1 - us-east-1 - ap-south-1 - serviceName: - title: 服务名 - type: string - default: unzip-oss - pattern: "^[a-zA-Z_][a-zA-Z0-9-_]{0,127}$" - description: 应用所属的函数计算服务,只能包含字母、数字、下划线和中划线。不能以数字、中划线开头。长度在 1-128 之间 functionName: title: 函数名 type: string - default: oss-invoke-fc - pattern: "^[a-zA-Z_][a-zA-Z0-9-_]{0,63}$" - description: 应用的函数名称, 只能包含字母、数字、下划线和中划线。不能以数字、中划线开头。长度在 1-64 之间 + default: oss-invoke-fc-${default-suffix} + pattern: "^[a-zA-Z_][a-zA-Z0-9-_]{0,127}$" + description: 应用的函数名称, 只能包含字母、数字、下划线和中划线。不能以数字、中划线开头。长度在 1-128 之间 roleArn: title: RAM角色ARN type: string default: "" - pattern: "^acs:ram::[0-9]*:role/.*$" - description: "函数计算访问其他云服务时使用的服务角色，需要填写具体的角色ARN，格式为acs:ram::$account-id>:role/$role-name。例如：acs:ram::14310000000:role/aliyunfcdefaultrole。 - \n如果您没有特殊要求，可以使用函数计算提供的默认的服务角色，即AliyunFCDefaultRole, 并增加 AliyunOSSFullAccess 权限。如果您首次使用函数计算，可以访问 https://fcnext.console.aliyun.com 进行授权。 - \n详细文档参考 https://help.aliyun.com/document_detail/181589.html#section-o93-dbr-z6o" + description: "函数计算访问对象存储OSS服务时使用的函数角色" required: true x-role: - name: fcunzipossrole + name: fcossrole service: fc authorities: - AliyunOSSFullAccess - - AliyunFCDefaultRolePolicy bucketName: title: OSS存储桶名 type: string @@ -112,7 +105,6 @@ Parameters: title: 触发器RAM角色ARN type: string default: AliyunOSSEventNotificationRole - pattern: "^acs:ram::[0-9]*:role/.*$" description: OSS使用此角色来发送事件通知来调用函数 required: true x-role: diff --git a/src/readme.md b/src/readme.md index 7ff5704..80c6ec0 100644 --- a/src/readme.md +++ b/src/readme.md @@ -1,16 +1,16 @@ -> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、服务名、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 +> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ${模版名称}` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 -# start-unzip-oss 帮助文档 +# start-unzip-oss-v3 帮助文档

- + - - + + - - + +

@@ -22,7 +22,7 @@ -- [:smiley_cat: 代码](https://github.com/devsapp/start-unzip-oss/tree/main/src) +- [:smiley_cat: 代码](https://github.com/devsapp/start-unzip-oss/tree/V3/src) @@ -34,7 +34,7 @@ ## 前期准备 -使用该项目，您需要有开通以下服务： +使用该项目，您需要有开通以下服务并拥有对应权限： @@ -75,16 +75,16 @@ -- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss) ， - [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss) 该应用。 +- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss-v3) ， + [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss-v3) 该应用。 - 通过 [Serverless Devs Cli](https://www.serverless-devs.com/serverless-devs/install) 进行部署： - [安装 Serverless Devs Cli 开发者工具](https://www.serverless-devs.com/serverless-devs/install) ，并进行[授权信息配置](https://docs.serverless-devs.com/fc/config) ； - - 初始化项目：`s init start-unzip-oss -d start-unzip-oss ` - - 进入项目，并进行项目部署：`cd start-unzip-oss && s deploy - y` + - 初始化项目：`s init start-unzip-oss-v3 -d start-unzip-oss-v3` + - 进入项目，并进行项目部署：`cd start-unzip-oss-v3 && s deploy -y` @@ -94,50 +94,51 @@ ![](http://image.editor.devsapp.cn/alibaba/4A5uks4sawFd26h9ksuc.png) -匹配解压规则的ZIP文件在上传到OSS后，会自动触发函数计算进行解压。文件解压完成后，会存储至OSS的指定目录中。 +匹配解压规则的 ZIP 文件在上传到 OSS 后，会自动触发函数计算进行解压。文件解压完成后，会存储至 OSS 的指定目录中。 -## 注意事项 +**注意事项** -- 建议使用UTF-8或GB 2312编码命名您的文件或文件夹，否则可能会出现解压后的文件或文件夹名称出现乱码、解压过程中断等问题。 +- 建议使用 UTF-8 或 GB 2312 编码命名您的文件或文件夹，否则可能会出现解压后的文件或文件夹名称出现乱码、解压过程中断等问题。 - 归档或冷归档类型的文件需先解冻再解压。 -- 解压单个压缩包的最大时间是2小时，超过2小时未完成的任务会解压失败。 +- 解压单个压缩包的最大时间是 2 小时，超过 2 小时未完成的任务会解压失败。 -- 建议ZIP包里面的单文件大小最好不超过1 GB，否则可能解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) +- 建议 ZIP 包里面的单文件大小最好不超过 1 GB，否则可能解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) -- 默认设置的函数执行时长为2h， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24h +- 默认设置的函数执行时长为 2h， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24h - - -## 使用文档 - - +**参数说明** +![](http://image.editor.devsapp.cn/alibaba/kD1lbEw48Er4s27212ri.png) -## 参数说明 +** 配置示例** -![](http://image.editor.devsapp.cn/alibaba/kD1lbEw48Er4s27212ri.png) +![](http://image.editor.devsapp.cn/alibaba/lASAfezjvifa9Cwawht6.png) -## 配置示例 -![](http://image.editor.devsapp.cn/alibaba/lASAfezjvifa9Cwawht6.png) +**二次开发示例** +上面的示例, 解压保存回去的都还是本身触发函数的 OSS， 如果是保存到其他 OSS， 直接将 `bucket.put_object(newKey + name, file_obj)` 这样上传回 oss 的代码修改下即可， 比如： -## 二次开发示例 -上面的示例, 解压保存回去的都还是本身触发函数的 OSS， 如果是保存到其他 OSS， 直接将 `bucket.put_object(newKey + name, file_obj)` 这样上传回 oss 的代码修改下即可， 比如： ```python auth = oss2.Auth('', '') dst_bucket_name = "xxx-bucket" -dst_bucket = oss2.Bucket(auth, endpoint, dst_bucket_name) +dst_bucket = oss2.Bucket(auth, endpoint, dst_bucket_name) dst_bucket.put_object(newKey + name, file_obj) ``` + + + +## 使用文档 + + @@ -155,3 +156,6 @@ dst_bucket.put_object(newKey + name, file_obj) |
微信公众号：`serverless`
|
微信小助手：`xiaojiangwh`
|
钉钉交流群：`33947367`
|

+ + + diff --git a/src/s.yaml b/src/s.yaml index cc94d42..bb5f93f 100644 --- a/src/s.yaml +++ b/src/s.yaml @@ -1,84 +1,66 @@ # ------------------------------------ -# ------------------------------------ # If you need English case, you can refer to [s_en.yaml] file # ------------------------------------ # 欢迎您使用阿里云函数计算 FC 组件进行项目开发 -# 组件仓库地址：https://github.com/devsapp/fc -# 组件帮助文档：https://www.serverless-devs.com/fc/readme -# Yaml参考文档：https://www.serverless-devs.com/fc/yaml/readme -# 关于： -# - Serverless Devs和FC组件的关系、如何声明/部署多个函数、超过50M的代码包如何部署 -# - 关于.fcignore使用方法、工具中.s目录是做什么、函数进行build操作之后如何处理build的产物 -# 等问题，可以参考文档：https://www.serverless-devs.com/fc/tips +# 组件仓库地址：https://github.com/devsapp/fc3 +# 组件帮助文档：https://github.com/devsapp/fc3/blob/master/docs/zh/readme.md +# Yaml参考文档：https://github.com/devsapp/fc3/blob/master/docs/zh/yaml/readme.md +# 关于Serverless Devs和FC组件的关系、如何声明/部署多个函数等问题，可以参考文档：https://github.com/devsapp/fc3/blob/master/docs/zh/tips.md # 关于如何做CICD等问题，可以参考：https://www.serverless-devs.com/serverless-devs/cicd # 关于如何进行环境划分等问题，可以参考：https://www.serverless-devs.com/serverless-devs/extend # 更多函数计算案例，可参考：https://github.com/devsapp/awesome/ # 有问题快来钉钉群问一下吧：33947367 -edition: 1.0.0 +# ------------------------------------ +edition: 3.0.0 name: UnZip OSS -# access 是当前应用所需要的密钥信息配置： -# 密钥配置可以参考：https://www.serverless-devs.com/serverless-devs/command/config -# 密钥使用顺序可以参考：https://www.serverless-devs.com/serverless-devs/tool#密钥使用顺序与规范 -access: "{{ access }}" - +access: '{{ access }}' vars: - region: "{{ region }}" # 关于变量的使用方法，可以参考：https://www.serverless-devs.com/serverless-devs/yaml#变量赋值 - service: - name: "{{ serviceName }}" - description: unzip oss sample - internetAccess: true - role: "{{ roleArn }}" - -services: + region: '{{ region }}' +resources: fc-zip-oss-service: - component: fc + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: "{{ functionName }}" - handler: index.handler - timeout: 7200 - runtime: python3 - codeUri: ./code - memorySize: 3072 - environmentVariables: - PROCESSED_DIR: "{{ processedDir }}" - RETAIN_FILE_NAME: "{{ retainFileName }}" - # 如果是很大的 ZIP 文件， 想使用异步调用，可以开启如下配置, 将每次函数调用看做一个任务执行 - asyncConfiguration: - destination: - # onSuccess: acs:fc:::services/${vars.service.name}/functions/dest-succ - onFailure: acs:fc:::services/${vars.service.name}/functions/dest-fail - maxAsyncEventAgeInSeconds: 7200 - maxAsyncRetryAttempts: 3 - statefulInvocation: true triggers: - - name: unzip-oss-trigger - type: oss - role: "{{ triggerRoleArn }}" - config: + - triggerName: unzip-oss-trigger + triggerType: oss + triggerConfig: events: - oss:ObjectCreated:PutObject - oss:ObjectCreated:PostObject - oss:ObjectCreated:CompleteMultipartUpload - oss:ObjectCreated:PutSymlink filter: - Key: - Prefix: "{{ prefix }}" - Suffix: .zip - bucketName: "{{ bucketName }}" - + key: + prefix: '{{ prefix }}' + suffix: .zip + invocationRole: '{{ triggerRoleArn }}' + sourceArn: acs:oss:${this.props.region}:${config("AccountID")}:{{ bucketName }} + handler: index.handler + timeout: 7200 + runtime: python3 + memorySize: 3072 + environmentVariables: + PROCESSED_DIR: '{{ processedDir }}' + RETAIN_FILE_NAME: '{{ retainFileName }}' + internetAccess: true + role: '{{ roleArn }}' + functionName: '{{ functionName }}' + code: ./code + asyncInvokeConfig: + destinationConfig: + onFailure: + destination: acs:fc:${this.props.region}::functions/dest-fail + maxAsyncEventAgeInSeconds: 7200 + maxAsyncRetryAttempts: 3 dest-fail: - component: fc + component: fc3 props: region: ${vars.region} - service: ${vars.service} - function: - name: dest-fail - description: "async task destination fail function by serverless devs" - runtime: python3 - codeUri: ./fail - handler: index.handler - memorySize: 512 - timeout: 60 + description: async task destination fail function by serverless devs + runtime: python3 + handler: index.handler + memorySize: 512 + timeout: 60 + functionName: dest-fail + code: ./fail From 7e966104a6ca50518e58fd679f58ce9df9fe83a4 Mon Sep 17 00:00:00 2001 From: mozhou52 Date: Fri, 29 Mar 2024 19:51:15 +0800 Subject: [PATCH 2/7] =?UTF-8?q?fix:unzip-oss=E7=9A=84readme,=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0github=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/registry-publish.yml | 28 +++++++++ publish.yaml | 4 +- src/readme.md | 78 +++++++++++++------------- 3 files changed, 70 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/registry-publish.yml diff --git a/.github/workflows/registry-publish.yml b/.github/workflows/registry-publish.yml new file mode 100644 index 0000000..c1c1c2a --- /dev/null +++ b/.github/workflows/registry-publish.yml @@ -0,0 +1,28 @@ +name: publish package to registry + +on: + release: + types: [created] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 16 + registry-url: https://registry.npmjs.org/ + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y git make jq curl wget + - name: install s + run: | + npm i @serverless-devs/s@v3 -g + - name: s registry login + run: | + s registry login --token ${{ secrets.alibaba_registry_v3_publish_token }} + - name: release prod + run: | + s registry publish \ No newline at end of file diff --git a/publish.yaml b/publish.yaml index 39603d3..074befe 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,10 +1,10 @@ Edition: 3.0.0 Type: Project Name: start-unzip-oss-v3 -Version: 0.0.9 +Version: 0.0.10 Provider: - 阿里云 -Description: 使用函数计算自动解压上传到OSS指定前缀目录的zip文件 +Description: 本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 HomePage: https://github.com/devsapp/start-unzip-oss Organization: 阿里云函数计算（FC） Effective: Public diff --git a/src/readme.md b/src/readme.md index 80c6ec0..cb459a3 100644 --- a/src/readme.md +++ b/src/readme.md @@ -16,7 +16,7 @@ -使用函数计算自动解压上传到OSS指定前缀目录的zip文件 +本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 @@ -40,25 +40,13 @@ -| 服务 | 备注 | -| --- | --- | -| 函数计算 FC | unzip解压函数部署在函数计算 | -| 对象存储 OSS | 待解压的zip文件和解压后的文件存放在对象存储 | +| 服务/业务 | 权限 | 相关文档 | +| --- | --- | --- | +| 函数计算 | AliyunFCFullAccess | [帮助文档](https://help.aliyun.com/product/2508973.html) [计费文档](https://help.aliyun.com/document_detail/2512928.html) | +| 对象存储 | AliyunOSSFullAccess | [帮助文档](https://help.aliyun.com/zh/oss) [计费文档](https://help.aliyun.com/zh/oss/product-overview/billing) | -推荐您拥有以下的产品权限 / 策略： - - - - -| 服务/业务 | 权限 | 备注 | -| --- | --- | --- | -| 函数计算 | AliyunFCFullAccess | 创建和更新 unzip 解压函数 | -| OSS | AliyunOSSFullAccess | 创建或更新 unzip 解压函数的 OSS 触发器 | - - - @@ -88,16 +76,13 @@ -## 应用详情 +## 案例介绍 -![](http://image.editor.devsapp.cn/alibaba/4A5uks4sawFd26h9ksuc.png) - -匹配解压规则的 ZIP 文件在上传到 OSS 后，会自动触发函数计算进行解压。文件解压完成后，会存储至 OSS 的指定目录中。 - +本案例是将zip文件进行 unzip 解压，快速创建并部署到阿里云函数计算 FC。 -**注意事项** +匹配解压规则的ZIP文件在上传到OSS后，会自动触发函数计算进行解压。文件解压完成后，会存储至OSS的指定目录中。 - 建议使用 UTF-8 或 GB 2312 编码命名您的文件或文件夹，否则可能会出现解压后的文件或文件夹名称出现乱码、解压过程中断等问题。 @@ -107,23 +92,44 @@ - 建议 ZIP 包里面的单文件大小最好不超过 1 GB，否则可能解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) -- 默认设置的函数执行时长为 2h， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24h +- 默认设置的函数执行时长为2h， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24h +![](http://image.editor.devsapp.cn/alibaba/4A5uks4sawFd26h9ksuc.png) -**参数说明** + -![](http://image.editor.devsapp.cn/alibaba/kD1lbEw48Er4s27212ri.png) +## 使用流程 + -** 配置示例** +### 查看部署的案例 + +部署时， 选择的OSS存储桶名，以及前缀名，如： + +![](https://img.alicdn.com/imgextra/i4/O1CN01KYL4lB1Obdl63LB39_!!6000000001724-0-tps-2336-1108.jpg) -![](http://image.editor.devsapp.cn/alibaba/lASAfezjvifa9Cwawht6.png) +部署完成后，在oss对象存储中找到相应的OSS存储桶名，向该存储桶上传zip文件，如： -**二次开发示例** +![](https://img.alicdn.com/imgextra/i2/O1CN01ew8GAi1pDomkNrcuC_!!6000000005327-0-tps-1874-850.jpg) -上面的示例, 解压保存回去的都还是本身触发函数的 OSS， 如果是保存到其他 OSS， 直接将 `bucket.put_object(newKey + name, file_obj)` 这样上传回 oss 的代码修改下即可， 比如： +在控制台就可以查看到调用日志，如： +![](https://img.alicdn.com/imgextra/i4/O1CN01Yv7Jsc1UJfur6G0WU_!!6000000002497-0-tps-2842-776.jpg) + +#### 参数说明 + +![](http://image.editor.devsapp.cn/alibaba/kD1lbEw48Er4s27212ri.png) + + +#### 配置示例 +![](http://image.editor.devsapp.cn/alibaba/lASAfezjvifa9Cwawht6.png) + + +### 二次开发 +您可以通过云端控制台的开发功能进行二次开发。如果您之前是在本地创建的项目案例，也可以在本地项目目录`start-unzip-oss-v3`文件夹下，对项目进行二次开发。开发完成后，可以通过`s deploy`进行快速部署。 + +上面的示例, 解压保存回去的都还是本身触发函数的 OSS， 如果是保存到其他 OSS， 直接将 `bucket.put_object(newKey + name, file_obj)` 这样上传回 oss 的代码修改下即可， 比如： ```python auth = oss2.Auth('', '') @@ -133,13 +139,12 @@ dst_bucket.put_object(newKey + name, file_obj) ``` + - +## 注意事项 -## 使用文档 - - - + + @@ -156,6 +161,3 @@ dst_bucket.put_object(newKey + name, file_obj) |
微信公众号：`serverless`
|
微信小助手：`xiaojiangwh`
|
钉钉交流群：`33947367`
|

- - - From eff88bf98b51b6476f51a199652defbc257f29f8 Mon Sep 17 00:00:00 2001 From: rsonghuster Date: Wed, 16 Oct 2024 15:55:15 +0800 Subject: [PATCH 3/7] optimize speed --- publish.yaml | 2 +- src/code/helper.py | 52 - src/code/index.py | 110 ++- src/code/ossZipfile.py | 2082 ---------------------------------------- src/code/speed.py | 101 ++ src/fail/index.py | 2 +- src/s.yaml | 23 +- 7 files changed, 175 insertions(+), 2197 deletions(-) delete mode 100644 src/code/helper.py delete mode 100644 src/code/ossZipfile.py create mode 100644 src/code/speed.py diff --git a/publish.yaml b/publish.yaml index 074befe..de4b688 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,7 +1,7 @@ Edition: 3.0.0 Type: Project Name: start-unzip-oss-v3 -Version: 0.0.10 +Version: 0.0.11 Provider: - 阿里云 Description: 本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 diff --git a/src/code/helper.py b/src/code/helper.py deleted file mode 100644 index d2fbb84..0000000 --- a/src/code/helper.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -import oss2 -from oss2 import utils, models -import ossZipfile as zipfile - -zipfile_support_oss = zipfile - -# support upload to oss as a file-like object - - -def make_crc_adapter(data, init_crc=0): - data = utils.to_bytes(data) - # file-like object - if hasattr(data, 'read'): - return utils._FileLikeAdapter(data, crc_callback=utils.Crc64(init_crc)) - - -utils.make_crc_adapter = make_crc_adapter - - -class OssStreamFileLikeObject(object): - def __init__(self, bucket, key): - super(OssStreamFileLikeObject, self).__init__() - self._bucket = bucket - self._key = key - self._meta_data = self._bucket.get_object_meta(self._key) - - @property - def bucket(self): - return self._bucket - - @property - def key(self): - return self._key - - @property - def filesize(self): - return self._meta_data.content_length - - def get_reader(self, begin, end): - begin = begin if begin >= 0 else 0 - end = end if end > 0 else self.filesize - 1 - end = end if end < self.filesize else self.filesize - 1 - begin = begin if begin < end else end - return self._bucket.get_object(self._key, byte_range=(begin, end)) - - def get_content_bytes(self, begin, end): - reader = self.get_reader(begin, end) - return reader.read() - - def get_last_content_bytes(self, offset): - return self.get_content_bytes(self.filesize-offset, self.filesize-1) diff --git a/src/code/index.py b/src/code/index.py index 0eba960..a4085d4 100644 --- a/src/code/index.py +++ b/src/code/index.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -''' +""" 声明： 这个函数针对文件和文件夹命名编码是如下格式： 1. mac/linux 系统， 默认是utf-8 @@ -16,39 +16,21 @@ For other encodings, we try to use the chardet library for coding judgment here, but this is not guaranteed to be 100% correct. If necessary to rewrite this function, and ensure that the debugging pass -''' +""" -import helper import oss2 import json import os -import time import logging +import zipfile import chardet -""" -When a source/ prefix object is placed in an OSS, it is hoped that the object will be decompressed and then stored in the OSS as processed/ prefixed. -For example, source/a.zip will be processed as processed/a/... -"Source /", "processed/" can be changed according to the user's requirements. -""" # Close the info log printed by the oss SDK logging.getLogger("oss2.api").setLevel(logging.ERROR) logging.getLogger("oss2.auth").setLevel(logging.ERROR) LOGGER = logging.getLogger() -# a decorator for print the excute time of a function - - -def print_excute_time(func): - def wrapper(*args, **kwargs): - local_time = time.time() - ret = func(*args, **kwargs) - LOGGER.info('current Function [%s] excute time is %.2f' % - (func.__name__, time.time() - local_time)) - return ret - return wrapper - def get_zipfile_name(origin_name): # 解决中文乱码问题 name = origin_name @@ -61,7 +43,9 @@ def get_zipfile_name(origin_name): # 解决中文乱码问题 detect = chardet.detect(name_bytes) confidence = detect["confidence"] detect_encoding = detect["encoding"] - if confidence > 0.75 and (detect_encoding.lower() in ["gb2312", "gbk", "gb18030", "ascii", "utf-8"]): + if confidence > 0.75 and ( + detect_encoding.lower() in ["gb2312", "gbk", "gb18030", "ascii", "utf-8"] + ): try: if detect_encoding.lower() in ["gb2312", "gbk", "gb18030"]: detect_encoding = "gb18030" @@ -78,7 +62,6 @@ def get_zipfile_name(origin_name): # 解决中文乱码问题 return name -@print_excute_time def handler(event, context): """ The object from OSS will be decompressed automatically . @@ -92,26 +75,38 @@ def handler(event, context): evt_lst = json.loads(event) creds = context.credentials auth = oss2.StsAuth( - creds.access_key_id, - creds.access_key_secret, - creds.security_token) + creds.access_key_id, creds.access_key_secret, creds.security_token + ) - evt = evt_lst['events'][0] - bucket_name = evt['oss']['bucket']['name'] - endpoint = 'oss-' + evt['region'] + '-internal.aliyuncs.com' + evt = evt_lst["events"][0] + bucket_name = evt["oss"]["bucket"]["name"] + endpoint = "oss-" + evt["region"] + "-internal.aliyuncs.com" bucket = oss2.Bucket(auth, endpoint, bucket_name) - object_name = evt['oss']['object']['key'] + object_name = evt["oss"]["object"]["key"] + object_sizeMB = evt["oss"]["object"]["size"] / 1024 / 1024 + LOGGER.info("{} size is = {}MB".format(object_name, object_sizeMB)) - if "ObjectCreated:PutSymlink" == evt['eventName']: + if object_sizeMB > 10240 * 0.9: + raise RuntimeError( + "{} size is too large; please use NAS, refer: https://github.com/zhaohang88/unzip-oss-nas".format( + object_name + ) + ) + + file_type = os.path.splitext(object_name)[1] + if file_type != ".zip": + raise RuntimeError("{} filetype is not zip".format(object_name)) + + if "ObjectCreated:PutSymlink" == evt["eventName"]: object_name = bucket.get_symlink(object_name).target_key if object_name == "": - raise RuntimeError('{} is invalid symlink file'.format( - evt['oss']['object']['key'])) + raise RuntimeError( + "{} is invalid symlink file".format(evt["oss"]["object"]["key"]) + ) file_type = os.path.splitext(object_name)[1] - if file_type != ".zip": - raise RuntimeError('{} filetype is not zip'.format(object_name)) + raise RuntimeError("{} filetype is not zip".format(object_name)) LOGGER.info("start to decompress zip file = {}".format(object_name)) @@ -119,18 +114,41 @@ def handler(event, context): zip_name = lst[-1] PROCESSED_DIR = os.environ.get("PROCESSED_DIR", "") RETAIN_FILE_NAME = os.environ.get("RETAIN_FILE_NAME", "") - if PROCESSED_DIR and PROCESSED_DIR[-1] != "/": - PROCESSED_DIR += "/" if RETAIN_FILE_NAME == "false": - newKey = PROCESSED_DIR + newKeyPrefix = PROCESSED_DIR else: - newKey = PROCESSED_DIR + zip_name + newKeyPrefix = os.path.join(PROCESSED_DIR, zip_name) + newKeyPrefix = newKeyPrefix.replace(".zip", "/") + + tmpWorkDir = "/tmp/{}".format(context.request_id) + if not os.path.exists(tmpWorkDir): + os.makedirs(tmpWorkDir) - zip_fp = helper.OssStreamFileLikeObject(bucket, object_name) - newKey = newKey.replace(".zip", "/") + tmpZipfile = os.path.join(tmpWorkDir, zip_name) + bucket.get_object_to_file(object_name, tmpZipfile) - with helper.zipfile_support_oss.ZipFile(zip_fp) as zip_file: - for name in zip_file.namelist(): - with zip_file.open(name) as file_obj: - name = get_zipfile_name(name) - bucket.put_object(newKey + name, file_obj) + try: + with zipfile.ZipFile(tmpZipfile) as zip_file: + for file_info in zip_file.infolist(): + if file_info.is_dir(): + continue + f_size = file_info.file_size + if ( + object_sizeMB + f_size / 1024 / 1024 > 10240 * 0.99 + ): # if zip file + one file size > 0.99G, skip extract and upload + LOGGER.error( + "{} size is too large; skip extract and upload".format(f) + ) + continue + zip_file.extract(file_info.filename, tmpWorkDir) + pathname = os.path.join(tmpWorkDir, file_info.filename) + newkey = os.path.join( + newKeyPrefix, get_zipfile_name(file_info.filename) + ) + LOGGER.info("upload to {}".format(newkey)) + bucket.put_object_from_file(newkey, pathname) + os.remove(pathname) + except Exception as e: + LOGGER.error(e) + finally: + os.remove(tmpZipfile) diff --git a/src/code/ossZipfile.py b/src/code/ossZipfile.py deleted file mode 100644 index a069e7b..0000000 --- a/src/code/ossZipfile.py +++ /dev/null @@ -1,2082 +0,0 @@ -""" -Read and write ZIP files. - -XXX references to utf-8 need further investigation. -""" -import io -import os -import importlib.util -import sys -import time -import stat -import shutil -import struct -import binascii -import threading - -try: - import zlib # We may need its compression method - crc32 = zlib.crc32 -except ImportError: - zlib = None - crc32 = binascii.crc32 - -try: - import bz2 # We may need its compression method -except ImportError: - bz2 = None - -try: - import lzma # We may need its compression method -except ImportError: - lzma = None - -__all__ = ["BadZipFile", "BadZipfile", "error", - "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile"] - - -class BadZipFile(Exception): - pass - - -class LargeZipFile(Exception): - """ - Raised when writing a zipfile, the zipfile requires ZIP64 extensions - and those extensions are disabled. - """ - - -error = BadZipfile = BadZipFile # Pre-3.2 compatibility names - - -ZIP64_LIMIT = (1 << 31) - 1 -ZIP_FILECOUNT_LIMIT = (1 << 16) - 1 -ZIP_MAX_COMMENT = (1 << 16) - 1 - -# constants for Zip file compression methods -ZIP_STORED = 0 -ZIP_DEFLATED = 8 -ZIP_BZIP2 = 12 -ZIP_LZMA = 14 -# Other ZIP compression methods not supported - -DEFAULT_VERSION = 20 -ZIP64_VERSION = 45 -BZIP2_VERSION = 46 -LZMA_VERSION = 63 -# we recognize (but not necessarily support) all features up to that version -MAX_EXTRACT_VERSION = 63 - -# Below are some formats and associated data for reading/writing headers using -# the struct module. The names and structures of headers/records are those used -# in the PKWARE description of the ZIP file format: -# http://www.pkware.com/documents/casestudies/APPNOTE.TXT -# (URL valid as of January 2008) - -# The "end of central directory" structure, magic number, size, and indices -# (section V.I in the format document) -structEndArchive = b"<4s4H2LH" -stringEndArchive = b"PK\005\006" -sizeEndCentDir = struct.calcsize(structEndArchive) - -_ECD_SIGNATURE = 0 -_ECD_DISK_NUMBER = 1 -_ECD_DISK_START = 2 -_ECD_ENTRIES_THIS_DISK = 3 -_ECD_ENTRIES_TOTAL = 4 -_ECD_SIZE = 5 -_ECD_OFFSET = 6 -_ECD_COMMENT_SIZE = 7 -# These last two indices are not part of the structure as defined in the -# spec, but they are used internally by this module as a convenience -_ECD_COMMENT = 8 -_ECD_LOCATION = 9 - -# The "central directory" structure, magic number, size, and indices -# of entries in the structure (section V.F in the format document) -structCentralDir = "<4s4B4HL2L5H2L" -stringCentralDir = b"PK\001\002" -sizeCentralDir = struct.calcsize(structCentralDir) - -# indexes of entries in the central directory structure -_CD_SIGNATURE = 0 -_CD_CREATE_VERSION = 1 -_CD_CREATE_SYSTEM = 2 -_CD_EXTRACT_VERSION = 3 -_CD_EXTRACT_SYSTEM = 4 -_CD_FLAG_BITS = 5 -_CD_COMPRESS_TYPE = 6 -_CD_TIME = 7 -_CD_DATE = 8 -_CD_CRC = 9 -_CD_COMPRESSED_SIZE = 10 -_CD_UNCOMPRESSED_SIZE = 11 -_CD_FILENAME_LENGTH = 12 -_CD_EXTRA_FIELD_LENGTH = 13 -_CD_COMMENT_LENGTH = 14 -_CD_DISK_NUMBER_START = 15 -_CD_INTERNAL_FILE_ATTRIBUTES = 16 -_CD_EXTERNAL_FILE_ATTRIBUTES = 17 -_CD_LOCAL_HEADER_OFFSET = 18 - -# The "local file header" structure, magic number, size, and indices -# (section V.A in the format document) -structFileHeader = "<4s2B4HL2L2H" -stringFileHeader = b"PK\003\004" -sizeFileHeader = struct.calcsize(structFileHeader) - -_FH_SIGNATURE = 0 -_FH_EXTRACT_VERSION = 1 -_FH_EXTRACT_SYSTEM = 2 -_FH_GENERAL_PURPOSE_FLAG_BITS = 3 -_FH_COMPRESSION_METHOD = 4 -_FH_LAST_MOD_TIME = 5 -_FH_LAST_MOD_DATE = 6 -_FH_CRC = 7 -_FH_COMPRESSED_SIZE = 8 -_FH_UNCOMPRESSED_SIZE = 9 -_FH_FILENAME_LENGTH = 10 -_FH_EXTRA_FIELD_LENGTH = 11 - -# The "Zip64 end of central directory locator" structure, magic number, and size -structEndArchive64Locator = "<4sLQL" -stringEndArchive64Locator = b"PK\x06\x07" -sizeEndCentDir64Locator = struct.calcsize(structEndArchive64Locator) - -# The "Zip64 end of central directory" record, magic number, size, and indices -# (section V.G in the format document) -structEndArchive64 = "<4sQ2H2L4Q" -stringEndArchive64 = b"PK\x06\x06" -sizeEndCentDir64 = struct.calcsize(structEndArchive64) - -_CD64_SIGNATURE = 0 -_CD64_DIRECTORY_RECSIZE = 1 -_CD64_CREATE_VERSION = 2 -_CD64_EXTRACT_VERSION = 3 -_CD64_DISK_NUMBER = 4 -_CD64_DISK_NUMBER_START = 5 -_CD64_NUMBER_ENTRIES_THIS_DISK = 6 -_CD64_NUMBER_ENTRIES_TOTAL = 7 -_CD64_DIRECTORY_SIZE = 8 -_CD64_OFFSET_START_CENTDIR = 9 - - -def _check_zipfile(fp): - try: - if _EndRecData(fp): - return True # file has correct magic number - except OSError: - pass - return False - - -def is_zipfile(filename): - """Quickly see if a file is a ZIP file by checking the magic number. - - The filename argument may be a file or file-like object too. - """ - result = False - try: - if hasattr(filename, "read"): - result = _check_zipfile(fp=filename) - else: - with open(filename, "rb") as fp: - result = _check_zipfile(fp) - except OSError: - pass - return result - - -def _EndRecData64(fpin, offset, endrec): - """ - Read the ZIP64 end-of-archive records and use that to update endrec - """ - if hasattr(fpin, 'bucket'): - data = fpin.get_content_bytes( - fpin.filesize+offset-sizeEndCentDir64Locator, fpin.filesize+offset-1) - else: - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 - # end-of-archive record, so just return the end record we were given. - return endrec - - data = fpin.read(sizeEndCentDir64Locator) - - if len(data) != sizeEndCentDir64Locator: - return endrec - sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) - if sig != stringEndArchive64Locator: - return endrec - - if diskno != 0 or disks != 1: - raise BadZipFile("zipfiles that span multiple disks are not supported") - - # Assume no 'zip64 extensible data' - if hasattr(fpin, 'bucket'): - data = fpin.get_content_bytes( - fpin.filesize + offset - sizeEndCentDir64Locator - sizeEndCentDir64, - fpin.filesize+offset-sizeEndCentDir64Locator-1) - else: - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) - data = fpin.read(sizeEndCentDir64) - if len(data) != sizeEndCentDir64: - return endrec - sig, sz, create_version, read_version, disk_num, disk_dir, \ - dircount, dircount2, dirsize, diroffset = \ - struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec - - # Update the original endrec using data from the ZIP64 record - endrec[_ECD_SIGNATURE] = sig - endrec[_ECD_DISK_NUMBER] = disk_num - endrec[_ECD_DISK_START] = disk_dir - endrec[_ECD_ENTRIES_THIS_DISK] = dircount - endrec[_ECD_ENTRIES_TOTAL] = dircount2 - endrec[_ECD_SIZE] = dirsize - endrec[_ECD_OFFSET] = diroffset - return endrec - - -def _EndRecData(fpin): - """Return data from the "End of Central Directory" record, or None. - - The data is a list of the nine items in the ZIP "End of central dir" - record followed by a tenth item, the file seek offset of this record.""" - - # Determine file size - if hasattr(fpin, 'bucket'): - filesize = fpin.filesize - data = fpin.get_last_content_bytes(sizeEndCentDir) - else: - fpin.seek(0, 2) - filesize = fpin.tell() - - # Check to see if this is ZIP file with no archive comment (the - # "end of central directory" structure should be the last item in the - # file if this is the case). - try: - fpin.seek(-sizeEndCentDir, 2) - except OSError: - return None - data = fpin.read() - - if (len(data) == sizeEndCentDir and - data[0:4] == stringEndArchive and - data[-2:] == b"\000\000"): - # the signature is correct and there's no comment, unpack structure - endrec = struct.unpack(structEndArchive, data) - endrec = list(endrec) - - # Append a blank comment and record start offset - endrec.append(b"") - endrec.append(filesize - sizeEndCentDir) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) - - # Either this is not a ZIP file, or it is a ZIP file with an archive - # comment. Search the end of the file for the "end of central directory" - # record signature. The comment is the last item in the ZIP file and may be - # up to 64K long. It is assumed that the "end of central directory" magic - # number does not appear in the comment. - maxCommentStart = max(filesize - (1 << 16) - sizeEndCentDir, 0) - - if hasattr(fpin, 'bucket'): - data = fpin.get_content_bytes(maxCommentStart, -1) - else: - fpin.seek(maxCommentStart, 0) - data = fpin.read() - - start = data.rfind(stringEndArchive) - if start >= 0: - # found the magic number; attempt to unpack and interpret - recData = data[start:start+sizeEndCentDir] - if len(recData) != sizeEndCentDir: - # Zip file is corrupted. - return None - endrec = list(struct.unpack(structEndArchive, recData)) - commentSize = endrec[_ECD_COMMENT_SIZE] # as claimed by the zip file - comment = data[start+sizeEndCentDir:start+sizeEndCentDir+commentSize] - endrec.append(comment) - endrec.append(maxCommentStart + start) - - # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) - - # Unable to find a valid end of central directory structure - return None - - -class ZipInfo (object): - """Class with attributes describing each file in the ZIP archive.""" - - __slots__ = ( - 'orig_filename', - 'filename', - 'date_time', - 'compress_type', - 'comment', - 'extra', - 'create_system', - 'create_version', - 'extract_version', - 'reserved', - 'flag_bits', - 'volume', - 'internal_attr', - 'external_attr', - 'header_offset', - 'CRC', - 'compress_size', - 'file_size', - '_raw_time', - ) - - def __init__(self, filename="NoName", date_time=(1980, 1, 1, 0, 0, 0)): - self.orig_filename = filename # Original file name in archive - - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") - - self.filename = filename # Normalized file name - self.date_time = date_time # year, month, day, hour, min, sec - - if date_time[0] < 1980: - raise ValueError('ZIP does not support timestamps before 1980') - - # Standard values: - self.compress_type = ZIP_STORED # Type of compression for the file - self.comment = b"" # Comment for each file - self.extra = b"" # ZIP extra data - if sys.platform == 'win32': - self.create_system = 0 # System which created ZIP archive - else: - # Assume everything else is unix-y - self.create_system = 3 # System which created ZIP archive - self.create_version = DEFAULT_VERSION # Version which created ZIP archive - self.extract_version = DEFAULT_VERSION # Version needed to extract archive - self.reserved = 0 # Must be zero - self.flag_bits = 0 # ZIP flag bits - self.volume = 0 # Volume number of file header - self.internal_attr = 0 # Internal attributes - self.external_attr = 0 # External file attributes - # Other attributes are set by class ZipFile: - # header_offset Byte offset to the file header - # CRC CRC-32 of the uncompressed file - # compress_size Size of the compressed file - # file_size Size of the uncompressed file - - def __repr__(self): - result = ['<%s filename=%r' % (self.__class__.__name__, self.filename)] - if self.compress_type != ZIP_STORED: - result.append(' compress_type=%s' % - compressor_names.get(self.compress_type, - self.compress_type)) - hi = self.external_attr >> 16 - lo = self.external_attr & 0xFFFF - if hi: - result.append(' filemode=%r' % stat.filemode(hi)) - if lo: - result.append(' external_attr=%#x' % lo) - isdir = self.is_dir() - if not isdir or self.file_size: - result.append(' file_size=%r' % self.file_size) - if ((not isdir or self.compress_size) and - (self.compress_type != ZIP_STORED or - self.file_size != self.compress_size)): - result.append(' compress_size=%r' % self.compress_size) - result.append('>') - return ''.join(result) - - def FileHeader(self, zip64=None): - """Return the per-file header as a string.""" - dt = self.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if self.flag_bits & 0x08: - # Set these to zero because we write them after the file data - CRC = compress_size = file_size = 0 - else: - CRC = self.CRC - compress_size = self.compress_size - file_size = self.file_size - - extra = self.extra - - min_version = 0 - if zip64 is None: - zip64 = file_size > ZIP64_LIMIT or compress_size > ZIP64_LIMIT - if zip64: - fmt = ' ZIP64_LIMIT or compress_size > ZIP64_LIMIT: - if not zip64: - raise LargeZipFile("Filesize would require ZIP64 extensions") - # File is larger than what fits into a 4 byte integer, - # fall back to the ZIP64 extension - file_size = 0xffffffff - compress_size = 0xffffffff - min_version = ZIP64_VERSION - - if self.compress_type == ZIP_BZIP2: - min_version = max(BZIP2_VERSION, min_version) - elif self.compress_type == ZIP_LZMA: - min_version = max(LZMA_VERSION, min_version) - - self.extract_version = max(min_version, self.extract_version) - self.create_version = max(min_version, self.create_version) - filename, flag_bits = self._encodeFilenameFlags() - header = struct.pack(structFileHeader, stringFileHeader, - self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, - compress_size, file_size, - len(filename), len(extra)) - return header + filename + extra - - def _encodeFilenameFlags(self): - try: - return self.filename.encode('ascii'), self.flag_bits - except UnicodeEncodeError: - return self.filename.encode('utf-8'), self.flag_bits | 0x800 - - def _decodeExtra(self): - # Try to decode the extra field. - extra = self.extra - unpack = struct.unpack - while len(extra) >= 4: - tp, ln = unpack(' len(extra): - raise BadZipFile( - "Corrupt extra field %04x (size=%d)" % (tp, ln)) - if tp == 0x0001: - if ln >= 24: - counts = unpack('> 1) ^ 0xEDB88320 - else: - crc >>= 1 - return crc - -# ZIP supports a password-based form of encryption. Even though known -# plaintext attacks have been found against it, it is still useful -# to be able to get data out of such a file. -# -# Usage: -# zd = _ZipDecrypter(mypwd) -# plain_bytes = zd(cypher_bytes) - - -def _ZipDecrypter(pwd): - key0 = 305419896 - key1 = 591751049 - key2 = 878082192 - - global _crctable - if _crctable is None: - _crctable = list(map(_gen_crc, range(256))) - crctable = _crctable - - def crc32(ch, crc): - """Compute the CRC32 primitive on one byte.""" - return (crc >> 8) ^ crctable[(crc ^ ch) & 0xFF] - - def update_keys(c): - nonlocal key0, key1, key2 - key0 = crc32(c, key0) - key1 = (key1 + (key0 & 0xFF)) & 0xFFFFFFFF - key1 = (key1 * 134775813 + 1) & 0xFFFFFFFF - key2 = crc32(key1 >> 24, key2) - - for p in pwd: - update_keys(p) - - def decrypter(data): - """Decrypt a bytes object.""" - result = bytearray() - append = result.append - for c in data: - k = key2 | 2 - c ^= ((k * (k ^ 1)) >> 8) & 0xFF - update_keys(c) - append(c) - return bytes(result) - - return decrypter - - -class LZMACompressor: - - def __init__(self): - self._comp = None - - def _init(self): - props = lzma._encode_filter_properties({'id': lzma.FILTER_LZMA1}) - self._comp = lzma.LZMACompressor(lzma.FORMAT_RAW, filters=[ - lzma._decode_filter_properties(lzma.FILTER_LZMA1, props) - ]) - return struct.pack('') - return ''.join(result) - - def readline(self, limit=-1): - """Read and return a line from the stream. - - If limit is specified, at most limit bytes will be read. - """ - - if limit < 0: - # Shortcut common case - newline found in buffer. - i = self._readbuffer.find(b'\n', self._offset) + 1 - if i > 0: - line = self._readbuffer[self._offset: i] - self._offset = i - return line - - return io.BufferedIOBase.readline(self, limit) - - def peek(self, n=1): - """Returns buffered bytes without advancing the position.""" - if n > len(self._readbuffer) - self._offset: - chunk = self.read(n) - if len(chunk) > self._offset: - self._readbuffer = chunk + self._readbuffer[self._offset:] - self._offset = 0 - else: - self._offset -= len(chunk) - - # Return up to 512 bytes to reduce allocation overhead for tight loops. - return self._readbuffer[self._offset: self._offset + 512] - - def readable(self): - return True - - def read(self, n=-1): - """Read and return up to n bytes. - If the argument is omitted, None, or negative, data is read and returned until EOF is reached.. - """ - if n is None or n < 0: - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while not self._eof: - buf += self._read1(self.MAX_N) - return buf - - end = n + self._offset - if end < len(self._readbuffer): - buf = self._readbuffer[self._offset:end] - self._offset = end - return buf - - n = end - len(self._readbuffer) - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while n > 0 and not self._eof: - data = self._read1(n) - if n < len(data): - self._readbuffer = data - self._offset = n - buf += data[:n] - break - buf += data - n -= len(data) - return buf - - def _update_crc(self, newdata): - # Update the CRC using the given data. - if self._expected_crc is None: - # No need to compute the CRC if we don't have a reference value - return - self._running_crc = crc32(newdata, self._running_crc) - # Check the CRC if we're at the end of the file - if self._eof and self._running_crc != self._expected_crc: - raise BadZipFile("Bad CRC-32 for file %r" % self.name) - - def read1(self, n): - """Read up to n bytes with at most one read() system call.""" - - if n is None or n < 0: - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - while not self._eof: - data = self._read1(self.MAX_N) - if data: - buf += data - break - return buf - - end = n + self._offset - if end < len(self._readbuffer): - buf = self._readbuffer[self._offset:end] - self._offset = end - return buf - - n = end - len(self._readbuffer) - buf = self._readbuffer[self._offset:] - self._readbuffer = b'' - self._offset = 0 - if n > 0: - while not self._eof: - data = self._read1(n) - if n < len(data): - self._readbuffer = data - self._offset = n - buf += data[:n] - break - if data: - buf += data - break - return buf - - def _read1(self, n): - # Read up to n compressed bytes with at most one read() system call, - # decrypt and decompress them. - if self._eof or n <= 0: - return b'' - - # Read from file. - if self._compress_type == ZIP_DEFLATED: - # Handle unconsumed data. - data = self._decompressor.unconsumed_tail - if n > len(data): - data += self._read2(n - len(data)) - else: - data = self._read2(n) - - if self._compress_type == ZIP_STORED: - self._eof = self._compress_left <= 0 - elif self._compress_type == ZIP_DEFLATED: - n = max(n, self.MIN_READ_SIZE) - data = self._decompressor.decompress(data, n) - self._eof = (self._decompressor.eof or - self._compress_left <= 0 and - not self._decompressor.unconsumed_tail) - if self._eof: - data += self._decompressor.flush() - else: - data = self._decompressor.decompress(data) - self._eof = self._decompressor.eof or self._compress_left <= 0 - - data = data[:self._left] - self._left -= len(data) - if self._left <= 0: - self._eof = True - self._update_crc(data) - return data - - def _read2(self, n): - if self._compress_left <= 0: - return b'' - - n = max(n, self.MIN_READ_SIZE) - n = min(n, self._compress_left) - - data = self._fileobj.read(n) - self._compress_left -= len(data) - if not data: - raise EOFError - - if self._decrypter is not None: - data = self._decrypter(data) - return data - - def close(self): - try: - if self._close_fileobj: - self._fileobj.close() - finally: - super().close() - - -class _ZipWriteFile(io.BufferedIOBase): - def __init__(self, zf, zinfo, zip64): - self._zinfo = zinfo - self._zip64 = zip64 - self._zipfile = zf - self._compressor = _get_compressor(zinfo.compress_type) - self._file_size = 0 - self._compress_size = 0 - self._crc = 0 - - @property - def _fileobj(self): - return self._zipfile.fp - - def writable(self): - return True - - def write(self, data): - if self.closed: - raise ValueError('I/O operation on closed file.') - nbytes = len(data) - self._file_size += nbytes - self._crc = crc32(data, self._crc) - if self._compressor: - data = self._compressor.compress(data) - self._compress_size += len(data) - self._fileobj.write(data) - return nbytes - - def close(self): - if self.closed: - return - super().close() - # Flush any data from the compressor, and update header info - if self._compressor: - buf = self._compressor.flush() - self._compress_size += len(buf) - self._fileobj.write(buf) - self._zinfo.compress_size = self._compress_size - else: - self._zinfo.compress_size = self._file_size - self._zinfo.CRC = self._crc - self._zinfo.file_size = self._file_size - - # Write updated header info - if self._zinfo.flag_bits & 0x08: - # Write CRC and file sizes after the file data - fmt = ' ZIP64_LIMIT: - raise RuntimeError('File size unexpectedly exceeded ZIP64 ' - 'limit') - if self._compress_size > ZIP64_LIMIT: - raise RuntimeError('Compressed size unexpectedly exceeded ' - 'ZIP64 limit') - # Seek backwards and write file header (which will now include - # correct CRC and file sizes) - - # Preserve current position in file - self._zipfile.start_dir = self._fileobj.tell() - self._fileobj.seek(self._zinfo.header_offset) - self._fileobj.write(self._zinfo.FileHeader(self._zip64)) - self._fileobj.seek(self._zipfile.start_dir) - - self._zipfile._writing = False - - # Successfully written: Add file to our caches - self._zipfile.filelist.append(self._zinfo) - self._zipfile.NameToInfo[self._zinfo.filename] = self._zinfo - - -class ZipFile: - """ Class with methods to open, read, write, close, list zip files. - - z = ZipFile(file, mode="r", compression=ZIP_STORED, allowZip64=True) - - file: Either the path to the file, or a file-like object. - If it is a path, the file will be opened and closed by ZipFile. - mode: The mode can be either read 'r', write 'w', exclusive create 'x', - or append 'a'. - compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - - """ - - fp = None # Set here since __del__ checks it - _windows_illegal_name_trans_table = None - - def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" - if mode not in ('r', 'w', 'x', 'a'): - raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") - - _check_compression(compression) - - self._allowZip64 = allowZip64 - self._didModify = False - self.debug = 0 # Level of printing: 0 through 3 - self.NameToInfo = {} # Find file info given name - self.filelist = [] # List of ZipInfo instances for archive - self.compression = compression # Method of compression - self.mode = mode - self.pwd = None - self._comment = b'' - - # Check if we were passed a file-like object - if isinstance(file, os.PathLike): - file = os.fspath(file) - if isinstance(file, str): - # No, it's a filename - self._filePassed = 0 - self.filename = file - modeDict = {'r': 'rb', 'w': 'w+b', 'x': 'x+b', 'a': 'r+b', - 'r+b': 'w+b', 'w+b': 'wb', 'x+b': 'xb'} - filemode = modeDict[mode] - while True: - try: - self.fp = io.open(file, filemode) - except OSError: - if filemode in modeDict: - filemode = modeDict[filemode] - continue - raise - break - else: - self._filePassed = 1 - self.fp = file - self.filename = getattr(file, 'name', None) - self._fileRefCnt = 1 - self._lock = threading.RLock() - self._seekable = True - self._writing = False - - try: - if mode == 'r': - self._RealGetContents() - elif mode in ('w', 'x'): - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - try: - self.start_dir = self.fp.tell() - except (AttributeError, OSError): - self.fp = _Tellable(self.fp) - self.start_dir = 0 - self._seekable = False - else: - # Some file-like objects can provide tell() but not seek() - try: - self.fp.seek(self.start_dir) - except (AttributeError, OSError): - self._seekable = False - elif mode == 'a': - try: - # See if file is a zip file - self._RealGetContents() - # seek to start of directory and overwrite - self.fp.seek(self.start_dir) - except BadZipFile: - # file is not a zip file, just append - self.fp.seek(0, 2) - - # set the modified flag so central directory gets written - # even if no files are added to the archive - self._didModify = True - self.start_dir = self.fp.tell() - else: - raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") - except: - fp = self.fp - self.fp = None - self._fpclose(fp) - raise - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def __repr__(self): - result = ['<%s.%s' % (self.__class__.__module__, - self.__class__.__qualname__)] - if self.fp is not None: - if self._filePassed: - result.append(' file=%r' % self.fp) - elif self.filename is not None: - result.append(' filename=%r' % self.filename) - result.append(' mode=%r' % self.mode) - else: - result.append(' [closed]') - result.append('>') - return ''.join(result) - - def _RealGetContents(self): - """Read in the table of contents for the ZIP file.""" - fp = self.fp - try: - endrec = _EndRecData(fp) - except OSError: - raise BadZipFile("File is not a zip file") - if not endrec: - raise BadZipFile("File is not a zip file") - if self.debug > 1: - print(endrec) - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory - self._comment = endrec[_ECD_COMMENT] # archive comment - - # "concat" is zero, unless zip was concatenated to another file - concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) - - if self.debug > 2: - inferred = concat + offset_cd - print("given, inferred, offset", offset_cd, inferred, concat) - # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + concat - - if hasattr(fp, "bucket"): - data = fp.get_content_bytes( - self.start_dir, self.start_dir+size_cd-1) - else: - fp.seek(self.start_dir, 0) - data = fp.read(size_cd) - - fp = io.BytesIO(data) - total = 0 - while total < size_cd: - centdir = fp.read(sizeCentralDir) - if len(centdir) != sizeCentralDir: - raise BadZipFile("Truncated central directory") - centdir = struct.unpack(structCentralDir, centdir) - if centdir[_CD_SIGNATURE] != stringCentralDir: - raise BadZipFile("Bad magic number for central directory") - if self.debug > 2: - print(centdir) - filename = fp.read(centdir[_CD_FILENAME_LENGTH]) - flags = centdir[5] - if flags & 0x800: - # UTF-8 file names extension - filename = filename.decode('utf-8') - else: - # Historical ZIP filename encoding - filename = filename.decode('cp437') - # Create ZipInfo instance to store file information - x = ZipInfo(filename) - x.extra = fp.read(centdir[_CD_EXTRA_FIELD_LENGTH]) - x.comment = fp.read(centdir[_CD_COMMENT_LENGTH]) - x.header_offset = centdir[_CD_LOCAL_HEADER_OFFSET] - (x.create_version, x.create_system, x.extract_version, x.reserved, - x.flag_bits, x.compress_type, t, d, - x.CRC, x.compress_size, x.file_size) = centdir[1:12] - if x.extract_version > MAX_EXTRACT_VERSION: - raise NotImplementedError("zip file version %.1f" % - (x.extract_version / 10)) - x.volume, x.internal_attr, x.external_attr = centdir[15:18] - # Convert date/time code to (year, month, day, hour, min, sec) - x._raw_time = t - x.date_time = ((d >> 9)+1980, (d >> 5) & 0xF, d & 0x1F, - t >> 11, (t >> 5) & 0x3F, (t & 0x1F) * 2) - - x._decodeExtra() - x.header_offset = x.header_offset + concat - self.filelist.append(x) - self.NameToInfo[x.filename] = x - - # update total bytes read from central directory - total = (total + sizeCentralDir + centdir[_CD_FILENAME_LENGTH] - + centdir[_CD_EXTRA_FIELD_LENGTH] - + centdir[_CD_COMMENT_LENGTH]) - - if self.debug > 2: - print("total", total) - - def namelist(self): - """Return a list of file names in the archive.""" - return [data.filename for data in self.filelist] - - def infolist(self): - """Return a list of class ZipInfo instances for files in the - archive.""" - return self.filelist - - def printdir(self, file=None): - """Print a table of contents for the zip file.""" - print("%-46s %19s %12s" % ("File Name", "Modified ", "Size"), - file=file) - for zinfo in self.filelist: - date = "%d-%02d-%02d %02d:%02d:%02d" % zinfo.date_time[:6] - print("%-46s %s %12d" % (zinfo.filename, date, zinfo.file_size), - file=file) - - def testzip(self): - """Read all the files and check the CRC.""" - chunk_size = 2 ** 20 - for zinfo in self.filelist: - try: - # Read by chunks, to avoid an OverflowError or a - # MemoryError with very large embedded files. - with self.open(zinfo.filename, "r") as f: - while f.read(chunk_size): # Check CRC-32 - pass - except BadZipFile: - return zinfo.filename - - def getinfo(self, name): - """Return the instance of ZipInfo given 'name'.""" - info = self.NameToInfo.get(name) - if info is None: - raise KeyError( - 'There is no item named %r in the archive' % name) - - return info - - def setpassword(self, pwd): - """Set default password for encrypted files.""" - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) - if pwd: - self.pwd = pwd - else: - self.pwd = None - - @property - def comment(self): - """The comment text associated with the ZIP file.""" - return self._comment - - @comment.setter - def comment(self, comment): - if not isinstance(comment, bytes): - raise TypeError("comment: expected bytes, got %s" % - type(comment).__name__) - # check for valid comment length - if len(comment) > ZIP_MAX_COMMENT: - import warnings - warnings.warn('Archive comment is too long; truncating to %d bytes' - % ZIP_MAX_COMMENT, stacklevel=2) - comment = comment[:ZIP_MAX_COMMENT] - self._comment = comment - self._didModify = True - - def read(self, name, pwd=None): - """Return file bytes (as a string) for name.""" - with self.open(name, "r", pwd) as fp: - return fp.read() - - def open(self, name, mode="r", pwd=None, *, force_zip64=False): - """Return file-like object for 'name'. - - name is a string for the file name within the ZIP file, or a ZipInfo - object. - - mode should be 'r' to read a file already in the ZIP file, or 'w' to - write to a file newly added to the archive. - - pwd is the password to decrypt files (only used for reading). - - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. - """ - if mode not in {"r", "w"}: - raise ValueError('open() requires mode "r" or "w"') - if pwd and not isinstance(pwd, bytes): - raise TypeError("pwd: expected bytes, got %s" % type(pwd).__name__) - if pwd and (mode == "w"): - raise ValueError("pwd is only supported for reading files") - if not self.fp: - raise ValueError( - "Attempt to use ZIP archive that was already closed") - - # Make sure we have an info object - if isinstance(name, ZipInfo): - # 'name' is already an info object - zinfo = name - elif mode == 'w': - zinfo = ZipInfo(name) - zinfo.compress_type = self.compression - else: - # Get info object for name - zinfo = self.getinfo(name) - - if mode == 'w': - return self._open_to_write(zinfo, force_zip64=force_zip64) - - if self._writing: - raise ValueError("Can't read from the ZIP file while there " - "is an open writing handle on it. " - "Close the writing handle before trying to read.") - - # Open for reading: - self._fileRefCnt += 1 - zef_file = _SharedFile(self.fp, zinfo.header_offset, - self._fpclose, self._lock, lambda: self._writing) - try: - # Skip the file header: - fheader = zef_file.read(sizeFileHeader) - if len(fheader) != sizeFileHeader: - raise BadZipFile("Truncated file header") - fheader = struct.unpack(structFileHeader, fheader) - if fheader[_FH_SIGNATURE] != stringFileHeader: - raise BadZipFile("Bad magic number for file header") - - fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) - if fheader[_FH_EXTRA_FIELD_LENGTH]: - zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) - - if zinfo.flag_bits & 0x20: - # Zip 2.7: compressed patched data - raise NotImplementedError( - "compressed patched data (flag bit 5)") - - if zinfo.flag_bits & 0x40: - # strong encryption - raise NotImplementedError("strong encryption (flag bit 6)") - - if zinfo.flag_bits & 0x800: - # UTF-8 filename - fname_str = fname.decode("utf-8") - else: - fname_str = fname.decode("cp437") - - if fname_str != zinfo.orig_filename: - raise BadZipFile( - 'File name in directory %r and header %r differ.' - % (zinfo.orig_filename, fname)) - - # check for encrypted flag & handle password - is_encrypted = zinfo.flag_bits & 0x1 - zd = None - if is_encrypted: - if not pwd: - pwd = self.pwd - if not pwd: - raise RuntimeError("File %r is encrypted, password " - "required for extraction" % name) - - zd = _ZipDecrypter(pwd) - # The first 12 bytes in the cypher stream is an encryption header - # used to strengthen the algorithm. The first 11 bytes are - # completely random, while the 12th contains the MSB of the CRC, - # or the MSB of the file time depending on the header type - # and is used to check the correctness of the password. - header = zef_file.read(12) - h = zd(header[0:12]) - if zinfo.flag_bits & 0x8: - # compare against the file type from extended local headers - check_byte = (zinfo._raw_time >> 8) & 0xff - else: - # compare against the CRC otherwise - check_byte = (zinfo.CRC >> 24) & 0xff - if h[11] != check_byte: - raise RuntimeError("Bad password for file %r" % name) - - return ZipExtFile(zef_file, mode, zinfo, zd, True) - except: - zef_file.close() - raise - - def _open_to_write(self, zinfo, force_zip64=False): - if force_zip64 and not self._allowZip64: - raise ValueError( - "force_zip64 is True, but allowZip64 was False when opening " - "the ZIP file." - ) - if self._writing: - raise ValueError("Can't write to the ZIP file while there is " - "another write handle open on it. " - "Close the first handle before opening another.") - - # Sizes and CRC are overwritten with correct data after processing the file - if not hasattr(zinfo, 'file_size'): - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - - zinfo.flag_bits = 0x00 - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - if not self._seekable: - zinfo.flag_bits |= 0x08 - - if not zinfo.external_attr: - zinfo.external_attr = 0o600 << 16 # permissions: ?rw------- - - # Compressed size can be larger than uncompressed size - zip64 = self._allowZip64 and \ - (force_zip64 or zinfo.file_size * 1.05 > ZIP64_LIMIT) - - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() - - self._writecheck(zinfo) - self._didModify = True - - self.fp.write(zinfo.FileHeader(zip64)) - - self._writing = True - return _ZipWriteFile(self, zinfo, zip64) - - def extract(self, member, path=None, pwd=None): - """Extract a member from the archive to the current working directory, - using its full name. Its file information is extracted as accurately - as possible. `member' may be a filename or a ZipInfo object. You can - specify a different directory using `path'. - """ - if path is None: - path = os.getcwd() - else: - path = os.fspath(path) - - return self._extract_member(member, path, pwd) - - def extractall(self, path=None, members=None, pwd=None): - """Extract all members from the archive to the current working - directory. `path' specifies a different directory to extract to. - `members' is optional and must be a subset of the list returned - by namelist(). - """ - if members is None: - members = self.namelist() - - if path is None: - path = os.getcwd() - else: - path = os.fspath(path) - - for zipinfo in members: - self._extract_member(zipinfo, path, pwd) - - @classmethod - def _sanitize_windows_name(cls, arcname, pathsep): - """Replace bad characters and remove trailing dots from parts.""" - table = cls._windows_illegal_name_trans_table - if not table: - illegal = ':<>|"?*' - table = str.maketrans(illegal, '_' * len(illegal)) - cls._windows_illegal_name_trans_table = table - arcname = arcname.translate(table) - # remove trailing dots - arcname = (x.rstrip('.') for x in arcname.split(pathsep)) - # rejoin, removing empty parts. - arcname = pathsep.join(x for x in arcname if x) - return arcname - - def _extract_member(self, member, targetpath, pwd): - """Extract the ZipInfo object 'member' to a physical - file on the path targetpath. - """ - if not isinstance(member, ZipInfo): - member = self.getinfo(member) - - # build the destination pathname, replacing - # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: - arcname = arcname.replace(os.path.altsep, os.path.sep) - # interpret absolute pathname as relative, remove drive letter or - # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] - invalid_path_parts = ('', os.path.curdir, os.path.pardir) - arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) - if x not in invalid_path_parts) - if os.path.sep == '\\': - # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) - - targetpath = os.path.join(targetpath, arcname) - targetpath = os.path.normpath(targetpath) - - # Create all upper directories if necessary. - upperdirs = os.path.dirname(targetpath) - if upperdirs and not os.path.exists(upperdirs): - os.makedirs(upperdirs) - - if member.is_dir(): - if not os.path.isdir(targetpath): - os.mkdir(targetpath) - return targetpath - - with self.open(member, pwd=pwd) as source, \ - open(targetpath, "wb") as target: - shutil.copyfileobj(source, target) - - return targetpath - - def _writecheck(self, zinfo): - """Check for errors before writing a file to the archive.""" - if zinfo.filename in self.NameToInfo: - import warnings - warnings.warn('Duplicate name: %r' % zinfo.filename, stacklevel=3) - if self.mode not in ('w', 'x', 'a'): - raise ValueError("write() requires mode 'w', 'x', or 'a'") - if not self.fp: - raise ValueError( - "Attempt to write ZIP archive that was already closed") - _check_compression(zinfo.compress_type) - if not self._allowZip64: - requires_zip64 = None - if len(self.filelist) >= ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif zinfo.file_size > ZIP64_LIMIT: - requires_zip64 = "Filesize" - elif zinfo.header_offset > ZIP64_LIMIT: - requires_zip64 = "Zipfile size" - if requires_zip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - - def write(self, filename, arcname=None, compress_type=None): - """Put the bytes from filename into the archive under the name - arcname.""" - if not self.fp: - raise ValueError( - "Attempt to write to ZIP archive that was already closed") - if self._writing: - raise ValueError( - "Can't write to ZIP archive while an open writing handle exists" - ) - - zinfo = ZipInfo.from_file(filename, arcname) - - if zinfo.is_dir(): - zinfo.compress_size = 0 - zinfo.CRC = 0 - else: - if compress_type is not None: - zinfo.compress_type = compress_type - else: - zinfo.compress_type = self.compression - - if zinfo.is_dir(): - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - zinfo.header_offset = self.fp.tell() # Start of header bytes - if zinfo.compress_type == ZIP_LZMA: - # Compressed data includes an end-of-stream (EOS) marker - zinfo.flag_bits |= 0x02 - - self._writecheck(zinfo) - self._didModify = True - - self.filelist.append(zinfo) - self.NameToInfo[zinfo.filename] = zinfo - self.fp.write(zinfo.FileHeader(False)) - self.start_dir = self.fp.tell() - else: - with open(filename, "rb") as src, self.open(zinfo, 'w') as dest: - shutil.copyfileobj(src, dest, 1024*8) - - def writestr(self, zinfo_or_arcname, data, compress_type=None): - """Write a file into the archive. The contents is 'data', which - may be either a 'str' or a 'bytes' instance; if it is a 'str', - it is encoded as UTF-8 first. - 'zinfo_or_arcname' is either a ZipInfo instance or - the name of the file in the archive.""" - if isinstance(data, str): - data = data.encode("utf-8") - if not isinstance(zinfo_or_arcname, ZipInfo): - zinfo = ZipInfo(filename=zinfo_or_arcname, - date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression - if zinfo.filename[-1] == '/': - zinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - zinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - zinfo.external_attr = 0o600 << 16 # ?rw------- - else: - zinfo = zinfo_or_arcname - - if not self.fp: - raise ValueError( - "Attempt to write to ZIP archive that was already closed") - if self._writing: - raise ValueError( - "Can't write to ZIP archive while an open writing handle exists." - ) - - if compress_type is not None: - zinfo.compress_type = compress_type - - zinfo.file_size = len(data) # Uncompressed size - with self._lock: - with self.open(zinfo, mode='w') as dest: - dest.write(data) - - def __del__(self): - """Call the "close()" method in case the user forgot.""" - self.close() - - def close(self): - """Close the file, and for mode 'w', 'x' and 'a' write the ending - records.""" - if self.fp is None: - return - - if self._writing: - raise ValueError("Can't close the ZIP file while there is " - "an open writing handle on it. " - "Close the writing handle before closing the zip.") - - try: - if self.mode in ('w', 'x', 'a') and self._didModify: # write ending records - with self._lock: - if self._seekable: - self.fp.seek(self.start_dir) - self._write_end_record() - finally: - fp = self.fp - self.fp = None - self._fpclose(fp) - - def _write_end_record(self): - for zinfo in self.filelist: # write central directory - dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - extra = [] - if zinfo.file_size > ZIP64_LIMIT \ - or zinfo.compress_size > ZIP64_LIMIT: - extra.append(zinfo.file_size) - extra.append(zinfo.compress_size) - file_size = 0xffffffff - compress_size = 0xffffffff - else: - file_size = zinfo.file_size - compress_size = zinfo.compress_size - - if zinfo.header_offset > ZIP64_LIMIT: - extra.append(zinfo.header_offset) - header_offset = 0xffffffff - else: - header_offset = zinfo.header_offset - - extra_data = zinfo.extra - min_version = 0 - if extra: - # Append a ZIP64 field to the extra's - extra_data = struct.pack( - ' ZIP_FILECOUNT_LIMIT: - requires_zip64 = "Files count" - elif centDirOffset > ZIP64_LIMIT: - requires_zip64 = "Central directory offset" - elif centDirSize > ZIP64_LIMIT: - requires_zip64 = "Central directory size" - if requires_zip64: - # Need to write the ZIP64 end-of-archive records - if not self._allowZip64: - raise LargeZipFile(requires_zip64 + - " would require ZIP64 extensions") - zip64endrec = struct.pack( - structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset) - self.fp.write(zip64endrec) - - zip64locrec = struct.pack( - structEndArchive64Locator, - stringEndArchive64Locator, 0, pos2, 1) - self.fp.write(zip64locrec) - centDirCount = min(centDirCount, 0xFFFF) - centDirSize = min(centDirSize, 0xFFFFFFFF) - centDirOffset = min(centDirOffset, 0xFFFFFFFF) - - endrec = struct.pack(structEndArchive, stringEndArchive, - 0, 0, centDirCount, centDirCount, - centDirSize, centDirOffset, len(self._comment)) - self.fp.write(endrec) - self.fp.write(self._comment) - self.fp.flush() - - def _fpclose(self, fp): - assert self._fileRefCnt > 0 - self._fileRefCnt -= 1 - if not self._fileRefCnt and not self._filePassed: - fp.close() - - -class PyZipFile(ZipFile): - """Class to create ZIP archives with Python library files and packages.""" - - def __init__(self, file, mode="r", compression=ZIP_STORED, - allowZip64=True, optimize=-1): - ZipFile.__init__(self, file, mode=mode, compression=compression, - allowZip64=allowZip64) - self._optimize = optimize - - def writepy(self, pathname, basename="", filterfunc=None): - """Add all files from "pathname" to the ZIP archive. - - If pathname is a package directory, search the directory and - all package subdirectories recursively for all *.py and enter - the modules into the archive. If pathname is a plain - directory, listdir *.py and enter all modules. Else, pathname - must be a Python *.py file and the module will be put into the - archive. Added modules are always module.pyc. - This method will compile the module.py into module.pyc if - necessary. - If filterfunc(pathname) is given, it is called with every argument. - When it is False, the file or directory is skipped. - """ - pathname = os.fspath(pathname) - if filterfunc and not filterfunc(pathname): - if self.debug: - label = 'path' if os.path.isdir(pathname) else 'file' - print('%s %r skipped by filterfunc' % (label, pathname)) - return - dir, name = os.path.split(pathname) - if os.path.isdir(pathname): - initname = os.path.join(pathname, "__init__.py") - if os.path.isfile(initname): - # This is a package directory, add it - if basename: - basename = "%s/%s" % (basename, name) - else: - basename = name - if self.debug: - print("Adding package in", pathname, "as", basename) - fname, arcname = self._get_codename(initname[0:-3], basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - dirlist = os.listdir(pathname) - dirlist.remove("__init__.py") - # Add all *.py files and package subdirectories - for filename in dirlist: - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if os.path.isdir(path): - if os.path.isfile(os.path.join(path, "__init__.py")): - # This is a package directory, add it - self.writepy(path, basename, - filterfunc=filterfunc) # Recursive call - elif ext == ".py": - if filterfunc and not filterfunc(path): - if self.debug: - print('file %r skipped by filterfunc' % path) - continue - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - else: - # This is NOT a package directory, add its files at top level - if self.debug: - print("Adding files from directory", pathname) - for filename in os.listdir(pathname): - path = os.path.join(pathname, filename) - root, ext = os.path.splitext(filename) - if ext == ".py": - if filterfunc and not filterfunc(path): - if self.debug: - print('file %r skipped by filterfunc' % path) - continue - fname, arcname = self._get_codename(path[0:-3], - basename) - if self.debug: - print("Adding", arcname) - self.write(fname, arcname) - else: - if pathname[-3:] != ".py": - raise RuntimeError( - 'Files added with writepy() must end with ".py"') - fname, arcname = self._get_codename(pathname[0:-3], basename) - if self.debug: - print("Adding file", arcname) - self.write(fname, arcname) - - def _get_codename(self, pathname, basename): - """Return (filename, archivename) for the path. - - Given a module name path, return the correct file path and - archive name, compiling if necessary. For example, given - /python/lib/string, return (/python/lib/string.pyc, string). - """ - def _compile(file, optimize=-1): - import py_compile - if self.debug: - print("Compiling", file) - try: - py_compile.compile(file, doraise=True, optimize=optimize) - except py_compile.PyCompileError as err: - print(err.msg) - return False - return True - - file_py = pathname + ".py" - file_pyc = pathname + ".pyc" - pycache_opt0 = importlib.util.cache_from_source( - file_py, optimization='') - pycache_opt1 = importlib.util.cache_from_source( - file_py, optimization=1) - pycache_opt2 = importlib.util.cache_from_source( - file_py, optimization=2) - if self._optimize == -1: - # legacy mode: use whatever file is present - if (os.path.isfile(file_pyc) and - os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): - # Use .pyc file. - arcname = fname = file_pyc - elif (os.path.isfile(pycache_opt0) and - os.stat(pycache_opt0).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt0 - arcname = file_pyc - elif (os.path.isfile(pycache_opt1) and - os.stat(pycache_opt1).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt1 - arcname = file_pyc - elif (os.path.isfile(pycache_opt2) and - os.stat(pycache_opt2).st_mtime >= os.stat(file_py).st_mtime): - # Use the __pycache__/*.pyc file, but write it to the legacy pyc - # file name in the archive. - fname = pycache_opt2 - arcname = file_pyc - else: - # Compile py into PEP 3147 pyc file. - if _compile(file_py): - if sys.flags.optimize == 0: - fname = pycache_opt0 - elif sys.flags.optimize == 1: - fname = pycache_opt1 - else: - fname = pycache_opt2 - arcname = file_pyc - else: - fname = arcname = file_py - else: - # new mode: use given optimization level - if self._optimize == 0: - fname = pycache_opt0 - arcname = file_pyc - else: - arcname = file_pyc - if self._optimize == 1: - fname = pycache_opt1 - elif self._optimize == 2: - fname = pycache_opt2 - else: - msg = "invalid value for 'optimize': {!r}".format( - self._optimize) - raise ValueError(msg) - if not (os.path.isfile(fname) and - os.stat(fname).st_mtime >= os.stat(file_py).st_mtime): - if not _compile(file_py, optimize=self._optimize): - fname = arcname = file_py - archivename = os.path.split(arcname)[1] - if basename: - archivename = "%s/%s" % (basename, archivename) - return (fname, archivename) - - -def main(args=None): - import argparse - - description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='', - help='Show listing of a zipfile') - group.add_argument('-e', '--extract', nargs=2, - metavar=('', ''), - help='Extract zipfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('', ''), - help='Create zipfile from sources') - group.add_argument('-t', '--test', metavar='', - help='Test if a zipfile is valid') - args = parser.parse_args(args) - - if args.test is not None: - src = args.test - with ZipFile(src, 'r') as zf: - badfile = zf.testzip() - if badfile: - print( - "The following enclosed file is corrupted: {!r}".format(badfile)) - print("Done testing") - - elif args.list is not None: - src = args.list - with ZipFile(src, 'r') as zf: - zf.printdir() - - elif args.extract is not None: - src, curdir = args.extract - with ZipFile(src, 'r') as zf: - zf.extractall(curdir) - - elif args.create is not None: - zip_name = args.create.pop(0) - files = args.create - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in os.listdir(path): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(zip_name, 'w') as zf: - for path in files: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - - -if __name__ == "__main__": - main() diff --git a/src/code/speed.py b/src/code/speed.py new file mode 100644 index 0000000..823c660 --- /dev/null +++ b/src/code/speed.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# zip 文件里面包含超多小文件的时候， 可以采用这个方案优化解压上传速度 + +import oss2 +import json +import os +import logging +import zipfile +import subprocess +from concurrent.futures import ThreadPoolExecutor + +# Close the info log printed by the oss SDK +logging.getLogger("oss2.api").setLevel(logging.ERROR) +logging.getLogger("oss2.auth").setLevel(logging.ERROR) + +LOGGER = logging.getLogger() + + +def handler(event, context): + """ + The object from OSS will be decompressed automatically . + param: event: The OSS event json string. Including oss object uri and other information. + For detail info, please refer https://help.aliyun.com/document_detail/70140.html?spm=a2c4g.11186623.6.578.5eb8cc74AJCA9p#OSS + + param: context: The function context, including credential and runtime info. + + For detail info, please refer to https://help.aliyun.com/document_detail/56316.html#using-context + """ + evt_lst = json.loads(event) + creds = context.credentials + auth = oss2.StsAuth( + creds.access_key_id, creds.access_key_secret, creds.security_token + ) + + evt = evt_lst["events"][0] + bucket_name = evt["oss"]["bucket"]["name"] + endpoint = "oss-" + evt["region"] + "-internal.aliyuncs.com" + bucket = oss2.Bucket(auth, endpoint, bucket_name) + object_name = evt["oss"]["object"]["key"] + + if "ObjectCreated:PutSymlink" == evt["eventName"]: + object_name = bucket.get_symlink(object_name).target_key + if object_name == "": + raise RuntimeError( + "{} is invalid symlink file".format(evt["oss"]["object"]["key"]) + ) + + file_type = os.path.splitext(object_name)[1] + + if file_type != ".zip": + raise RuntimeError("{} filetype is not zip".format(object_name)) + + LOGGER.info("start to decompress zip file = {}".format(object_name)) + + lst = object_name.split("/") + zip_name = lst[-1] + PROCESSED_DIR = os.environ.get("PROCESSED_DIR", "") + RETAIN_FILE_NAME = os.environ.get("RETAIN_FILE_NAME", "") + if RETAIN_FILE_NAME == "false": + newKeyPrefix = PROCESSED_DIR + else: + newKeyPrefix = os.path.join(PROCESSED_DIR, zip_name) + newKeyPrefix = newKeyPrefix.replace(".zip", "/") + + tmpWorkDir = "/tmp/{}".format(context.request_id) + if not os.path.exists(tmpWorkDir): + os.makedirs(tmpWorkDir) + + tmpZipfile = "{}/{}".format(tmpWorkDir, zip_name) + bucket.get_object_to_file(object_name, tmpZipfile) + + with zipfile.ZipFile(tmpZipfile) as zip_file: + zip_list = zip_file.namelist() + for f in zip_list: + zip_file.extract(f, tmpWorkDir) + + os.remove(tmpZipfile) + try: + listDir(tmpWorkDir, bucket, newKeyPrefix) + except Exception as e: + LOGGER.error(e) + finally: + subprocess.check_call("rm -rf {}".format(tmpWorkDir), shell=True) + + +def upload_file(bucket, newkey, pathname): + """用于上传单个文件的辅助函数""" + bucket.put_object_from_file(newkey, pathname) + + +def listDir(destDir, bucket, newKeyPrefix): + # 创建一个最大线程数为3的线程池 + with ThreadPoolExecutor(max_workers=3) as executor: + for filename in os.listdir(destDir): + pathname = os.path.join(destDir, filename) + if os.path.isdir(pathname): + listDir(pathname, bucket, newKeyPrefix) + else: + newkey = os.path.join(newKeyPrefix, "/".join(pathname.split("/")[3:])) + # 提交任务到线程池 + executor.submit(upload_file, bucket, newkey, pathname) diff --git a/src/fail/index.py b/src/fail/index.py index b3e86ac..07b4508 100644 --- a/src/fail/index.py +++ b/src/fail/index.py @@ -4,7 +4,7 @@ def handler(event, context): logger = logging.getLogger() - logger.info('destnation fail: {}'.format(event)) + logger.info("destnation fail: {}".format(event)) # do your things # ... return {} diff --git a/src/s.yaml b/src/s.yaml index bb5f93f..e25eeb9 100644 --- a/src/s.yaml +++ b/src/s.yaml @@ -1,15 +1,6 @@ # ------------------------------------ -# If you need English case, you can refer to [s_en.yaml] file -# ------------------------------------ -# 欢迎您使用阿里云函数计算 FC 组件进行项目开发 -# 组件仓库地址：https://github.com/devsapp/fc3 -# 组件帮助文档：https://github.com/devsapp/fc3/blob/master/docs/zh/readme.md -# Yaml参考文档：https://github.com/devsapp/fc3/blob/master/docs/zh/yaml/readme.md -# 关于Serverless Devs和FC组件的关系、如何声明/部署多个函数等问题，可以参考文档：https://github.com/devsapp/fc3/blob/master/docs/zh/tips.md -# 关于如何做CICD等问题，可以参考：https://www.serverless-devs.com/serverless-devs/cicd -# 关于如何进行环境划分等问题，可以参考：https://www.serverless-devs.com/serverless-devs/extend -# 更多函数计算案例，可参考：https://github.com/devsapp/awesome/ -# 有问题快来钉钉群问一下吧：33947367 +# https://docs.serverless-devs.com/user-guide/aliyun/#fc3 +# 有问题快来钉钉群问一下吧：33947367 # ------------------------------------ edition: 3.0.0 name: UnZip OSS @@ -37,9 +28,11 @@ resources: invocationRole: '{{ triggerRoleArn }}' sourceArn: acs:oss:${this.props.region}:${config("AccountID")}:{{ bucketName }} handler: index.handler - timeout: 7200 - runtime: python3 - memorySize: 3072 + timeout: 21600 + runtime: python3.10 + cpu: 2 + memorySize: 4096 + diskSize: 10240 environmentVariables: PROCESSED_DIR: '{{ processedDir }}' RETAIN_FILE_NAME: '{{ retainFileName }}' @@ -51,7 +44,7 @@ resources: destinationConfig: onFailure: destination: acs:fc:${this.props.region}::functions/dest-fail - maxAsyncEventAgeInSeconds: 7200 + maxAsyncEventAgeInSeconds: 86400 maxAsyncRetryAttempts: 3 dest-fail: component: fc3 From 0a87b69aa5cd20365abc4ff5f4b64fa5acb22540 Mon Sep 17 00:00:00 2001 From: mozhou52 Date: Wed, 23 Oct 2024 18:21:43 +0800 Subject: [PATCH 4/7] fix:readme bug --- publish.yaml | 2 +- readme.md | 150 +++++++++++++++++++++++++++++++++++++++++++++++++- src/readme.md | 36 ++++-------- 3 files changed, 161 insertions(+), 27 deletions(-) diff --git a/publish.yaml b/publish.yaml index de4b688..e02da6c 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,7 +1,7 @@ Edition: 3.0.0 Type: Project Name: start-unzip-oss-v3 -Version: 0.0.11 +Version: 0.0.13 Provider: - 阿里云 Description: 本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 diff --git a/readme.md b/readme.md index 1105835..11e2b0a 120000 --- a/readme.md +++ b/readme.md @@ -1 +1,149 @@ -src/readme.md \ No newline at end of file + +> 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ${模版名称}` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 + +# start-unzip-oss-v3 帮助文档 + + + +本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 + + + + + + + + + + + + + + + +## 前期准备 + +使用该项目，您需要有开通以下服务并拥有对应权限： + + + + + +| 服务/业务 | 权限 | 相关文档 | +| --- | --- | --- | +| 函数计算 | AliyunFCFullAccess | [帮助文档](https://help.aliyun.com/product/2508973.html) [计费文档](https://help.aliyun.com/document_detail/2512928.html) | +| 对象存储 | AliyunOSSFullAccess | [帮助文档](https://help.aliyun.com/zh/oss) [计费文档](https://help.aliyun.com/zh/oss/product-overview/billing) | + + + + + + + + + + + + + + + +## 部署 & 体验 + + + +- :fire: 通过 [Serverless 应用中心](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss-v3) ， + [![Deploy with Severless Devs](https://img.alicdn.com/imgextra/i1/O1CN01w5RFbX1v45s8TIXPz_!!6000000006118-55-tps-95-28.svg)](https://fcnext.console.aliyun.com/applications/create?template=start-unzip-oss-v3) 该应用。 + + + + +- 通过 [Serverless Devs Cli](https://docs.serverless-devs.com/user-guide/install) 进行部署： + - [安装 Serverless Devs Cli 开发者工具](https://docs.serverless-devs.com/user-guide/install) ，并进行[授权信息配置]( https://docs.serverless-devs.com/user-guide/config) ； + - 初始化项目：`s init start-unzip-oss-v3 -d start-unzip-oss-v3` + - 进入项目，并进行项目部署：`cd start-unzip-oss-v3 && s deploy -y` + + + +## 案例介绍 + + + +本案例是将zip文件进行 unzip 解压，快速创建并部署到阿里云函数计算 FC。 + +匹配解压规则的ZIP文件在上传到OSS后，会自动触发函数计算进行解压。文件解压完成后，会存储至OSS的指定目录中。 + +- 建议使用 UTF-8 或 GB 2312 编码命名您的文件或文件夹，否则可能会出现解压后的文件或文件夹名称出现乱码、解压过程中断等问题。 + +- 归档或冷归档类型的文件需先解冻再解压。 + +- 建议 ZIP 包 + zip包里面最大单文件的大小之和不要超过 10 GB，否则解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) + +- 默认设置的函数执行时长为6小时， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24小时 + + ![](https://help-static-aliyun-doc.aliyuncs.com/assets/img/zh-CN/5901957271/CAEQNBiBgMCKx9z24RgiIGMyMTE2ZDczNmNkYjQ2ZmI4MDNkYmYwMDIyZWE0ODkx3963382_20230830144006.372.svg) + + + +## 使用流程 + + + +### 查看部署的案例 + +部署时， 选择的OSS存储桶名，以及前缀名，如： + +![](https://img.alicdn.com/imgextra/i4/O1CN01KYL4lB1Obdl63LB39_!!6000000001724-0-tps-2336-1108.jpg) + + +部署完成后，在oss对象存储中找到相应的OSS存储桶名，向该存储桶上传zip文件，如： + +![](https://img.alicdn.com/imgextra/i2/O1CN01ew8GAi1pDomkNrcuC_!!6000000005327-0-tps-1874-850.jpg) + +在控制台就可以查看到调用日志，如： + +![](https://img.alicdn.com/imgextra/i4/O1CN01Yv7Jsc1UJfur6G0WU_!!6000000002497-0-tps-2842-776.jpg) + +#### 参数说明 + +![](https://img.alicdn.com/imgextra/i3/O1CN014DqNwc1jgmBgXBzqW_!!6000000004578-0-tps-1592-760.jpg) + +#### 配置示例 +![](https://img.alicdn.com/imgextra/i4/O1CN01WYPc2X1wcjhUAXa65_!!6000000006329-0-tps-1596-876.jpg) + + +### 二次开发 +您可以通过云端控制台的开发功能进行二次开发。如果您之前是在本地创建的项目案例，也可以在本地项目目录`start-unzip-oss-v3`文件夹下，对项目进行二次开发。开发完成后，可以通过`s deploy`进行快速部署。 + +上面的示例, 解压保存回去的都还是本身触发函数的 OSS， 如果是保存到其他 OSS， 直接将 `bucket.put_object(newKey + name, file_obj)` 这样上传回 oss 的代码修改下即可， 比如： + +```python +auth = oss2.Auth('', '') +dst_bucket_name = "xxx-bucket" +dst_bucket = oss2.Bucket(auth, endpoint, dst_bucket_name) +dst_bucket.put_object(newKey + name, file_obj) + +``` + + + +## 注意事项 + + + + + + + + +## 开发者社区 + +您如果有关于错误的反馈或者未来的期待，您可以在 [Serverless Devs repo Issues](https://github.com/serverless-devs/serverless-devs/issues) 中进行反馈和交流。如果您想要加入我们的讨论组或者了解 FC 组件的最新动态，您可以通过以下渠道进行： + +

+ +| | | +| --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +|

微信公众号：`serverless`
|
钉钉交流群：`33947367`
| +

+ diff --git a/src/readme.md b/src/readme.md index cb459a3..11e2b0a 100644 --- a/src/readme.md +++ b/src/readme.md @@ -2,17 +2,6 @@ > 注：当前项目为 Serverless Devs 应用，由于应用中会存在需要初始化才可运行的变量（例如应用部署地区、函数名等等），所以**不推荐**直接 Clone 本仓库到本地进行部署或直接复制 s.yaml 使用，**强烈推荐**通过 `s init ${模版名称}` 的方法或应用中心进行初始化，详情可参考[部署 & 体验](#部署--体验) 。 # start-unzip-oss-v3 帮助文档 -

- - - - - - - - - -

@@ -22,7 +11,7 @@ -- [:smiley_cat: 代码](https://github.com/devsapp/start-unzip-oss/tree/V3/src) + @@ -69,8 +58,8 @@ -- 通过 [Serverless Devs Cli](https://www.serverless-devs.com/serverless-devs/install) 进行部署： - - [安装 Serverless Devs Cli 开发者工具](https://www.serverless-devs.com/serverless-devs/install) ，并进行[授权信息配置](https://docs.serverless-devs.com/fc/config) ； +- 通过 [Serverless Devs Cli](https://docs.serverless-devs.com/user-guide/install) 进行部署： + - [安装 Serverless Devs Cli 开发者工具](https://docs.serverless-devs.com/user-guide/install) ，并进行[授权信息配置]( https://docs.serverless-devs.com/user-guide/config) ； - 初始化项目：`s init start-unzip-oss-v3 -d start-unzip-oss-v3` - 进入项目，并进行项目部署：`cd start-unzip-oss-v3 && s deploy -y` @@ -88,13 +77,11 @@ - 归档或冷归档类型的文件需先解冻再解压。 -- 解压单个压缩包的最大时间是 2 小时，超过 2 小时未完成的任务会解压失败。 +- 建议 ZIP 包 + zip包里面最大单文件的大小之和不要超过 10 GB，否则解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) -- 建议 ZIP 包里面的单文件大小最好不超过 1 GB，否则可能解压失败。如果出现这个场景， 请参考：[unzip-oss-with-nas](https://github.com/zhaohang88/unzip-oss-nas) +- 默认设置的函数执行时长为6小时， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24小时 -- 默认设置的函数执行时长为2h， 如果不满足需求， 自己直接调整函数的 timeout, 最大可到 24h - -![](http://image.editor.devsapp.cn/alibaba/4A5uks4sawFd26h9ksuc.png) + ![](https://help-static-aliyun-doc.aliyuncs.com/assets/img/zh-CN/5901957271/CAEQNBiBgMCKx9z24RgiIGMyMTE2ZDczNmNkYjQ2ZmI4MDNkYmYwMDIyZWE0ODkx3963382_20230830144006.372.svg) @@ -119,11 +106,10 @@ #### 参数说明 -![](http://image.editor.devsapp.cn/alibaba/kD1lbEw48Er4s27212ri.png) - +![](https://img.alicdn.com/imgextra/i3/O1CN014DqNwc1jgmBgXBzqW_!!6000000004578-0-tps-1592-760.jpg) #### 配置示例 -![](http://image.editor.devsapp.cn/alibaba/lASAfezjvifa9Cwawht6.png) +![](https://img.alicdn.com/imgextra/i4/O1CN01WYPc2X1wcjhUAXa65_!!6000000006329-0-tps-1596-876.jpg) ### 二次开发 @@ -156,8 +142,8 @@ dst_bucket.put_object(newKey + name, file_obj)

-| | | | -| --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -|

微信公众号：`serverless`
|
微信小助手：`xiaojiangwh`
|
钉钉交流群：`33947367`
| +| | | +| --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +|
微信公众号：`serverless`
|
钉钉交流群：`33947367`
|

From 029fc03a9476824b009a5bba0924967dcbdf9975 Mon Sep 17 00:00:00 2001 From: xiliu Date: Thu, 24 Oct 2024 19:28:57 +0800 Subject: [PATCH 5/7] Update registry-publish.yml --- .github/workflows/registry-publish.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/registry-publish.yml b/.github/workflows/registry-publish.yml index c1c1c2a..1665e86 100644 --- a/.github/workflows/registry-publish.yml +++ b/.github/workflows/registry-publish.yml @@ -1,14 +1,17 @@ name: publish package to registry on: - release: - types: [created] + push: + branches: + - V3 jobs: - deploy: + release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - run: git config --global core.symlinks false + shell: bash + - uses: actions/checkout@v4 - uses: actions/setup-node@v2 with: node-version: 16 @@ -25,4 +28,4 @@ jobs: s registry login --token ${{ secrets.alibaba_registry_v3_publish_token }} - name: release prod run: | - s registry publish \ No newline at end of file + s registry publish From 611ffa020f5c6c5a3aeaab019c47e1f6097392e5 Mon Sep 17 00:00:00 2001 From: rsonghuster Date: Thu, 14 Nov 2024 11:00:17 +0800 Subject: [PATCH 6/7] add support nas tips --- .gitignore | 3 ++- publish.yaml | 6 +++--- src/code/index.py | 17 ++++++++++++----- src/code/speed.py | 3 ++- src/s.yaml | 3 ++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index b5f940e..5b26172 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.s \ No newline at end of file +.s +git-statistics.sh \ No newline at end of file diff --git a/publish.yaml b/publish.yaml index e02da6c..c2fbf26 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,7 +1,7 @@ Edition: 3.0.0 Type: Project Name: start-unzip-oss-v3 -Version: 0.0.13 +Version: 0.0.14 Provider: - 阿里云 Description: 本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 @@ -86,12 +86,12 @@ Parameters: prefix: title: 前缀 type: string - default: src + default: src/ description: 配置触发函数计算的文件前缀。您上传指定前缀的ZIP文件或将ZIP文件上传至指定目录会触发函数计算。置空此项则匹配所有上传的ZIP文件，置空此项可能会触发循环执行，建议您配置文件前缀。 processedDir: title: 解压目标目录 type: string - default: dst + default: dst/ description: 将匹配到的文件解压到此目标目录下。为防止循环触发产生不必要的费用，建议您设置不同于前缀的目标目录。 retainFileName: title: 是否保留压缩文件名为路径目录 diff --git a/src/code/index.py b/src/code/index.py index a4085d4..edf78b4 100644 --- a/src/code/index.py +++ b/src/code/index.py @@ -24,6 +24,7 @@ import logging import zipfile import chardet +import shutil # Close the info log printed by the oss SDK logging.getLogger("oss2.api").setLevel(logging.ERROR) @@ -86,9 +87,11 @@ def handler(event, context): object_sizeMB = evt["oss"]["object"]["size"] / 1024 / 1024 LOGGER.info("{} size is = {}MB".format(object_name, object_sizeMB)) - if object_sizeMB > 10240 * 0.9: + WORK_DIR = os.environ.get("WORK_DIR", "/tmp") + + if WORK_DIR == "/tmp" and object_sizeMB > 10240 * 0.9: raise RuntimeError( - "{} size is too large; please use NAS, refer: https://github.com/zhaohang88/unzip-oss-nas".format( + "{} size is too large; Please use NAS and set the WORK_DIR environment variable to specify the NAS mount directory. For reference, see: https://help.aliyun.com/zh/functioncompute/fc-3-0/user-guide/configure-a-nas-file-system-1".format( object_name ) ) @@ -120,7 +123,7 @@ def handler(event, context): newKeyPrefix = os.path.join(PROCESSED_DIR, zip_name) newKeyPrefix = newKeyPrefix.replace(".zip", "/") - tmpWorkDir = "/tmp/{}".format(context.request_id) + tmpWorkDir = "{}/{}".format(WORK_DIR, context.request_id) if not os.path.exists(tmpWorkDir): os.makedirs(tmpWorkDir) @@ -134,10 +137,13 @@ def handler(event, context): continue f_size = file_info.file_size if ( - object_sizeMB + f_size / 1024 / 1024 > 10240 * 0.99 + WORK_DIR == "/tmp" + and object_sizeMB + f_size / 1024 / 1024 > 10240 * 0.99 ): # if zip file + one file size > 0.99G, skip extract and upload LOGGER.error( - "{} size is too large; skip extract and upload".format(f) + "{} size is too large; skip extract and upload. Please use NAS and set the WORK_DIR environment variable to specify the NAS mount directory. For reference, see: https://help.aliyun.com/zh/functioncompute/fc-3-0/user-guide/configure-a-nas-file-system-1".format( + file_info.filename + ) ) continue zip_file.extract(file_info.filename, tmpWorkDir) @@ -152,3 +158,4 @@ def handler(event, context): LOGGER.error(e) finally: os.remove(tmpZipfile) + shutil.rmtree(tmpWorkDir) diff --git a/src/code/speed.py b/src/code/speed.py index 823c660..959a397 100644 --- a/src/code/speed.py +++ b/src/code/speed.py @@ -62,7 +62,8 @@ def handler(event, context): newKeyPrefix = os.path.join(PROCESSED_DIR, zip_name) newKeyPrefix = newKeyPrefix.replace(".zip", "/") - tmpWorkDir = "/tmp/{}".format(context.request_id) + WORK_DIR = os.environ.get("WORK_DIR", "/tmp") + tmpWorkDir = "{}/{}".format(WORK_DIR, context.request_id) if not os.path.exists(tmpWorkDir): os.makedirs(tmpWorkDir) diff --git a/src/s.yaml b/src/s.yaml index e25eeb9..9ec2eed 100644 --- a/src/s.yaml +++ b/src/s.yaml @@ -8,7 +8,7 @@ access: '{{ access }}' vars: region: '{{ region }}' resources: - fc-zip-oss-service: + fc-unzip-oss-service: component: fc3 props: region: ${vars.region} @@ -36,6 +36,7 @@ resources: environmentVariables: PROCESSED_DIR: '{{ processedDir }}' RETAIN_FILE_NAME: '{{ retainFileName }}' + WORK_DIR: "/tmp" internetAccess: true role: '{{ roleArn }}' functionName: '{{ functionName }}' From 3e63a181c7a709c44b1a9806bdf90fc9be1163f3 Mon Sep 17 00:00:00 2001 From: mozhou52 Date: Thu, 3 Jul 2025 16:02:19 +0800 Subject: [PATCH 7/7] fix: support me-central-1 --- publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/publish.yaml b/publish.yaml index c2fbf26..cefb7f0 100644 --- a/publish.yaml +++ b/publish.yaml @@ -1,7 +1,7 @@ Edition: 3.0.0 Type: Project Name: start-unzip-oss-v3 -Version: 0.0.14 +Version: 0.0.15 Provider: - 阿里云 Description: 本案例是将解压zip文件工具 unzip，快速创建并部署到阿里云函数计算 FC。 @@ -58,6 +58,7 @@ Parameters: - us-west-1 - us-east-1 - ap-south-1 + - me-central-1 functionName: title: 函数名 type: string