Added crc64 webassembly crc64 c++ addon

.eslintrc.js

@@ -3,33 +3,38 @@ module.exports = {
   extends: ['airbnb', 'eslint-config-ali/typescript'],
   parserOptions: {
     ecmaFeatures: {
-      experimentalObjectRestSpread: true,
-    },
+      experimentalObjectRestSpread: true
+    }
   },
   env: {
     browser: true,
     node: true,
     es6: true,
     mocha: true,
     jasmine: true,
-    jest: true,
+    jest: true
   },
   rules: {
     indent: ['error', 2],
     // override default options
     'no-underscore-dangle': [0],
     'no-plusplus': [0],
-    'no-return-await':[0],
+    'no-return-await': [0],
     'no-param-reassign': [0],
-    'max-len': ['warn', 120, 2, {
-      ignoreUrls: true,
-      ignoreComments: false,
-      ignoreRegExpLiterals: true,
-      ignoreStrings: true,
-      ignoreTemplateLiterals: true,
-    }],
+    'max-len': [
+      'warn',
+      120,
+      2,
+      {
+        ignoreUrls: true,
+        ignoreComments: false,
+        ignoreRegExpLiterals: true,
+        ignoreStrings: true,
+        ignoreTemplateLiterals: true
+      }
+    ],
     'no-buffer-constructor': [2],
-    "comma-dangle": [0],
-    'import/prefer-default-export': [0],
+    'comma-dangle': [0],
+    'import/prefer-default-export': [0]
   }
 };

README.md

@@ -183,6 +183,10 @@ All operation use es7 async/await to implement. All api is async function.
   - [imgClient.listStyle([options])](#imgclientliststyleoptions)
   - [imgClient.deleteStyle(name[, options])](#imgclientdeletestylename-options)
   - [imgClient.signatureUrl(name)](#imgclientsignatureurlname)
+- [Utils](#utils)
+  - crc
+    - [.checkCrc64(content, result)](#checkcrc64)
+    - [.checkCrc64File(filePath,callback)](#checkcrc64file)
 - [Known Errors](#known-errors)
 
 ## Node Usage
@@ -1665,6 +1669,7 @@ parameters:
     - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT`
     - See more: [PutObject](https://help.aliyun.com/document_detail/31978.html#title-yxe-96d-x61)
   - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process
+  - [crc64] {Boolean} whether to enable crc64. If you are using a browser environment please check the oss-crc64-plug plugin
 
 Success will return the object information.
 
@@ -1788,6 +1793,7 @@ parameters:
     - 'Content-Disposition' object name for download, e.g.: `Content-Disposition: somename`
     - 'Content-Encoding' object content encoding for download, e.g.: `Content-Encoding: gzip`
     - 'Expires' expires time for download, an absolute date and time. e.g.: `Tue, 08 Dec 2020 13:49:43 GMT`
+  - [crc64] {Boolean} whether to enable crc64. If you are using a browser environment please check the oss-crc64-plug plugin
 
 Success will return the object information.
 
@@ -2023,6 +2029,7 @@ parameters:
         otherwise throw PreconditionFailedError
     - 'If-None-Match' object etag not equal this will return 200 and object meta,
         otherwise return 304 not modified
+  - [crc64] {Boolean} whether to enable crc64 . If you are using a browser environment please check the oss-crc64-plug plugin
 
 Success will return the info contains response.
 
@@ -3210,6 +3217,7 @@ parameters:
     - **NOTE**: Some headers are [disabled in browser][disabled-browser-headers]
   - [timeout] {Number} Milliseconds before a request is considered to be timed out
   - [disabledMD5] {Boolean} default true, it just work in Browser. if false,it means that MD5 is automatically calculated for uploaded files. **_NOTE:_** Synchronous computing tasks will block the main process
+  - [crc64] {Boolean} whether to enable crc64 . If you are using a browser environment please check the oss-crc64-plug plugin
 
 Success will return:
 
@@ -4448,6 +4456,44 @@ const url = imgClient.signatureUrl('
 // http://thumbnail.myimageservice.com/demo.jpg@200w_200h?OSSAccessKeyId=uZxyLARzYZtGwHKY&Expires=1427803849&Signature=JSPRe06%2FjQpQSj5zlx2ld1V%2B35I%3D
 ```
 
+
+## Utils
+
+### .checkCrc64
+Whether the calculation is consistent with the results
+If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
+
+parameters:
+
+- content {String|Buffer} what needs to be calculated
+- result  {String} results to be compared
+
+Success will return boolean 
+
+```js
+store.checkCrc64(123456789, '11051210869376104954')
+
+store.checkCrc64(Buffer.from('123456789','11051210869376104954'))
+```
+
+### .checkCrc64File
+Compare Documents
+If you are using a browser environment please check the [oss-crc64-plug](https://github.com/taotao7/oss-crc64-plugin) plugin
+
+parameters:
+
+- filePath {String} file path
+- callback {Function} callback functions
+
+```js
+store.checkCrc64File(path.join(__dirname,'test.txt'),(err,res)=>{
+  if(!err){
+    // the returned crc64 value
+    console.log(res)
+  }
+})
+```
+
 ## Cluster Mode
 
 Cluster mode now only support object operations.

lib/browser/client.js

@@ -43,7 +43,7 @@ function Client(options, ctx) {
     this.options = Client.initOptions(options);
   }
 
-  this.options.cancelFlag = false;// cancel flag: if true need to be cancelled, default false
+  this.options.cancelFlag = false; // cancel flag: if true need to be cancelled, default false
 
   // support custom agent and urllib client
   if (this.options.urllib) {
@@ -68,18 +68,22 @@ module.exports = Client;
 
 Client.initOptions = function initOptions(options) {
   if (!options.stsToken) {
-    console.warn('Please use STS Token for safety, see more details at https://help.aliyun.com/document_detail/32077.html');
+    console.warn(
+      'Please use STS Token for safety, see more details at https://help.aliyun.com/document_detail/32077.html'
+    );
   }
-  const opts = Object.assign({
-    secure: isHttpsWebProtocol(),
-    // for browser compatibility disable fetch.
-    useFetch: false,
-  }, options);
+  const opts = Object.assign(
+    {
+      secure: isHttpsWebProtocol(),
+      // for browser compatibility disable fetch.
+      useFetch: false
+    },
+    options
+  );
 
   return _initOptions(opts);
 };
 
-
 /**
  * prototype
  */
@@ -139,6 +143,11 @@ merge(proto, require('../common/multipart'));
  */
 merge(proto, require('../common/parallel'));
 
+/**
+ * Crc64 method
+ */
+merge(proto, require('../common/utils/crc64'));
+
 /**
  * get OSS signature
  * @param {String} stringToSign
@@ -179,7 +188,12 @@ proto.authorization = function authorization(method, resource, subres, headers)
     parameters: subres
   });
 
-  return signUtils.authorization(this.options.accessKeyId, this.options.accessKeySecret, stringToSign, this.options.headerEncoding);
+  return signUtils.authorization(
+    this.options.accessKeyId,
+    this.options.accessKeySecret,
+    stringToSign,
+    this.options.headerEncoding
+  );
 };
 
 /**
@@ -204,8 +218,8 @@ proto.authorization = function authorization(method, resource, subres, headers)
 proto.request = async function (params) {
   if (this.options.retryMax) {
     return await retry(request.bind(this), this.options.retryMax, {
-      errorHandler: (err) => {
-        const _errHandle = (_err) => {
+      errorHandler: err => {
+        const _errHandle = _err => {
           if (params.stream) return false;
           const statusErr = [-1, -2].includes(_err.status);
           const requestErrorRetryHandle = this.options.requestErrorRetryHandle || (() => true);
@@ -233,7 +247,14 @@ async function request(params) {
   const useStream = !!params.stream;
   try {
     result = await this.urllib.request(reqParams.url, reqParams.params);
-    this.debug('response %s %s, got %s, headers: %j', params.method, reqParams.url, result.status, result.headers, 'info');
+    this.debug(
+      'response %s %s, got %s, headers: %j',
+      params.method,
+      reqParams.url,
+      result.status,
+      result.headers,
+      'info'
+    );
   } catch (err) {
     reqErr = err;
   }
@@ -282,7 +303,7 @@ proto._escape = function _escape(name) {
  */
 
 proto._getUserAgent = function _getUserAgent() {
-  const agent = (process && process.browser) ? 'js' : 'nodejs';
+  const agent = process && process.browser ? 'js' : 'nodejs';
   const sdk = `aliyun-sdk-${agent}/${pkg.version}`;
   let plat = platform.description;
   if (!plat && process) {
@@ -306,7 +327,7 @@ proto._checkUserAgent = function _checkUserAgent(ua) {
  */
 
 proto.checkBrowserAndVersion = function checkBrowserAndVersion(name, version) {
-  return ((bowser.name === name) && (bowser.version.split('.')[0] === version));
+  return bowser.name === name && bowser.version.split('.')[0] === version;
 };
 
 /**
@@ -321,16 +342,20 @@ proto.parseXML = function parseXMLThunk(str) {
     if (Buffer.isBuffer(str)) {
       str = str.toString();
     }
-    xml.parseString(str, {
-      explicitRoot: false,
-      explicitArray: false
-    }, (err, result) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(result);
+    xml.parseString(
+      str,
+      {
+        explicitRoot: false,
+        explicitArray: false
+      },
+      (err, result) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(result);
+        }
       }
-    });
+    );
   });
 };
 
@@ -344,7 +369,8 @@ proto.parseXML = function parseXMLThunk(str) {
 proto.requestError = async function requestError(result) {
   let err = null;
   if (!result.data || !result.data.length) {
-    if (result.status === -1 || result.status === -2) { // -1 is net error , -2 is timeout
+    if (result.status === -1 || result.status === -2) {
+      // -1 is net error , -2 is timeout
       err = new Error(result.message);
       err.name = result.name;
       err.status = result.status;
@@ -375,7 +401,7 @@ proto.requestError = async function requestError(result) {
 
     let info;
     try {
-      info = await this.parseXML(message) || {};
+      info = (await this.parseXML(message)) || {};
     } catch (error) {
       this.debug(message, 'error');
       error.message += `\nraw xml: ${message}`;
@@ -384,7 +410,7 @@ proto.requestError = async function requestError(result) {
       return error;
     }
 
-    let msg = info.Message || (`unknow request error, status: ${result.status}`);
+    let msg = info.Message || `unknow request error, status: ${result.status}`;
     if (info.Condition) {
       msg += ` (condition: ${info.Condition})`;
     }
@@ -400,4 +426,3 @@ proto.requestError = async function requestError(result) {
   this.debug('generate error %j', err, 'error');
   return err;
 };
-

lib/browser/managed-upload.js

@@ -143,7 +143,8 @@ proto._resumeMultipart = async function _resumeMultipart(checkpoint, options) {
           try {
             result = await self._uploadPart(name, uploadId, partNo, data, {
               timeout: options.timeout,
-              disabledMD5: options.disabledMD5
+              disabledMD5: options.disabledMD5,
+              crc64: options.crc64
             });
           } catch (error) {
             if (error.status === 404) {

lib/browser/object.js

@@ -90,6 +90,11 @@ proto.put = async function put(name, file, options) {
   params.successStatuses = [200];
 
   const result = await this.request(params);
+  if (options.crc64) {
+    if (options.crc64(content) !== result.res.headers['x-oss-hash-crc64ecma']) {
+      throw new Error('crc64 check fail');
+    }
+  }
 
   const ret = {
     name,
@@ -251,12 +256,7 @@ proto.listV2 = async function listV2(query, options = {}) {
       type: obj.Type,
       size: Number(obj.Size),
       storageClass: obj.StorageClass,
-      owner: obj.Owner
-        ? {
-            id: obj.Owner.ID,
-            displayName: obj.Owner.DisplayName
-          }
-        : null
+      owner: obj.Owner ? { id: obj.Owner.ID, displayName: obj.Owner.DisplayName } : null
     }));
   }
   let prefixes = result.data.CommonPrefixes || null;

lib/client.js

@@ -95,6 +95,10 @@ merge(proto, require('./common/parallel'));
  * Multipart operations
  */
 merge(proto, require('./common/multipart'));
+/**
+ * Crc64 method
+ */
+merge(proto, require('./common/utils/crc64'));
 /**
  * ImageClient class
  */

lib/common/multipart.js

@@ -3,6 +3,7 @@ const callback = require('./callback');
 const { deepCopyWith } = require('./utils/deepCopy');
 const { isBuffer } = require('./utils/isBuffer');
 const { omit } = require('./utils/omit');
+const { checkCrc64 } = require('./utils/crc64');
 
 const proto = exports;
 
@@ -250,7 +251,29 @@ proto._uploadPart = async function _uploadPart(name, uploadId, partNo, data, opt
   params.successStatuses = [200];
   params.disabledMD5 = options.disabledMD5;
 
+  // current part buffer to calculator
+  let buffers = [];
+  if (opt.crc64) {
+    if (isBrowserEnv) {
+      buffers = params.content;
+    } else {
+      params.stream.on('data', d => {
+        buffers.push(d);
+      });
+    }
+  }
+
   const result = await this.request(params);
+  // check crc64
+  if (opt.crc64) {
+    if (isBrowserEnv) {
+      if (options.crc64(buffers) !== result.res.headers['x-oss-hash-crc64ecma']) {
+        throw new Error('crc64 check fail');
+      }
+    } else if (!checkCrc64(Buffer.concat(buffers), result.res.headers['x-oss-hash-crc64ecma'])) {
+      throw new Error('crc64 check fail');
+    }
+  }
 
   if (!result.res.headers.etag) {
     throw new Error(