CLDSRV-848: add missing checksum algorithms

lib/Config.js

@@ -556,11 +556,15 @@ function parseIntegrityChecks(config) {
         'bucketPutReplication': true,
         'bucketPutVersioning': true,
         'bucketPutWebsite': true,
+        'bucketPutLogging': true,
+        'bucketPutTagging': true,
         'multiObjectDelete': true,
         'objectPutACL': true,
         'objectPutLegalHold': true,
         'objectPutTagging': true,
         'objectPutRetention': true,
+        'objectRestore': true,
+        'completeMultipartUpload': true,
     };
 
     if (config && config.integrityChecks) {

lib/api/api.js

@@ -301,15 +301,20 @@ const api = {
                     }
 
                     const buff = Buffer.concat(post, bodyLength);
+                    return validateMethodChecksumNoChunking(request, buff, log)
+                        .then(error => {
+                            if (error) {
+                                return next(error);
+                            }
 
-                    const err = validateMethodChecksumNoChunking(request, buff, log);
-                    if (err) {
-                        return next(err);
-                    }
-
-                    // Convert array of post buffers into one string
-                    request.post = buff.toString();
-                    return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                            // Convert array of post buffers into one string
+                            request.post = buff.toString();
+                            return next(null, userInfo, authorizationResults, streamingV4Params, infos);
+                        })
+                        .catch(error => {
+                            log.error('error validating checksums', { error });
+                            next(error);
+                        });
                 });
                 return undefined;
             },

lib/api/apiUtils/integrity/validateChecksums.js

@@ -1,42 +1,238 @@
 const crypto = require('crypto');
+const { Crc32 } = require('@aws-crypto/crc32');
+const { Crc32c } = require('@aws-crypto/crc32c');
+const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
 const { errors: ArsenalErrors } = require('arsenal');
 const { config } = require('../../../Config');
 
 const ChecksumError = Object.freeze({
     MD5Mismatch: 'MD5Mismatch',
+    MD5Invalid: 'MD5Invalid',
+    XAmzMismatch: 'XAmzMismatch',
     MissingChecksum: 'MissingChecksum',
+    AlgoNotSupported: 'AlgoNotSupported',
+    AlgoNotSupportedSDK: 'AlgoNotSupportedSDK',
+    MultipleChecksumTypes: 'MultipleChecksumTypes',
+    MissingCorresponding: 'MissingCorresponding',
+    InvalidAlgoValue: 'InvalidAlgoValue',
 });
 
+const algorithms = {
+    'crc64nvme': {
+        'digest': async data => {
+            const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+            const crc = new CrtCrc64Nvme();
+            crc.update(input);
+            const result = await crc.digest();
+            return Buffer.from(result).toString('base64');
+        },
+        'checkExpected': expected => {
+            if (typeof expected !== 'string') {
+                return false;
+            }
+            return expected.length === 12;
+        },
+    },
+    'crc32': {
+        'digest': data => {
+            const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+            return uint32ToBase64(new Crc32().update(input).digest() >>> 0);
+        },
+        'checkExpected': expected => {
+            if (typeof expected !== 'string') {
+                return false;
+            }
+            return expected.length === 8;
+        },
+    },
+    'crc32c': {
+        'digest': data => {
+            const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+            return uint32ToBase64(new Crc32c().update(input).digest() >>> 0);
+        },
+        'checkExpected': expected => {
+            if (typeof expected !== 'string') {
+                return false;
+            }
+            return expected.length === 8;
+        },
+    },
+    'sha1': {
+        'digest': data => {
+            const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+            return crypto.createHash('sha1').update(input).digest('base64');
+        },
+        'checkExpected': expected => {
+            if (typeof expected !== 'string') {
+                return false;
+            }
+            return expected.length === 28;
+        },
+    },
+    'sha256': {
+        'digest': data => {
+            const input = Buffer.isBuffer(data) ? data : Buffer.from(data);
+            return crypto.createHash('sha256').update(input).digest('base64');
+        },
+        'checkExpected': expected => {
+            if (typeof expected !== 'string') {
+                return false;
+            }
+            return expected.length === 44;
+        },
+    }
+};
+
+function uint32ToBase64(num) {
+    const buf = Buffer.alloc(4);
+    buf.writeUInt32BE(num, 0);
+    return buf.toString('base64');
+}
+
+async function validateXAmzChecksums(headers, body) {
+    const checksumHeaders = Object.keys(headers).filter(header => header.startsWith('x-amz-checksum-'));
+    const xAmzCheckumCnt = checksumHeaders.length;
+    if (xAmzCheckumCnt > 1) {
+        return { error: ChecksumError.MultipleChecksumTypes, details: { algorithms: checksumHeaders } };
+    }
+
+    if (xAmzCheckumCnt === 0 && 'x-amz-sdk-checksum-algorithm' in headers) {
+        return {
+            error: ChecksumError.MissingCorresponding,
+            details: { expected: headers['x-amz-sdk-checksum-algorithm'] }
+        };
+    } else if (xAmzCheckumCnt === 0) {
+        return { error: ChecksumError.MissingChecksum, details: null };
+    }
+
+    // No x-amz-sdk-checksum-algorithm we expect one x-amz-checksum-[crc64nvme, crc32, crc32C, sha1, sha256].
+    const algo = checksumHeaders[0].split('-')[3];
+    if (typeof algo !== 'string') {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };
+    }
+
+    if (algo in algorithms === false) {
+        return { error: ChecksumError.AlgoNotSupported, details: { algorithm: algo } };;
+    }
+
+    const expected = headers[`x-amz-checksum-${algo}`];
+    if (!algorithms[algo].checkExpected(expected)) {
+        return { error: ChecksumError.InvalidAlgoValue, details: { algorithm: algo, expected } };
+    }
+
+    const calculated = await algorithms[algo].digest(body);
+    if (expected !== calculated) {
+        return { error: ChecksumError.XAmzMismatch, details: { algorithm: algo, calculated, expected } };
+    }
+
+    // AWS checks x-amz-checksum- first and then x-amz-sdk-checksum-algorithm
+    if ('x-amz-sdk-checksum-algorithm' in headers) {
+        const sdkAlgo = headers['x-amz-sdk-checksum-algorithm'];
+        if (typeof sdkAlgo !== 'string') {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        const sdkLowerAlgo = sdkAlgo.toLowerCase();
+        if (sdkLowerAlgo in algorithms === false) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+
+        // If AWS there is a mismatch, AWS returns the same error as if the algo was invalid.
+        if (sdkLowerAlgo !== algo) {
+            return { error: ChecksumError.AlgoNotSupportedSDK, details: { algorithm: sdkAlgo } };
+        }
+    }
+
+    return null;
+}
+
 /**
  * validateChecksumsNoChunking - Validate the checksums of a request.
  * @param {object} headers - http headers
  * @param {Buffer} body - http request body
  * @return {object} - error
  */
-function validateChecksumsNoChunking(headers, body) {
-    if (headers && 'content-md5' in headers) {
+async function validateChecksumsNoChunking(headers, body) {
+    if (!headers) {
+        return { error: ChecksumError.MissingChecksum, details: null };
+    }
+
+    let md5Present = false;
+    if ('content-md5' in headers) {
+        if (typeof headers['content-md5'] !== 'string') {
+            return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } };
+        }
+
+        if (headers['content-md5'].length !== 24) {
+            return { error: ChecksumError.MD5Invalid, details: { expected: headers['content-md5'] } };
+        }
+
         const md5 = crypto.createHash('md5').update(body).digest('base64');
         if (md5 !== headers['content-md5']) {
             return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } };
         }
 
+        md5Present = true;
+    }
+
+    const err = await validateXAmzChecksums(headers, body);
+    if (err && err.error === ChecksumError.MissingChecksum && md5Present) {
+        // Don't return MissingChecksum if MD5 is present.
         return null;
     }
 
-    return { error: ChecksumError.MissingChecksum, details: null };
+    return err;
 }
 
-function defaultValidationFunc(request, body, log) {
-    const err = validateChecksumsNoChunking(request.headers, body);
-    if (err && err.error !== ChecksumError.MissingChecksum) {
+async function defaultValidationFunc(request, body, log) {
+    const err = await validateChecksumsNoChunking(request.headers, body);
+    if (!err) {
+        return null;
+    }
+
+    if (err.error !== ChecksumError.MissingChecksum) {
         log.debug('failed checksum validation', { method: request.apiMethod }, err);
-        return ArsenalErrors.BadDigest;
     }
 
-    return null;
+    switch (err.error) {
+        case ChecksumError.MissingChecksum:
+            return null;
+        case ChecksumError.XAmzMismatch: {
+            const algoUpper = err.details.algorithm.toUpperCase();
+            return ArsenalErrors.BadDigest.customizeDescription(
+                `The ${algoUpper} you specified did not match the calculated checksum.`
+            );
+        }
+        case ChecksumError.AlgoNotSupported:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'The algorithm type you specified in x-amz-checksum- header is invalid.'
+            );
+        case ChecksumError.AlgoNotSupportedSDK:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'Value for x-amz-sdk-checksum-algorithm header is invalid.'
+            );
+        case ChecksumError.MissingCorresponding:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'x-amz-sdk-checksum-algorithm specified, but no corresponding x-amz-checksum-* ' +
+                'or x-amz-trailer headers were found.'
+            );
+        case ChecksumError.MultipleChecksumTypes:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                'Expecting a single x-amz-checksum- header. Multiple checksum Types are not allowed.'
+            );
+        case ChecksumError.InvalidAlgoValue:
+            return ArsenalErrors.InvalidRequest.customizeDescription(
+                `Value for x-amz-checksum-${err.details.algorithm} header is invalid.`
+            );
+        case ChecksumError.MD5Invalid:
+            return ArsenalErrors.InvalidDigest;
+        default:
+            return ArsenalErrors.BadDigest;
+    }
 }
 
 const methodValidationFunc = Object.freeze({
+    'completeMultipartUpload': defaultValidationFunc,
     'bucketPutACL': defaultValidationFunc,
     'bucketPutCors': defaultValidationFunc,
     'bucketPutEncryption': defaultValidationFunc,
@@ -47,12 +243,15 @@ const methodValidationFunc = Object.freeze({
     'bucketPutReplication': defaultValidationFunc,
     'bucketPutVersioning': defaultValidationFunc,
     'bucketPutWebsite': defaultValidationFunc,
+    'bucketPutLogging': defaultValidationFunc,
+    'bucketPutTagging': defaultValidationFunc,
     // TODO: DeleteObjects requires a checksum. Should return an error if ChecksumError.MissingChecksum.
     'multiObjectDelete': defaultValidationFunc,
     'objectPutACL': defaultValidationFunc,
     'objectPutLegalHold': defaultValidationFunc,
     'objectPutTagging': defaultValidationFunc,
     'objectPutRetention': defaultValidationFunc,
+    'objectRestore': defaultValidationFunc,
 });
 
 /**
@@ -62,14 +261,13 @@ const methodValidationFunc = Object.freeze({
  * @param {object} log - logger
  * @return {object} - error
  */
-function validateMethodChecksumNoChunking(request, body, log) {
+async function validateMethodChecksumNoChunking(request, body, log) {
     if (config.integrityChecks[request.apiMethod]) {
         const validationFunc = methodValidationFunc[request.apiMethod];
         if (!validationFunc) {
-            return null;
+            return null; //await defaultValidationFunc2(request, body, log);
         }
-
-        return validationFunc(request, body, log);
+        return await validationFunc(request, body, log);
     }
 
     return null;

package.json

@@ -27,6 +27,9 @@
     "@aws-sdk/protocol-http": "^3.374.0",
     "@aws-sdk/s3-request-presigner": "^3.901.0",
     "@aws-sdk/signature-v4": "^3.374.0",
+    "@aws-crypto/crc32": "^5.2.0",
+    "@aws-crypto/crc32c": "^5.2.0",
+    "@aws-sdk/crc64-nvme-crt": "^3.989.0",
     "@azure/storage-blob": "^12.28.0",
     "@hapi/joi": "^17.1.1",
     "@smithy/node-http-handler": "^3.0.0",