From 578111c83b59636a83d97485aaea0f4690ea30e4 Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 11 Apr 2017 10:17:28 +1200 Subject: [PATCH 01/11] Update index.js added hash lookup --- index.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 6bab15c..43878a5 100644 --- a/index.js +++ b/index.js @@ -10,10 +10,12 @@ var gutil = require('gulp-util'); var PluginError = gutil.PluginError; var path = require('path'); var defaults = require('lodash.defaults'); - +var hasher = require('hash-files'); module.exports = function(options) { - var filesMap = {}; + + var filesMap = {}, + filesHashes = {}; options = defaults(options || {}, { error: false, // Throw an error in case of duplicate. @@ -26,9 +28,15 @@ module.exports = function(options) { if (file.isStream()) { return this.emit('error', new PluginError('gulp-dedupe', 'Streaming not supported')); } var fullpath = path.resolve(file.path), + hash = hashFiles.sync(fullpath), + h, f; - if ((f = filesMap[fullpath])) { + if ((f = filesMap[fullpath]) || (h = filesHashes[hash])) { + + // fall back to hash lookup + if (!f && h) f = h; + if (options.error) { this.emit('error', new PluginError('gulp-dedupe', 'Duplicate `' + file.path + '`')); } else if (options.same && file.contents.toString() !== f.contents.toString()) { @@ -56,7 +64,9 @@ module.exports = function(options) { return; } else { filesMap[fullpath] = file; + filesHashes[hash] = file; } + this.emit('data', file); } From 9e8e22506c92238a6e61fd7e68f56acc2b6914fd Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 11 Apr 2017 10:18:23 +1200 Subject: [PATCH 02/11] Update updated deps --- package.json | 115 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 31 deletions(-) diff --git a/package.json b/package.json index 746f54f..3bdfa73 100644 --- a/package.json +++ b/package.json @@ -1,35 +1,88 @@ { + "_args": [ + [ + "gulp-dedupe", + "/home/azt3k/dev/catch/ctas" + ] + ], + "_from": "gulp-dedupe@latest", + "_id": "gulp-dedupe@0.0.2", + "_inCache": true, + "_installable": true, + "_location": "/gulp-dedupe", + "_npmUser": { + "email": "dakota@brokenpipe.ru", + "name": "hoho" + }, + "_npmVersion": "1.4.3", + "_phantomChildren": {}, + "_requested": { "name": "gulp-dedupe", - "description": "Check for duplicates in the stream and filter them or throw an error", - "version": "0.0.2", - "homepage": "https://github.com/hoho/gulp-dedupe", - "repository": "git://github.com/hoho/gulp-dedupe.git", - "author": "Marat Abdullin ", - "license": "MIT", - "main": "./index.js", - "keywords": [ - "gulpplugin", - "dedupe", - "duplicates" - ], - "bugs": { - "url": "https://github.com/hoho/gulp-dedupe/issues" - }, - "dependencies": { - "gulp-util": "~3.0.1", - "through": "~2.3.6", - "lodash.defaults": "~2.4.1", - "colors": "~1.0.2", - "diff": "~1.0.8" - }, - "devDependencies": { - "mocha": "*", - "should": "*" - }, - "scripts": { - "test": "mocha" - }, - "engines": { - "node": ">=0.10" + "raw": "gulp-dedupe", + "rawSpec": "", + "scope": null, + "spec": "latest", + "type": "tag" + }, + "_requiredBy": [ + "#DEV:/" + ], + "_resolved": "https://registry.npmjs.org/gulp-dedupe/-/gulp-dedupe-0.0.2.tgz", + "_shasum": "36ef92adf7fcf53e2f096fbe9665d988f9e1ca7e", + "_shrinkwrap": null, + "_spec": "gulp-dedupe", + "_where": "/home/azt3k/dev/catch/ctas", + "author": { + "email": "dakota@brokenpipe.ru", + "name": "Marat Abdullin" + }, + "bugs": { + "url": "https://github.com/hoho/gulp-dedupe/issues" + }, + "dependencies": { + "colors": "~1.0.2", + "diff": "~1.0.8", + "gulp-util": "~3.0.1", + "lodash.defaults": "~2.4.1", + "through": "~2.3.6", + "hash-files": "~1.1.1" + }, + "description": "Check for duplicates in the stream and filter them or throw an error", + "devDependencies": { + "mocha": "*", + "should": "*" + }, + "directories": {}, + "dist": { + "shasum": "36ef92adf7fcf53e2f096fbe9665d988f9e1ca7e", + "tarball": "https://registry.npmjs.org/gulp-dedupe/-/gulp-dedupe-0.0.2.tgz" + }, + "engines": { + "node": ">=0.10" + }, + "homepage": "https://github.com/hoho/gulp-dedupe", + "keywords": [ + "gulpplugin", + "dedupe", + "duplicates" + ], + "license": "MIT", + "main": "./index.js", + "maintainers": [ + { + "email": "dakota@brokenpipe.ru", + "name": "hoho" } + ], + "name": "gulp-dedupe", + "optionalDependencies": {}, + "readme": "ERROR: No README data found!", + "repository": { + "type": "git", + "url": "git://github.com/hoho/gulp-dedupe.git" + }, + "scripts": { + "test": "mocha" + }, + "version": "0.0.2" } From 599e64d57365721a01da7bccc6b39b611eef17a7 Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 11 Apr 2017 10:32:44 +1200 Subject: [PATCH 03/11] Update index.js --- index.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 43878a5..1d37a1b 100644 --- a/index.js +++ b/index.js @@ -24,21 +24,29 @@ module.exports = function(options) { }); function bufferContents(file) { + if (file.isNull()) { return; } if (file.isStream()) { return this.emit('error', new PluginError('gulp-dedupe', 'Streaming not supported')); } var fullpath = path.resolve(file.path), - hash = hashFiles.sync(fullpath), + hash = hasher.sync({files: [fullpath]}), + dupeType = null, h, f; if ((f = filesMap[fullpath]) || (h = filesHashes[hash])) { // fall back to hash lookup - if (!f && h) f = h; + if (!f && h) { + dupeType = 'hash'; + f = h; + } + else { + dupeType = 'path'; + } if (options.error) { - this.emit('error', new PluginError('gulp-dedupe', 'Duplicate `' + file.path + '`')); + this.emit('error', new PluginError('gulp-dedupe', 'Duplicate files (' + dupeType + ') ' + file.path + ' and ' + f.path)); } else if (options.same && file.contents.toString() !== f.contents.toString()) { var errorDiff = []; From e29ad17012476a00aced13a4618d6e15564dd355 Mon Sep 17 00:00:00 2001 From: azt3k Date: Mon, 24 Apr 2017 10:49:52 +1200 Subject: [PATCH 04/11] Update package.json restored original package.json with new deps --- package.json | 116 ++++++++++++++------------------------------------- 1 file changed, 32 insertions(+), 84 deletions(-) diff --git a/package.json b/package.json index 3bdfa73..64a3533 100644 --- a/package.json +++ b/package.json @@ -1,88 +1,36 @@ { - "_args": [ - [ - "gulp-dedupe", - "/home/azt3k/dev/catch/ctas" - ] - ], - "_from": "gulp-dedupe@latest", - "_id": "gulp-dedupe@0.0.2", - "_inCache": true, - "_installable": true, - "_location": "/gulp-dedupe", - "_npmUser": { - "email": "dakota@brokenpipe.ru", - "name": "hoho" - }, - "_npmVersion": "1.4.3", - "_phantomChildren": {}, - "_requested": { "name": "gulp-dedupe", - "raw": "gulp-dedupe", - "rawSpec": "", - "scope": null, - "spec": "latest", - "type": "tag" - }, - "_requiredBy": [ - "#DEV:/" - ], - "_resolved": "https://registry.npmjs.org/gulp-dedupe/-/gulp-dedupe-0.0.2.tgz", - "_shasum": "36ef92adf7fcf53e2f096fbe9665d988f9e1ca7e", - "_shrinkwrap": null, - "_spec": "gulp-dedupe", - "_where": "/home/azt3k/dev/catch/ctas", - "author": { - "email": "dakota@brokenpipe.ru", - "name": "Marat Abdullin" - }, - "bugs": { - "url": "https://github.com/hoho/gulp-dedupe/issues" - }, - "dependencies": { - "colors": "~1.0.2", - "diff": "~1.0.8", - "gulp-util": "~3.0.1", - "lodash.defaults": "~2.4.1", - "through": "~2.3.6", - "hash-files": "~1.1.1" - }, - "description": "Check for duplicates in the stream and filter them or throw an error", - "devDependencies": { - "mocha": "*", - "should": "*" - }, - "directories": {}, - "dist": { - "shasum": "36ef92adf7fcf53e2f096fbe9665d988f9e1ca7e", - "tarball": "https://registry.npmjs.org/gulp-dedupe/-/gulp-dedupe-0.0.2.tgz" - }, - "engines": { - "node": ">=0.10" - }, - "homepage": "https://github.com/hoho/gulp-dedupe", - "keywords": [ - "gulpplugin", - "dedupe", - "duplicates" - ], - "license": "MIT", - "main": "./index.js", - "maintainers": [ - { - "email": "dakota@brokenpipe.ru", - "name": "hoho" + "description": "Check for duplicates in the stream and filter them or throw an error", + "version": "0.0.2", + "homepage": "https://github.com/hoho/gulp-dedupe", + "repository": "git://github.com/hoho/gulp-dedupe.git", + "author": "Marat Abdullin ", + "license": "MIT", + "main": "./index.js", + "keywords": [ + "gulpplugin", + "dedupe", + "duplicates" + ], + "bugs": { + "url": "https://github.com/hoho/gulp-dedupe/issues" + }, + "dependencies": { + "gulp-util": "~3.0.1", + "through": "~2.3.6", + "lodash.defaults": "~2.4.1", + "colors": "~1.0.2", + "diff": "~1.0.8", + "hash-files": "~1.1.1" + }, + "devDependencies": { + "mocha": "*", + "should": "*" + }, + "scripts": { + "test": "mocha" + }, + "engines": { + "node": ">=0.10" } - ], - "name": "gulp-dedupe", - "optionalDependencies": {}, - "readme": "ERROR: No README data found!", - "repository": { - "type": "git", - "url": "git://github.com/hoho/gulp-dedupe.git" - }, - "scripts": { - "test": "mocha" - }, - "version": "0.0.2" } From 41bacf751f6131018a86684883350520b331d9ba Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 09:35:25 +1200 Subject: [PATCH 05/11] change hash module --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 64a3533..adb7d69 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "lodash.defaults": "~2.4.1", "colors": "~1.0.2", "diff": "~1.0.8", - "hash-files": "~1.1.1" + "md5": "~2.2.1", }, "devDependencies": { "mocha": "*", From fcb90bca4036cde58a711181080ed6c83276823a Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 09:45:31 +1200 Subject: [PATCH 06/11] hash from buffer --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 1d37a1b..39c6b9d 100644 --- a/index.js +++ b/index.js @@ -10,7 +10,7 @@ var gutil = require('gulp-util'); var PluginError = gutil.PluginError; var path = require('path'); var defaults = require('lodash.defaults'); -var hasher = require('hash-files'); +var md5 = require('md5'); module.exports = function(options) { @@ -29,7 +29,7 @@ module.exports = function(options) { if (file.isStream()) { return this.emit('error', new PluginError('gulp-dedupe', 'Streaming not supported')); } var fullpath = path.resolve(file.path), - hash = hasher.sync({files: [fullpath]}), + hash = md5(file._contents), dupeType = null, h, f; From bf0aad70dedcc8d9feec2eba420b341fbb4a44d1 Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 09:53:04 +1200 Subject: [PATCH 07/11] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index adb7d69..96583d1 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "lodash.defaults": "~2.4.1", "colors": "~1.0.2", "diff": "~1.0.8", - "md5": "~2.2.1", + "md5": "~2.2.1" }, "devDependencies": { "mocha": "*", From 2ebaa320a0559cd3ed9cd1b088110768b26ad6f9 Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 10:07:10 +1200 Subject: [PATCH 08/11] update tests for new behaviour --- test/main.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/main.js b/test/main.js index 7d6c14e..73099aa 100644 --- a/test/main.js +++ b/test/main.js @@ -28,7 +28,6 @@ describe('gulp-dedupe', function() { 'file2.txt', 'Contents2', 'file3.txt', 'Contents3', 'file4.txt', 'Contents4', - 'test/file1.txt', 'Contents1' ] ); @@ -82,8 +81,10 @@ describe('gulp-dedupe', function() { it('should dedupe files', function(done) { stream.on('data', function (file) { + var expectedFilename = results.shift(), expectedHead = results.shift(); + should.exist(file); should.exist(file.relative); should.exist(file.contents); @@ -98,12 +99,17 @@ describe('gulp-dedupe', function() { file.contents.toString().substring(0, expectedHead.length).should.equal(expectedHead); if (results && !results.length) { + console.log('all g'); results = null; done(); } + else { + console.log('wtf', results); + } }); stream.on('error', function(err) { + console.log(err); var expected = results.shift(); var msg = (err.message || '').substring(0, expected.length); msg.should.equal(expected); From d375989615bda20d56d0c3cf03053c59ad3eae6a Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 10:07:58 +1200 Subject: [PATCH 09/11] make tests pass --- index.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 39c6b9d..2c3fed7 100644 --- a/index.js +++ b/index.js @@ -46,8 +46,10 @@ module.exports = function(options) { } if (options.error) { - this.emit('error', new PluginError('gulp-dedupe', 'Duplicate files (' + dupeType + ') ' + file.path + ' and ' + f.path)); - } else if (options.same && file.contents.toString() !== f.contents.toString()) { + this.emit('error', new PluginError('gulp-dedupe', 'Duplicate `' + file.path + '` - Duplicate files (' + dupeType + ') ' + file.path + ' and ' + f.path)); + } + + else if (options.same && file.contents.toString() !== f.contents.toString()) { var errorDiff = []; if (options.diff) { @@ -69,8 +71,11 @@ module.exports = function(options) { this.emit('error', new PluginError('gulp-dedupe', 'Duplicate file `' + file.path + '` with different contents' + errorDiff)); } + return; - } else { + + } + else { filesMap[fullpath] = file; filesHashes[hash] = file; } From 51ac499462ae257b3700df7105ceee3743589170 Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 10:20:19 +1200 Subject: [PATCH 10/11] modified output --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 2c3fed7..df83830 100644 --- a/index.js +++ b/index.js @@ -46,7 +46,7 @@ module.exports = function(options) { } if (options.error) { - this.emit('error', new PluginError('gulp-dedupe', 'Duplicate `' + file.path + '` - Duplicate files (' + dupeType + ') ' + file.path + ' and ' + f.path)); + this.emit('error', new PluginError('gulp-dedupe', 'Duplicate `' + file.path + '` has same ' + dupeType + ' as `' + f.path + '`')); } else if (options.same && file.contents.toString() !== f.contents.toString()) { From 7f4fb0a292a89aa441cda819c06c23acf58f4cbb Mon Sep 17 00:00:00 2001 From: azt3k Date: Tue, 9 May 2017 10:21:26 +1200 Subject: [PATCH 11/11] removed some console logs --- test/main.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/main.js b/test/main.js index 73099aa..ed9a9a7 100644 --- a/test/main.js +++ b/test/main.js @@ -99,17 +99,12 @@ describe('gulp-dedupe', function() { file.contents.toString().substring(0, expectedHead.length).should.equal(expectedHead); if (results && !results.length) { - console.log('all g'); results = null; done(); } - else { - console.log('wtf', results); - } }); stream.on('error', function(err) { - console.log(err); var expected = results.shift(); var msg = (err.message || '').substring(0, expected.length); msg.should.equal(expected);