Skip to content
This repository was archived by the owner on Dec 17, 2025. It is now read-only.

Commit f2ce2ca

Browse files
alexander-akaitalexander-akait
committed
refactor!: use xxhash64 by default for [hash]/[contenthash] and getHashDigest API
1 parent 10503d6 commit f2ce2ca

File tree

4 files changed

+222
-43
lines changed

4 files changed

+222
-43
lines changed

‎README.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,14 @@ The following tokens are replaced in the `name` parameter:
7777
- `[path]` the path of the resource relative to the `context` query parameter or option.
7878
- `[folder]` the folder the resource is in
7979
- `[query]` the queryof the resource, i.e. `?foo=bar`
80-
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
80+
- `[contenthash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
8181
- `[<hashType>:contenthash:<digestType>:<length>]` optionally one can configure
82-
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
82+
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
8383
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
8484
- and `length` the length in chars
85-
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the md4 hash)
85+
- `[hash]` the hash of `options.content` (Buffer) (by default it's the hex digest of the `xxhash64` hash)
8686
- `[<hashType>:hash:<digestType>:<length>]` optionally one can configure
87-
- other `hashType`s, i. e. `sha1`, `md4`, `md5`, `sha256`, `sha512`
87+
- other `hashType`s, i. e. `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512`
8888
- other `digestType`s, i. e. `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
8989
- and `length` the length in chars
9090
- `[N]` the N-th match obtained from matching the current file name against `options.regExp`
@@ -118,7 +118,7 @@ loaderUtils.interpolateName(loaderContext, "[hash]", { content: ... });
118118
// loaderContext.resourcePath = "/absolute/path/to/app/img/image.png"
119119
loaderUtils.interpolateName(loaderContext, "[sha512:hash:base64:7].[ext]", { content: ... });
120120
// => 2BKDTjl.png
121-
// use sha512 hash instead of md4 and with only 7 chars of base64
121+
// use sha512 hash instead of xxhash64 and with only 7 chars of base64
122122

123123
// loaderContext.resourcePath = "/absolute/path/to/app/img/myself.png"
124124
// loaderContext.query.name =
@@ -160,7 +160,7 @@ const digestString = loaderUtils.getHashDigest(
160160
```
161161

162162
- `buffer` the content that should be hashed
163-
- `hashType` one of `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
163+
- `hashType` one of `xxhash64`, `sha1`, `md4`, `md5`, `sha256`, `sha512` or any other node.js supported hash type
164164
- `digestType` one of `hex`, `base26`, `base32`, `base36`, `base49`, `base52`, `base58`, `base62`, `base64`
165165
- `maxLength` the maximum length in chars
166166

‎lib/getHashDigest.js‎

Lines changed: 190 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,168 @@
11
"use strict";
22

3+
// Copied from `webpack`
4+
//#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1
5+
const xxhash64 = new WebAssembly.Module(
6+
Buffer.from(
7+
// 1180 bytes
8+
"AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==",
9+
"base64"
10+
)
11+
);
12+
//#endregion
13+
14+
class XxHash64 {
15+
/**
16+
* @param {WebAssembly.Instance} instance wasm instance
17+
*/
18+
constructor(instance) {
19+
const exports = /** @type {any} */ (instance.exports);
20+
21+
exports.init();
22+
23+
this.exports = exports;
24+
this.mem = Buffer.from(exports.memory.buffer, 0, 65536);
25+
this.buffered = 0;
26+
}
27+
28+
reset() {
29+
this.buffered = 0;
30+
this.exports.init();
31+
}
32+
33+
/**
34+
* @param {Buffer | string} data data
35+
* @param {BufferEncoding=} encoding encoding
36+
* @returns {this} itself
37+
*/
38+
update(data, encoding) {
39+
if (typeof data === "string") {
40+
if (data.length < 21845) {
41+
this._updateWithShortString(data, encoding);
42+
43+
return this;
44+
} else {
45+
data = Buffer.from(data, encoding);
46+
}
47+
}
48+
49+
this._updateWithBuffer(data);
50+
51+
return this;
52+
}
53+
54+
/**
55+
* @param {string} data data
56+
* @param {BufferEncoding=} encoding encoding
57+
* @returns {void}
58+
*/
59+
_updateWithShortString(data, encoding) {
60+
const { exports, buffered, mem } = this;
61+
62+
let endPos;
63+
64+
if (data.length < 70) {
65+
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
66+
endPos = buffered;
67+
68+
for (let i = 0; i < data.length; i++) {
69+
const cc = data.charCodeAt(i);
70+
71+
if (cc < 0x80) {
72+
mem[endPos++] = cc;
73+
} else if (cc < 0x800) {
74+
mem[endPos] = (cc >> 6) | 0xc0;
75+
mem[endPos + 1] = (cc & 0x3f) | 0x80;
76+
endPos += 2;
77+
} else {
78+
// bail-out for weird chars
79+
endPos += mem.write(data.slice(endPos), endPos, encoding);
80+
break;
81+
}
82+
}
83+
} else if (encoding === "latin1") {
84+
endPos = buffered;
85+
86+
for (let i = 0; i < data.length; i++) {
87+
const cc = data.charCodeAt(i);
88+
89+
mem[endPos++] = cc;
90+
}
91+
} else {
92+
endPos = buffered + mem.write(data, buffered, encoding);
93+
}
94+
} else {
95+
endPos = buffered + mem.write(data, buffered, encoding);
96+
}
97+
98+
if (endPos < 32) {
99+
this.buffered = endPos;
100+
} else {
101+
const l = (endPos >> 5) << 5;
102+
103+
exports.update(l);
104+
105+
const newBuffered = endPos - l;
106+
107+
this.buffered = newBuffered;
108+
109+
if (newBuffered > 0) {
110+
mem.copyWithin(0, l, endPos);
111+
}
112+
}
113+
}
114+
115+
/**
116+
* @param {Buffer} data data
117+
* @returns {void}
118+
*/
119+
_updateWithBuffer(data) {
120+
const { exports, buffered, mem } = this;
121+
const length = data.length;
122+
if (buffered + length < 32) {
123+
data.copy(mem, buffered, 0, length);
124+
this.buffered += length;
125+
} else {
126+
const l = ((buffered + length) >> 5) << 5;
127+
if (l > 65536) {
128+
let i = 65536 - buffered;
129+
data.copy(mem, buffered, 0, i);
130+
exports.update(65536);
131+
const stop = l - buffered - 65536;
132+
while (i < stop) {
133+
data.copy(mem, 0, i, i + 65536);
134+
exports.update(65536);
135+
i += 65536;
136+
}
137+
data.copy(mem, 0, i, l - buffered);
138+
exports.update(l - buffered - i);
139+
} else {
140+
data.copy(mem, buffered, 0, l - buffered);
141+
exports.update(l);
142+
}
143+
144+
const newBuffered = length + buffered - l;
145+
146+
this.buffered = newBuffered;
147+
148+
if (newBuffered > 0) {
149+
data.copy(mem, 0, length - newBuffered, length);
150+
}
151+
}
152+
}
153+
154+
digest() {
155+
const { exports, buffered, mem } = this;
156+
157+
exports.final(buffered);
158+
instancesPool.push(this);
159+
160+
return mem.toString("latin1", 0, 16);
161+
}
162+
}
163+
164+
const instancesPool = [];
165+
3166
const baseEncodeTables = {
4167
26: "abcdefghijklmnopqrstuvwxyz",
5168
32: "123456789abcdefghjkmnpqrstuvwxyz", // no 0lio
@@ -13,6 +176,7 @@ const baseEncodeTables = {
13176

14177
function encodeBufferToBase(buffer, base) {
15178
const encodeTable = baseEncodeTables[base];
179+
16180
if (!encodeTable) {
17181
throw new Error("Unknown encoding base" + base);
18182
}
@@ -21,13 +185,15 @@ function encodeBufferToBase(buffer, base) {
21185
const Big = require("big.js");
22186

23187
Big.RM = Big.DP = 0;
188+
24189
let b = new Big(0);
25190

26191
for (let i = readLength - 1; i >= 0; i--) {
27192
b = b.times(256).plus(buffer[i]);
28193
}
29194

30195
let output = "";
196+
31197
while (b.gt(0)) {
32198
output = encodeTable[b.mod(base)] + output;
33199
b = b.div(base);
@@ -39,11 +205,29 @@ function encodeBufferToBase(buffer, base) {
39205
return output;
40206
}
41207

208+
const create = () => {
209+
if (instancesPool.length > 0) {
210+
const old = instancesPool.pop();
211+
212+
old.reset();
213+
214+
return old;
215+
} else {
216+
return new XxHash64(new WebAssembly.Instance(xxhash64));
217+
}
218+
};
219+
42220
function getHashDigest(buffer, hashType, digestType, maxLength) {
43-
hashType = hashType || "md4";
221+
hashType = hashType || "xxhash64";
44222
maxLength = maxLength || 9999;
45223

46-
const hash = require("crypto").createHash(hashType);
224+
let hash;
225+
226+
if (hashType === "xxhash64") {
227+
hash = create(maxLength);
228+
} else {
229+
hash = require("crypto").createHash(hashType);
230+
}
47231

48232
hash.update(buffer);
49233

@@ -57,10 +241,10 @@ function getHashDigest(buffer, hashType, digestType, maxLength) {
57241
digestType === "base62" ||
58242
digestType === "base64"
59243
) {
60-
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
61-
0,
62-
maxLength
63-
);
244+
return encodeBufferToBase(
245+
hashType === "xxhash64" ? Buffer.from(hash.digest()) : hash.digest(),
246+
digestType.substr(4)
247+
).substr(0, maxLength);
64248
} else {
65249
return hash.digest(digestType || "hex").substr(0, maxLength);
66250
}

‎test/getHashDigest.test.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,13 @@ describe("getHashDigest()", () => {
1111
undefined,
1212
"6f8db599de986fab7a21625b7916589c",
1313
],
14-
["test string", "md5", "hex", 4, "6f8d"],
1514
["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"],
15+
["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"],
16+
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
17+
["test string", "xxhash64", "base64", undefined, "Uej5ydCcPpj4RcScOpjBB"],
18+
["test string", "xxhash64", "base52", undefined, "bqOwublJwrBqLcKHCVpojCL"],
19+
["test string", "xxhash64", "base64url", undefined, "e9e2c351e3c6b198"],
20+
["test string", "md5", "hex", 4, "6f8d"],
1621
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
1722
["test string", "md5", "base26", 6, "bhtsgu"],
1823
[

0 commit comments

Comments
 (0)