Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace Buffer with Uint8Array #3323

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 37 additions & 36 deletions lib/snapshot-manager.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import {Buffer} from 'node:buffer';
import crypto from 'node:crypto';
import fs from 'node:fs';
import {findSourceMap} from 'node:module';
Expand All @@ -16,17 +15,17 @@
import slash from './slash.cjs';

// Increment if encoding layout or Concordance serialization versions change. Previous AVA versions will not be able to
// decode buffers generated by a newer version, so changing this value will require a major version bump of AVA itself.
// decode byte arrays generated by a newer version, so changing this value will require a major version bump of AVA itself.
// The version is encoded as an unsigned 16 bit integer.
const VERSION = 3;

const VERSION_HEADER = Buffer.alloc(2);
VERSION_HEADER.writeUInt16LE(VERSION);
const VERSION_HEADER = new Uint8Array(2);
new DataView(VERSION_HEADER).setUint16(0, VERSION, true);

// The decoder matches on the trailing newline byte (0x0A).
const READABLE_PREFIX = Buffer.from(`AVA Snapshot v${VERSION}\n`, 'ascii');
const REPORT_SEPARATOR = Buffer.from('\n\n', 'ascii');
const REPORT_TRAILING_NEWLINE = Buffer.from('\n', 'ascii');
const READABLE_PREFIX = new TextEncoder().encode(`AVA Snapshot v${VERSION}\n`);
const REPORT_SEPARATOR = new TextEncoder().encode('\n\n');
const REPORT_TRAILING_NEWLINE = new TextEncoder().encode('\n');

const SHA_256_HASH_LENGTH = 32;

Expand Down Expand Up @@ -61,9 +60,9 @@
}
}

const LEGACY_SNAPSHOT_HEADER = Buffer.from('// Jest Snapshot v1');
function isLegacySnapshot(buffer) {
return LEGACY_SNAPSHOT_HEADER.equals(buffer.slice(0, LEGACY_SNAPSHOT_HEADER.byteLength));
const LEGACY_SNAPSHOT_HEADER = new TextEncoder().encode('// Jest Snapshot v1');
function isLegacySnapshot(array) {
return LEGACY_SNAPSHOT_HEADER.equals(array.subarray(0, LEGACY_SNAPSHOT_HEADER.byteLength));
}

export class LegacyError extends SnapshotError {
Expand Down Expand Up @@ -101,7 +100,7 @@
}

function combineEntries({blocks}) {
const combined = new BufferBuilder();
const combined = new Uint8ArrayBuilder();

for (const {title, snapshots} of blocks) {
const last = snapshots.at(-1);
Expand All @@ -120,42 +119,43 @@
}

function generateReport(relFile, snapFile, snapshots) {
return new BufferBuilder()
return new Uint8ArrayBuilder()
.write(`# Snapshot report for \`${slash(relFile)}\`

The actual snapshot is saved in \`${snapFile}\`.

Generated by [AVA](https://avajs.dev).`)
.append(combineEntries(snapshots))
.write(REPORT_TRAILING_NEWLINE)
.toBuffer();
.toUint8Array();
}

class BufferBuilder {
class Uint8ArrayBuilder {
constructor() {
this.buffers = [];
this.arrays = [];
this.byteOffset = 0;
}

append(builder) {
this.buffers.push(...builder.buffers);
this.arrays.push(...builder.arrays);
this.byteOffset += builder.byteOffset;
return this;
}

write(data) {
if (typeof data === 'string') {
this.write(Buffer.from(data, 'utf8'));
const encoder = new TextEncoder();
this.write(encoder.encode(data));
} else {
this.buffers.push(data);
this.arrays.push(data);
this.byteOffset += data.byteLength;
}

return this;
}

toBuffer() {
return Buffer.concat(this.buffers, this.byteOffset);
toUint8Array() {
return concatUint8Arrays(this.arrays, this.byteOffset);

Check failure on line 158 in lib/snapshot-manager.js

View workflow job for this annotation

GitHub Actions / Lint source files

'concatUint8Arrays' is not defined.
}
}

Expand Down Expand Up @@ -190,46 +190,47 @@
const compressed = zlib.gzipSync(encoded);
compressed[9] = 0x03; // Override the GZip header containing the OS to always be Linux
const sha256sum = crypto.createHash('sha256').update(compressed).digest();
return Buffer.concat([
return concatUint8Arrays([

Check failure on line 193 in lib/snapshot-manager.js

View workflow job for this annotation

GitHub Actions / Lint source files

'concatUint8Arrays' is not defined.
READABLE_PREFIX,
VERSION_HEADER,
sha256sum,
compressed,
], READABLE_PREFIX.byteLength + VERSION_HEADER.byteLength + SHA_256_HASH_LENGTH + compressed.byteLength);
}

export function extractCompressedSnapshot(buffer, snapPath) {
if (isLegacySnapshot(buffer)) {
export function extractCompressedSnapshot(array, snapPath) {
if (isLegacySnapshot(array)) {
throw new LegacyError(snapPath);
}

// The version starts after the readable prefix, which is ended by a newline
// byte (0x0A).
const newline = buffer.indexOf(0x0A);
const newline = array.indexOf(0x0A);
if (newline === -1) {
throw new InvalidSnapshotError(snapPath);
}

const view = new DataView(array);
const versionOffset = newline + 1;
const version = buffer.readUInt16LE(versionOffset);
const version = view.getUint16(versionOffset, true);
if (version !== VERSION) {
throw new VersionMismatchError(snapPath, version);
}

const sha256sumOffset = versionOffset + 2;
const compressedOffset = sha256sumOffset + SHA_256_HASH_LENGTH;
const compressed = buffer.slice(compressedOffset);
const compressed = array.subarray(compressedOffset);

return {
version, compressed, sha256sumOffset, compressedOffset,
};
}

function decodeSnapshots(buffer, snapPath) {
const {compressed, sha256sumOffset, compressedOffset} = extractCompressedSnapshot(buffer, snapPath);
function decodeSnapshots(array, snapPath) {
const {compressed, sha256sumOffset, compressedOffset} = extractCompressedSnapshot(array, snapPath);

const sha256sum = crypto.createHash('sha256').update(compressed).digest();
const expectedSum = buffer.slice(sha256sumOffset, compressedOffset);
const expectedSum = array.subarray(sha256sumOffset, compressedOffset);
if (!sha256sum.equals(expectedSum)) {
throw new ChecksumError(snapPath);
}
Expand Down Expand Up @@ -373,16 +374,16 @@
),
};

const buffer = await encodeSnapshots(snapshots);
const reportBuffer = generateReport(relFile, snapFile, snapshots);
const array = await encodeSnapshots(snapshots);
const reportArray = generateReport(relFile, snapFile, snapshots);

await fs.promises.mkdir(dir, {recursive: true});

const temporaryFiles = [];
const tmpfileCreated = file => temporaryFiles.push(file);
await Promise.all([
writeFileAtomic(snapPath, buffer, {tmpfileCreated}),
writeFileAtomic(reportPath, reportBuffer, {tmpfileCreated}),
writeFileAtomic(snapPath, array, {tmpfileCreated}),
writeFileAtomic(reportPath, reportArray, {tmpfileCreated}),
]);
return {
changedFiles: [snapPath, reportPath],
Expand Down Expand Up @@ -470,9 +471,9 @@
}

const paths = determineSnapshotPaths({file, fixedLocation, projectDir});
const buffer = tryRead(paths.snapPath);
const array = tryRead(paths.snapPath);

if (!buffer) {
if (!array) {
return new Manager({
recordNewSnapshots,
updating,
Expand All @@ -486,7 +487,7 @@
let snapshotError;

try {
const data = decodeSnapshots(buffer, paths.snapPath);
const data = decodeSnapshots(array, paths.snapPath);
blocksByTitle = new Map(data.blocks.map(({title, ...block}) => [title, block]));
} catch (error) {
blocksByTitle = new Map();
Expand Down
5 changes: 3 additions & 2 deletions media/screenshot-fixtures/magic-assert-buffers.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import test from 'ava';
import {hexToUint8Array} from 'uint8array-extras';

test('buffers', t => {
const actual = Buffer.from('decafbadcab00d1e'.repeat(4), 'hex')
const expected = Buffer.from('cab00d1edecafbad' + 'decafbadcab00d1e'.repeat(3), 'hex')
const actual = hexToUint8Array('decafbadcab00d1e'.repeat(4))
const expected = hexToUint8Array('cab00d1edecafbad' + 'decafbadcab00d1e'.repeat(3))
t.deepEqual(actual, expected)
});
12 changes: 12 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@
"strip-ansi": "^7.1.0",
"supertap": "^3.0.1",
"temp-dir": "^3.0.0",
"uint8array-extras": "^1.1.0",
"write-file-atomic": "^5.0.1",
"yargs": "^17.7.2"
},
Expand Down
11 changes: 6 additions & 5 deletions test-tap/helper/report.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,27 @@
import Api from '../../lib/api.js';
import {_testOnlyReplaceWorkerPath} from '../../lib/fork.js';
import {normalizeGlobs} from '../../lib/globs.js';
import pkg from '../../lib/pkg.cjs';

Check failure on line 10 in test-tap/helper/report.js

View workflow job for this annotation

GitHub Actions / Lint source files

There should be at least one empty line between import groups
import {uint8ArrayToString} from 'uint8array-extras';

Check failure on line 11 in test-tap/helper/report.js

View workflow job for this annotation

GitHub Actions / Lint source files

`uint8array-extras` import should occur before import of `../../lib/api.js`

_testOnlyReplaceWorkerPath(new URL('report-worker.js', import.meta.url));

const exports = {};
export default exports;

exports.assert = (t, logFile, buffer) => {
exports.assert = (t, logFile, array) => {
let existing = null;
try {
existing = fs.readFileSync(logFile);
} catch {}

if (existing === null || process.env.UPDATE_REPORTER_LOG) {
fs.writeFileSync(logFile, buffer);
existing = buffer;
fs.writeFileSync(logFile, array);
existing = array;
}

const expected = existing.toString('utf8');
const actual = buffer.toString('utf8');
const expected = uint8ArrayToString(existing);
const actual = uint8ArrayToString(array);
if (actual === expected) {
t.pass();
} else {
Expand Down
26 changes: 15 additions & 11 deletions test-tap/helper/tty-stream.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {Buffer} from 'node:buffer';
import stream from 'node:stream';

import ansiEscapes from 'ansi-escapes';
import {stringToUint8Array, concatUint8Arrays, uint8ArrayToString} from 'uint8array-extras';

export default class TTYStream extends stream.Writable {
constructor(options) {
Expand All @@ -17,7 +17,7 @@

_write(chunk, encoding, callback) {
if (this.spinnerActivity.length > 0) {
this.chunks.push(Buffer.concat(this.spinnerActivity), TTYStream.SEPARATOR);
this.chunks.push(concatUint8Arrays(this.spinnerActivity), TTYStream.SEPARATOR);
this.spinnerActivity = [];
}

Expand All @@ -26,7 +26,7 @@
// chunks.
if (string !== '' || chunk.length === 0) {
this.chunks.push(
Buffer.from(string, 'utf8'),
stringToUint8Array(string),
TTYStream.SEPARATOR,
);
}
Expand All @@ -36,33 +36,37 @@

_writev(chunks, callback) {
if (this.spinnerActivity.length > 0) {
this.chunks.push(Buffer.concat(this.spinnerActivity), TTYStream.SEPARATOR);
this.chunks.push(concatUint8Arrays(this.spinnerActivity), TTYStream.SEPARATOR);
this.spinnerActivity = [];
}

for (const object of chunks) {
this.chunks.push(Buffer.from(this.sanitizers.reduce((string, sanitizer) => sanitizer(string), object.chunk.toString('utf8')), 'utf8')); // eslint-disable-line unicorn/no-array-reduce
this.chunks.push(stringToUint8Array(this.sanitizers.reduce((string, sanitizer) => sanitizer(string), uint8ArrayToString(object.chunk)))); // eslint-disable-line unicorn/no-array-reduce
}

this.chunks.push(TTYStream.SEPARATOR);
callback();
}

asBuffer() {
return Buffer.concat(this.chunks);
asUint8Array() {
return concatUint8Arrays(this.chunks);
}

toString() {
return uint8ArrayToString(array);

Check failure on line 56 in test-tap/helper/tty-stream.js

View workflow job for this annotation

GitHub Actions / Lint source files

'array' is not defined.
}

clearLine() {
this.spinnerActivity.push(Buffer.from(ansiEscapes.eraseLine, 'ascii'));
this.spinnerActivity.push(stringToUint8Array(ansiEscapes.eraseLine));
}

cursorTo(x, y) {
this.spinnerActivity.push(Buffer.from(ansiEscapes.cursorTo(x, y), 'ascii'));
this.spinnerActivity.push(stringToUint8Array(ansiEscapes.cursorTo(x, y)));
}

moveCursor(dx, dy) {
this.spinnerActivity.push(Buffer.from(ansiEscapes.cursorMove(dx, dy), 'ascii'));
this.spinnerActivity.push(stringToUint8Array(ansiEscapes.cursorMove(dx, dy)));
}
}

TTYStream.SEPARATOR = Buffer.from('---tty-stream-chunk-separator\n', 'utf8');
TTYStream.SEPARATOR = stringToUint8Array('---tty-stream-chunk-separator\n');
Loading
Loading