Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

node:zlib: Brotli #10722

Merged
merged 34 commits into from
May 7, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
3434cdc
node:zlib: implement brotliCompress and brotliDecompress
nektro May 1, 2024
b52beb0
update docs
nektro May 1, 2024
8ea8bcc
update docs
nektro May 1, 2024
b7ef27b
only load these once
nektro May 1, 2024
2dc1959
reduce memory usage
nektro May 2, 2024
459edfe
move collecting the output buffer into a JSValue into a helper
nektro May 2, 2024
bfe9c1b
tidy
nektro May 2, 2024
56083a9
add brotliCompressSync and brotliDecompressSync
nektro May 2, 2024
37ee737
fix encoder output management
nektro May 2, 2024
7537c2a
this assert was invalid
nektro May 2, 2024
8e55934
destroy in deinit
nektro May 2, 2024
2af09ad
add a todo
nektro May 2, 2024
33db4e6
use has_pending_activity instead of ref
nektro May 2, 2024
3035031
node:stream: add _events props and test
nektro May 4, 2024
4794736
dead code remove
nektro May 4, 2024
978f1b1
add missing constants
nektro May 4, 2024
ed830a5
implement createBrotliCompress and createBrotliDecompress
nektro May 4, 2024
931ce90
add another test
nektro May 4, 2024
1520afa
compress stream fix
nektro May 4, 2024
605cc80
use the harness here
nektro May 4, 2024
7670c9c
add an end-to-end test
nektro May 4, 2024
37bec3a
oops didnt actually need that ig
nektro May 4, 2024
5bf78c2
avoid this dupe
nektro May 6, 2024
9cd48ef
if true bad
nektro May 6, 2024
68fde62
postpone this until EventEmitter is refactored
nektro May 6, 2024
7a3f3a8
we always have brotli now
nektro May 7, 2024
e9f6775
node:zlib: expose BrotliCompress and BrotliDecompress constructors
nektro May 7, 2024
bcdc345
move these imports to the top of the file
nektro May 7, 2024
e41c69f
add more brotli e2e tests with node:http
nektro May 7, 2024
ac4c4b6
avoid heap allocation in sync case
nektro May 7, 2024
f3b106e
Merge branch 'main' into nektro-patch-15305
nektro May 7, 2024
8e920db
fix constructor check
nektro May 7, 2024
a264d59
Update src/js/node/zlib.ts
Jarred-Sumner May 7, 2024
0a5cbe1
Update src/js/node/zlib.ts
Jarred-Sumner May 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
127 changes: 111 additions & 16 deletions src/brotli.zig
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
const bun = @import("root").bun;
const std = @import("std");
const c = @import("./deps/brotli_decoder.zig");
const c = struct {
pub usingnamespace @import("./deps/brotli_decoder.zig");
pub usingnamespace @import("./deps/brotli_encoder.zig");
};
const BrotliDecoder = c.BrotliDecoder;
const BrotliEncoder = c.BrotliEncoder;

const mimalloc = bun.Mimalloc;

Expand All @@ -15,7 +19,7 @@ const BrotliAllocator = struct {
return mimalloc.mi_malloc(len) orelse unreachable;
}

pub fn free(_: ?*anyopaque, data: *anyopaque) callconv(.C) void {
pub fn free(_: ?*anyopaque, data: ?*anyopaque) callconv(.C) void {
if (comptime bun.is_heap_breakdown_enabled) {
const zone = bun.HeapBreakdown.malloc_zone_t.get(BrotliAllocator);
zone.malloc_zone_free(data);
Expand All @@ -26,7 +30,7 @@ const BrotliAllocator = struct {
}
};

pub const Options = struct {
pub const DecoderOptions = struct {
pub const Params = std.enums.EnumFieldStruct(c.BrotliDecoderParameter, bool, false);

params: Params = Params{
Expand Down Expand Up @@ -54,7 +58,11 @@ pub const BrotliReaderArrayList = struct {

pub usingnamespace bun.New(BrotliReaderArrayList);

pub fn initWithOptions(input: []const u8, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, options: Options) !*BrotliReaderArrayList {
pub fn newWithOptions(input: []const u8, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, options: DecoderOptions) !*BrotliReaderArrayList {
return BrotliReaderArrayList.new(try initWithOptions(input, list, allocator, options));
}

pub fn initWithOptions(input: []const u8, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, options: DecoderOptions) !BrotliReaderArrayList {
if (!BrotliDecoder.initializeBrotli()) {
return error.BrotliFailedToLoad;
}
Expand All @@ -67,25 +75,21 @@ pub const BrotliReaderArrayList = struct {

bun.assert(list.items.ptr != input.ptr);

return BrotliReaderArrayList.new(
.{
.input = input,
.list_ptr = list,
.list = list.*,
.list_allocator = allocator,
.brotli = brotli,
},
);
return .{
.input = input,
.list_ptr = list,
.list = list.*,
.list_allocator = allocator,
.brotli = brotli,
};
}

pub fn end(this: *BrotliReaderArrayList) void {
this.state = .End;
}

pub fn readAll(this: *BrotliReaderArrayList, is_done: bool) !void {
defer {
this.list_ptr.* = this.list;
}
defer this.list_ptr.* = this.list;

if (this.state == .End or this.state == .Error) {
return;
Expand Down Expand Up @@ -166,3 +170,94 @@ pub const BrotliReaderArrayList = struct {
this.destroy();
}
};

pub const BrotliCompressionStream = struct {
pub const State = enum {
Inflating,
End,
Error,
};

brotli: *BrotliEncoder,
state: State = State.Inflating,
total_out: usize = 0,
total_in: usize = 0,

pub fn init() !BrotliCompressionStream {
const instance = BrotliEncoder.createInstance(&BrotliAllocator.alloc, &BrotliAllocator.free, null) orelse return error.BrotliFailedToCreateInstance;

return BrotliCompressionStream{
.brotli = instance,
};
}

pub fn writeChunk(this: *BrotliCompressionStream, input: []const u8, last: bool) ![]const u8 {
const result = this.brotli.compressStream(if (last) BrotliEncoder.Operation.finish else .process, input);

if (!result.success) {
this.state = .Error;
return error.BrotliCompressionError;
}

return result.output;
}

pub fn write(this: *BrotliCompressionStream, input: []const u8, last: bool) ![]const u8 {
if (this.state == .End or this.state == .Error) {
return "";
}

return this.writeChunk(input, last);
}

pub fn end(this: *BrotliCompressionStream) ![]const u8 {
defer this.state = .End;

return try this.write("", true);
}

pub fn deinit(this: *BrotliCompressionStream) void {
this.brotli.destroyInstance();
}

fn NewWriter(comptime InputWriter: type) type {
return struct {
compressor: *BrotliCompressionStream,
input_writer: InputWriter,

const Self = @This();
pub const WriteError = error{BrotliCompressionError} || InputWriter.Error;
pub const Writer = std.io.Writer(@This(), WriteError, Self.write);

pub fn init(compressor: *BrotliCompressionStream, input_writer: InputWriter) Self {
return Self{
.compressor = compressor,
.input_writer = input_writer,
};
}

pub fn write(self: Self, to_compress: []const u8) WriteError!usize {
const decompressed = try self.compressor.write(to_compress, false);
try self.input_writer.writeAll(decompressed);
return to_compress.len;
}

pub fn end(self: Self) !usize {
const decompressed = try self.compressor.end();
try self.input_writer.writeAll(decompressed);
}

pub fn writer(self: Self) Writer {
return Writer{ .context = self };
}
};
}

pub fn writerContext(this: *BrotliCompressionStream, writable: anytype) NewWriter(@TypeOf(writable)) {
return NewWriter(@TypeOf(writable)).init(this, writable);
}

pub fn writer(this: *BrotliCompressionStream, writable: anytype) NewWriter(@TypeOf(writable)).Writer {
return this.writerContext(writable).writer();
}
};
63 changes: 63 additions & 0 deletions src/bun.js/api/brotli.classes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { define } from "../../codegen/class-definitions";

export default [
define({
name: "BrotliEncoder",
construct: true,
noConstructor: true,
finalize: true,
configurable: false,
hasPendingActivity: true,
klass: {},
JSType: "0b11101110",
values: ["callback"],
proto: {
encode: {
fn: "encode",
length: 2,
},
encodeSync: {
fn: "encodeSync",
length: 2,
},
end: {
fn: "end",
length: 2,
},
endSync: {
fn: "endSync",
length: 2,
},
},
}),
define({
name: "BrotliDecoder",
construct: true,
noConstructor: true,
finalize: true,
configurable: false,
hasPendingActivity: true,
klass: {},
JSType: "0b11101110",
values: ["callback"],

proto: {
decode: {
fn: "decode",
length: 2,
},
decodeSync: {
fn: "decodeSync",
length: 2,
},
end: {
fn: "end",
length: 2,
},
endSync: {
fn: "endSync",
length: 2,
},
},
}),
];