Generator - chung-leong/zigar GitHub Wiki

The zigar.function.Generation parametric struct provides an interface to a JavaScript Async Generator object. It's analogous to a Promise except that it yields data multiple times.

JavaScript-to-Zig function calls

When a Zig function accepts a Generator as an argument, on the JavaScript side it returns an async generator:

const std = @import("std");
const zigar = @import("zigar");

const Generator = zigar.function.Generator(?error{OutOfMemory}![]u8);

pub fn start() !void {
    try zigar.thread.use();
}

pub fn stop() void {
    zigar.thread.end();
}

pub fn getStrings(a: std.mem.Allocator, generator: Generator) !void {
    const thread = try std.Thread.spawn(.{}, generateStrings, .{ a, generator });
    thread.detach();
}

fn generateStrings(a: std.mem.Allocator, generator: Generator) void {
    for (0..10) |i| {
        if (!generator.yield(std.fmt.allocPrint(a, "string {d}", .{i}))) break;
    } else generator.end();
}

import { getStrings, start, stop } from './generator-example-1.zig';

try {
    start();
    for await (const s of getStrings()) {
        console.log(s.string);
        await new Promise(r => setTimeout(r, 100));
    }
} finally {
    stop();
}

string 0
string 1
string 2
string 3
string 4
string 5
string 6
string 7
string 8
string 9

The code above spawns a thread which sends strings to the JavaScript side, one after another. Generation would pause when result previously sent hasn't been processed yet.

The yield() method returns false when the for async ()loop on the JavaScript side is broken by abreak, return, or throw` statement:

const std = @import("std");
const zigar = @import("zigar");

const Generator = zigar.function.Generator(?error{OutOfMemory}![]u8);

pub fn start() !void {
    try zigar.thread.use();
}

pub fn stop() void {
    zigar.thread.end();
}

pub fn getStrings(a: std.mem.Allocator, generator: Generator) !void {
    const thread = try std.Thread.spawn(.{}, generateStrings, .{ a, generator });
    thread.detach();
}

fn generateStrings(a: std.mem.Allocator, generator: Generator) void {
    for (0..10) |i| {
        std.debug.print("generating item {d}\n", .{i});
        if (!generator.yield(std.fmt.allocPrint(a, "string {d}", .{i}))) break;
    } else generator.end();
}

import { getStrings, start, stop } from './generator-example-2.zig';

try {
    start();
    for await (const s of getStrings()) {
        console.log(s.string);
        break;
    }
} finally {
    console.log('finally');
    stop();
}

generating item 0
string 0
generating item 1
finally

Note the order of the printed lines. The finally clause is reached only after yield() has returned false (i.e. after the stoppage of content generation on the Zig side).

end() is simply yield(null) with the return value ignored. It should not be called when yield() has previously returned false.

If the function returns an error, the async generator would throw it on the JavaScript side:

const std = @import("std");
const zigar = @import("zigar");

const Generator = zigar.function.Generator(?error{OutOfMemory}![]u8);

pub fn start() !void {
    try zigar.thread.use();
}

pub fn stop() void {
    zigar.thread.end();
}

pub fn getStrings(_: std.mem.Allocator, _: Generator) !void {
    return error.OutOfMemory;
}

import { getStrings, start, stop } from './generator-example-3.zig';

try {
    start();
    for await (const s of getStrings());
} catch (err) {
    console.error(err);
} finally {
    stop();
}

Error: Out of memory
    at getStrings (<anonymous>:1:26192)
    at file:///home/cleong/zigar.wiki/examples/generator-example-3.js:5:25 {
  number: 1
}

You can use the pipe() method to generate results from a Zig iterator:

const std = @import("std");
const zigar = @import("zigar");

var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();

const Generator = zigar.function.Generator(?[]const u8);

pub fn start() !void {
    try zigar.thread.use();
}

pub fn stop() void {
    zigar.thread.end();
}

pub fn splitSequence(text: []const u8, delimiter: []const u8, generator: Generator) !void {
    const thread = try std.Thread.spawn(.{}, generateSequence, .{
        text,
        delimiter,
        generator,
    });
    thread.detach();
}

fn generateSequence(text: []const u8, delimiter: []const u8, generator: Generator) void {
    const iter = std.mem.splitSequence(u8, text, delimiter);
    generator.pipe(iter);
}

import { splitSequence, start, stop } from './generator-example-4.zig';

try {
    start();
    for await (const s of splitSequence('hello||world||123||chicken', '||')) {
        console.log(s.string);
    }
} finally {
    stop();
}

hello
world
123
chicken

As in case of Promise, you can use zigar.thread.WorkQueue to handle the asynchronous generation of results:

const std = @import("std");
const zigar = @import("zigar");

var work_queue: zigar.thread.WorkQueue(thread_ns) = .{};
const allocator = zigar.mem.getDefaultAllocator();

pub fn start(threads: usize) !void {
    try work_queue.init(.{
        .allocator = allocator,
        .n_jobs = threads,
    });
}

pub fn stop(promise: zigar.function.Promise(void)) void {
    work_queue.deinitAsync(promise);
}

const Generator = zigar.function.GeneratorOf(thread_ns.scanDir);

pub fn scanDir(path: []const u8, generator: Generator) !void {
    try work_queue.push(thread_ns.scanDir, .{path}, generator);
}

const thread_ns = struct {
    pub fn scanDir(path: []const u8) !std.fs.Dir.Iterator {
        const dir = try std.fs.openDirAbsolute(path, .{ .iterate = true });
        return dir.iterate();
    }
};

import { availableParallelism } from 'os';
import { scanDir, start, stop } from './generator-example-5.zig';

try {
    start(availableParallelism());
    for await (const file of scanDir(process.cwd())) {
        console.log(`${file.name.string} (${file.kind})`);
    }
} finally {
    await stop();
}

generator-example-1.js (file)
generator-example-1.zig (file)
generator-example-2.js (file)
generator-example-2.zig (file)
...

Because scanDir() returns an iterator, the third argument to push() is expected to be a compatible Generator type. We use zigar.function.GeneratorOf to define it, based on the return value of the iterator's next() method (IteratorError ! Entry in this case) with the error set of the function itself (OpenError from openDirAbsolute()) added in. If WorkQueue manages to obtain an iterator, pipe() is used to send its output through to the JavaScript generator.

Zig-to-JavaScript function calls

Generator can be used to return a series of results from JavaScript to Zig code:

const std = @import("std");
const zigar = @import("zigar");

const Generator = zigar.function.Generator(?[]const u8);

pub fn call(cb: *const fn (Generator) void) void {
    cb(.{ .ptr = null, .callback = &callback });
}

fn callback(_: ?*anyopaque, payload: ?[]const u8) bool {
    if (payload) |s| {
        std.debug.print("received = {s}\n", .{s});
        return true;
    } else {
        return false;
    }
}

import { call } from './generator-example-6.zig';

call(async function*() {
    for (let i = 0; i < 10; i++) {
        yield `string ${i}`;
    }
});

received = string 0
received = string 1
received = string 2
received = string 3
received = string 4

As in the case of Promise, you can use callback instead:

import { call } from './generator-example-6.zig';

call(function({ callback }) {
    for (let i = 0; i < 5; i++) {
        if (!callback(`string ${i}`)) {
            break;
        }
    }
    callback(null);
});

received = string 0
received = string 1
received = string 2
received = string 3
received = string 4

If the callback returns false, then the generator is terminated early. It would behave as though a return had been inserted just below the yield statement:

const std = @import("std");
const zigar = @import("zigar");

const Generator = zigar.function.Generator(?u32);

pub fn call(cb: *const fn (Generator) void) void {
    cb(.{ .ptr = null, .callback = &callback });
}

fn callback(_: ?*anyopaque, payload: ?u32) bool {
    if (payload) |num| {
        std.debug.print("received = {d}\n", .{num});
        return num < 3;
    } else {
        return false;
    }
}

import { call } from './generator-example-7.zig';

call(async function*() {
    try  {
        for (let i = 0; i < 10; i++) {
            console.log(`generating: ${i}`);
            yield i;
        }
    } finally {
        console.log(`finally`);
    }
    console.log(`the end`);
});

generating: 0
received = 0
generating: 1
received = 1
generating: 2
received = 2
generating: 3
received = 3
finally

You can pass an Allocator to the JavaScript function. The generator will allocate memory from it when it converts regular JavaScript values to Zig objects:

const std = @import("std");
const zigar = @import("zigar");

var gpa = std.heap.GeneralPurposeAllocator(.{}){};
const allocator = gpa.allocator();

const Avenger = struct {
    real_name: []const u8,
    superhero_name: []const u8,
    age: u32,

    fn deinit(self: *const @This(), a: std.mem.Allocator) void {
        a.free(self.real_name);
        a.free(self.superhero_name);
        a.destroy(self);
    }
};
const ErrorSet = error{Unexpected};
const Generator = zigar.function.Generator(?ErrorSet!*const Avenger);

pub fn call(cb: *const fn (std.mem.Allocator, Generator) void) void {
    const generator = Generator.init(&allocator, callback);
    cb(allocator, generator);
}

fn callback(a: *std.mem.Allocator, payload: Generator.payload) bool {
    if (payload == null) return false;
    if (payload.?) |avenger| {
        defer avenger.deinit(a.*);
        std.debug.print("real_name = {s}, superhero_name = {s}, age = {d}\n", .{
            avenger.real_name,
            avenger.superhero_name,
            avenger.age,
        });
        return true;
    } else |err| {
        std.debug.print("error = {s}", .{@errorName(err)});
        return false;
    }
}

import { call } from './generator-example-8.zig';

call(async function*() {
    const avengers = [
        {
            real_name: 'Tony Stack',
            superhero_name: 'Ironman',
            age: 53,
        },
        {
            real_name: 'Natasha Romanoff',
            superhero_name: 'Black Widow',
            age: 37,
        }
    ];
    for (const avenger of avengers) {
        yield avenger;
    }
    throw new Error('Dog ate soul stone');
});

real_name = Tony Stack, superhero_name = Ironman, age = 53
real_name = Natasha Romanoff, superhero_name = Black Widow, age = 37
error = Unexpected

init() of Generator, like its counterpart in Promise, will cast any compatible callback function to the required type. Instead of ?*anyopaque, for convenience's sake we want the first argument to be *std.mem.Allocator.

error{Unexpected} is used here to capture all possible errors.

Generator - chung-leong/zigar GitHub Wiki

JavaScript-to-Zig function calls

Zig-to-JavaScript function calls

⚠️ **GitHub.com Fallback** ⚠️

⚠️ GitHub.com Fallback ⚠️