From ad5a1e94bc468499c3cb363cdb9db6ca6ad930c2 Mon Sep 17 00:00:00 2001 From: Hugo Mano Date: Tue, 6 Jan 2026 16:44:56 +0100 Subject: [PATCH] wip --- zml2/io/vfs.zig | 983 +++++++++------------------------------- zml2/io/vfs/BUILD.bazel | 11 +- zml2/io/vfs/base.zig | 43 -- zml2/io/vfs/hf2.zig | 583 ++++++++++++++++-------- zml2/io/vfs/index.zig | 3 +- zml2/io/vfs/proxy.zig | 546 ---------------------- zml2/main_io.zig | 193 ++++---- 7 files changed, 714 insertions(+), 1648 deletions(-) delete mode 100644 zml2/io/vfs/proxy.zig diff --git a/zml2/io/vfs.zig b/zml2/io/vfs.zig index 413761a7..43950c36 100644 --- a/zml2/io/vfs.zig +++ b/zml2/io/vfs.zig @@ -4,856 +4,297 @@ const vfs = @import("vfs"); const log = std.log.scoped(.@"zml/io/vfs"); pub const VFS = struct { - pub const Error = FileEntryError || DirEntryError || ResolveAbsolutePathError; - - pub const FileEntryError = error{ - MissingVirtualFileHandle, - } || ResolveAbsolutePathError; - - pub const DirEntryError = error{ - MissingVirtualDirHandle, - } || ResolveAbsolutePathError; - - pub const ResolveAbsolutePathError = error{ - SchemeNotRegistered, - InvalidURI, - }; - pub const File = vfs.File; pub const HTTP = vfs.HTTP; pub const HF = vfs.HF; + const VFSBase = vfs.VFSBase; - const FsFile = struct { - backend: std.Io, - handle: std.Io.File.Handle, - }; + const Handle = struct { + pub const Type = enum { + file, + directory, + }; - const FsDir = struct { + type: Type, + handle: u32, + scheme: []const u8, backend: std.Io, - handle: std.Io.Dir.Handle, }; allocator: std.mem.Allocator, - mutex: std.Io.Mutex, + mutex: std.Io.Mutex = .init, - backends: std.StringHashMapUnmanaged(std.Io), - // Maps virtual file and dir handles to backend handles - files: std.AutoHashMapUnmanaged(std.Io.File.Handle, *FsFile), - dirs: std.AutoHashMapUnmanaged(std.Io.Dir.Handle, *FsDir), - next_file_handle: std.Io.File.Handle, - next_dir_handle: std.Io.Dir.Handle, + backends: std.StringHashMapUnmanaged(std.Io) = .{}, + handles: std.ArrayList(Handle) = .{}, + closed_handles: std.ArrayList(u32) = .{}, - vtable: std.Io.VTable, - base_io: std.Io, + base: VFSBase, - pub fn init(allocator: std.mem.Allocator, base_io: std.Io) VFS { - const vtable: std.Io.VTable = .{ - .async = async, - .concurrent = concurrent, - .await = await, - .cancel = cancel, - .cancelRequested = cancelRequested, - .groupAsync = groupAsync, - .groupConcurrent = groupConcurrent, - .groupWait = groupWait, - .groupCancel = groupCancel, - .select = select, - .mutexLock = mutexLock, - .mutexLockUncancelable = mutexLockUncancelable, - .mutexUnlock = mutexUnlock, - .conditionWait = conditionWait, - .conditionWaitUncancelable = conditionWaitUncancelable, - .conditionWake = conditionWake, - .now = now, - .sleep = sleep, - .netListenIp = netListenIp, - .netAccept = netAccept, - .netBindIp = netBindIp, - .netConnectIp = netConnectIp, - .netListenUnix = netListenUnix, - .netConnectUnix = netConnectUnix, - .netSend = netSend, - .netReceive = netReceive, - .netRead = netRead, - .netWrite = netWrite, - .netClose = netClose, - .netInterfaceNameResolve = netInterfaceNameResolve, - .netInterfaceName = netInterfaceName, - .netLookup = netLookup, - .dirMake = dirMake, - .dirMakePath = dirMakePath, - .dirMakeOpenPath = dirMakeOpenPath, - .dirStat = dirStat, - .dirStatPath = dirStatPath, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirOpenDir = dirOpenDir, - .dirClose = dirClose, - .fileStat = fileStat, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .openSelfExe = openSelfExe, - }; + pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: std.ArrayList(Handle) = .{}; + // cwd handle + try handles.append(allocator, .{ + .type = .directory, + .handle = 0, + .scheme = "", + .backend = base.inner, + }); return .{ .allocator = allocator, - .mutex = .init, - .backends = .{}, - .files = .{}, - .dirs = .{}, - .next_file_handle = 0, - .next_dir_handle = 0, - .vtable = vtable, - .base_io = base_io, + .handles = handles, + .base = base, }; } pub fn deinit(self: *VFS) void { - var file_it = self.files.valueIterator(); - while (file_it.next()) |entry_ptr| { - self.allocator.destroy(entry_ptr.*); - } - self.files.deinit(self.allocator); - - var dir_it = self.dirs.valueIterator(); - while (dir_it.next()) |entry_ptr| { - self.allocator.destroy(entry_ptr.*); - } - self.dirs.deinit(self.allocator); - + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); self.backends.deinit(self.allocator); } pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); try self.backends.put(self.allocator, scheme, backend); } pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); return self.backends.remove(scheme); } pub fn io(self: *VFS) std.Io { return .{ - .vtable = &self.vtable, - .userdata = self, + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), }; } - fn backendFromAbsolutePath(self: *VFS, path: []const u8) ResolveAbsolutePathError!std.Io { - if (std.fs.path.isAbsolutePosix(path)) return self.base_io; - const uri = std.Uri.parse(path) catch return ResolveAbsolutePathError.InvalidURI; - return self.backends.get(uri.scheme) orelse return ResolveAbsolutePathError.SchemeNotRegistered; - } - - fn registerFile(self: *VFS, backend: std.Io, handle: std.Io.File.Handle) std.mem.Allocator.Error!std.Io.File.Handle { - const entry = try self.allocator.create(FsFile); - errdefer self.allocator.destroy(entry); - - entry.* = .{ - .backend = backend, - .handle = handle, - }; - - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const virtual_handle = self.next_file_handle; - self.next_file_handle += 1; - - try self.files.put(self.allocator, virtual_handle, entry); - - return virtual_handle; - } - - fn closeFile(self: *VFS, virtual_handle: std.Io.File.Handle) void { - self.mutex.lockUncancelable(self.base_io); - const entry = self.files.fetchRemove(virtual_handle); - self.mutex.unlock(self.base_io); - - if (entry) |kv| { - const fs_file = kv.value; - fs_file.backend.vtable.fileClose(fs_file.backend.userdata, .{ .handle = fs_file.handle }); - self.allocator.destroy(fs_file); + fn openHandle(self: *VFS) !struct { u32, *Handle } { + if (self.closed_handles.pop()) |idx| { + return .{ idx, &self.handles.items[idx] }; } + return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; } - fn fsFile(self: *VFS, virtual_handle: std.Io.File.Handle) FileEntryError!FsFile { - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const entry = self.files.get(virtual_handle) orelse return FileEntryError.MissingVirtualFileHandle; - return entry.*; + fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == 0) return; // cwd + try self.closed_handles.append(self.allocator, idx); } - fn registerDir(self: *VFS, backend: std.Io, handle: std.Io.Dir.Handle) std.mem.Allocator.Error!std.Io.Dir.Handle { - const entry = try self.allocator.create(FsDir); - errdefer self.allocator.destroy(entry); - - entry.* = .{ - .backend = backend, - .handle = handle, - }; - - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const virtual_handle = self.next_dir_handle; - self.next_dir_handle += 1; - - try self.dirs.put(self.allocator, virtual_handle, entry); - return virtual_handle; + fn getFileHandle(self: *VFS, file: std.Io.File) *Handle { + return &self.handles.items[@intCast(file.handle)]; } - fn closeDir(self: *VFS, virtual_handle: std.Io.Dir.Handle) void { - self.mutex.lockUncancelable(self.base_io); - const entry = self.dirs.fetchRemove(virtual_handle); - self.mutex.unlock(self.base_io); - - if (entry) |kv| { - const fs_dir = kv.value; - fs_dir.backend.vtable.dirClose(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); - self.allocator.destroy(fs_dir); - } - } - - fn fsDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) DirEntryError!FsDir { + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { if (std.meta.eql(dir, std.Io.Dir.cwd())) { - std.debug.assert(sub_path != null); - const backend = self.backendFromAbsolutePath(sub_path.?) catch blk: { - log.warn("Unable to resolve backend for relative path {s} in cwd, falling back to base_io", .{sub_path.?}); - break :blk self.base_io; - }; - return .{ .backend = backend, .handle = dir.handle }; + return &self.handles.items[0]; } - self.mutex.lockUncancelable(self.base_io); - defer self.mutex.unlock(self.base_io); - - const entry = self.dirs.get(dir.handle) orelse return DirEntryError.MissingVirtualDirHandle; - return entry.*; + return &self.handles.items[@intCast(dir.handle)]; } - // VTable implementations + fn lookupFromDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { []const u8, std.Io.Dir, std.Io } { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) { + return .{ "", dir, self.base.inner }; + } - fn dirMake( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + if (std.fs.path.isAbsolutePosix(sub_path.?)) { + return .{ "", dir, self.base.inner }; + } - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, - }; + const uri = try std.Uri.parse(sub_path.?); + return .{ uri.scheme, std.Io.Dir.cwd(), self.backends.get(uri.scheme).? }; + } else { + const handle = self.getDirHandle(dir); + return .{ handle.scheme, .{ .handle = @intCast(handle.handle) }, handle.backend }; + } + } + + fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + + const scheme, const dir_, const backend = self.lookupFromDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .type = .directory, + .handle = @intCast(fs_dir.handle), + .scheme = scheme, + .backend = backend, }; - return fs_dir.backend.vtable.dirMake(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + return .{ .handle = @intCast(idx) }; } - fn dirMakePath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - mode: std.Io.Dir.Mode, - ) std.Io.Dir.MakeError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupFromDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); + } - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeError.FileNotFound, - }; + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupFromDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, sub_path, options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupFromDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, sub_path, options); + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookupFromDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .type = .file, + .handle = @intCast(file.handle), + .scheme = scheme, + .backend = backend, }; - return fs_dir.backend.vtable.dirMakePath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, mode); + return .{ .handle = @intCast(idx) }; } - fn dirMakeOpenPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.MakeOpenPathError!std.Io.Dir { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.MakeOpenPathError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.MakeOpenPathError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.MakeOpenPathError.FileNotFound, + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookupFromDir(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir_.handle)) catch unreachable; + } + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookupFromDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; }; + const prefix_len = scheme.len + 3; + @memcpy(out_buffer[0..scheme.len], scheme); + @memcpy(out_buffer[scheme.len..prefix_len], "://"); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix_len..]); + return prefix_len + path_len; + } - const fs_open_dir = try fs_dir.backend.vtable.dirMakeOpenPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); - - const vfs_handle = self.registerDir(fs_dir.backend, fs_open_dir.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, - }; + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const scheme, const dir_, const backend = self.lookupFromDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; }; - - return .{ .handle = vfs_handle }; + const prefix_len = scheme.len + 3; + @memcpy(out_buffer[0..scheme.len], scheme); + @memcpy(out_buffer[scheme.len..prefix_len], "://"); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix_len..]); + return prefix_len + path_len; } - fn dirStat( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, null) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.StatError.Unexpected, - DirEntryError.InvalidURI => std.Io.Dir.StatError.Unexpected, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatError.Unexpected, - }; - }; - - return fs_dir.backend.vtable.dirStat(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }); + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileStat(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }); } - fn dirStatPath( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.StatPathOptions, - ) std.Io.Dir.StatPathError!std.Io.Dir.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.StatPathError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.StatPathError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.StatPathError.FileNotFound, - }; - }; - - return fs_dir.backend.vtable.dirStatPath(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileLength(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }); } - fn dirAccess( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.AccessOptions, - ) std.Io.Dir.AccessError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.AccessError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.AccessError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.AccessError.FileNotFound, - }; - }; - - return fs_dir.backend.vtable.dirAccess(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle = self.getFileHandle(file); + handle.backend.vtable.fileClose(handle.backend.userdata, &.{.{ .handle = @intCast(handle.handle) }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } } - fn dirCreateFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.CreateFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, - }; - }; - - const fs_file = try fs_dir.backend.vtable.dirCreateFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); - - const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileReadStreaming(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }, data); } - fn dirOpenFile( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - flags: std.Io.File.OpenFlags, - ) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.File.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.File.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.File.OpenError.FileNotFound, - }; - }; - - const fs_file = try fs_dir.backend.vtable.dirOpenFile(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, flags); - - const vfs_handle = self.registerFile(fs_dir.backend, fs_file.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.File.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileReadPositional(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }, data, offset); } - fn dirOpenDir( - userdata: ?*anyopaque, - dir: std.Io.Dir, - sub_path: []const u8, - options: std.Io.Dir.OpenOptions, - ) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_dir = self.fsDir(dir, sub_path) catch |err| { - return switch (err) { - DirEntryError.SchemeNotRegistered => std.Io.Dir.OpenError.BadPathName, - DirEntryError.InvalidURI => std.Io.Dir.OpenError.BadPathName, - DirEntryError.MissingVirtualDirHandle => std.Io.Dir.OpenError.FileNotFound, - }; - }; - - const fs_sub_dir = try fs_dir.backend.vtable.dirOpenDir(fs_dir.backend.userdata, .{ .handle = fs_dir.handle }, sub_path, options); - - const vfs_handle = self.registerDir(fs_dir.backend, fs_sub_dir.handle) catch |err| { - return switch (err) { - std.mem.Allocator.Error.OutOfMemory => std.Io.Dir.OpenError.SystemFdQuotaExceeded, - }; - }; - - return .{ .handle = vfs_handle }; + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileSeekBy(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }, relative_offset); } - fn dirClose( - userdata: ?*anyopaque, - dir: std.Io.Dir, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeDir(dir.handle); + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return handle.backend.vtable.fileSeekTo(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }, absolute_offset); } - fn fileStat( - userdata: ?*anyopaque, - file: std.Io.File, - ) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.StatError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.StatError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.StatError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileStat(fs_file.backend.userdata, .{ .handle = fs_file.handle }); - } - - fn fileClose( - userdata: ?*anyopaque, - file: std.Io.File, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - self.closeFile(file.handle); - } - - fn fileWriteStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - ) std.Io.File.WriteStreamingError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.WriteStreamingError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.WriteStreamingError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.WriteStreamingError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileWriteStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer); - } - - fn fileWritePositional( - userdata: ?*anyopaque, - file: std.Io.File, - buffer: [][]const u8, - offset: u64, - ) std.Io.File.WritePositionalError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.WritePositionalError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.WritePositionalError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.WritePositionalError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileWritePositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, buffer, offset); - } - - fn fileReadStreaming( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - ) std.Io.File.Reader.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.Reader.Error.Unexpected, - FileEntryError.InvalidURI => std.Io.File.Reader.Error.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.Reader.Error.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileReadStreaming(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data); - } - - fn fileReadPositional( - userdata: ?*anyopaque, - file: std.Io.File, - data: [][]u8, - offset: u64, - ) std.Io.File.ReadPositionalError!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.ReadPositionalError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.ReadPositionalError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.ReadPositionalError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileReadPositional(fs_file.backend.userdata, .{ .handle = fs_file.handle }, data, offset); - } - - fn fileSeekBy( - userdata: ?*anyopaque, - file: std.Io.File, - relative_offset: i64, - ) std.Io.File.SeekError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileSeekBy(fs_file.backend.userdata, .{ .handle = fs_file.handle }, relative_offset); - } - - fn fileSeekTo( - userdata: ?*anyopaque, - file: std.Io.File, - absolute_offset: u64, - ) std.Io.File.SeekError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - - const fs_file = self.fsFile(file.handle) catch |err| { - return switch (err) { - FileEntryError.SchemeNotRegistered => std.Io.File.SeekError.Unexpected, - FileEntryError.InvalidURI => std.Io.File.SeekError.Unexpected, - FileEntryError.MissingVirtualFileHandle => std.Io.File.SeekError.Unexpected, - }; - }; - - return fs_file.backend.vtable.fileSeekTo(fs_file.backend.userdata, .{ .handle = fs_file.handle }, absolute_offset); - } - - fn openSelfExe(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.Io.File.OpenSelfExeError!std.Io.File { - _ = userdata; - _ = flags; - return std.Io.File.OpenSelfExeError.NotSupported; - } - - // Forwarding wrappers - - fn async( - userdata: ?*anyopaque, - result: []u8, - result_alignment: std.mem.Alignment, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (context: *const anyopaque, result: *anyopaque) void, - ) ?*std.Io.AnyFuture { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.async; - return f(self.base_io.userdata, result, result_alignment, context, context_alignment, start); - } - - fn concurrent( - userdata: ?*anyopaque, - result_len: usize, - result_alignment: std.mem.Alignment, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (context: *const anyopaque, result: *anyopaque) void, - ) std.Io.ConcurrentError!*std.Io.AnyFuture { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.concurrent; - return f(self.base_io.userdata, result_len, result_alignment, context, context_alignment, start); - } - - fn await( - userdata: ?*anyopaque, - any_future: *std.Io.AnyFuture, - result: []u8, - result_alignment: std.mem.Alignment, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.await; - f(self.base_io.userdata, any_future, result, result_alignment); - } - - fn cancel( - userdata: ?*anyopaque, - any_future: *std.Io.AnyFuture, - result: []u8, - result_alignment: std.mem.Alignment, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.cancel; - f(self.base_io.userdata, any_future, result, result_alignment); - } - - fn cancelRequested(userdata: ?*anyopaque) bool { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.cancelRequested; - return f(self.base_io.userdata); - } - - fn groupAsync( - userdata: ?*anyopaque, - group: *std.Io.Group, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (*std.Io.Group, context: *const anyopaque) void, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupAsync; - f(self.base_io.userdata, group, context, context_alignment, start); - } - - fn groupConcurrent( - userdata: ?*anyopaque, - group: *std.Io.Group, - context: []const u8, - context_alignment: std.mem.Alignment, - start: *const fn (*std.Io.Group, context: *const anyopaque) void, - ) std.Io.ConcurrentError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.groupConcurrent(self.base_io.userdata, group, context, context_alignment, start); - } - - fn groupWait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupWait; - f(self.base_io.userdata, group, token); - } - - fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.groupCancel; - f(self.base_io.userdata, group, token); - } - - fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.select(self.base_io.userdata, futures); - } - - fn mutexLock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) std.Io.Cancelable!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.mutexLock(self.base_io.userdata, prev_state, mutex); - } - - fn mutexLockUncancelable(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.mutexLockUncancelable; - f(self.base_io.userdata, prev_state, mutex); - } - - fn mutexUnlock(userdata: ?*anyopaque, prev_state: std.Io.Mutex.State, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.mutexUnlock; - f(self.base_io.userdata, prev_state, mutex); - } - - fn conditionWait(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) std.Io.Cancelable!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.conditionWait(self.base_io.userdata, cond, mutex); - } - - fn conditionWaitUncancelable(userdata: ?*anyopaque, cond: *std.Io.Condition, mutex: *std.Io.Mutex) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.conditionWaitUncancelable; - f(self.base_io.userdata, cond, mutex); - } - - fn conditionWake(userdata: ?*anyopaque, cond: *std.Io.Condition, wake: std.Io.Condition.Wake) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.conditionWake; - f(self.base_io.userdata, cond, wake); - } - - fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.now(self.base_io.userdata, clock); - } - - fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.sleep(self.base_io.userdata, timeout); - } - - fn netListenIp( - userdata: ?*anyopaque, - address: std.Io.net.IpAddress, - options: std.Io.net.IpAddress.ListenOptions, - ) std.Io.net.IpAddress.ListenError!std.Io.net.Server { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netListenIp(self.base_io.userdata, address, options); - } - - fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netAccept(self.base_io.userdata, server); - } - - fn netBindIp( - userdata: ?*anyopaque, - address: *const std.Io.net.IpAddress, - options: std.Io.net.IpAddress.BindOptions, - ) std.Io.net.IpAddress.BindError!std.Io.net.Socket { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netBindIp(self.base_io.userdata, address, options); - } - - fn netConnectIp( - userdata: ?*anyopaque, - address: *const std.Io.net.IpAddress, - options: std.Io.net.IpAddress.ConnectOptions, - ) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netConnectIp(self.base_io.userdata, address, options); - } - - fn netListenUnix( - userdata: ?*anyopaque, - address: *const std.Io.net.UnixAddress, - options: std.Io.net.UnixAddress.ListenOptions, - ) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netListenUnix(self.base_io.userdata, address, options); - } - - fn netConnectUnix( - userdata: ?*anyopaque, - address: *const std.Io.net.UnixAddress, - ) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netConnectUnix(self.base_io.userdata, address); - } - - fn netSend( - userdata: ?*anyopaque, - handle: std.Io.net.Socket.Handle, - msgs: []std.Io.net.OutgoingMessage, - flags: std.Io.net.SendFlags, - ) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netSend(self.base_io.userdata, handle, msgs, flags); - } - - fn netReceive( - userdata: ?*anyopaque, - handle: std.Io.net.Socket.Handle, - message_buffer: []std.Io.net.IncomingMessage, - data_buffer: []u8, - flags: std.Io.net.ReceiveFlags, - timeout: std.Io.Timeout, - ) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netReceive(self.base_io.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - - fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netRead(self.base_io.userdata, src, data); - } - - fn netWrite( - userdata: ?*anyopaque, - dest: std.Io.net.Socket.Handle, - header: []const u8, - data: []const []const u8, - splat: usize, - ) std.Io.net.Stream.Writer.Error!usize { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netWrite(self.base_io.userdata, dest, header, data, splat); - } - - fn netClose(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.netClose; - f(self.base_io.userdata, handle); - } - - fn netInterfaceNameResolve( - userdata: ?*anyopaque, - name: *const std.Io.net.Interface.Name, - ) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netInterfaceNameResolve(self.base_io.userdata, name); - } - - fn netInterfaceName( - userdata: ?*anyopaque, - iface: std.Io.net.Interface, - ) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.base_io.vtable.netInterfaceName(self.base_io.userdata, iface); - } - - fn netLookup( - userdata: ?*anyopaque, - host: std.Io.net.HostName, - q: *std.Io.Queue(std.Io.net.HostName.LookupResult), - options: std.Io.net.HostName.LookupOptions, - ) void { - const self: *VFS = @ptrCast(@alignCast(userdata orelse unreachable)); - const f = self.base_io.vtable.netLookup; - f(self.base_io.userdata, host, q, options); + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const prefix_len = handle.scheme.len + 3; + @memcpy(out_buffer[0..handle.scheme.len], handle.scheme); + @memcpy(out_buffer[handle.scheme.len..prefix_len], "://"); + const path_len = try handle.backend.vtable.fileRealPath(handle.backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix_len..]); + return prefix_len + path_len; } }; diff --git a/zml2/io/vfs/BUILD.bazel b/zml2/io/vfs/BUILD.bazel index 0a0bc3c4..6cc11617 100644 --- a/zml2/io/vfs/BUILD.bazel +++ b/zml2/io/vfs/BUILD.bazel @@ -5,18 +5,9 @@ zig_library( srcs = [ "base.zig", "file.zig", - "hf.zig", - "proxy.zig", + "hf2.zig", "http.zig", ], main = "index.zig", visibility = ["//visibility:public"], ) - -zig_binary( - name = "hf2", - main = "hf2.zig", - srcs = ["base.zig"], - deps = ["//stdx"], - visibility = ["//visibility:public"], -) diff --git a/zml2/io/vfs/base.zig b/zml2/io/vfs/base.zig index d2d8f30e..e586d10b 100644 --- a/zml2/io/vfs/base.zig +++ b/zml2/io/vfs/base.zig @@ -3,49 +3,6 @@ const std = @import("std"); const stdx = @import("stdx"); pub const VFSBase = struct { - pub fn HandleRegistry(comptime InnerType: type) type { - return struct { - const Self = @This(); - - const Node = struct { - value: InnerType, - next_closed: ?usize = null, - }; - - mutex: std.Io.Mutex = .init, - handles: stdx.SegmentedList(InnerType, 0) = .{}, - next_closed: ?usize = null, - - pub fn open(self: *Self, allocator: std.mem.Allocator, io: std.Io) !struct { usize, *InnerType } { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - if (self.next_closed) |handle| { - const node = self.handles.at(handle); - self.next_closed = node.next_closed; - return .{ handle, node.value }; - } - return .{ self.handles.count(), try self.handles.addOne(allocator) }; - } - - pub fn close(self: *Self, io: std.Io, handle: usize) void { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - const node = self.handles.at(handle); - node.next_closed = self.next_closed; - self.next_closed = handle; - } - - pub fn get(self: *Self, io: std.Io, handle: usize) *InnerType { - self.mutex.lockUncancelable(io); - defer self.mutex.unlock(io); - - return &self.handles.at(handle).value; - } - }; - } - inner: std.Io, pub fn init(io: std.Io) VFSBase { diff --git a/zml2/io/vfs/hf2.zig b/zml2/io/vfs/hf2.zig index 1092ab98..6c2ce855 100644 --- a/zml2/io/vfs/hf2.zig +++ b/zml2/io/vfs/hf2.zig @@ -1,38 +1,89 @@ const std = @import("std"); -const stdx = @import("stdx"); - const VFSBase = @import("base.zig").VFSBase; const log = std.log.scoped(.@"zml/io/vfs/hf"); pub const API = struct { - const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}"; const TREESIZE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/treesize/{[rev]s}/{[path]s}"; - const RAW_FILE_URL_REMPLATE = "https://huggingface.co/{model}/raw/{rev}/{path}"; const LFS_FILE_URL_TEMPLATE = "https://huggingface.co/{[repo]s}/{[model]s}/resolve/{[rev]s}/{[path]s}"; - pub const Entry = struct { - pub const Type = enum { - file, - directory, - }; - - pub const LFS = struct { - oid: []const u8, - size: u64, - pointerSize: u64, - }; - - type: Type, - oid: []const u8, - size: u64, + pub const TreeSize = struct { path: []const u8, - lfs: ?LFS = null, + size: u64, }; }; pub const HF = struct { + pub const Auth = union(enum) { + none, + hf_token: []const u8, + + pub const AuthError = error{ + MissingHFToken, + InvalidHFToken, + MissingHomePath, + MissingConfigFile, + Unexpected, + }; + + pub fn deinit(self: *Auth, allocator: std.mem.Allocator) void { + switch (self.*) { + .none => {}, + .hf_token => |token| { + allocator.free(token); + }, + } + } + + pub fn auto(allocator: std.mem.Allocator, io_: std.Io) AuthError!Auth { + return fromHomeConfig(allocator, io_) catch |err| switch (err) { + AuthError.MissingHomePath, AuthError.MissingConfigFile => return fromEnv(allocator) catch |e| switch (e) { + AuthError.MissingHFToken => { + log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); + return .none; + }, + else => return AuthError.Unexpected, + }, + else => return AuthError.Unexpected, + }; + } + + pub fn fromEnv(allocator: std.mem.Allocator) AuthError!Auth { + const token = std.process.getEnvVarOwned(allocator, "HF_TOKEN") catch |err| switch (err) { + error.EnvironmentVariableNotFound => return AuthError.MissingHFToken, + else => return AuthError.Unexpected, + }; + defer allocator.free(token); + + return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{token}) catch return AuthError.Unexpected }; + } + + pub fn fromHomeConfig(allocator: std.mem.Allocator, base_io: std.Io) AuthError!Auth { + const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch |err| switch (err) { + error.EnvironmentVariableNotFound => return AuthError.MissingHomePath, + else => return AuthError.Unexpected, + }; + defer allocator.free(home_path); + + const token_path = std.fmt.allocPrint(allocator, "{s}/.cache/huggingface/token", .{home_path}) catch return AuthError.Unexpected; + defer allocator.free(token_path); + + const file = std.Io.Dir.openFileAbsolute(base_io, token_path, .{ .mode = .read_only }) catch |err| switch (err) { + error.FileNotFound => return AuthError.MissingConfigFile, + else => return AuthError.Unexpected, + }; + + var reader = file.reader(base_io, &.{}); + const size = reader.getSize() catch return AuthError.Unexpected; + + const token = reader.interface.readAlloc(allocator, size) catch return AuthError.Unexpected; + defer allocator.free(token); + + return .{ .hf_token = std.fmt.allocPrint(allocator, "Bearer {s}", .{std.mem.trim(u8, token, "\n")}) catch return AuthError.Unexpected }; + } + }; + pub const Repo = struct { repo: []const u8, model: []const u8, @@ -55,79 +106,98 @@ pub const HF = struct { } }; - pub const Handle = struct { + const Handle = struct { pub const Type = enum { file, directory, }; type: Type, - url: []const u8, - uri: std.Uri, + uri: []const u8, pos: u64, size: u64, - pub fn init(allocator: std.mem.Allocator, type_: Type, sub_path: []const u8, size: u64) Handle { - const repo = Repo.parse(sub_path) catch unreachable; - const lfs_url = std.fmt.allocPrint(allocator, API.LFS_FILE_URL_TEMPLATE, repo) catch unreachable; - errdefer allocator.free(lfs_url); - + pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) Handle { return .{ .type = type_, - .url = lfs_url, - .uri = std.Uri.parse(lfs_url) catch unreachable, + .uri = allocator.dupe(u8, path) catch unreachable, .pos = 0, .size = size, }; } pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { - allocator.free(self.url); + allocator.free(self.uri); } }; - base: VFSBase, - authorization: std.http.Client.Request.Headers.Value = .default, + const Config = struct { + http_client: *std.http.Client, + auth: Auth = .none, + }; + allocator: std.mem.Allocator, client: *std.http.Client, + authorization: std.http.Client.Request.Headers.Value = .default, handles: std.ArrayList(Handle) = .{}, closed_handles: std.ArrayList(u32) = .{}, + base: VFSBase, - pub fn init(allocator: std.mem.Allocator, io_: std.Io, http_client_: *std.http.Client) !HF { - return .{ + pub fn init(allocator: std.mem.Allocator, base_io: std.Io, config: Config) !HF { + var hf: HF = .{ .allocator = allocator, - .base = .init(io_), - .client = http_client_, - .authorization = .{ - .override = "Bearer xxxx", - }, + .base = .init(base_io), + .client = config.http_client, }; + + switch (config.auth) { + .hf_token => |token| { + hf.authorization = .{ .override = token }; + }, + .none => {}, + } + + return hf; } pub fn deinit(self: *HF) void { - _ = self; // autofix + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); } pub fn io(self: *HF) std.Io { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, .dirOpenFile = dirOpenFile, - .fileReadPositional = fileReadPositional, - .fileReadStreaming = fileReadStreaming, + .dirClose = dirClose, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, }), }; } - pub fn openHandle(self: *HF) !struct { u32, *Handle } { + fn openHandle(self: *HF) !struct { u32, *Handle } { if (self.closed_handles.pop()) |idx| { return .{ idx, &self.handles.items[idx] }; } return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; } - pub fn closeHandle(self: *HF, idx: u32) !void { + fn closeHandle(self: *HF, idx: u32) !void { + self.handles.items[idx].deinit(self.allocator); try self.closed_handles.append(self.allocator, idx); } @@ -135,132 +205,129 @@ pub const HF = struct { return &self.handles.items[@intCast(file.handle)]; } - fn parseContentRange(value: []const u8) struct { u64, u64, u64 } { - const space = std.mem.indexOfScalar(u8, value, ' ').?; - const range = std.mem.indexOfScalar(u8, value, '-').?; - const slash = std.mem.indexOfScalar(u8, value, '/').?; + fn getDirHandle(self: *HF, dir: std.Io.Dir) *Handle { + return &self.handles.items[@intCast(dir.handle)]; + } + + fn resolvePath(self: *HF, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + + const handle = self.getDirHandle(dir); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ handle.uri, sub_path }); + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + + const size = self.fetchSize(dir, sub_path) catch |err| { + log.err("Failed to fetch size for directory with path {s}: {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .init(self.allocator, .directory, path, size); + + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); return .{ - std.fmt.parseInt(u64, value[space + 1 .. range], 10) catch 0, - std.fmt.parseInt(u64, value[range + 1 .. slash], 10) catch 0, - std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch 0, + .inode = @intCast(dir.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, }; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - _ = dir; // autofix - _ = flags; // autofix - // _ = flags; // autofix + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + + return .{ + .inode = @intCast(dir.handle), + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + _ = self.fetchSize(dir, sub_path) catch return std.Io.Dir.AccessError.FileNotFound; + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - var url_buffer: [1024]u8 = undefined; - var redirect_buffer: [8 * 1024]u8 = undefined; - var aux_buffer: []u8 = &redirect_buffer; + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; - const repo = Repo.parse(sub_path) catch return std.Io.File.OpenError.BadPathName; - const url: []const u8 = std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo) catch return std.Io.File.OpenError.BadPathName; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - log.info("Opening url: {s}", .{url}); + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = .init(self.allocator, .file, path, size); - var uri = std.Uri.parse(url) catch return std.Io.File.OpenError.BadPathName; - while (true) { - var req = self.client.request(.HEAD, uri, .{ - .headers = .{ .authorization = self.authorization }, - .redirect_behavior = .not_allowed, - }) catch return std.Io.File.Reader.Error.Unexpected; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - var resp = req.receiveHead(&.{}) catch { - log.err("Failed to receive HTTP response for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - if (resp.head.status.class() == .redirect) { - const location = resp.head.location.?; - @memcpy(aux_buffer[0..location.len], location); - uri = uri.resolveInPlace(location.len, &aux_buffer) catch unreachable; - log.info("Redirect {f}", .{uri}); - continue; - } - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .init(self.allocator, .file, sub_path, resp.head.content_length.?); - - return .{ .handle = @intCast(idx) }; - } + return .{ .handle = @intCast(idx) }; } - pub fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) std.Io.File.Reader.Error!usize { - if (offset >= handle.size) { - log.info("EOF", .{}); - return 0; - } - - var range_buf: [64]u8 = undefined; - const range_header = blk: { - var total_bytes: u64 = 0; - for (data) |buf| { - total_bytes += buf.len; - } - break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, @min(handle.size - offset, offset + total_bytes) }) catch unreachable; - }; - - var req = self.client.request(.GET, handle.uri, .{ - .headers = .{ .authorization = self.authorization }, - .extra_headers = &.{ - .{ .name = "Range", .value = range_header }, - }, - }) catch return std.Io.File.Reader.Error.Unexpected; - defer req.deinit(); - - req.sendBodiless() catch { - log.err("Failed to send HTTP request for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var resp = req.receiveHead(&redirect_buffer) catch { - log.err("Failed to receive HTTP response for HF model size API URL", .{}); - return std.Io.File.Reader.Error.Unexpected; - }; - - const range = blk: { - var it = resp.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parseContentRange(header.value); - } - } else break :blk .{ 0, 0, 0 }; - }; - - log.info(">>>>> {any}", .{range}); - - const reader = resp.reader(&.{}); - - if (range.@"0" < offset) { - reader.discardAll(offset - range.@"0") catch return std.Io.File.Reader.Error.Unexpected; - } - - return reader.readSliceShort(data[0]) catch return std.Io.File.Reader.Error.Unexpected; - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const len = handle.uri.len; + @memcpy(out_buffer[0..len], handle.uri); + return len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const full_path = std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ handle.uri, path_name }) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return full_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); - return try self.performRead(handle, data, offset); + + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; } - fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const total = try self.performRead(handle, data, handle.pos); - handle.pos += @intCast(total); - return total; + return self.getFileHandle(file).size; } fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { @@ -269,48 +336,168 @@ pub const HF = struct { self.closeHandle(@intCast(file.handle)) catch unreachable; } } + + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + handle.pos = if (relative_offset >= 0) + handle.pos + @as(u64, @intCast(relative_offset)) + else + handle.pos - @as(u64, @intCast(-relative_offset)); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const len = handle.uri.len; + @memcpy(out_buffer[0..len], handle.uri); + return len; + } + + fn fetchSize(self: *HF, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + var url_buffer: [8 * 1024]u8 = undefined; + var redirect_buffer: [8 * 1024]u8 = undefined; + + const path = try self.resolvePath(dir, sub_path, &path_buffer); + + const repo = Repo.parse(path) catch return std.Io.File.OpenError.BadPathName; + const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.TREESIZE_URL_TEMPLATE, repo); + const uri = std.Uri.parse(url) catch return std.Io.Dir.OpenError.BadPathName; + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .authorization = self.authorization }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .ok) { + log.err("Failed to fetch tree size for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + } + + const body = try res.reader(&.{}).readAlloc(self.allocator, res.head.content_length.?); + defer self.allocator.free(body); + + if (res.head.status != .ok) { + log.err("{s}", .{body}); + return error.RequestFailed; + } + + const parsed = try std.json.parseFromSlice( + API.TreeSize, + self.allocator, + body, + .{ .ignore_unknown_fields = true }, + ); + defer parsed.deinit(); + + return parsed.value.size; + } + + fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; + + const repo: Repo = try .parse(handle.uri); + var url_buffer: [8 * 1024]u8 = undefined; + const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + + const uri: std.Uri = try .parse(url); + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .authorization = self.authorization }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content) { + log.err("Failed to perform read for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; + } + + const reader = res.reader(&.{}); + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + if (content_range) |range| { + if (range.start < offset) { + try reader.discardAll(offset - range.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } }; - -pub fn main() !void { - const allocator = std.heap.smp_allocator; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - - var client: std.http.Client = .{ - .allocator = allocator, - .io = threaded.io(), - }; - - try client.initDefaultProxies(allocator); - defer client.deinit(); - - var hf: HF = try .init(allocator, threaded.io(), &client); - defer hf.deinit(); - - const io = hf.io(); - - const file = std.Io.Dir.cwd().openFile(io, "Qwen/Qwen3-8B/model-00001-of-00005.safetensors", .{}) catch |err| { - log.err("Failed to open HF file: {any}", .{err}); - unreachable; - }; - - log.info(">>>>>>> {any}", .{file}); - - var reader = file.reader(io, &.{}); - reader.mode = reader.mode.toSimple(); - defer log.info("DONE", .{}); - - // const data = try reader.interface.readAlloc(allocator, 1024); - // defer allocator.free(data); - // log.info("READ {s}", .{data}); - - const buffer = try allocator.alloc(u8, 256 * 1024 * 1024); - var stdout_writer = std.Io.File.stdout().writer(io, buffer); - defer stdout_writer.interface.flush() catch {}; - - _ = reader.interface.streamRemaining(&stdout_writer.interface) catch |err| { - log.err("Failed to stream HF file: {any}", .{err}); - unreachable; - }; -} diff --git a/zml2/io/vfs/index.zig b/zml2/io/vfs/index.zig index fc44f6b6..3c4c145e 100644 --- a/zml2/io/vfs/index.zig +++ b/zml2/io/vfs/index.zig @@ -2,6 +2,7 @@ const std = @import("std"); const log = std.log.scoped(.@"zml/io/vfs/vfs"); +pub const VFSBase = @import("base.zig").VFSBase; pub const File = @import("file.zig").File; pub const HTTP = @import("http.zig").HTTP; -pub const HF = @import("hf.zig").HF; +pub const HF = @import("hf2.zig").HF; diff --git a/zml2/io/vfs/proxy.zig b/zml2/io/vfs/proxy.zig deleted file mode 100644 index 0382d5ac..00000000 --- a/zml2/io/vfs/proxy.zig +++ /dev/null @@ -1,546 +0,0 @@ -const std = @import("std"); - -pub const IoProxy = struct { - inner: std.Io, - - pub fn init(io: std.Io) IoProxy { - return .{ .inner = io }; - } - - pub fn partial(overrides: anytype) std.Io.VTable { - var new_vtable = vtable; - for (std.meta.fieldNames(@TypeOf(overrides))) |field_name| { - @field(new_vtable, field_name) = @field(overrides, field_name); - } - return new_vtable; - } - - pub fn as(userdata: ?*anyopaque) *IoProxy { - return @ptrCast(@alignCast(userdata orelse unreachable)); - } - - pub const vtable: std.Io.VTable = .{ - .async = async, - .concurrent = concurrent, - .await = await, - .cancel = cancel, - .groupAsync = groupAsync, - .groupConcurrent = groupConcurrent, - .groupAwait = groupAwait, - .groupCancel = groupCancel, - .recancel = recancel, - .swapCancelProtection = swapCancelProtection, - .checkCancel = checkCancel, - .select = select, - .futexWait = futexWait, - .futexWaitUncancelable = futexWaitUncancelable, - .futexWake = futexWake, - .dirCreateDir = dirCreateDir, - .dirCreateDirPath = dirCreateDirPath, - .dirCreateDirPathOpen = dirCreateDirPathOpen, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .dirDeleteFile = dirDeleteFile, - .dirDeleteDir = dirDeleteDir, - .dirRename = dirRename, - .dirSymLink = dirSymLink, - .dirReadLink = dirReadLink, - .dirSetOwner = dirSetOwner, - .dirSetFileOwner = dirSetFileOwner, - .dirSetPermissions = dirSetPermissions, - .dirSetFilePermissions = dirSetFilePermissions, - .dirSetTimestamps = dirSetTimestamps, - .dirHardLink = dirHardLink, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWriteStreaming = fileWriteStreaming, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadStreaming = fileReadStreaming, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileSync = fileSync, - .fileIsTty = fileIsTty, - .fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes, - .fileSupportsAnsiEscapeCodes = fileSupportsAnsiEscapeCodes, - .fileSetLength = fileSetLength, - .fileSetOwner = fileSetOwner, - .fileSetPermissions = fileSetPermissions, - .fileSetTimestamps = fileSetTimestamps, - .fileLock = fileLock, - .fileTryLock = fileTryLock, - .fileUnlock = fileUnlock, - .fileDowngradeLock = fileDowngradeLock, - .fileRealPath = fileRealPath, - .processExecutableOpen = processExecutableOpen, - .processExecutablePath = processExecutablePath, - .lockStderr = lockStderr, - .tryLockStderr = tryLockStderr, - .unlockStderr = unlockStderr, - .processSetCurrentDir = processSetCurrentDir, - .now = now, - .sleep = sleep, - .netListenIp = netListenIp, - .netAccept = netAccept, - .netBindIp = netBindIp, - .netConnectIp = netConnectIp, - .netListenUnix = netListenUnix, - .netConnectUnix = netConnectUnix, - .netSend = netSend, - .netReceive = netReceive, - .netRead = netRead, - .netWrite = netWrite, - .netWriteFile = netWriteFile, - .netClose = netClose, - .netShutdown = netShutdown, - .netInterfaceNameResolve = netInterfaceNameResolve, - .netInterfaceName = netInterfaceName, - .netLookup = netLookup, - }; - - pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); - } - - pub fn concurrent(userdata: ?*anyopaque, result_len: usize, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) std.Io.ConcurrentError!*std.Io.AnyFuture { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.concurrent(self.inner.userdata, result_len, result_alignment, context, context_alignment, start); - } - - pub fn await(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.await(self.inner.userdata, any_future, result, result_alignment); - } - - pub fn cancel(userdata: ?*anyopaque, any_future: *std.Io.AnyFuture, result: []u8, result_alignment: std.mem.Alignment) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); - } - - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); - } - - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (*std.Io.Group, context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); - } - - pub fn groupAwait(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupAwait(self.inner.userdata, group, token); - } - - pub fn groupCancel(userdata: ?*anyopaque, group: *std.Io.Group, token: *anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.groupCancel(self.inner.userdata, group, token); - } - - pub fn recancel(userdata: ?*anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.recancel(self.inner.userdata); - } - - pub fn swapCancelProtection(userdata: ?*anyopaque, new: std.Io.CancelProtection) std.Io.CancelProtection { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.swapCancelProtection(self.inner.userdata, new); - } - - pub fn checkCancel(userdata: ?*anyopaque) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.checkCancel(self.inner.userdata); - } - - pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.select(self.inner.userdata, futures); - } - - pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); - } - - pub fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWaitUncancelable(self.inner.userdata, ptr, expected); - } - - pub fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); - } - - pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); - } - - pub fn dirCreateDirPath(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirPathError!std.Io.Dir.CreatePathStatus { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDirPath(self.inner.userdata, dir, sub_path, permissions); - } - - pub fn dirCreateDirPathOpen(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions, options: std.Io.Dir.OpenOptions) std.Io.Dir.CreateDirPathOpenError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateDirPathOpen(self.inner.userdata, dir, sub_path, permissions, options); - } - - pub fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenDir(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStat(self.inner.userdata, dir); - } - - pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirStatFile(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); - } - - pub fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.dirClose(self.inner.userdata, dirs); - } - - pub fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRead(self.inner.userdata, reader, entries); - } - - pub fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRealPath(self.inner.userdata, dir, out_buffer); - } - - pub fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRealPathFile(self.inner.userdata, dir, path_name, out_buffer); - } - - pub fn dirDeleteFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteFileError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirDeleteFile(self.inner.userdata, dir, sub_path); - } - - pub fn dirDeleteDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8) std.Io.Dir.DeleteDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirDeleteDir(self.inner.userdata, dir, sub_path); - } - - pub fn dirRename(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenameError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); - } - - pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); - } - - pub fn dirReadLink(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, buffer: []u8) std.Io.Dir.ReadLinkError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirReadLink(self.inner.userdata, dir, sub_path, buffer); - } - - pub fn dirSetOwner(userdata: ?*anyopaque, dir: std.Io.Dir, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.Dir.SetOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetOwner(self.inner.userdata, dir, uid, gid); - } - - pub fn dirSetFileOwner(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid, options: std.Io.Dir.SetFileOwnerOptions) std.Io.Dir.SetFileOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetFileOwner(self.inner.userdata, dir, sub_path, uid, gid, options); - } - - pub fn dirSetPermissions(userdata: ?*anyopaque, dir: std.Io.Dir, permissions: std.Io.Dir.Permissions) std.Io.Dir.SetPermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetPermissions(self.inner.userdata, dir, permissions); - } - - pub fn dirSetFilePermissions(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.File.Permissions, options: std.Io.Dir.SetFilePermissionsOptions) std.Io.Dir.SetFilePermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetFilePermissions(self.inner.userdata, dir, sub_path, permissions, options); - } - - pub fn dirSetTimestamps(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.SetTimestampsOptions) std.Io.Dir.SetTimestampsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirSetTimestamps(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirHardLink(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8, options: std.Io.Dir.HardLinkOptions) std.Io.Dir.HardLinkError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.dirHardLink(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path, options); - } - - pub fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileStat(self.inner.userdata, file); - } - - pub fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileLength(self.inner.userdata, file); - } - - pub fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.fileClose(self.inner.userdata, files); - } - - pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); - } - - pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); - } - - pub fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteFileStreaming(self.inner.userdata, file, header, reader, limit); - } - - pub fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); - } - - pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); - } - - pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); - } - - pub fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekBy(self.inner.userdata, file, relative_offset); - } - - pub fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSeekTo(self.inner.userdata, file, absolute_offset); - } - - pub fn fileSync(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.SyncError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSync(self.inner.userdata, file); - } - - pub fn fileIsTty(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileIsTty(self.inner.userdata, file); - } - - pub fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.EnableAnsiEscapeCodesError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileEnableAnsiEscapeCodes(self.inner.userdata, file); - } - - pub fn fileSupportsAnsiEscapeCodes(userdata: ?*anyopaque, file: std.Io.File) std.Io.Cancelable!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSupportsAnsiEscapeCodes(self.inner.userdata, file); - } - - pub fn fileSetLength(userdata: ?*anyopaque, file: std.Io.File, length: u64) std.Io.File.SetLengthError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetLength(self.inner.userdata, file, length); - } - - pub fn fileSetOwner(userdata: ?*anyopaque, file: std.Io.File, uid: ?std.Io.File.Uid, gid: ?std.Io.File.Gid) std.Io.File.SetOwnerError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetOwner(self.inner.userdata, file, uid, gid); - } - - pub fn fileSetPermissions(userdata: ?*anyopaque, file: std.Io.File, permissions: std.Io.File.Permissions) std.Io.File.SetPermissionsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetPermissions(self.inner.userdata, file, permissions); - } - - pub fn fileSetTimestamps(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.SetTimestampsOptions) std.Io.File.SetTimestampsError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileSetTimestamps(self.inner.userdata, file, options); - } - - pub fn fileLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileLock(self.inner.userdata, file, lock); - } - - pub fn fileTryLock(userdata: ?*anyopaque, file: std.Io.File, lock: std.Io.File.Lock) std.Io.File.LockError!bool { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileTryLock(self.inner.userdata, file, lock); - } - - pub fn fileUnlock(userdata: ?*anyopaque, file: std.Io.File) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileUnlock(self.inner.userdata, file); - } - - pub fn fileDowngradeLock(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.DowngradeLockError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileDowngradeLock(self.inner.userdata, file); - } - - pub fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); - } - - pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); - } - - pub fn processExecutablePath(userdata: ?*anyopaque, buffer: []u8) std.process.ExecutablePathError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processExecutablePath(self.inner.userdata, buffer); - } - - pub fn lockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!std.Io.LockedStderr { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.lockStderr(self.inner.userdata, mode); - } - - pub fn tryLockStderr(userdata: ?*anyopaque, mode: ?std.Io.Terminal.Mode) std.Io.Cancelable!?std.Io.LockedStderr { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.tryLockStderr(self.inner.userdata, mode); - } - - pub fn unlockStderr(userdata: ?*anyopaque) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.unlockStderr(self.inner.userdata); - } - - pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); - } - - pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.now(self.inner.userdata, clock); - } - - pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.sleep(self.inner.userdata, timeout); - } - - pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netListenIp(self.inner.userdata, address, options); - } - - pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netAccept(self.inner.userdata, server); - } - - pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netBindIp(self.inner.userdata, address, options); - } - - pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); - } - - pub fn netListenUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress, options: std.Io.net.UnixAddress.ListenOptions) std.Io.net.UnixAddress.ListenError!std.Io.net.Socket.Handle { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netListenUnix(self.inner.userdata, address, options); - } - - pub fn netConnectUnix(userdata: ?*anyopaque, address: *const std.Io.net.UnixAddress) std.Io.net.UnixAddress.ConnectError!std.Io.net.Socket.Handle { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netConnectUnix(self.inner.userdata, address); - } - - pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); - } - - pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); - } - - pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netRead(self.inner.userdata, src, data); - } - - pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); - } - - pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); - } - - pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - self.inner.vtable.netClose(self.inner.userdata, handles); - } - - pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netShutdown(self.inner.userdata, handle, how); - } - - pub fn netInterfaceNameResolve(userdata: ?*anyopaque, name: *const std.Io.net.Interface.Name) std.Io.net.Interface.Name.ResolveError!std.Io.net.Interface { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netInterfaceNameResolve(self.inner.userdata, name); - } - - pub fn netInterfaceName(userdata: ?*anyopaque, iface: std.Io.net.Interface) std.Io.net.Interface.NameError!std.Io.net.Interface.Name { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netInterfaceName(self.inner.userdata, iface); - } - - pub fn netLookup(userdata: ?*anyopaque, host: std.Io.net.HostName, q: *std.Io.Queue(std.Io.net.HostName.LookupResult), options: std.Io.net.HostName.LookupOptions) std.Io.net.HostName.LookupError!void { - const self: *IoProxy = @ptrCast(@alignCast(userdata orelse unreachable)); - return self.inner.vtable.netLookup(self.inner.userdata, host, q, options); - } -}; diff --git a/zml2/main_io.zig b/zml2/main_io.zig index f9a612df..f6278a9b 100644 --- a/zml2/main_io.zig +++ b/zml2/main_io.zig @@ -23,7 +23,7 @@ pub fn main() !void { // const allocator = std.heap.smp_allocator; - var threaded: std.Io.Threaded = .init(allocator); + var threaded: std.Io.Threaded = .init(allocator, .{}); defer threaded.deinit(); if (try std.process.hasEnvVar(allocator, "ASYNC_LIMIT")) { @@ -46,18 +46,18 @@ pub fn main() !void { try http_client.initDefaultProxies(allocator); defer http_client.deinit(); - var vfs_file: zml.io.VFS.File = .init( - allocator, - threaded.io(), - .{ - .direct_io = true, - .direct_io_alignment = .fromByteUnits(4 * 1024), - }, - ); - defer vfs_file.deinit(); + // var vfs_file: zml.io.VFS.File = .init( + // allocator, + // threaded.io(), + // .{ + // .direct_io = true, + // .direct_io_alignment = .fromByteUnits(4 * 1024), + // }, + // ); + // defer vfs_file.deinit(); - var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); - defer vfs_http.deinit(); + // var vfs_http: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .{}); + // defer vfs_http.deinit(); var hf_auth: zml.io.VFS.HF.Auth = try .auto(allocator, threaded.io()); defer hf_auth.deinit(allocator); @@ -65,22 +65,19 @@ pub fn main() !void { var hf_vfs: zml.io.VFS.HF = try .init( allocator, threaded.io(), - &http_client, .{ - .request_range_min = 8 * 1024, - .request_range_max = 128 * 1024 * 1024, - .hf_pagination_limit = 100, + .http_client = &http_client, .auth = hf_auth, }, ); defer hf_vfs.deinit(); - var vfs: zml.io.VFS = .init(allocator, threaded.io()); + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); defer vfs.deinit(); - try vfs.register("file", vfs_file.io()); - try vfs.register("http", vfs_http.io()); - try vfs.register("https", vfs_http.io()); + // try vfs.register("file", vfs_file.io()); + // try vfs.register("http", vfs_http.io()); + // try vfs.register("https", vfs_http.io()); try vfs.register("hf", hf_vfs.io()); const io = vfs.io(); @@ -100,12 +97,17 @@ pub fn main() !void { log.info("Using default repo URI: {s}", .{default_uri}); break :blk default_uri; }; + _ = uri; // autofix { const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + // const local_uri = "hf://meta-llama/Llama-3.1-8B-Instruct@main"; const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); defer repo.close(io); + const stat_from_dir = try repo.statFile(io, "model.safetensors.index.json", .{}); + log.info("Stat local index file size from dir: {d} bytes", .{stat_from_dir.size}); + const index_file = try repo.openFile(io, "model.safetensors.index.json", .{}); defer index_file.close(io); @@ -115,41 +117,74 @@ pub fn main() !void { const original = try repo.openDir(io, "original", .{}); defer original.close(io); - const original_config = try original.openFile(io, "config.json", .{}); - defer original_config.close(io); + var origin_real_path_buf: [512]u8 = undefined; + const origin_real_path_len = try original.realPath(io, &origin_real_path_buf); + log.info("Original real path: {s}", .{origin_real_path_buf[0..origin_real_path_len]}); - // dir access / stat + repo.access(io, "non_existing", .{}) catch |err| { + log.info("Accessing non_existing dir failed as expected: {any}", .{err}); + }; - const stat_config = try original_config.stat(io); + var config_real_path_from_dir_buf: [512]u8 = undefined; + const config_real_path_from_dir = try repo.realPathFile(io, "config.json", &config_real_path_from_dir_buf); + log.info("Config real path from dir: {s}", .{config_real_path_from_dir_buf[0..config_real_path_from_dir]}); + + const config = try original.openFile(io, "config.json", .{}); + defer config.close(io); + + var config_real_path_buf: [512]u8 = undefined; + const config_real_path_len = try config.realPath(io, &config_real_path_buf); + log.info("Config real path: {s}", .{config_real_path_buf[0..config_real_path_len]}); + + const stat_config = try config.stat(io); log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); + + const safetensors_file = try repo.openFile(io, "model-00000-of-00002.safetensors", .{}); + defer safetensors_file.close(io); + + const safetensors_file_length = try safetensors_file.length(io); + log.info("Opened local model-00000-of-00004.safetensors file with length: {d} bytes", .{safetensors_file_length}); + + var reader_buf: [256]u8 = undefined; + var reader = safetensors_file.reader(io, &reader_buf); + + try reader.seekTo(safetensors_file_length); + try reader.seekBy(-128); + + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + + const writer_read = try reader.interface.streamRemaining(&writer.writer); + log.info("Read last {d} bytes of safetensors file", .{writer_read}); + log.info("Reader pos: {d}", .{reader.pos}); } - { - const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; - const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); - defer repo.close(io); + // { + // const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + // const repo = try std.Io.Dir.openDir(.cwd(), io, local_uri, .{}); + // defer repo.close(io); - const original_config = try repo.openFile(io, "original/config.json", .{}); - defer original_config.close(io); + // const original_config = try repo.openFile(io, "original/config.json", .{}); + // defer original_config.close(io); - const stat_config = try original_config.stat(io); - log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); - } + // const stat_config = try original_config.stat(io); + // log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); + // } - { - const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; - const original_uri = try std.fmt.allocPrint(allocator, "{s}/original", .{local_uri}); - defer allocator.free(original_uri); + // { + // const local_uri = "hf://openai/gpt-oss-20b@6cee5e8"; + // const original_uri = try std.fmt.allocPrint(allocator, "{s}/original", .{local_uri}); + // defer allocator.free(original_uri); - const repo = try std.Io.Dir.openDir(.cwd(), io, original_uri, .{}); - defer repo.close(io); + // const repo = try std.Io.Dir.openDir(.cwd(), io, original_uri, .{}); + // defer repo.close(io); - const original_config = try repo.openFile(io, "config.json", .{}); - defer original_config.close(io); + // const original_config = try repo.openFile(io, "config.json", .{}); + // defer original_config.close(io); - const stat_config = try original_config.stat(io); - log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); - } + // const stat_config = try original_config.stat(io); + // log.info("Opened local original/config.json file with size: {d} bytes", .{stat_config.size}); + // } // { // const repo = try std.Io.Dir.openDir(.cwd(), io, "file:///home/hugo/Llama-3.1-8B-Instruct/", .{}); @@ -200,51 +235,51 @@ pub fn main() !void { // } // } - { - var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); - defer pool.deinit(io); + // { + // var pool: MemoryPool = try .init(allocator, threaded.async_limit.toInt() orelse 16, 256 * 1024 * 1024); + // defer pool.deinit(io); - var timer: std.time.Timer = try .start(); - defer { - const elapsed = timer.read(); - log.info("Completed in {d} ms", .{elapsed / std.time.ns_per_ms}); - } + // var timer: std.time.Timer = try .start(); + // defer { + // const elapsed = timer.read(); + // log.info("Completed in {d} ms", .{elapsed / std.time.ns_per_ms}); + // } - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, uri); - defer registry.deinit(); + // var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, uri); + // defer registry.deinit(); - log.info("Parsed {d} tensors", .{registry.tensors.count()}); + // log.info("Parsed {d} tensors", .{registry.tensors.count()}); - var err: ?anyerror = null; + // var err: ?anyerror = null; - var group: std.Io.Group = .init; - defer group.cancel(io); + // var group: std.Io.Group = .init; + // defer group.cancel(io); - var timer_async_read: std.time.Timer = try .start(); - var it = registry.iterator(); + // var timer_async_read: std.time.Timer = try .start(); + // var it = registry.iterator(); - while (it.next()) |entry| { - const tensor_name = entry.key_ptr.*; - group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); - } + // while (it.next()) |entry| { + // const tensor_name = entry.key_ptr.*; + // group.async(io, readTensor, .{ io, &pool, ®istry, tensor_name, &err }); + // } - group.wait(io); + // group.wait(io); - if (err) |e| { - log.err("Error reading tensor: {any}", .{e}); - return e; - } + // if (err) |e| { + // log.err("Error reading tensor: {any}", .{e}); + // return e; + // } - const elapsed = timer_async_read.read(); - const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); - const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); - log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ - read_mb, - read_time_s, - read_mb / read_time_s, - (read_mb * 8.0) / 1024.0 / read_time_s, - }); - } + // const elapsed = timer_async_read.read(); + // const read_mb = @as(f64, @floatFromInt(registry.totalBytes())) / (1024.0 * 1024.0); + // const read_time_s = @as(f64, @floatFromInt(elapsed)) / @as(f64, std.time.ns_per_s); + // log.info("Throughput: {d:.2} MB in {d:.2} s = {d:.2} MB/s | {d:.2} Gbps", .{ + // read_mb, + // read_time_s, + // read_mb / read_time_s, + // (read_mb * 8.0) / 1024.0 / read_time_s, + // }); + // } // { // const root_dir = try vfs.openAbsoluteDir(io, "https://storage.googleapis.com", .{});