From e3e68c0cb921ef2fe1c038b199ba5ce095123cad Mon Sep 17 00:00:00 2001 From: Samuel Mbabhazi Date: Wed, 14 Oct 2026 10:40:03 +0200 Subject: [PATCH] fs: let glob walk a mounted virtual file system The native glob walks the file system with libuv, which cannot see a mounted virtual file system: fs.glob(), fs.globSync() and fsPromises.glob() find nothing under a mount point, where the JavaScript walk they replaced read through the fs hooks. Give the walk a reader. When its cwd is under a mount, or a pattern names one absolutely, the hooked readdirSync(), lstatSync(), statSync() and realpathSync.native() serve its reads, so a mount and the real file system are walked alike, with the same patterns, exclusions, depth limit and symbolic link rules. A walk with a reader runs on the main thread; the asynchronous APIs advance it one batch per pull, from a later loop turn as a thread-pool slice would settle in. A virtual file system need not have inodes, so such a walk tells two spellings of a directory apart by realpath, as on Windows. The reserved root sits under os.devNull, a character device, and the walk descends an absolute pattern from the root one segment at a time, so to the walk that one path is a directory holding the reserved root. Refs: https://github.com/nodejs/node/pull/65392 Signed-off-by: Samuel Mbabhazi --- lib/internal/fs/glob.js | 15 +- lib/internal/vfs/setup.js | 93 +++++++++++- src/glob/glob_walker.cc | 77 ++++++---- src/glob/glob_walker.h | 32 +++++ src/glob/node_glob.cc | 230 ++++++++++++++++++++++++++---- src/glob/node_glob.h | 48 ++++++- test/parallel/test-vfs-fs-glob.js | 151 ++++++++++++++++++++ 7 files changed, 581 insertions(+), 65 deletions(-) create mode 100644 test/parallel/test-vfs-fs-glob.js diff --git a/lib/internal/fs/glob.js b/lib/internal/fs/glob.js index 12165cf42194..1f3ea6092b54 100644 --- a/lib/internal/fs/glob.js +++ b/lib/internal/fs/glob.js @@ -20,7 +20,7 @@ const { validateString, validateStringArray, } = require('internal/validators'); -const { Dirent, DirentFromStats } = require('internal/fs/utils'); +const { Dirent, DirentFromStats, vfsState } = require('internal/fs/utils'); const { codes: { ERR_INVALID_ARG_TYPE, @@ -126,7 +126,7 @@ class Glob { globSync() { const result = globBinding.globSync( this.#root, this.#patterns, this.#excludePatterns, this.#flags(), - this.#excludeAdapter(), this.#maxDepth); + this.#excludeAdapter(), this.#maxDepth, this.#reader()); if (!this.#withFileTypes) { return result; } @@ -141,7 +141,7 @@ class Glob { async globAll() { const handle = globBinding.globStart( this.#root, this.#patterns, this.#excludePatterns, this.#flags(), - this.#excludeAdapter(), this.#maxDepth); + this.#excludeAdapter(), this.#maxDepth, this.#reader()); try { const { 0: paths, 1: types } = await handle.all(); if (!this.#withFileTypes) { @@ -161,7 +161,7 @@ class Glob { async * glob() { const handle = globBinding.globStart( this.#root, this.#patterns, this.#excludePatterns, this.#flags(), - this.#excludeAdapter(), this.#maxDepth); + this.#excludeAdapter(), this.#maxDepth, this.#reader()); try { let done = false; while (!done) { @@ -181,6 +181,13 @@ class Glob { (this.#withFileTypes ? kFlagWithFileTypes : 0); } + // The native walk reads the file system with libuv, which cannot see a + // mounted virtual file system: a walk that can reach one reads through + // the fs hooks instead. + #reader() { + return vfsState.handlers?.globReader(this.#root, this.#patterns); + } + // Shapes the argument an `exclude` callback receives to // work both with and without Dirents #excludeAdapter() { diff --git a/lib/internal/vfs/setup.js b/lib/internal/vfs/setup.js index a5f1b42659b0..5d6ad9dcbdcd 100644 --- a/lib/internal/vfs/setup.js +++ b/lib/internal/vfs/setup.js @@ -2,6 +2,7 @@ const { ArrayPrototypeForEach, + ArrayPrototypePush, MapPrototypeForEach, ObjectKeys, PromiseResolve, @@ -13,7 +14,7 @@ const { const { Buffer } = require('buffer'); const { isArrayBufferView } = require('internal/util/types'); -const { dirname, join, sep } = require('path'); +const { basename, dirname, isAbsolute, join, resolve, sep } = require('path'); const { fileURLToPath, pathToFileURL, URL } = require('internal/url'); const { kEmptyObject } = require('internal/util'); const { validateObject } = require('internal/validators'); @@ -42,6 +43,22 @@ const permission = require('internal/process/permission'); const { getOptionValue } = require('internal/options'); const nativeModulesBinding = internalBinding('modules'); const { UV_ENOENT } = internalBinding('uv'); +const { + kReadLstat, + kReadStat, + kReadRealpath, + kReadScandir, +} = internalBinding('glob'); +const { + UV_DIRENT_UNKNOWN, + UV_DIRENT_FILE, + UV_DIRENT_DIR, + UV_DIRENT_LINK, + UV_DIRENT_FIFO, + UV_DIRENT_SOCKET, + UV_DIRENT_CHAR, + UV_DIRENT_BLOCK, +} = internalBinding('constants').fs; let debug = require('internal/util/debuglog').debuglog('vfs', (fn) => { debug = fn; }); @@ -385,6 +402,69 @@ function rmRootSync(path, options) { throw createEROFS('rm', path); } +/** + * The uv_dirent_type_t of a Dirent or Stats + * @param {object} entry + * @returns {number} + */ +function direntType(entry) { + if (entry.isSymbolicLink()) return UV_DIRENT_LINK; + if (entry.isDirectory()) return UV_DIRENT_DIR; + if (entry.isFile()) return UV_DIRENT_FILE; + if (entry.isFIFO()) return UV_DIRENT_FIFO; + if (entry.isSocket()) return UV_DIRENT_SOCKET; + if (entry.isCharacterDevice()) return UV_DIRENT_CHAR; + if (entry.isBlockDevice()) return UV_DIRENT_BLOCK; + return UV_DIRENT_UNKNOWN; +} + +/** + * Serves the reads of a glob walk that can reach a mount, which the + * native walk cannot see: the fs functions dispatch to the mount, and to + * the real file system for every other path. A read that fails answers + * undefined, as the walk's own reads do. The parent of the reserved root + * (os.devNull) is a directory to the walk, which descends an absolute + * pattern from the root one segment at a time. + * @param {number} op + * @param {string} path + * @returns {number|boolean|string|[string[], number[]]|undefined} + */ +let lazyFs; +function readForGlob(op, path) { + const fs = lazyFs ??= require('fs'); + try { + if (normalizeMountedPath(path) === dirname(normalizedVfsRoot)) { + switch (op) { + case kReadLstat: return UV_DIRENT_DIR; + case kReadStat: return true; + case kReadRealpath: return path; + case kReadScandir: + return [[basename(normalizedVfsRoot)], [UV_DIRENT_DIR]]; + } + } + switch (op) { + case kReadLstat: + return direntType(fs.lstatSync(path)); + case kReadStat: + return fs.statSync(path).isDirectory(); + case kReadRealpath: + return fs.realpathSync.native(path); + case kReadScandir: { + const dirents = fs.readdirSync(path, { withFileTypes: true }); + const names = []; + const types = []; + for (let i = 0; i < dirents.length; i++) { + ArrayPrototypePush(names, dirents[i].name); + ArrayPrototypePush(types, direntType(dirents[i])); + } + return [names, types]; + } + } + } catch { + return undefined; + } +} + function createVfsHandlers() { return { __proto__: null, @@ -430,6 +510,17 @@ function createVfsHandlers() { statSync(path, options) { return vfsRead(path, 'stat', (vfs, n) => vfs.statSync(n, options)); }, + // The reads of a glob walk that can reach a mount: it starts under + // one, or a pattern names one absolutely. See internal/fs/glob. + globReader(cwd, patterns) { + if (findVFS(resolve(cwd)) !== null) return readForGlob; + for (let i = 0; i < patterns.length; i++) { + if (isAbsolute(patterns[i]) && findVFS(patterns[i]) !== null) { + return readForGlob; + } + } + return undefined; + }, realpathSync(path, options) { const result = vfsRead(path, 'realpath', (vfs, n) => vfs.realpathSync(n)); if (result !== undefined && options?.encoding === 'buffer') { diff --git a/src/glob/glob_walker.cc b/src/glob/glob_walker.cc index dda6e7c1c1f2..017b19a85ffe 100644 --- a/src/glob/glob_walker.cc +++ b/src/glob/glob_walker.cc @@ -268,13 +268,6 @@ bool MatchFullPath(const CompiledPattern& pattern, return MatchPattern(pattern, PatternView(*scratch)); } -struct StatInfo { - bool exists = false; - bool is_dir = false; - bool is_link = false; - int type = UV_DIRENT_UNKNOWN; -}; - // The dirent type a stat's mode describes; special files keep their real // type, as the previous implementation's lstat-built Dirents did. int TypeOfMode(uint64_t mode) { @@ -311,11 +304,6 @@ int DirentTypeOf(const StatInfo& info) { : UV_DIRENT_FILE; } -struct DirEntry { - std::string name; - int type = UV_DIRENT_UNKNOWN; -}; - // One blocking scandir with no Environment access, callable from any thread. std::vector ScanDir(const char* path) { std::vector entries; @@ -652,6 +640,7 @@ class WalkerState { bool RunSlice(size_t max_results, bool on_main_thread, std::vector* out) { + CHECK_IMPLIES(options_.fs_reader != nullptr, on_main_thread); on_main_thread_ = on_main_thread; if (!started_) { started_ = true; @@ -706,11 +695,13 @@ class WalkerState { // visited once and nothing can observe the traversal while it runs: // one pattern row, no '..' hops or absolute heads (revisits), no // symlink following (realpath chains), no exclude pattern or callback - // (pruning), and no permission model (per-path checks). + // (pruning), no reader (main-thread reads), and no permission model + // (per-path checks). bool ClassifyFree() const { if (rows_.size() != 1) return false; if (!excludes_.empty() || options_.exclude_filter != nullptr) return false; if (options_.follow_symlinks) return false; + if (options_.fs_reader != nullptr) return false; if (env_->permission()->enabled()) return false; const CompiledPattern::Row& row = *rows_[0]; if (row.parts.empty()) return false; @@ -922,18 +913,28 @@ class WalkerState { return fn(nullptr, &req->req, args..., nullptr); } + // A reader call that threw ends the walk, its exception pending. + void CheckReader() { + if (options_.fs_reader->failed()) aborted_ = true; + } + StatInfo StatSync(const std::string& path) { auto it = stat_cache_.find(path); if (it != stat_cache_.end()) return it->second; StatInfo info; if (PermissionGranted(path)) { - fs::FSReqWrapSync req("lstat", path.c_str()); - if (FsCall(&req, uv_fs_lstat, path.c_str()) == 0) { - const uint64_t mode = req.req.statbuf.st_mode; - info.exists = true; - info.is_dir = (mode & S_IFMT) == S_IFDIR; - info.is_link = (mode & S_IFMT) == S_IFLNK; - info.type = TypeOfMode(mode); + if (options_.fs_reader != nullptr) { + info = options_.fs_reader->Lstat(path); + CheckReader(); + } else { + fs::FSReqWrapSync req("lstat", path.c_str()); + if (FsCall(&req, uv_fs_lstat, path.c_str()) == 0) { + const uint64_t mode = req.req.statbuf.st_mode; + info.exists = true; + info.is_dir = (mode & S_IFMT) == S_IFDIR; + info.is_link = (mode & S_IFMT) == S_IFLNK; + info.type = TypeOfMode(mode); + } } } stat_cache_.emplace(path, info); @@ -949,9 +950,14 @@ class WalkerState { if (it != follow_stat_cache_.end()) return it->second; bool is_dir = false; if (PermissionGranted(path)) { - fs::FSReqWrapSync req("stat", path.c_str()); - if (FsCall(&req, uv_fs_stat, path.c_str()) == 0) { - is_dir = (req.req.statbuf.st_mode & S_IFMT) == S_IFDIR; + if (options_.fs_reader != nullptr) { + is_dir = options_.fs_reader->IsDirectory(path); + CheckReader(); + } else { + fs::FSReqWrapSync req("stat", path.c_str()); + if (FsCall(&req, uv_fs_stat, path.c_str()) == 0) { + is_dir = (req.req.statbuf.st_mode & S_IFMT) == S_IFDIR; + } } } follow_stat_cache_.emplace(path, is_dir); @@ -963,10 +969,15 @@ class WalkerState { if (it != realpath_cache_.end()) return it->second; std::string real; if (PermissionGranted(path)) { - fs::FSReqWrapSync req("realpath", path.c_str()); - if (FsCall(&req, uv_fs_realpath, path.c_str()) == 0 && - req.req.ptr != nullptr) { - real = static_cast(req.req.ptr); + if (options_.fs_reader != nullptr) { + real = options_.fs_reader->Realpath(path); + CheckReader(); + } else { + fs::FSReqWrapSync req("realpath", path.c_str()); + if (FsCall(&req, uv_fs_realpath, path.c_str()) == 0 && + req.req.ptr != nullptr) { + real = static_cast(req.req.ptr); + } } } return realpath_cache_.emplace(path, std::move(real)).first->second; @@ -977,7 +988,10 @@ class WalkerState { if (it != readdir_cache_.end()) return it->second; std::vector entries; if (PermissionGranted(path)) { - if (!prefetcher_.Take(path, &entries)) { + if (options_.fs_reader != nullptr) { + entries = options_.fs_reader->Scandir(path); + CheckReader(); + } else if (!prefetcher_.Take(path, &entries)) { fs::FSReqWrapSync req("scandir", path.c_str()); if (FsCall(&req, uv_fs_scandir, path.c_str(), 0) >= 0) { uv_dirent_t ent; @@ -1003,6 +1017,7 @@ class WalkerState { } void MaybePrefetch(const QueueItem& item) { + if (options_.fs_reader != nullptr) return; const bool needs_listing = std::ranges::any_of(item.patterns, [](const PatternState& pattern) { return pattern.indexes.size() != 1 || @@ -1108,10 +1123,12 @@ class WalkerState { return depth; } - // Whether two spellings name one directory. + // Whether two spellings name one directory. A reader's stats carry no + // identity (a virtual file system need not have inodes), so its paths + // are told apart by realpath, as on Windows. bool SamePath(const std::string& a, const std::string& b) { if (a == b) return true; - if (kWindows) { + if (kWindows || options_.fs_reader != nullptr) { const std::string real = RealpathSync(a); return !real.empty() && real == RealpathSync(b); } else { diff --git a/src/glob/glob_walker.h b/src/glob/glob_walker.h index 9a8ec0647a09..5244b226987d 100644 --- a/src/glob/glob_walker.h +++ b/src/glob/glob_walker.h @@ -34,6 +34,37 @@ class ExcludeFilter { virtual bool failed() const = 0; }; +// What lstat reports about a path. +struct StatInfo { + bool exists = false; + bool is_dir = false; + bool is_link = false; + int type = UV_DIRENT_UNKNOWN; +}; + +struct DirEntry { + std::string name; + int type = UV_DIRENT_UNKNOWN; +}; + +// Reads the filesystem for a walk in place of libuv, for paths libuv +// cannot see: a mounted virtual file system only answers through the +// JavaScript fs hooks. A walk with a reader runs on the main thread, and +// a read that fails answers as a missing path, as the walk's own reads do. +class FsReader { + public: + virtual ~FsReader() = default; + virtual StatInfo Lstat(const std::string& path) = 0; + // Whether the path, with symbolic links followed, is a directory. + virtual bool IsDirectory(const std::string& path) = 0; + // Empty when the path cannot be resolved. + virtual std::string Realpath(const std::string& path) = 0; + virtual std::vector Scandir(const std::string& path) = 0; + // Set once a call has thrown. The walk stops and the exception is left + // for the caller to propagate. + virtual bool failed() const = 0; +}; + struct WalkOptions { // options.cwd, or ".": the string paths are resolved against. std::string cwd = "."; @@ -45,6 +76,7 @@ struct WalkOptions { size_t max_depth = kNoDepthLimit; ExcludeFilter* exclude_filter = nullptr; + FsReader* fs_reader = nullptr; }; struct WalkEntry { diff --git a/src/glob/node_glob.cc b/src/glob/node_glob.cc index 9337775c79da..bad3a5aa049e 100644 --- a/src/glob/node_glob.cc +++ b/src/glob/node_glob.cc @@ -3,6 +3,7 @@ #include #include +#include "base_object-inl.h" #include "env-inl.h" #include "glob/glob_matcher.h" #include "glob/glob_parser.h" @@ -18,11 +19,13 @@ namespace node::glob { using v8::Array; using v8::Boolean; using v8::Context; +using v8::EscapableHandleScope; using v8::Exception; using v8::Function; using v8::FunctionCallbackInfo; using v8::FunctionTemplate; using v8::HandleScope; +using v8::Int32; using v8::Integer; using v8::Isolate; using v8::Just; @@ -282,18 +285,20 @@ bool BindingData::ReadWalkArguments(const FunctionCallbackInfo& args, WalkOptions* options, std::vector* includes, std::vector* excludes, - std::unique_ptr* filter) { + std::unique_ptr* filter, + std::unique_ptr* reader) { Realm* realm = Realm::GetCurrent(args); Isolate* isolate = realm->isolate(); Local context = realm->context(); - CHECK_EQ(args.Length(), 6); + CHECK_EQ(args.Length(), 7); CHECK(args[0]->IsString()); // cwd CHECK(args[1]->IsArray()); // include patterns CHECK(args[2]->IsArray()); // exclude patterns CHECK(args[3]->IsUint32()); // packed flags CHECK(args[4]->IsFunction() || args[4]->IsUndefined()); // exclude callback CHECK(args[5]->IsNumber()); // maxDepth + CHECK(args[6]->IsFunction() || args[6]->IsUndefined()); // reader const uint32_t packed = args[3].As()->Value() | kHostWalkFlags; const CompileFlags flags = UnpackFlags(packed); @@ -310,6 +315,11 @@ bool BindingData::ReadWalkArguments(const FunctionCallbackInfo& args, std::make_unique(realm->env(), args[4].As()); options->exclude_filter = filter->get(); } + if (args[6]->IsFunction()) { + *reader = + std::make_unique(realm->env(), args[6].As()); + options->fs_reader = reader->get(); + } { Utf8Value cwd(isolate, args[0]); options->cwd = cwd.ToString(); @@ -346,7 +356,7 @@ bool BindingData::ReadWalkArguments(const FunctionCallbackInfo& args, } // globSync(cwd, includePatterns[], excludePatterns[], flags, excludeFn, -// maxDepth) +// maxDepth, reader) void BindingData::GlobSync(const FunctionCallbackInfo& args) { Realm* realm = Realm::GetCurrent(args); Environment* env = realm->env(); @@ -357,15 +367,17 @@ void BindingData::GlobSync(const FunctionCallbackInfo& args) { std::vector includes; std::vector excludes; std::unique_ptr filter; + std::unique_ptr reader; if (!binding->ReadWalkArguments( - args, &options, &includes, &excludes, &filter)) { + args, &options, &includes, &excludes, &filter, &reader)) { return; } std::vector entries; glob::GlobSync(env, options, includes, excludes, &entries); - // An exclude callback that threw leaves its exception pending. + // An exclude callback or a reader that threw leaves its exception pending. if (filter != nullptr && filter->failed()) return; + if (reader != nullptr && reader->failed()) return; Local paths; Local types; @@ -424,15 +436,117 @@ bool JsExcludeFilter::ExcludesEntry(std::string_view name, return excluded; } +JsFsReader::JsFsReader(Environment* env, Local read) + : env_(env), read_(env->isolate(), read) {} + +// (op, path) -> what the read found, or undefined for a path it could +// not read, see internal/fs/glob. +MaybeLocal JsFsReader::Call(ReadOp op, const std::string& path) { + Isolate* isolate = env_->isolate(); + EscapableHandleScope scope(isolate); + Local context = env_->context(); + Local argv[2]; + argv[0] = Integer::New(isolate, op); + Local result; + if (!ToV8Value(context, path, isolate).ToLocal(&argv[1]) || + !read_.Get(isolate) + ->Call(context, Undefined(isolate), arraysize(argv), argv) + .ToLocal(&result)) { + failed_ = true; + return MaybeLocal(); + } + return scope.Escape(result); +} + +// The entry's uv_dirent_type_t +StatInfo JsFsReader::Lstat(const std::string& path) { + StatInfo info; + if (failed_) return info; + HandleScope scope(env_->isolate()); + Local result; + if (!Call(kReadLstat, path).ToLocal(&result) || !result->IsInt32()) { + return info; + } + info.exists = true; + info.type = result.As()->Value(); + info.is_dir = info.type == UV_DIRENT_DIR; + info.is_link = info.type == UV_DIRENT_LINK; + return info; +} + +bool JsFsReader::IsDirectory(const std::string& path) { + if (failed_) return false; + HandleScope scope(env_->isolate()); + Local result; + return Call(kReadStat, path).ToLocal(&result) && result->IsTrue(); +} + +std::string JsFsReader::Realpath(const std::string& path) { + if (failed_) return std::string(); + Isolate* isolate = env_->isolate(); + HandleScope scope(isolate); + Local result; + if (!Call(kReadRealpath, path).ToLocal(&result) || !result->IsString()) { + return std::string(); + } + return Utf8Value(isolate, result).ToString(); +} + +// [names, types] +std::vector JsFsReader::Scandir(const std::string& path) { + std::vector entries; + if (failed_) return entries; + Isolate* isolate = env_->isolate(); + HandleScope scope(isolate); + Local context = env_->context(); + Local result; + if (!Call(kReadScandir, path).ToLocal(&result) || !result->IsArray()) { + return entries; + } + Local listing = result.As(); + CHECK_EQ(listing->Length(), 2); + Local names_value; + Local types_value; + if (!listing->Get(context, 0).ToLocal(&names_value) || + !listing->Get(context, 1).ToLocal(&types_value)) { + failed_ = true; + return entries; + } + CHECK(names_value->IsArray()); + CHECK(types_value->IsArray()); + Local names = names_value.As(); + Local types = types_value.As(); + const uint32_t length = names->Length(); + CHECK_EQ(types->Length(), length); + entries.reserve(length); + for (uint32_t i = 0; i < length; i++) { + Local name; + Local type; + if (!names->Get(context, i).ToLocal(&name) || + !types->Get(context, i).ToLocal(&type)) { + failed_ = true; + entries.clear(); + return entries; + } + CHECK(name->IsString()); + CHECK(type->IsInt32()); + entries.push_back( + {Utf8Value(isolate, name).ToString(), type.As()->Value()}); + } + return entries; +} + GlobRequest::GlobRequest(Environment* env, Local object, const WalkOptions& options, const std::vector& includes, const std::vector& excludes, - std::unique_ptr filter) + std::unique_ptr filter, + std::unique_ptr reader) : AsyncWrap(env, object, AsyncWrap::PROVIDER_GLOBREQUEST), ThreadPoolWork(env, "glob"), filter_(std::move(filter)), + reader_(std::move(reader)), walk_(env, options, includes, excludes), with_file_types_(options.with_file_types) { MakeWeak(); @@ -445,15 +559,16 @@ void GlobRequest::MemoryInfo(MemoryTracker* tracker) const { } } +// A pull spans several slices when the exclude callback is consulted +// in between; the batch keeps filling across them (and a directory +// visited before a stop may already have filled it). +size_t GlobRequest::MaxResults() const { + if (drain_) return std::numeric_limits::max(); + return batch_.size() < kBatchSize ? kBatchSize - batch_.size() : 0; +} + void GlobRequest::DoThreadPoolWork() { - // A pull spans several slices when the exclude callback is consulted - // in between; the batch keeps filling across them (and a directory - // visited before a stop may already have filled it). - size_t max_results = std::numeric_limits::max(); - if (!drain_) { - max_results = batch_.size() < kBatchSize ? kBatchSize - batch_.size() : 0; - } - done_ = walk_.RunSlice(max_results, /*on_main_thread=*/false, &batch_); + done_ = walk_.RunSlice(MaxResults(), /*on_main_thread=*/false, &batch_); } void GlobRequest::PublishDenials() { @@ -534,16 +649,7 @@ void GlobRequest::AfterThreadPoolWork(int status) { errors::TryCatchScope try_catch(env); AnswerQuestions(); if (try_catch.HasCaught() || filter_->failed()) { - // The exclude callback threw: the pull rejects with its exception. - done_ = true; - walk_.Stop(); - batch_.clear(); - Local resolver = resolver_.Get(env->isolate()); - resolver_.Reset(); - MakeWeak(); - // A terminating isolate (worker shutdown) has nothing to pass on. - if (try_catch.HasTerminated() || !try_catch.HasCaught()) return; - USE(resolver->Reject(context, try_catch.Exception())); + Reject(try_catch); return; } // The walk resumes with the answers. @@ -552,6 +658,41 @@ void GlobRequest::AfterThreadPoolWork(int status) { return; } + Resolve(); +} + +void GlobRequest::RunMainThreadSlice() { + in_flight_ = false; + Environment* env = AsyncWrap::env(); + if (!env->can_call_into_js()) { + resolver_.Reset(); + done_ = true; + walk_.Stop(); + MakeWeak(); + return; + } + if (cancelled_) done_ = true; + + HandleScope scope(env->isolate()); + Local context = env->context(); + Context::Scope context_scope(context); + InternalCallbackScope callback_scope(this); + + if (!done_) { + errors::TryCatchScope try_catch(env); + done_ = walk_.RunSlice(MaxResults(), /*on_main_thread=*/true, &batch_); + if (try_catch.HasCaught() || reader_->failed() || + (filter_ != nullptr && filter_->failed())) { + Reject(try_catch); + return; + } + } + + Resolve(); +} + +void GlobRequest::Resolve() { + Environment* env = AsyncWrap::env(); // A finished walk keeps no threads waiting for garbage collection. if (done_) walk_.Stop(); Local resolver = resolver_.Get(env->isolate()); @@ -561,7 +702,21 @@ void GlobRequest::AfterThreadPoolWork(int status) { Local value; if (!Settle().ToLocal(&value)) return; - USE(resolver->Resolve(context, value)); + USE(resolver->Resolve(env->context(), value)); +} + +// A callback threw: the pull rejects with its exception. +void GlobRequest::Reject(const errors::TryCatchScope& try_catch) { + Environment* env = AsyncWrap::env(); + done_ = true; + walk_.Stop(); + batch_.clear(); + Local resolver = resolver_.Get(env->isolate()); + resolver_.Reset(); + MakeWeak(); + // A terminating isolate (worker shutdown) has nothing to pass on. + if (try_catch.HasTerminated() || !try_catch.HasCaught()) return; + USE(resolver->Reject(env->context(), try_catch.Exception())); } // next() -> Promise<[paths, types, done]>: one batch of results. @@ -605,6 +760,15 @@ void GlobRequest::Pull(const FunctionCallbackInfo& args, bool drain) { request->in_flight_ = true; // Keep the request alive for as long as the worker holds a pointer to it. request->ClearWeak(); + if (request->reader_ != nullptr) { + // The reader is JavaScript, so the slice runs on the main thread, from + // a later loop turn as a thread-pool slice would settle in. + env->SetImmediate( + [self = BaseObjectPtr(request)](Environment* env) { + self->RunMainThreadSlice(); + }); + return; + } request->ScheduleWork(); } @@ -627,8 +791,9 @@ void BindingData::GlobStart(const FunctionCallbackInfo& args) { std::vector includes; std::vector excludes; std::unique_ptr filter; + std::unique_ptr reader; if (!binding->ReadWalkArguments( - args, &options, &includes, &excludes, &filter)) { + args, &options, &includes, &excludes, &filter, &reader)) { return; } @@ -639,7 +804,13 @@ void BindingData::GlobStart(const FunctionCallbackInfo& args) { .ToLocal(&object)) { return; } - new GlobRequest(env, object, options, includes, excludes, std::move(filter)); + new GlobRequest(env, + object, + options, + includes, + excludes, + std::move(filter), + std::move(reader)); args.GetReturnValue().Set(object); } @@ -674,6 +845,11 @@ void BindingData::CreatePerContextProperties(Local target, NODE_DEFINE_CONSTANT(target, kFlagFollowSymlinks); NODE_DEFINE_CONSTANT(target, kFlagWithFileTypes); NODE_DEFINE_CONSTANT(target, kFlagMatchBase); + // The reads a reader serves + NODE_DEFINE_CONSTANT(target, kReadLstat); + NODE_DEFINE_CONSTANT(target, kReadStat); + NODE_DEFINE_CONSTANT(target, kReadRealpath); + NODE_DEFINE_CONSTANT(target, kReadScandir); } void BindingData::RegisterExternalReferences( diff --git a/src/glob/node_glob.h b/src/glob/node_glob.h index cf106a888eba..ee5cb8583ded 100644 --- a/src/glob/node_glob.h +++ b/src/glob/node_glob.h @@ -11,6 +11,7 @@ #include "glob/glob_program.h" #include "glob/glob_walker.h" #include "lru_cache-inl.h" +#include "node_errors.h" #include "node_internals.h" #include "node_snapshotable.h" #include "threadpoolwork-inl.h" @@ -40,6 +41,35 @@ class JsExcludeFilter final : public ExcludeFilter { bool failed_ = false; }; +// The reads a JavaScript reader serves, see internal/fs/glob. +enum ReadOp : int { + kReadLstat = 0, + kReadStat = 1, + kReadRealpath = 2, + kReadScandir = 3, +}; + +// Reads the filesystem through a JavaScript function: the fs hooks of a +// mounted virtual file system. +class JsFsReader final : public FsReader { + public: + JsFsReader(Environment* env, v8::Local read); + + StatInfo Lstat(const std::string& path) override; + bool IsDirectory(const std::string& path) override; + std::string Realpath(const std::string& path) override; + std::vector Scandir(const std::string& path) override; + bool failed() const override { return failed_; } + + private: + // Nothing when the function threw; the exception stays pending. + v8::MaybeLocal Call(ReadOp op, const std::string& path); + + Environment* env_; + v8::Global read_; + bool failed_ = false; +}; + class BindingData : public SnapshotableObject { public: BindingData(Realm* realm, @@ -72,7 +102,8 @@ class BindingData : public SnapshotableObject { WalkOptions* options, std::vector* includes, std::vector* excludes, - std::unique_ptr* filter); + std::unique_ptr* filter, + std::unique_ptr* reader); // Compiles (or fetches) `pattern`; returns nullptr on a compile error, // with the reason written to *error. @@ -103,7 +134,8 @@ class GlobRequest : public AsyncWrap, public ThreadPoolWork { const WalkOptions& options, const std::vector& includes, const std::vector& excludes, - std::unique_ptr filter); + std::unique_ptr filter, + std::unique_ptr reader); // How many results an iterator pull returns at a time. // See the explanation in this header's .cc file for @@ -123,6 +155,15 @@ class GlobRequest : public AsyncWrap, public ThreadPoolWork { private: static void Pull(const v8::FunctionCallbackInfo& args, bool drain); + // How many results the pull in flight still wants. + size_t MaxResults() const; + // A slice of a walk that reads through JavaScript, run on the main + // thread from a later loop turn. + void RunMainThreadSlice(); + // Settles the pull in flight: with the batch, or with the exception a + // callback threw. + void Resolve(); + void Reject(const errors::TryCatchScope& try_catch); v8::MaybeLocal Settle(); // The main-thread half of a slice: publishes the permission denials the // walk met and puts the questions of a directory waiting on the exclude @@ -130,8 +171,9 @@ class GlobRequest : public AsyncWrap, public ThreadPoolWork { void PublishDenials(); void AnswerQuestions(); - // Declared before the walk, which points at it. + // Declared before the walk, which points at them. std::unique_ptr filter_; + std::unique_ptr reader_; Walk walk_; const bool with_file_types_; // Set by all(): the pull consumes the rest of the walk in one batch. diff --git a/test/parallel/test-vfs-fs-glob.js b/test/parallel/test-vfs-fs-glob.js new file mode 100644 index 000000000000..3ceb7a6a17d1 --- /dev/null +++ b/test/parallel/test-vfs-fs-glob.js @@ -0,0 +1,151 @@ +// Flags: --experimental-vfs +'use strict'; + +// fs.glob, fs.globSync and fs.promises.glob walk a mounted VFS: a walk +// whose cwd is under a mount, or whose pattern names one absolutely, reads +// the file system through the fs hooks. A walk of the real file system is +// unchanged while a VFS is mounted. + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { promisify } = require('util'); +const vfs = require('node:vfs'); +const tmpdir = require('../common/tmpdir'); + +tmpdir.refresh(); + +const myVfs = vfs.create(); +const files = ['a.txt', 'sub/b.txt', 'sub/c.md', 'sub/deep/d.txt', '.hidden/e.txt']; +for (const file of files) { + myVfs.mkdirSync(path.posix.dirname(`/data/${file}`), { recursive: true }); + myVfs.writeFileSync(`/data/${file}`, file); +} +myVfs.symlinkSync('/data/sub', '/data/link'); +const mountPoint = myVfs.mount(); +const cwd = path.join(mountPoint, 'data'); + +// The same tree on the real file system +const realDir = tmpdir.resolve('real'); +for (const file of files) { + fs.mkdirSync(path.join(realDir, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(realDir, file), file); +} + +const native = (paths) => paths.map((p) => p.replaceAll('/', path.sep)).sort(); + +async function collect(pattern, options) { + const results = []; + for await (const entry of fs.promises.glob(pattern, options)) { + results.push(entry); + } + return results; +} + +const apis = [ + ['globSync', async (pattern, options) => fs.globSync(pattern, options)], + ['glob', promisify(fs.glob)], + ['promises.glob', collect], +]; + +async function run(name, glob) { + // Relative patterns from a cwd under the mount. `**` neither enters + // dot directories nor follows the symbolic link. + assert.deepStrictEqual( + (await glob('**/*.txt', { cwd })).sort(), + native(['a.txt', 'sub/b.txt', 'sub/deep/d.txt']), name); + assert.deepStrictEqual( + (await glob(['*.txt', 'sub/*.md'], { cwd })).sort(), + native(['a.txt', 'sub/c.md']), name); + assert.deepStrictEqual( + (await glob('**/*.txt', { cwd, followSymlinks: true })).sort(), + native(['a.txt', 'link/b.txt', 'link/deep/d.txt', 'sub/b.txt', 'sub/deep/d.txt']), name); + assert.deepStrictEqual( + (await glob('**', { cwd, maxDepth: 1 })).sort(), + native(['.', 'a.txt', 'link', 'sub']), name); + + // Exclusion by pattern and by callback, with and without Dirents. + assert.deepStrictEqual( + (await glob('**/*.txt', { cwd, exclude: ['sub/**'] })).sort(), + native(['a.txt']), name); + assert.deepStrictEqual( + (await glob('**/*.txt', { cwd, exclude: (entry) => entry === 'sub' })).sort(), + native(['a.txt']), name); + const dirents = await glob('**/*.txt', { + cwd, + withFileTypes: true, + exclude: common.mustCallAtLeast((dirent) => { + assert.ok(dirent instanceof fs.Dirent, name); + return dirent.isDirectory() && dirent.name === 'deep'; + }, 1), + }); + assert.deepStrictEqual( + dirents.map((dirent) => path.join(dirent.parentPath, dirent.name)).sort(), + native(['a.txt', 'sub/b.txt']).map((p) => path.join(cwd, p)), name); + assert.ok(dirents.every((dirent) => dirent.isFile()), name); + + // An absolute pattern into the mount, from a cwd on the real file + // system, and a relative one that leaves its directory within the mount. + assert.deepStrictEqual( + await glob(path.join(cwd, '**', '*.md'), { cwd: tmpdir.path }), + [path.join(cwd, 'sub', 'c.md')], name); + assert.deepStrictEqual( + await glob('../*.txt', { cwd: path.join(cwd, 'sub') }), + [path.join('..', 'a.txt')], name); + + // The reserved root lists the mount points, from within and from above. + assert.ok( + (await glob('*', { cwd: vfs.vfsBase() })).includes(path.basename(mountPoint)), name); + assert.ok( + (await glob(path.join(vfs.vfsBase(), '*'), { cwd: tmpdir.path })).includes(mountPoint), name); + assert.deepStrictEqual( + await glob(cwd, { cwd, maxDepth: 0 }), [cwd], name); + + // The real file system, while the VFS is mounted + assert.deepStrictEqual( + (await glob('**/*.txt', { cwd: realDir })).sort(), + native(['a.txt', 'sub/b.txt', 'sub/deep/d.txt']), name); + assert.deepStrictEqual( + (await glob(path.join(realDir, '**', '*.md'), { cwd })).sort(), + [path.join(realDir, 'sub', 'c.md')], name); +} + +(async () => { + for (const [name, glob] of apis) { + await run(name, glob); + } + + // An exclude callback that throws fails the walk with its error. + const error = new Error('exclude failed'); + const throwing = () => { throw error; }; + assert.throws(() => fs.globSync('**', { cwd, exclude: throwing }), (err) => err === error); + await assert.rejects(promisify(fs.glob)('**', { cwd, exclude: throwing }), (err) => err === error); + await assert.rejects(collect('**', { cwd, exclude: throwing }), (err) => err === error); + + // Nothing of an async walk runs before the call returns, and leaving + // the iterator early ends it. + let calls = 0; + const exclude = () => { calls++; return false; }; + const iterator = fs.promises.glob('**', { cwd, exclude }); + assert.strictEqual(calls, 0); + for await (const entry of iterator) { + assert.strictEqual(typeof entry, 'string'); + break; + } + const callsAtBreak = calls; + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.strictEqual(calls, callsAtBreak); + + // A tree that fills several batches of the iterator + const manyDir = path.join(mountPoint, 'many'); + fs.mkdirSync(manyDir); + for (let i = 0; i < 600; i++) { + fs.writeFileSync(path.join(manyDir, `${i}.txt`), ''); + } + const many = await collect('*.txt', { cwd: manyDir }); + assert.strictEqual(many.length, 600); + + myVfs.unmount(); + assert.deepStrictEqual(fs.globSync('**', { cwd }), []); +})().then(common.mustCall());