opusdev/vector-similarity-api

1'use strict';2 3const fs = require('fs');4const { Readable } = require('stream');5const sysPath = require('path');6const { promisify } = require('util');7const picomatch = require('picomatch');8 9const readdir = promisify(fs.readdir);10const stat = promisify(fs.stat);11const lstat = promisify(fs.lstat);12const realpath = promisify(fs.realpath);13 14/**15 * @typedef {Object} EntryInfo16 * @property {String} path17 * @property {String} fullPath18 * @property {fs.Stats=} stats19 * @property {fs.Dirent=} dirent20 * @property {String} basename21 */22 23const BANG = '!';24const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';25const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);26const FILE_TYPE = 'files';27const DIR_TYPE = 'directories';28const FILE_DIR_TYPE = 'files_directories';29const EVERYTHING_TYPE = 'all';30const ALL_TYPES = [FILE_TYPE, DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE];31 32const isNormalFlowError = error => NORMAL_FLOW_ERRORS.has(error.code);33const [maj, min] = process.versions.node.split('.').slice(0, 2).map(n => Number.parseInt(n, 10));34const wantBigintFsStats = process.platform === 'win32' && (maj > 10 || (maj === 10 && min >= 5));35 36const normalizeFilter = filter => {37  if (filter === undefined) return;38  if (typeof filter === 'function') return filter;39 40  if (typeof filter === 'string') {41    const glob = picomatch(filter.trim());42    return entry => glob(entry.basename);43  }44 45  if (Array.isArray(filter)) {46    const positive = [];47    const negative = [];48    for (const item of filter) {49      const trimmed = item.trim();50      if (trimmed.charAt(0) === BANG) {51        negative.push(picomatch(trimmed.slice(1)));52      } else {53        positive.push(picomatch(trimmed));54      }55    }56 57    if (negative.length > 0) {58      if (positive.length > 0) {59        return entry =>60          positive.some(f => f(entry.basename)) && !negative.some(f => f(entry.basename));61      }62      return entry => !negative.some(f => f(entry.basename));63    }64    return entry => positive.some(f => f(entry.basename));65  }66};67 68class ReaddirpStream extends Readable {69  static get defaultOptions() {70    return {71      root: '.',72      /* eslint-disable no-unused-vars */73      fileFilter: (path) => true,74      directoryFilter: (path) => true,75      /* eslint-enable no-unused-vars */76      type: FILE_TYPE,77      lstat: false,78      depth: 2147483648,79      alwaysStat: false80    };81  }82 83  constructor(options = {}) {84    super({85      objectMode: true,86      autoDestroy: true,87      highWaterMark: options.highWaterMark || 409688    });89    const opts = { ...ReaddirpStream.defaultOptions, ...options };90    const { root, type } = opts;91 92    this._fileFilter = normalizeFilter(opts.fileFilter);93    this._directoryFilter = normalizeFilter(opts.directoryFilter);94 95    const statMethod = opts.lstat ? lstat : stat;96    // Use bigint stats if it's windows and stat() supports options (node 10+).97    if (wantBigintFsStats) {98      this._stat = path => statMethod(path, { bigint: true });99    } else {100      this._stat = statMethod;101    }102 103    this._maxDepth = opts.depth;104    this._wantsDir = [DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);105    this._wantsFile = [FILE_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);106    this._wantsEverything = type === EVERYTHING_TYPE;107    this._root = sysPath.resolve(root);108    this._isDirent = ('Dirent' in fs) && !opts.alwaysStat;109    this._statsProp = this._isDirent ? 'dirent' : 'stats';110    this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };111 112    // Launch stream with one parent, the root dir.113    this.parents = [this._exploreDir(root, 1)];114    this.reading = false;115    this.parent = undefined;116  }117 118  async _read(batch) {119    if (this.reading) return;120    this.reading = true;121 122    try {123      while (!this.destroyed && batch > 0) {124        const { path, depth, files = [] } = this.parent || {};125 126        if (files.length > 0) {127          const slice = files.splice(0, batch).map(dirent => this._formatEntry(dirent, path));128          for (const entry of await Promise.all(slice)) {129            if (this.destroyed) return;130 131            const entryType = await this._getEntryType(entry);132            if (entryType === 'directory' && this._directoryFilter(entry)) {133              if (depth <= this._maxDepth) {134                this.parents.push(this._exploreDir(entry.fullPath, depth + 1));135              }136 137              if (this._wantsDir) {138                this.push(entry);139                batch--;140              }141            } else if ((entryType === 'file' || this._includeAsFile(entry)) && this._fileFilter(entry)) {142              if (this._wantsFile) {143                this.push(entry);144                batch--;145              }146            }147          }148        } else {149          const parent = this.parents.pop();150          if (!parent) {151            this.push(null);152            break;153          }154          this.parent = await parent;155          if (this.destroyed) return;156        }157      }158    } catch (error) {159      this.destroy(error);160    } finally {161      this.reading = false;162    }163  }164 165  async _exploreDir(path, depth) {166    let files;167    try {168      files = await readdir(path, this._rdOptions);169    } catch (error) {170      this._onError(error);171    }172    return { files, depth, path };173  }174 175  async _formatEntry(dirent, path) {176    let entry;177    try {178      const basename = this._isDirent ? dirent.name : dirent;179      const fullPath = sysPath.resolve(sysPath.join(path, basename));180      entry = { path: sysPath.relative(this._root, fullPath), fullPath, basename };181      entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);182    } catch (err) {183      this._onError(err);184    }185    return entry;186  }187 188  _onError(err) {189    if (isNormalFlowError(err) && !this.destroyed) {190      this.emit('warn', err);191    } else {192      this.destroy(err);193    }194  }195 196  async _getEntryType(entry) {197    // entry may be undefined, because a warning or an error were emitted198    // and the statsProp is undefined199    const stats = entry && entry[this._statsProp];200    if (!stats) {201      return;202    }203    if (stats.isFile()) {204      return 'file';205    }206    if (stats.isDirectory()) {207      return 'directory';208    }209    if (stats && stats.isSymbolicLink()) {210      const full = entry.fullPath;211      try {212        const entryRealPath = await realpath(full);213        const entryRealPathStats = await lstat(entryRealPath);214        if (entryRealPathStats.isFile()) {215          return 'file';216        }217        if (entryRealPathStats.isDirectory()) {218          const len = entryRealPath.length;219          if (full.startsWith(entryRealPath) && full.substr(len, 1) === sysPath.sep) {220            const recursiveError = new Error(221              `Circular symlink detected: "${full}" points to "${entryRealPath}"`222            );223            recursiveError.code = RECURSIVE_ERROR_CODE;224            return this._onError(recursiveError);225          }226          return 'directory';227        }228      } catch (error) {229        this._onError(error);230      }231    }232  }233 234  _includeAsFile(entry) {235    const stats = entry && entry[this._statsProp];236 237    return stats && this._wantsEverything && !stats.isDirectory();238  }239}240 241/**242 * @typedef {Object} ReaddirpArguments243 * @property {Function=} fileFilter244 * @property {Function=} directoryFilter245 * @property {String=} type246 * @property {Number=} depth247 * @property {String=} root248 * @property {Boolean=} lstat249 * @property {Boolean=} bigint250 */251 252/**253 * Main function which ends up calling readdirRec and reads all files and directories in given root recursively.254 * @param {String} root Root directory255 * @param {ReaddirpArguments=} options Options to specify root (start directory), filters and recursion depth256 */257const readdirp = (root, options = {}) => {258  let type = options.entryType || options.type;259  if (type === 'both') type = FILE_DIR_TYPE; // backwards-compatibility260  if (type) options.type = type;261  if (!root) {262    throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');263  } else if (typeof root !== 'string') {264    throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');265  } else if (type && !ALL_TYPES.includes(type)) {266    throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);267  }268 269  options.root = root;270  return new ReaddirpStream(options);271};272 273const readdirpPromise = (root, options = {}) => {274  return new Promise((resolve, reject) => {275    const files = [];276    readdirp(root, options)277      .on('data', entry => files.push(entry))278      .on('end', () => resolve(files))279      .on('error', error => reject(error));280  });281};282 283readdirp.promise = readdirpPromise;284readdirp.ReaddirpStream = ReaddirpStream;285readdirp.default = readdirp;286 287module.exports = readdirp;288