AK-21/Graphite-Industrial-Intelligence

1'use strict';2 3const fs = require('fs');4const sysPath = require('path');5const { promisify } = require('util');6const isBinaryPath = require('is-binary-path');7const {8  isWindows,9  isLinux,10  EMPTY_FN,11  EMPTY_STR,12  KEY_LISTENERS,13  KEY_ERR,14  KEY_RAW,15  HANDLER_KEYS,16  EV_CHANGE,17  EV_ADD,18  EV_ADD_DIR,19  EV_ERROR,20  STR_DATA,21  STR_END,22  BRACE_START,23  STAR24} = require('./constants');25 26const THROTTLE_MODE_WATCH = 'watch';27 28const open = promisify(fs.open);29const stat = promisify(fs.stat);30const lstat = promisify(fs.lstat);31const close = promisify(fs.close);32const fsrealpath = promisify(fs.realpath);33 34const statMethods = { lstat, stat };35 36// TODO: emit errors properly. Example: EMFILE on Macos.37const foreach = (val, fn) => {38  if (val instanceof Set) {39    val.forEach(fn);40  } else {41    fn(val);42  }43};44 45const addAndConvert = (main, prop, item) => {46  let container = main[prop];47  if (!(container instanceof Set)) {48    main[prop] = container = new Set([container]);49  }50  container.add(item);51};52 53const clearItem = cont => key => {54  const set = cont[key];55  if (set instanceof Set) {56    set.clear();57  } else {58    delete cont[key];59  }60};61 62const delFromSet = (main, prop, item) => {63  const container = main[prop];64  if (container instanceof Set) {65    container.delete(item);66  } else if (container === item) {67    delete main[prop];68  }69};70 71const isEmptySet = (val) => val instanceof Set ? val.size === 0 : !val;72 73/**74 * @typedef {String} Path75 */76 77// fs_watch helpers78 79// object to hold per-process fs_watch instances80// (may be shared across chokidar FSWatcher instances)81 82/**83 * @typedef {Object} FsWatchContainer84 * @property {Set} listeners85 * @property {Set} errHandlers86 * @property {Set} rawEmitters87 * @property {fs.FSWatcher=} watcher88 * @property {Boolean=} watcherUnusable89 */90 91/**92 * @type {Map<String,FsWatchContainer>}93 */94const FsWatchInstances = new Map();95 96/**97 * Instantiates the fs_watch interface98 * @param {String} path to be watched99 * @param {Object} options to be passed to fs_watch100 * @param {Function} listener main event handler101 * @param {Function} errHandler emits info about errors102 * @param {Function} emitRaw emits raw event data103 * @returns {fs.FSWatcher} new fsevents instance104 */105function createFsWatchInstance(path, options, listener, errHandler, emitRaw) {106  const handleEvent = (rawEvent, evPath) => {107    listener(path);108    emitRaw(rawEvent, evPath, {watchedPath: path});109 110    // emit based on events occurring for files from a directory's watcher in111    // case the file's watcher misses it (and rely on throttling to de-dupe)112    if (evPath && path !== evPath) {113      fsWatchBroadcast(114        sysPath.resolve(path, evPath), KEY_LISTENERS, sysPath.join(path, evPath)115      );116    }117  };118  try {119    return fs.watch(path, options, handleEvent);120  } catch (error) {121    errHandler(error);122  }123}124 125/**126 * Helper for passing fs_watch event data to a collection of listeners127 * @param {Path} fullPath absolute path bound to fs_watch instance128 * @param {String} type listener type129 * @param {*=} val1 arguments to be passed to listeners130 * @param {*=} val2131 * @param {*=} val3132 */133const fsWatchBroadcast = (fullPath, type, val1, val2, val3) => {134  const cont = FsWatchInstances.get(fullPath);135  if (!cont) return;136  foreach(cont[type], (listener) => {137    listener(val1, val2, val3);138  });139};140 141/**142 * Instantiates the fs_watch interface or binds listeners143 * to an existing one covering the same file system entry144 * @param {String} path145 * @param {String} fullPath absolute path146 * @param {Object} options to be passed to fs_watch147 * @param {Object} handlers container for event listener functions148 */149const setFsWatchListener = (path, fullPath, options, handlers) => {150  const {listener, errHandler, rawEmitter} = handlers;151  let cont = FsWatchInstances.get(fullPath);152 153  /** @type {fs.FSWatcher=} */154  let watcher;155  if (!options.persistent) {156    watcher = createFsWatchInstance(157      path, options, listener, errHandler, rawEmitter158    );159    return watcher.close.bind(watcher);160  }161  if (cont) {162    addAndConvert(cont, KEY_LISTENERS, listener);163    addAndConvert(cont, KEY_ERR, errHandler);164    addAndConvert(cont, KEY_RAW, rawEmitter);165  } else {166    watcher = createFsWatchInstance(167      path,168      options,169      fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS),170      errHandler, // no need to use broadcast here171      fsWatchBroadcast.bind(null, fullPath, KEY_RAW)172    );173    if (!watcher) return;174    watcher.on(EV_ERROR, async (error) => {175      const broadcastErr = fsWatchBroadcast.bind(null, fullPath, KEY_ERR);176      cont.watcherUnusable = true; // documented since Node 10.4.1177      // Workaround for https://github.com/joyent/node/issues/4337178      if (isWindows && error.code === 'EPERM') {179        try {180          const fd = await open(path, 'r');181          await close(fd);182          broadcastErr(error);183        } catch (err) {}184      } else {185        broadcastErr(error);186      }187    });188    cont = {189      listeners: listener,190      errHandlers: errHandler,191      rawEmitters: rawEmitter,192      watcher193    };194    FsWatchInstances.set(fullPath, cont);195  }196  // const index = cont.listeners.indexOf(listener);197 198  // removes this instance's listeners and closes the underlying fs_watch199  // instance if there are no more listeners left200  return () => {201    delFromSet(cont, KEY_LISTENERS, listener);202    delFromSet(cont, KEY_ERR, errHandler);203    delFromSet(cont, KEY_RAW, rawEmitter);204    if (isEmptySet(cont.listeners)) {205      // Check to protect against issue gh-730.206      // if (cont.watcherUnusable) {207      cont.watcher.close();208      // }209      FsWatchInstances.delete(fullPath);210      HANDLER_KEYS.forEach(clearItem(cont));211      cont.watcher = undefined;212      Object.freeze(cont);213    }214  };215};216 217// fs_watchFile helpers218 219// object to hold per-process fs_watchFile instances220// (may be shared across chokidar FSWatcher instances)221const FsWatchFileInstances = new Map();222 223/**224 * Instantiates the fs_watchFile interface or binds listeners225 * to an existing one covering the same file system entry226 * @param {String} path to be watched227 * @param {String} fullPath absolute path228 * @param {Object} options options to be passed to fs_watchFile229 * @param {Object} handlers container for event listener functions230 * @returns {Function} closer231 */232const setFsWatchFileListener = (path, fullPath, options, handlers) => {233  const {listener, rawEmitter} = handlers;234  let cont = FsWatchFileInstances.get(fullPath);235 236  /* eslint-disable no-unused-vars, prefer-destructuring */237  let listeners = new Set();238  let rawEmitters = new Set();239 240  const copts = cont && cont.options;241  if (copts && (copts.persistent < options.persistent || copts.interval > options.interval)) {242    // "Upgrade" the watcher to persistence or a quicker interval.243    // This creates some unlikely edge case issues if the user mixes244    // settings in a very weird way, but solving for those cases245    // doesn't seem worthwhile for the added complexity.246    listeners = cont.listeners;247    rawEmitters = cont.rawEmitters;248    fs.unwatchFile(fullPath);249    cont = undefined;250  }251 252  /* eslint-enable no-unused-vars, prefer-destructuring */253 254  if (cont) {255    addAndConvert(cont, KEY_LISTENERS, listener);256    addAndConvert(cont, KEY_RAW, rawEmitter);257  } else {258    // TODO259    // listeners.add(listener);260    // rawEmitters.add(rawEmitter);261    cont = {262      listeners: listener,263      rawEmitters: rawEmitter,264      options,265      watcher: fs.watchFile(fullPath, options, (curr, prev) => {266        foreach(cont.rawEmitters, (rawEmitter) => {267          rawEmitter(EV_CHANGE, fullPath, {curr, prev});268        });269        const currmtime = curr.mtimeMs;270        if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {271          foreach(cont.listeners, (listener) => listener(path, curr));272        }273      })274    };275    FsWatchFileInstances.set(fullPath, cont);276  }277  // const index = cont.listeners.indexOf(listener);278 279  // Removes this instance's listeners and closes the underlying fs_watchFile280  // instance if there are no more listeners left.281  return () => {282    delFromSet(cont, KEY_LISTENERS, listener);283    delFromSet(cont, KEY_RAW, rawEmitter);284    if (isEmptySet(cont.listeners)) {285      FsWatchFileInstances.delete(fullPath);286      fs.unwatchFile(fullPath);287      cont.options = cont.watcher = undefined;288      Object.freeze(cont);289    }290  };291};292 293/**294 * @mixin295 */296class NodeFsHandler {297 298/**299 * @param {import("../index").FSWatcher} fsW300 */301constructor(fsW) {302  this.fsw = fsW;303  this._boundHandleError = (error) => fsW._handleError(error);304}305 306/**307 * Watch file for changes with fs_watchFile or fs_watch.308 * @param {String} path to file or dir309 * @param {Function} listener on fs change310 * @returns {Function} closer for the watcher instance311 */312_watchWithNodeFs(path, listener) {313  const opts = this.fsw.options;314  const directory = sysPath.dirname(path);315  const basename = sysPath.basename(path);316  const parent = this.fsw._getWatchedDir(directory);317  parent.add(basename);318  const absolutePath = sysPath.resolve(path);319  const options = {persistent: opts.persistent};320  if (!listener) listener = EMPTY_FN;321 322  let closer;323  if (opts.usePolling) {324    options.interval = opts.enableBinaryInterval && isBinaryPath(basename) ?325      opts.binaryInterval : opts.interval;326    closer = setFsWatchFileListener(path, absolutePath, options, {327      listener,328      rawEmitter: this.fsw._emitRaw329    });330  } else {331    closer = setFsWatchListener(path, absolutePath, options, {332      listener,333      errHandler: this._boundHandleError,334      rawEmitter: this.fsw._emitRaw335    });336  }337  return closer;338}339 340/**341 * Watch a file and emit add event if warranted.342 * @param {Path} file Path343 * @param {fs.Stats} stats result of fs_stat344 * @param {Boolean} initialAdd was the file added at watch instantiation?345 * @returns {Function} closer for the watcher instance346 */347_handleFile(file, stats, initialAdd) {348  if (this.fsw.closed) {349    return;350  }351  const dirname = sysPath.dirname(file);352  const basename = sysPath.basename(file);353  const parent = this.fsw._getWatchedDir(dirname);354  // stats is always present355  let prevStats = stats;356 357  // if the file is already being watched, do nothing358  if (parent.has(basename)) return;359 360  const listener = async (path, newStats) => {361    if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5)) return;362    if (!newStats || newStats.mtimeMs === 0) {363      try {364        const newStats = await stat(file);365        if (this.fsw.closed) return;366        // Check that change event was not fired because of changed only accessTime.367        const at = newStats.atimeMs;368        const mt = newStats.mtimeMs;369        if (!at || at <= mt || mt !== prevStats.mtimeMs) {370          this.fsw._emit(EV_CHANGE, file, newStats);371        }372        if (isLinux && prevStats.ino !== newStats.ino) {373          this.fsw._closeFile(path)374          prevStats = newStats;375          this.fsw._addPathCloser(path, this._watchWithNodeFs(file, listener));376        } else {377          prevStats = newStats;378        }379      } catch (error) {380        // Fix issues where mtime is null but file is still present381        this.fsw._remove(dirname, basename);382      }383      // add is about to be emitted if file not already tracked in parent384    } else if (parent.has(basename)) {385      // Check that change event was not fired because of changed only accessTime.386      const at = newStats.atimeMs;387      const mt = newStats.mtimeMs;388      if (!at || at <= mt || mt !== prevStats.mtimeMs) {389        this.fsw._emit(EV_CHANGE, file, newStats);390      }391      prevStats = newStats;392    }393  }394  // kick off the watcher395  const closer = this._watchWithNodeFs(file, listener);396 397  // emit an add event if we're supposed to398  if (!(initialAdd && this.fsw.options.ignoreInitial) && this.fsw._isntIgnored(file)) {399    if (!this.fsw._throttle(EV_ADD, file, 0)) return;400    this.fsw._emit(EV_ADD, file, stats);401  }402 403  return closer;404}405 406/**407 * Handle symlinks encountered while reading a dir.408 * @param {Object} entry returned by readdirp409 * @param {String} directory path of dir being read410 * @param {String} path of this item411 * @param {String} item basename of this item412 * @returns {Promise<Boolean>} true if no more processing is needed for this entry.413 */414async _handleSymlink(entry, directory, path, item) {415  if (this.fsw.closed) {416    return;417  }418  const full = entry.fullPath;419  const dir = this.fsw._getWatchedDir(directory);420 421  if (!this.fsw.options.followSymlinks) {422    // watch symlink directly (don't follow) and detect changes423    this.fsw._incrReadyCount();424 425    let linkPath;426    try {427      linkPath = await fsrealpath(path);428    } catch (e) {429      this.fsw._emitReady();430      return true;431    }432 433    if (this.fsw.closed) return;434    if (dir.has(item)) {435      if (this.fsw._symlinkPaths.get(full) !== linkPath) {436        this.fsw._symlinkPaths.set(full, linkPath);437        this.fsw._emit(EV_CHANGE, path, entry.stats);438      }439    } else {440      dir.add(item);441      this.fsw._symlinkPaths.set(full, linkPath);442      this.fsw._emit(EV_ADD, path, entry.stats);443    }444    this.fsw._emitReady();445    return true;446  }447 448  // don't follow the same symlink more than once449  if (this.fsw._symlinkPaths.has(full)) {450    return true;451  }452 453  this.fsw._symlinkPaths.set(full, true);454}455 456_handleRead(directory, initialAdd, wh, target, dir, depth, throttler) {457  // Normalize the directory name on Windows458  directory = sysPath.join(directory, EMPTY_STR);459 460  if (!wh.hasGlob) {461    throttler = this.fsw._throttle('readdir', directory, 1000);462    if (!throttler) return;463  }464 465  const previous = this.fsw._getWatchedDir(wh.path);466  const current = new Set();467 468  let stream = this.fsw._readdirp(directory, {469    fileFilter: entry => wh.filterPath(entry),470    directoryFilter: entry => wh.filterDir(entry),471    depth: 0472  }).on(STR_DATA, async (entry) => {473    if (this.fsw.closed) {474      stream = undefined;475      return;476    }477    const item = entry.path;478    let path = sysPath.join(directory, item);479    current.add(item);480 481    if (entry.stats.isSymbolicLink() && await this._handleSymlink(entry, directory, path, item)) {482      return;483    }484 485    if (this.fsw.closed) {486      stream = undefined;487      return;488    }489    // Files that present in current directory snapshot490    // but absent in previous are added to watch list and491    // emit `add` event.492    if (item === target || !target && !previous.has(item)) {493      this.fsw._incrReadyCount();494 495      // ensure relativeness of path is preserved in case of watcher reuse496      path = sysPath.join(dir, sysPath.relative(dir, path));497 498      this._addToNodeFs(path, initialAdd, wh, depth + 1);499    }500  }).on(EV_ERROR, this._boundHandleError);501 502  return new Promise(resolve =>503    stream.once(STR_END, () => {504      if (this.fsw.closed) {505        stream = undefined;506        return;507      }508      const wasThrottled = throttler ? throttler.clear() : false;509 510      resolve();511 512      // Files that absent in current directory snapshot513      // but present in previous emit `remove` event514      // and are removed from @watched[directory].515      previous.getChildren().filter((item) => {516        return item !== directory &&517          !current.has(item) &&518          // in case of intersecting globs;519          // a path may have been filtered out of this readdir, but520          // shouldn't be removed because it matches a different glob521          (!wh.hasGlob || wh.filterPath({522            fullPath: sysPath.resolve(directory, item)523          }));524      }).forEach((item) => {525        this.fsw._remove(directory, item);526      });527 528      stream = undefined;529 530      // one more time for any missed in case changes came in extremely quickly531      if (wasThrottled) this._handleRead(directory, false, wh, target, dir, depth, throttler);532    })533  );534}535 536/**537 * Read directory to add / remove files from `@watched` list and re-read it on change.538 * @param {String} dir fs path539 * @param {fs.Stats} stats540 * @param {Boolean} initialAdd541 * @param {Number} depth relative to user-supplied path542 * @param {String} target child path targeted for watch543 * @param {Object} wh Common watch helpers for this path544 * @param {String} realpath545 * @returns {Promise<Function>} closer for the watcher instance.546 */547async _handleDir(dir, stats, initialAdd, depth, target, wh, realpath) {548  const parentDir = this.fsw._getWatchedDir(sysPath.dirname(dir));549  const tracked = parentDir.has(sysPath.basename(dir));550  if (!(initialAdd && this.fsw.options.ignoreInitial) && !target && !tracked) {551    if (!wh.hasGlob || wh.globFilter(dir)) this.fsw._emit(EV_ADD_DIR, dir, stats);552  }553 554  // ensure dir is tracked (harmless if redundant)555  parentDir.add(sysPath.basename(dir));556  this.fsw._getWatchedDir(dir);557  let throttler;558  let closer;559 560  const oDepth = this.fsw.options.depth;561  if ((oDepth == null || depth <= oDepth) && !this.fsw._symlinkPaths.has(realpath)) {562    if (!target) {563      await this._handleRead(dir, initialAdd, wh, target, dir, depth, throttler);564      if (this.fsw.closed) return;565    }566 567    closer = this._watchWithNodeFs(dir, (dirPath, stats) => {568      // if current directory is removed, do nothing569      if (stats && stats.mtimeMs === 0) return;570 571      this._handleRead(dirPath, false, wh, target, dir, depth, throttler);572    });573  }574  return closer;575}576 577/**578 * Handle added file, directory, or glob pattern.579 * Delegates call to _handleFile / _handleDir after checks.580 * @param {String} path to file or ir581 * @param {Boolean} initialAdd was the file added at watch instantiation?582 * @param {Object} priorWh depth relative to user-supplied path583 * @param {Number} depth Child path actually targeted for watch584 * @param {String=} target Child path actually targeted for watch585 * @returns {Promise}586 */587async _addToNodeFs(path, initialAdd, priorWh, depth, target) {588  const ready = this.fsw._emitReady;589  if (this.fsw._isIgnored(path) || this.fsw.closed) {590    ready();591    return false;592  }593 594  const wh = this.fsw._getWatchHelpers(path, depth);595  if (!wh.hasGlob && priorWh) {596    wh.hasGlob = priorWh.hasGlob;597    wh.globFilter = priorWh.globFilter;598    wh.filterPath = entry => priorWh.filterPath(entry);599    wh.filterDir = entry => priorWh.filterDir(entry);600  }601 602  // evaluate what is at the path we're being asked to watch603  try {604    const stats = await statMethods[wh.statMethod](wh.watchPath);605    if (this.fsw.closed) return;606    if (this.fsw._isIgnored(wh.watchPath, stats)) {607      ready();608      return false;609    }610 611    const follow = this.fsw.options.followSymlinks && !path.includes(STAR) && !path.includes(BRACE_START);612    let closer;613    if (stats.isDirectory()) {614      const absPath = sysPath.resolve(path);615      const targetPath = follow ? await fsrealpath(path) : path;616      if (this.fsw.closed) return;617      closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);618      if (this.fsw.closed) return;619      // preserve this symlink's target path620      if (absPath !== targetPath && targetPath !== undefined) {621        this.fsw._symlinkPaths.set(absPath, targetPath);622      }623    } else if (stats.isSymbolicLink()) {624      const targetPath = follow ? await fsrealpath(path) : path;625      if (this.fsw.closed) return;626      const parent = sysPath.dirname(wh.watchPath);627      this.fsw._getWatchedDir(parent).add(wh.watchPath);628      this.fsw._emit(EV_ADD, wh.watchPath, stats);629      closer = await this._handleDir(parent, stats, initialAdd, depth, path, wh, targetPath);630      if (this.fsw.closed) return;631 632      // preserve this symlink's target path633      if (targetPath !== undefined) {634        this.fsw._symlinkPaths.set(sysPath.resolve(path), targetPath);635      }636    } else {637      closer = this._handleFile(wh.watchPath, stats, initialAdd);638    }639    ready();640 641    this.fsw._addPathCloser(path, closer);642    return false;643 644  } catch (error) {645    if (this.fsw._handleError(error)) {646      ready();647      return path;648    }649  }650}651 652}653 654module.exports = NodeFsHandler;655