basant307/AI_Governance_Project

1'use strict'2 3const crypto = require('crypto')4const {5  appendFile,6  mkdir,7  readFile,8  readdir,9  rm,10  writeFile,11} = require('fs/promises')12const { Minipass } = require('minipass')13const path = require('path')14const ssri = require('ssri')15const uniqueFilename = require('unique-filename')16 17const contentPath = require('./content/path')18const hashToSegments = require('./util/hash-to-segments')19const indexV = require('../package.json')['cache-version'].index20const { moveFile } = require('@npmcli/fs')21 22const lsStreamConcurrency = 523 24module.exports.NotFoundError = class NotFoundError extends Error {25  constructor (cache, key) {26    super(`No cache entry for ${key} found in ${cache}`)27    this.code = 'ENOENT'28    this.cache = cache29    this.key = key30  }31}32 33module.exports.compact = compact34 35async function compact (cache, key, matchFn, opts = {}) {36  const bucket = bucketPath(cache, key)37  const entries = await bucketEntries(bucket)38  const newEntries = []39  // we loop backwards because the bottom-most result is the newest40  // since we add new entries with appendFile41  for (let i = entries.length - 1; i >= 0; --i) {42    const entry = entries[i]43    // a null integrity could mean either a delete was appended44    // or the user has simply stored an index that does not map45    // to any content. we determine if the user wants to keep the46    // null integrity based on the validateEntry function passed in options.47    // if the integrity is null and no validateEntry is provided, we break48    // as we consider the null integrity to be a deletion of everything49    // that came before it.50    if (entry.integrity === null && !opts.validateEntry) {51      break52    }53 54    // if this entry is valid, and it is either the first entry or55    // the newEntries array doesn't already include an entry that56    // matches this one based on the provided matchFn, then we add57    // it to the beginning of our list58    if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&59      (newEntries.length === 0 ||60        !newEntries.find((oldEntry) => matchFn(oldEntry, entry)))) {61      newEntries.unshift(entry)62    }63  }64 65  const newIndex = '\n' + newEntries.map((entry) => {66    const stringified = JSON.stringify(entry)67    const hash = hashEntry(stringified)68    return `${hash}\t${stringified}`69  }).join('\n')70 71  const setup = async () => {72    const target = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)73    await mkdir(path.dirname(target), { recursive: true })74    return {75      target,76      moved: false,77    }78  }79 80  const teardown = async (tmp) => {81    if (!tmp.moved) {82      return rm(tmp.target, { recursive: true, force: true })83    }84  }85 86  const write = async (tmp) => {87    await writeFile(tmp.target, newIndex, { flag: 'wx' })88    await mkdir(path.dirname(bucket), { recursive: true })89    // we use @npmcli/move-file directly here because we90    // want to overwrite the existing file91    await moveFile(tmp.target, bucket)92    tmp.moved = true93  }94 95  // write the file atomically96  const tmp = await setup()97  try {98    await write(tmp)99  } finally {100    await teardown(tmp)101  }102 103  // we reverse the list we generated such that the newest104  // entries come first in order to make looping through them easier105  // the true passed to formatEntry tells it to keep null106  // integrity values, if they made it this far it's because107  // validateEntry returned true, and as such we should return it108  return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))109}110 111module.exports.insert = insert112 113async function insert (cache, key, integrity, opts = {}) {114  const { metadata, size, time } = opts115  const bucket = bucketPath(cache, key)116  const entry = {117    key,118    integrity: integrity && ssri.stringify(integrity),119    time: time || Date.now(),120    size,121    metadata,122  }123  try {124    await mkdir(path.dirname(bucket), { recursive: true })125    const stringified = JSON.stringify(entry)126    // NOTE - Cleverness ahoy!127    //128    // This works because it's tremendously unlikely for an entry to corrupt129    // another while still preserving the string length of the JSON in130    // question. So, we just slap the length in there and verify it on read.131    //132    // Thanks to @isaacs for the whiteboarding session that ended up with133    // this.134    await appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)135  } catch (err) {136    if (err.code === 'ENOENT') {137      return undefined138    }139 140    throw err141  }142  return formatEntry(cache, entry)143}144 145module.exports.find = find146 147async function find (cache, key) {148  const bucket = bucketPath(cache, key)149  try {150    const entries = await bucketEntries(bucket)151    return entries.reduce((latest, next) => {152      if (next && next.key === key) {153        return formatEntry(cache, next)154      } else {155        return latest156      }157    }, null)158  } catch (err) {159    if (err.code === 'ENOENT') {160      return null161    } else {162      throw err163    }164  }165}166 167module.exports.delete = del168 169function del (cache, key, opts = {}) {170  if (!opts.removeFully) {171    return insert(cache, key, null, opts)172  }173 174  const bucket = bucketPath(cache, key)175  return rm(bucket, { recursive: true, force: true })176}177 178module.exports.lsStream = lsStream179 180function lsStream (cache) {181  const indexDir = bucketDir(cache)182  const stream = new Minipass({ objectMode: true })183 184  // Set all this up to run on the stream and then just return the stream185  Promise.resolve().then(async () => {186    const { default: pMap } = await import('p-map')187    const buckets = await readdirOrEmpty(indexDir)188    await pMap(buckets, async (bucket) => {189      const bucketPath = path.join(indexDir, bucket)190      const subbuckets = await readdirOrEmpty(bucketPath)191      await pMap(subbuckets, async (subbucket) => {192        const subbucketPath = path.join(bucketPath, subbucket)193 194        // "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"195        const subbucketEntries = await readdirOrEmpty(subbucketPath)196        await pMap(subbucketEntries, async (entry) => {197          const entryPath = path.join(subbucketPath, entry)198          try {199            const entries = await bucketEntries(entryPath)200            // using a Map here prevents duplicate keys from showing up201            // twice, I guess?202            const reduced = entries.reduce((acc, entry) => {203              acc.set(entry.key, entry)204              return acc205            }, new Map())206            // reduced is a map of key => entry207            for (const entry of reduced.values()) {208              const formatted = formatEntry(cache, entry)209              if (formatted) {210                stream.write(formatted)211              }212            }213          } catch (err) {214            if (err.code === 'ENOENT') {215              return undefined216            }217            throw err218          }219        },220        { concurrency: lsStreamConcurrency })221      },222      { concurrency: lsStreamConcurrency })223    },224    { concurrency: lsStreamConcurrency })225    stream.end()226    return stream227  }).catch(err => stream.emit('error', err))228 229  return stream230}231 232module.exports.ls = ls233 234async function ls (cache) {235  const entries = await lsStream(cache).collect()236  return entries.reduce((acc, xs) => {237    acc[xs.key] = xs238    return acc239  }, {})240}241 242module.exports.bucketEntries = bucketEntries243 244async function bucketEntries (bucket, filter) {245  const data = await readFile(bucket, 'utf8')246  return _bucketEntries(data, filter)247}248 249function _bucketEntries (data) {250  const entries = []251  data.split('\n').forEach((entry) => {252    if (!entry) {253      return254    }255 256    const pieces = entry.split('\t')257    if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {258      // Hash is no good! Corruption or malice? Doesn't matter!259      // EJECT EJECT260      return261    }262    let obj263    try {264      obj = JSON.parse(pieces[1])265    } catch (_) {266      // eslint-ignore-next-line no-empty-block267    }268    // coverage disabled here, no need to test with an entry that parses to something falsey269    // istanbul ignore else270    if (obj) {271      entries.push(obj)272    }273  })274  return entries275}276 277module.exports.bucketDir = bucketDir278 279function bucketDir (cache) {280  return path.join(cache, `index-v${indexV}`)281}282 283module.exports.bucketPath = bucketPath284 285function bucketPath (cache, key) {286  const hashed = hashKey(key)287  return path.join.apply(288    path,289    [bucketDir(cache)].concat(hashToSegments(hashed))290  )291}292 293module.exports.hashKey = hashKey294 295function hashKey (key) {296  return hash(key, 'sha256')297}298 299module.exports.hashEntry = hashEntry300 301function hashEntry (str) {302  return hash(str, 'sha1')303}304 305function hash (str, digest) {306  return crypto307    .createHash(digest)308    .update(str)309    .digest('hex')310}311 312function formatEntry (cache, entry, keepAll) {313  // Treat null digests as deletions. They'll shadow any previous entries.314  if (!entry.integrity && !keepAll) {315    return null316  }317 318  return {319    key: entry.key,320    integrity: entry.integrity,321    path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,322    size: entry.size,323    time: entry.time,324    metadata: entry.metadata,325  }326}327 328function readdirOrEmpty (dir) {329  return readdir(dir).catch((err) => {330    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {331      return []332    }333 334    throw err335  })336}337