basant307/AI_Governance_Project

1/**2 * @import {3 *   Chunk,4 *   Code,5 *   ConstructRecord,6 *   Construct,7 *   Effects,8 *   InitialConstruct,9 *   ParseContext,10 *   Point,11 *   State,12 *   TokenizeContext,13 *   Token14 * } from 'micromark-util-types'15 */16 17/**18 * @callback Restore19 *   Restore the state.20 * @returns {undefined}21 *   Nothing.22 *23 * @typedef Info24 *   Info.25 * @property {Restore} restore26 *   Restore.27 * @property {number} from28 *   From.29 *30 * @callback ReturnHandle31 *   Handle a successful run.32 * @param {Construct} construct33 *   Construct.34 * @param {Info} info35 *   Info.36 * @returns {undefined}37 *   Nothing.38 */39 40import { markdownLineEnding } from 'micromark-util-character';41import { push, splice } from 'micromark-util-chunked';42import { resolveAll } from 'micromark-util-resolve-all';43/**44 * Create a tokenizer.45 * Tokenizers deal with one type of data (e.g., containers, flow, text).46 * The parser is the object dealing with it all.47 * `initialize` works like other constructs, except that only its `tokenize`48 * function is used, in which case it doesn’t receive an `ok` or `nok`.49 * `from` can be given to set the point before the first character, although50 * when further lines are indented, they must be set with `defineSkip`.51 *52 * @param {ParseContext} parser53 *   Parser.54 * @param {InitialConstruct} initialize55 *   Construct.56 * @param {Omit<Point, '_bufferIndex' | '_index'> | undefined} [from]57 *   Point (optional).58 * @returns {TokenizeContext}59 *   Context.60 */61export function createTokenizer(parser, initialize, from) {62  /** @type {Point} */63  let point = {64    _bufferIndex: -1,65    _index: 0,66    line: from && from.line || 1,67    column: from && from.column || 1,68    offset: from && from.offset || 069  };70  /** @type {Record<string, number>} */71  const columnStart = {};72  /** @type {Array<Construct>} */73  const resolveAllConstructs = [];74  /** @type {Array<Chunk>} */75  let chunks = [];76  /** @type {Array<Token>} */77  let stack = [];78  /** @type {boolean | undefined} */79  let consumed = true;80 81  /**82   * Tools used for tokenizing.83   *84   * @type {Effects}85   */86  const effects = {87    attempt: constructFactory(onsuccessfulconstruct),88    check: constructFactory(onsuccessfulcheck),89    consume,90    enter,91    exit,92    interrupt: constructFactory(onsuccessfulcheck, {93      interrupt: true94    })95  };96 97  /**98   * State and tools for resolving and serializing.99   *100   * @type {TokenizeContext}101   */102  const context = {103    code: null,104    containerState: {},105    defineSkip,106    events: [],107    now,108    parser,109    previous: null,110    sliceSerialize,111    sliceStream,112    write113  };114 115  /**116   * The state function.117   *118   * @type {State | undefined}119   */120  let state = initialize.tokenize.call(context, effects);121 122  /**123   * Track which character we expect to be consumed, to catch bugs.124   *125   * @type {Code}126   */127  let expectedCode;128  if (initialize.resolveAll) {129    resolveAllConstructs.push(initialize);130  }131  return context;132 133  /** @type {TokenizeContext['write']} */134  function write(slice) {135    chunks = push(chunks, slice);136    main();137 138    // Exit if we’re not done, resolve might change stuff.139    if (chunks[chunks.length - 1] !== null) {140      return [];141    }142    addResult(initialize, 0);143 144    // Otherwise, resolve, and exit.145    context.events = resolveAll(resolveAllConstructs, context.events, context);146    return context.events;147  }148 149  //150  // Tools.151  //152 153  /** @type {TokenizeContext['sliceSerialize']} */154  function sliceSerialize(token, expandTabs) {155    return serializeChunks(sliceStream(token), expandTabs);156  }157 158  /** @type {TokenizeContext['sliceStream']} */159  function sliceStream(token) {160    return sliceChunks(chunks, token);161  }162 163  /** @type {TokenizeContext['now']} */164  function now() {165    // This is a hot path, so we clone manually instead of `Object.assign({}, point)`166    const {167      _bufferIndex,168      _index,169      line,170      column,171      offset172    } = point;173    return {174      _bufferIndex,175      _index,176      line,177      column,178      offset179    };180  }181 182  /** @type {TokenizeContext['defineSkip']} */183  function defineSkip(value) {184    columnStart[value.line] = value.column;185    accountForPotentialSkip();186  }187 188  //189  // State management.190  //191 192  /**193   * Main loop (note that `_index` and `_bufferIndex` in `point` are modified by194   * `consume`).195   * Here is where we walk through the chunks, which either include strings of196   * several characters, or numerical character codes.197   * The reason to do this in a loop instead of a call is so the stack can198   * drain.199   *200   * @returns {undefined}201   *   Nothing.202   */203  function main() {204    /** @type {number} */205    let chunkIndex;206    while (point._index < chunks.length) {207      const chunk = chunks[point._index];208 209      // If we’re in a buffer chunk, loop through it.210      if (typeof chunk === 'string') {211        chunkIndex = point._index;212        if (point._bufferIndex < 0) {213          point._bufferIndex = 0;214        }215        while (point._index === chunkIndex && point._bufferIndex < chunk.length) {216          go(chunk.charCodeAt(point._bufferIndex));217        }218      } else {219        go(chunk);220      }221    }222  }223 224  /**225   * Deal with one code.226   *227   * @param {Code} code228   *   Code.229   * @returns {undefined}230   *   Nothing.231   */232  function go(code) {233    consumed = undefined;234    expectedCode = code;235    state = state(code);236  }237 238  /** @type {Effects['consume']} */239  function consume(code) {240    if (markdownLineEnding(code)) {241      point.line++;242      point.column = 1;243      point.offset += code === -3 ? 2 : 1;244      accountForPotentialSkip();245    } else if (code !== -1) {246      point.column++;247      point.offset++;248    }249 250    // Not in a string chunk.251    if (point._bufferIndex < 0) {252      point._index++;253    } else {254      point._bufferIndex++;255 256      // At end of string chunk.257      if (point._bufferIndex ===258      // Points w/ non-negative `_bufferIndex` reference259      // strings.260      /** @type {string} */261      chunks[point._index].length) {262        point._bufferIndex = -1;263        point._index++;264      }265    }266 267    // Expose the previous character.268    context.previous = code;269 270    // Mark as consumed.271    consumed = true;272  }273 274  /** @type {Effects['enter']} */275  function enter(type, fields) {276    /** @type {Token} */277    // @ts-expect-error Patch instead of assign required fields to help GC.278    const token = fields || {};279    token.type = type;280    token.start = now();281    context.events.push(['enter', token, context]);282    stack.push(token);283    return token;284  }285 286  /** @type {Effects['exit']} */287  function exit(type) {288    const token = stack.pop();289    token.end = now();290    context.events.push(['exit', token, context]);291    return token;292  }293 294  /**295   * Use results.296   *297   * @type {ReturnHandle}298   */299  function onsuccessfulconstruct(construct, info) {300    addResult(construct, info.from);301  }302 303  /**304   * Discard results.305   *306   * @type {ReturnHandle}307   */308  function onsuccessfulcheck(_, info) {309    info.restore();310  }311 312  /**313   * Factory to attempt/check/interrupt.314   *315   * @param {ReturnHandle} onreturn316   *   Callback.317   * @param {{interrupt?: boolean | undefined} | undefined} [fields]318   *   Fields.319   */320  function constructFactory(onreturn, fields) {321    return hook;322 323    /**324     * Handle either an object mapping codes to constructs, a list of325     * constructs, or a single construct.326     *327     * @param {Array<Construct> | ConstructRecord | Construct} constructs328     *   Constructs.329     * @param {State} returnState330     *   State.331     * @param {State | undefined} [bogusState]332     *   State.333     * @returns {State}334     *   State.335     */336    function hook(constructs, returnState, bogusState) {337      /** @type {ReadonlyArray<Construct>} */338      let listOfConstructs;339      /** @type {number} */340      let constructIndex;341      /** @type {Construct} */342      let currentConstruct;343      /** @type {Info} */344      let info;345      return Array.isArray(constructs) ? /* c8 ignore next 1 */346      handleListOfConstructs(constructs) : 'tokenize' in constructs ?347      // Looks like a construct.348      handleListOfConstructs([(/** @type {Construct} */constructs)]) : handleMapOfConstructs(constructs);349 350      /**351       * Handle a list of construct.352       *353       * @param {ConstructRecord} map354       *   Constructs.355       * @returns {State}356       *   State.357       */358      function handleMapOfConstructs(map) {359        return start;360 361        /** @type {State} */362        function start(code) {363          const left = code !== null && map[code];364          const all = code !== null && map.null;365          const list = [366          // To do: add more extension tests.367          /* c8 ignore next 2 */368          ...(Array.isArray(left) ? left : left ? [left] : []), ...(Array.isArray(all) ? all : all ? [all] : [])];369          return handleListOfConstructs(list)(code);370        }371      }372 373      /**374       * Handle a list of construct.375       *376       * @param {ReadonlyArray<Construct>} list377       *   Constructs.378       * @returns {State}379       *   State.380       */381      function handleListOfConstructs(list) {382        listOfConstructs = list;383        constructIndex = 0;384        if (list.length === 0) {385          return bogusState;386        }387        return handleConstruct(list[constructIndex]);388      }389 390      /**391       * Handle a single construct.392       *393       * @param {Construct} construct394       *   Construct.395       * @returns {State}396       *   State.397       */398      function handleConstruct(construct) {399        return start;400 401        /** @type {State} */402        function start(code) {403          // To do: not needed to store if there is no bogus state, probably?404          // Currently doesn’t work because `inspect` in document does a check405          // w/o a bogus, which doesn’t make sense. But it does seem to help perf406          // by not storing.407          info = store();408          currentConstruct = construct;409          if (!construct.partial) {410            context.currentConstruct = construct;411          }412 413          // Always populated by defaults.414 415          if (construct.name && context.parser.constructs.disable.null.includes(construct.name)) {416            return nok(code);417          }418          return construct.tokenize.call(419          // If we do have fields, create an object w/ `context` as its420          // prototype.421          // This allows a “live binding”, which is needed for `interrupt`.422          fields ? Object.assign(Object.create(context), fields) : context, effects, ok, nok)(code);423        }424      }425 426      /** @type {State} */427      function ok(code) {428        consumed = true;429        onreturn(currentConstruct, info);430        return returnState;431      }432 433      /** @type {State} */434      function nok(code) {435        consumed = true;436        info.restore();437        if (++constructIndex < listOfConstructs.length) {438          return handleConstruct(listOfConstructs[constructIndex]);439        }440        return bogusState;441      }442    }443  }444 445  /**446   * @param {Construct} construct447   *   Construct.448   * @param {number} from449   *   From.450   * @returns {undefined}451   *   Nothing.452   */453  function addResult(construct, from) {454    if (construct.resolveAll && !resolveAllConstructs.includes(construct)) {455      resolveAllConstructs.push(construct);456    }457    if (construct.resolve) {458      splice(context.events, from, context.events.length - from, construct.resolve(context.events.slice(from), context));459    }460    if (construct.resolveTo) {461      context.events = construct.resolveTo(context.events, context);462    }463  }464 465  /**466   * Store state.467   *468   * @returns {Info}469   *   Info.470   */471  function store() {472    const startPoint = now();473    const startPrevious = context.previous;474    const startCurrentConstruct = context.currentConstruct;475    const startEventsIndex = context.events.length;476    const startStack = Array.from(stack);477    return {478      from: startEventsIndex,479      restore480    };481 482    /**483     * Restore state.484     *485     * @returns {undefined}486     *   Nothing.487     */488    function restore() {489      point = startPoint;490      context.previous = startPrevious;491      context.currentConstruct = startCurrentConstruct;492      context.events.length = startEventsIndex;493      stack = startStack;494      accountForPotentialSkip();495    }496  }497 498  /**499   * Move the current point a bit forward in the line when it’s on a column500   * skip.501   *502   * @returns {undefined}503   *   Nothing.504   */505  function accountForPotentialSkip() {506    if (point.line in columnStart && point.column < 2) {507      point.column = columnStart[point.line];508      point.offset += columnStart[point.line] - 1;509    }510  }511}512 513/**514 * Get the chunks from a slice of chunks in the range of a token.515 *516 * @param {ReadonlyArray<Chunk>} chunks517 *   Chunks.518 * @param {Pick<Token, 'end' | 'start'>} token519 *   Token.520 * @returns {Array<Chunk>}521 *   Chunks.522 */523function sliceChunks(chunks, token) {524  const startIndex = token.start._index;525  const startBufferIndex = token.start._bufferIndex;526  const endIndex = token.end._index;527  const endBufferIndex = token.end._bufferIndex;528  /** @type {Array<Chunk>} */529  let view;530  if (startIndex === endIndex) {531    // @ts-expect-error `_bufferIndex` is used on string chunks.532    view = [chunks[startIndex].slice(startBufferIndex, endBufferIndex)];533  } else {534    view = chunks.slice(startIndex, endIndex);535    if (startBufferIndex > -1) {536      const head = view[0];537      if (typeof head === 'string') {538        view[0] = head.slice(startBufferIndex);539        /* c8 ignore next 4 -- used to be used, no longer */540      } else {541        view.shift();542      }543    }544    if (endBufferIndex > 0) {545      // @ts-expect-error `_bufferIndex` is used on string chunks.546      view.push(chunks[endIndex].slice(0, endBufferIndex));547    }548  }549  return view;550}551 552/**553 * Get the string value of a slice of chunks.554 *555 * @param {ReadonlyArray<Chunk>} chunks556 *   Chunks.557 * @param {boolean | undefined} [expandTabs=false]558 *   Whether to expand tabs (default: `false`).559 * @returns {string}560 *   Result.561 */562function serializeChunks(chunks, expandTabs) {563  let index = -1;564  /** @type {Array<string>} */565  const result = [];566  /** @type {boolean | undefined} */567  let atTab;568  while (++index < chunks.length) {569    const chunk = chunks[index];570    /** @type {string} */571    let value;572    if (typeof chunk === 'string') {573      value = chunk;574    } else switch (chunk) {575      case -5:576        {577          value = "\r";578          break;579        }580      case -4:581        {582          value = "\n";583          break;584        }585      case -3:586        {587          value = "\r" + "\n";588          break;589        }590      case -2:591        {592          value = expandTabs ? " " : "\t";593          break;594        }595      case -1:596        {597          if (!expandTabs && atTab) continue;598          value = " ";599          break;600        }601      default:602        {603          // Currently only replacement character.604          value = String.fromCharCode(chunk);605        }606    }607    atTab = chunk === -2;608    result.push(value);609  }610  return result.join('');611}