AK-21/Graphite-Industrial-Intelligence

1/**2 * @import {Chunk, Event, Token} from 'micromark-util-types'3 */4 5import { splice } from 'micromark-util-chunked';6import { SpliceBuffer } from './lib/splice-buffer.js';7 8// Hidden API exposed for testing.9export { SpliceBuffer } from './lib/splice-buffer.js';10 11/**12 * Tokenize subcontent.13 *14 * @param {Array<Event>} eventsArray15 *   List of events.16 * @returns {boolean}17 *   Whether subtokens were found.18 */19// eslint-disable-next-line complexity20export function subtokenize(eventsArray) {21  /** @type {Record<string, number>} */22  const jumps = {};23  let index = -1;24  /** @type {Event} */25  let event;26  /** @type {number | undefined} */27  let lineIndex;28  /** @type {number} */29  let otherIndex;30  /** @type {Event} */31  let otherEvent;32  /** @type {Array<Event>} */33  let parameters;34  /** @type {Array<Event>} */35  let subevents;36  /** @type {boolean | undefined} */37  let more;38  const events = new SpliceBuffer(eventsArray);39  while (++index < events.length) {40    while (index in jumps) {41      index = jumps[index];42    }43    event = events.get(index);44 45    // Add a hook for the GFM tasklist extension, which needs to know if text46    // is in the first content of a list item.47    if (index && event[1].type === "chunkFlow" && events.get(index - 1)[1].type === "listItemPrefix") {48      subevents = event[1]._tokenizer.events;49      otherIndex = 0;50      if (otherIndex < subevents.length && subevents[otherIndex][1].type === "lineEndingBlank") {51        otherIndex += 2;52      }53      if (otherIndex < subevents.length && subevents[otherIndex][1].type === "content") {54        while (++otherIndex < subevents.length) {55          if (subevents[otherIndex][1].type === "content") {56            break;57          }58          if (subevents[otherIndex][1].type === "chunkText") {59            subevents[otherIndex][1]._isInFirstContentOfListItem = true;60            otherIndex++;61          }62        }63      }64    }65 66    // Enter.67    if (event[0] === 'enter') {68      if (event[1].contentType) {69        Object.assign(jumps, subcontent(events, index));70        index = jumps[index];71        more = true;72      }73    }74    // Exit.75    else if (event[1]._container) {76      otherIndex = index;77      lineIndex = undefined;78      while (otherIndex--) {79        otherEvent = events.get(otherIndex);80        if (otherEvent[1].type === "lineEnding" || otherEvent[1].type === "lineEndingBlank") {81          if (otherEvent[0] === 'enter') {82            if (lineIndex) {83              events.get(lineIndex)[1].type = "lineEndingBlank";84            }85            otherEvent[1].type = "lineEnding";86            lineIndex = otherIndex;87          }88        } else if (otherEvent[1].type === "linePrefix" || otherEvent[1].type === "listItemIndent") {89          // Move past.90        } else {91          break;92        }93      }94      if (lineIndex) {95        // Fix position.96        event[1].end = {97          ...events.get(lineIndex)[1].start98        };99 100        // Switch container exit w/ line endings.101        parameters = events.slice(lineIndex, index);102        parameters.unshift(event);103        events.splice(lineIndex, index - lineIndex + 1, parameters);104      }105    }106  }107 108  // The changes to the `events` buffer must be copied back into the eventsArray109  splice(eventsArray, 0, Number.POSITIVE_INFINITY, events.slice(0));110  return !more;111}112 113/**114 * Tokenize embedded tokens.115 *116 * @param {SpliceBuffer<Event>} events117 *   Events.118 * @param {number} eventIndex119 *   Index.120 * @returns {Record<string, number>}121 *   Gaps.122 */123function subcontent(events, eventIndex) {124  const token = events.get(eventIndex)[1];125  const context = events.get(eventIndex)[2];126  let startPosition = eventIndex - 1;127  /** @type {Array<number>} */128  const startPositions = [];129  let tokenizer = token._tokenizer;130  if (!tokenizer) {131    tokenizer = context.parser[token.contentType](token.start);132    if (token._contentTypeTextTrailing) {133      tokenizer._contentTypeTextTrailing = true;134    }135  }136  const childEvents = tokenizer.events;137  /** @type {Array<[number, number]>} */138  const jumps = [];139  /** @type {Record<string, number>} */140  const gaps = {};141  /** @type {Array<Chunk>} */142  let stream;143  /** @type {Token | undefined} */144  let previous;145  let index = -1;146  /** @type {Token | undefined} */147  let current = token;148  let adjust = 0;149  let start = 0;150  const breaks = [start];151 152  // Loop forward through the linked tokens to pass them in order to the153  // subtokenizer.154  while (current) {155    // Find the position of the event for this token.156    while (events.get(++startPosition)[1] !== current) {157      // Empty.158    }159    startPositions.push(startPosition);160    if (!current._tokenizer) {161      stream = context.sliceStream(current);162      if (!current.next) {163        stream.push(null);164      }165      if (previous) {166        tokenizer.defineSkip(current.start);167      }168      if (current._isInFirstContentOfListItem) {169        tokenizer._gfmTasklistFirstContentOfListItem = true;170      }171      tokenizer.write(stream);172      if (current._isInFirstContentOfListItem) {173        tokenizer._gfmTasklistFirstContentOfListItem = undefined;174      }175    }176 177    // Unravel the next token.178    previous = current;179    current = current.next;180  }181 182  // Now, loop back through all events (and linked tokens), to figure out which183  // parts belong where.184  current = token;185  while (++index < childEvents.length) {186    if (187    // Find a void token that includes a break.188    childEvents[index][0] === 'exit' && childEvents[index - 1][0] === 'enter' && childEvents[index][1].type === childEvents[index - 1][1].type && childEvents[index][1].start.line !== childEvents[index][1].end.line) {189      start = index + 1;190      breaks.push(start);191      // Help GC.192      current._tokenizer = undefined;193      current.previous = undefined;194      current = current.next;195    }196  }197 198  // Help GC.199  tokenizer.events = [];200 201  // If there’s one more token (which is the cases for lines that end in an202  // EOF), that’s perfect: the last point we found starts it.203  // If there isn’t then make sure any remaining content is added to it.204  if (current) {205    // Help GC.206    current._tokenizer = undefined;207    current.previous = undefined;208  } else {209    breaks.pop();210  }211 212  // Now splice the events from the subtokenizer into the current events,213  // moving back to front so that splice indices aren’t affected.214  index = breaks.length;215  while (index--) {216    const slice = childEvents.slice(breaks[index], breaks[index + 1]);217    const start = startPositions.pop();218    jumps.push([start, start + slice.length - 1]);219    events.splice(start, 2, slice);220  }221  jumps.reverse();222  index = -1;223  while (++index < jumps.length) {224    gaps[adjust + jumps[index][0]] = adjust + jumps[index][1];225    adjust += jumps[index][1] - jumps[index][0] - 1;226  }227  return gaps;228}