AK-21/Graphite-Industrial-Intelligence

1/**2 * @import {Chunk, Event, Token} from 'micromark-util-types'3 */4 5import {ok as assert} from 'devlop'6import {splice} from 'micromark-util-chunked'7import {codes, types} from 'micromark-util-symbol'8import {SpliceBuffer} from './lib/splice-buffer.js'9 10// Hidden API exposed for testing.11export {SpliceBuffer} from './lib/splice-buffer.js'12 13/**14 * Tokenize subcontent.15 *16 * @param {Array<Event>} eventsArray17 *   List of events.18 * @returns {boolean}19 *   Whether subtokens were found.20 */21// eslint-disable-next-line complexity22export function subtokenize(eventsArray) {23  /** @type {Record<string, number>} */24  const jumps = {}25  let index = -126  /** @type {Event} */27  let event28  /** @type {number | undefined} */29  let lineIndex30  /** @type {number} */31  let otherIndex32  /** @type {Event} */33  let otherEvent34  /** @type {Array<Event>} */35  let parameters36  /** @type {Array<Event>} */37  let subevents38  /** @type {boolean | undefined} */39  let more40  const events = new SpliceBuffer(eventsArray)41 42  while (++index < events.length) {43    while (index in jumps) {44      index = jumps[index]45    }46 47    event = events.get(index)48 49    // Add a hook for the GFM tasklist extension, which needs to know if text50    // is in the first content of a list item.51    if (52      index &&53      event[1].type === types.chunkFlow &&54      events.get(index - 1)[1].type === types.listItemPrefix55    ) {56      assert(event[1]._tokenizer, 'expected `_tokenizer` on subtokens')57      subevents = event[1]._tokenizer.events58      otherIndex = 059 60      if (61        otherIndex < subevents.length &&62        subevents[otherIndex][1].type === types.lineEndingBlank63      ) {64        otherIndex += 265      }66 67      if (68        otherIndex < subevents.length &&69        subevents[otherIndex][1].type === types.content70      ) {71        while (++otherIndex < subevents.length) {72          if (subevents[otherIndex][1].type === types.content) {73            break74          }75 76          if (subevents[otherIndex][1].type === types.chunkText) {77            subevents[otherIndex][1]._isInFirstContentOfListItem = true78            otherIndex++79          }80        }81      }82    }83 84    // Enter.85    if (event[0] === 'enter') {86      if (event[1].contentType) {87        Object.assign(jumps, subcontent(events, index))88        index = jumps[index]89        more = true90      }91    }92    // Exit.93    else if (event[1]._container) {94      otherIndex = index95      lineIndex = undefined96 97      while (otherIndex--) {98        otherEvent = events.get(otherIndex)99 100        if (101          otherEvent[1].type === types.lineEnding ||102          otherEvent[1].type === types.lineEndingBlank103        ) {104          if (otherEvent[0] === 'enter') {105            if (lineIndex) {106              events.get(lineIndex)[1].type = types.lineEndingBlank107            }108 109            otherEvent[1].type = types.lineEnding110            lineIndex = otherIndex111          }112        } else if (113          otherEvent[1].type === types.linePrefix ||114          otherEvent[1].type === types.listItemIndent115        ) {116          // Move past.117        } else {118          break119        }120      }121 122      if (lineIndex) {123        // Fix position.124        event[1].end = {...events.get(lineIndex)[1].start}125 126        // Switch container exit w/ line endings.127        parameters = events.slice(lineIndex, index)128        parameters.unshift(event)129        events.splice(lineIndex, index - lineIndex + 1, parameters)130      }131    }132  }133 134  // The changes to the `events` buffer must be copied back into the eventsArray135  splice(eventsArray, 0, Number.POSITIVE_INFINITY, events.slice(0))136  return !more137}138 139/**140 * Tokenize embedded tokens.141 *142 * @param {SpliceBuffer<Event>} events143 *   Events.144 * @param {number} eventIndex145 *   Index.146 * @returns {Record<string, number>}147 *   Gaps.148 */149function subcontent(events, eventIndex) {150  const token = events.get(eventIndex)[1]151  const context = events.get(eventIndex)[2]152  let startPosition = eventIndex - 1153  /** @type {Array<number>} */154  const startPositions = []155  assert(token.contentType, 'expected `contentType` on subtokens')156 157  let tokenizer = token._tokenizer158 159  if (!tokenizer) {160    tokenizer = context.parser[token.contentType](token.start)161 162    if (token._contentTypeTextTrailing) {163      tokenizer._contentTypeTextTrailing = true164    }165  }166 167  const childEvents = tokenizer.events168  /** @type {Array<[number, number]>} */169  const jumps = []170  /** @type {Record<string, number>} */171  const gaps = {}172  /** @type {Array<Chunk>} */173  let stream174  /** @type {Token | undefined} */175  let previous176  let index = -1177  /** @type {Token | undefined} */178  let current = token179  let adjust = 0180  let start = 0181  const breaks = [start]182 183  // Loop forward through the linked tokens to pass them in order to the184  // subtokenizer.185  while (current) {186    // Find the position of the event for this token.187    while (events.get(++startPosition)[1] !== current) {188      // Empty.189    }190 191    assert(192      !previous || current.previous === previous,193      'expected previous to match'194    )195    assert(!previous || previous.next === current, 'expected next to match')196 197    startPositions.push(startPosition)198 199    if (!current._tokenizer) {200      stream = context.sliceStream(current)201 202      if (!current.next) {203        stream.push(codes.eof)204      }205 206      if (previous) {207        tokenizer.defineSkip(current.start)208      }209 210      if (current._isInFirstContentOfListItem) {211        tokenizer._gfmTasklistFirstContentOfListItem = true212      }213 214      tokenizer.write(stream)215 216      if (current._isInFirstContentOfListItem) {217        tokenizer._gfmTasklistFirstContentOfListItem = undefined218      }219    }220 221    // Unravel the next token.222    previous = current223    current = current.next224  }225 226  // Now, loop back through all events (and linked tokens), to figure out which227  // parts belong where.228  current = token229 230  while (++index < childEvents.length) {231    if (232      // Find a void token that includes a break.233      childEvents[index][0] === 'exit' &&234      childEvents[index - 1][0] === 'enter' &&235      childEvents[index][1].type === childEvents[index - 1][1].type &&236      childEvents[index][1].start.line !== childEvents[index][1].end.line237    ) {238      assert(current, 'expected a current token')239      start = index + 1240      breaks.push(start)241      // Help GC.242      current._tokenizer = undefined243      current.previous = undefined244      current = current.next245    }246  }247 248  // Help GC.249  tokenizer.events = []250 251  // If there’s one more token (which is the cases for lines that end in an252  // EOF), that’s perfect: the last point we found starts it.253  // If there isn’t then make sure any remaining content is added to it.254  if (current) {255    // Help GC.256    current._tokenizer = undefined257    current.previous = undefined258    assert(!current.next, 'expected no next token')259  } else {260    breaks.pop()261  }262 263  // Now splice the events from the subtokenizer into the current events,264  // moving back to front so that splice indices aren’t affected.265  index = breaks.length266 267  while (index--) {268    const slice = childEvents.slice(breaks[index], breaks[index + 1])269    const start = startPositions.pop()270    assert(start !== undefined, 'expected a start position when splicing')271    jumps.push([start, start + slice.length - 1])272    events.splice(start, 2, slice)273  }274 275  jumps.reverse()276  index = -1277 278  while (++index < jumps.length) {279    gaps[adjust + jumps[index][0]] = adjust + jumps[index][1]280    adjust += jumps[index][1] - jumps[index][0] - 1281  }282 283  return gaps284}285