opusdev/vector-similarity-api

1 2export const streamChunk = function* (chunk, chunkSize) {3  let len = chunk.byteLength;4 5  if (!chunkSize || len < chunkSize) {6    yield chunk;7    return;8  }9 10  let pos = 0;11  let end;12 13  while (pos < len) {14    end = pos + chunkSize;15    yield chunk.slice(pos, end);16    pos = end;17  }18}19 20export const readBytes = async function* (iterable, chunkSize) {21  for await (const chunk of readStream(iterable)) {22    yield* streamChunk(chunk, chunkSize);23  }24}25 26const readStream = async function* (stream) {27  if (stream[Symbol.asyncIterator]) {28    yield* stream;29    return;30  }31 32  const reader = stream.getReader();33  try {34    for (;;) {35      const {done, value} = await reader.read();36      if (done) {37        break;38      }39      yield value;40    }41  } finally {42    await reader.cancel();43  }44}45 46export const trackStream = (stream, chunkSize, onProgress, onFinish) => {47  const iterator = readBytes(stream, chunkSize);48 49  let bytes = 0;50  let done;51  let _onFinish = (e) => {52    if (!done) {53      done = true;54      onFinish && onFinish(e);55    }56  }57 58  return new ReadableStream({59    async pull(controller) {60      try {61        const {done, value} = await iterator.next();62 63        if (done) {64         _onFinish();65          controller.close();66          return;67        }68 69        let len = value.byteLength;70        if (onProgress) {71          let loadedBytes = bytes += len;72          onProgress(loadedBytes);73        }74        controller.enqueue(new Uint8Array(value));75      } catch (err) {76        _onFinish(err);77        throw err;78      }79    },80    cancel(reason) {81      _onFinish(reason);82      return iterator.return();83    }84  }, {85    highWaterMark: 286  })87}88