opusdev/vector-similarity-api

1module.exports = Pager2 3function Pager (pageSize, opts) {4  if (!(this instanceof Pager)) return new Pager(pageSize, opts)5 6  this.length = 07  this.updates = []8  this.path = new Uint16Array(4)9  this.pages = new Array(32768)10  this.maxPages = this.pages.length11  this.level = 012  this.pageSize = pageSize || 102413  this.deduplicate = opts ? opts.deduplicate : null14  this.zeros = this.deduplicate ? alloc(this.deduplicate.length) : null15}16 17Pager.prototype.updated = function (page) {18  while (this.deduplicate && page.buffer[page.deduplicate] === this.deduplicate[page.deduplicate]) {19    page.deduplicate++20    if (page.deduplicate === this.deduplicate.length) {21      page.deduplicate = 022      if (page.buffer.equals && page.buffer.equals(this.deduplicate)) page.buffer = this.deduplicate23      break24    }25  }26  if (page.updated || !this.updates) return27  page.updated = true28  this.updates.push(page)29}30 31Pager.prototype.lastUpdate = function () {32  if (!this.updates || !this.updates.length) return null33  var page = this.updates.pop()34  page.updated = false35  return page36}37 38Pager.prototype._array = function (i, noAllocate) {39  if (i >= this.maxPages) {40    if (noAllocate) return41    grow(this, i)42  }43 44  factor(i, this.path)45 46  var arr = this.pages47 48  for (var j = this.level; j > 0; j--) {49    var p = this.path[j]50    var next = arr[p]51 52    if (!next) {53      if (noAllocate) return54      next = arr[p] = new Array(32768)55    }56 57    arr = next58  }59 60  return arr61}62 63Pager.prototype.get = function (i, noAllocate) {64  var arr = this._array(i, noAllocate)65  var first = this.path[0]66  var page = arr && arr[first]67 68  if (!page && !noAllocate) {69    page = arr[first] = new Page(i, alloc(this.pageSize))70    if (i >= this.length) this.length = i + 171  }72 73  if (page && page.buffer === this.deduplicate && this.deduplicate && !noAllocate) {74    page.buffer = copy(page.buffer)75    page.deduplicate = 076  }77 78  return page79}80 81Pager.prototype.set = function (i, buf) {82  var arr = this._array(i, false)83  var first = this.path[0]84 85  if (i >= this.length) this.length = i + 186 87  if (!buf || (this.zeros && buf.equals && buf.equals(this.zeros))) {88    arr[first] = undefined89    return90  }91 92  if (this.deduplicate && buf.equals && buf.equals(this.deduplicate)) {93    buf = this.deduplicate94  }95 96  var page = arr[first]97  var b = truncate(buf, this.pageSize)98 99  if (page) page.buffer = b100  else arr[first] = new Page(i, b)101}102 103Pager.prototype.toBuffer = function () {104  var list = new Array(this.length)105  var empty = alloc(this.pageSize)106  var ptr = 0107 108  while (ptr < list.length) {109    var arr = this._array(ptr, true)110    for (var i = 0; i < 32768 && ptr < list.length; i++) {111      list[ptr++] = (arr && arr[i]) ? arr[i].buffer : empty112    }113  }114 115  return Buffer.concat(list)116}117 118function grow (pager, index) {119  while (pager.maxPages < index) {120    var old = pager.pages121    pager.pages = new Array(32768)122    pager.pages[0] = old123    pager.level++124    pager.maxPages *= 32768125  }126}127 128function truncate (buf, len) {129  if (buf.length === len) return buf130  if (buf.length > len) return buf.slice(0, len)131  var cpy = alloc(len)132  buf.copy(cpy)133  return cpy134}135 136function alloc (size) {137  if (Buffer.alloc) return Buffer.alloc(size)138  var buf = new Buffer(size)139  buf.fill(0)140  return buf141}142 143function copy (buf) {144  var cpy = Buffer.allocUnsafe ? Buffer.allocUnsafe(buf.length) : new Buffer(buf.length)145  buf.copy(cpy)146  return cpy147}148 149function Page (i, buf) {150  this.offset = i * buf.length151  this.buffer = buf152  this.updated = false153  this.deduplicate = 0154}155 156function factor (n, out) {157  n = (n - (out[0] = (n & 32767))) / 32768158  n = (n - (out[1] = (n & 32767))) / 32768159  out[3] = ((n - (out[2] = (n & 32767))) / 32768) & 32767160}161