opusdev/vector-similarity-api
1
1import { Binary, UUID } from '../binary';2import type { Document } from '../bson';3import { Code } from '../code';4import * as constants from '../constants';5import { DBRef, type DBRefLike, isDBRefLike } from '../db_ref';6import { Decimal128 } from '../decimal128';7import { Double } from '../double';8import { BSONError } from '../error';9import { Int32 } from '../int_32';10import { Long } from '../long';11import { MaxKey } from '../max_key';12import { MinKey } from '../min_key';13import { ObjectId } from '../objectid';14import { BSONRegExp } from '../regexp';15import { BSONSymbol } from '../symbol';16import { Timestamp } from '../timestamp';17import { ByteUtils } from '../utils/byte_utils';18import { NumberUtils } from '../utils/number_utils';19 20/** @public */21export interface DeserializeOptions {22 /**23 * when deserializing a Long return as a BigInt.24 * @defaultValue `false`25 */26 useBigInt64?: boolean;27 /**28 * when deserializing a Long will fit it into a Number if it's smaller than 53 bits.29 * @defaultValue `true`30 */31 promoteLongs?: boolean;32 /**33 * when deserializing a Binary will return it as a node.js Buffer instance.34 * @defaultValue `false`35 */36 promoteBuffers?: boolean;37 /**38 * when deserializing will promote BSON values to their Node.js closest equivalent types.39 * @defaultValue `true`40 */41 promoteValues?: boolean;42 /**43 * allow to specify if there what fields we wish to return as unserialized raw buffer.44 * @defaultValue `null`45 */46 fieldsAsRaw?: Document;47 /**48 * return BSON regular expressions as BSONRegExp instances.49 * @defaultValue `false`50 */51 bsonRegExp?: boolean;52 /**53 * allows the buffer to be larger than the parsed BSON object.54 * @defaultValue `false`55 */56 allowObjectSmallerThanBufferSize?: boolean;57 /**58 * Offset into buffer to begin reading document from59 * @defaultValue `0`60 */61 index?: number;62 63 raw?: boolean;64 /** Allows for opt-out utf-8 validation for all keys or65 * specified keys. Must be all true or all false.66 *67 * @example68 * ```js69 * // disables validation on all keys70 * validation: { utf8: false }71 *72 * // enables validation only on specified keys a, b, and c73 * validation: { utf8: { a: true, b: true, c: true } }74 *75 * // disables validation only on specified keys a, b76 * validation: { utf8: { a: false, b: false } }77 * ```78 */79 validation?: { utf8: boolean | Record<string, true> | Record<string, false> };80}81 82// Internal long versions83const JS_INT_MAX_LONG = Long.fromNumber(constants.JS_INT_MAX);84const JS_INT_MIN_LONG = Long.fromNumber(constants.JS_INT_MIN);85 86export function internalDeserialize(87 buffer: Uint8Array,88 options: DeserializeOptions,89 isArray?: boolean90): Document {91 options = options == null ? {} : options;92 const index = options && options.index ? options.index : 0;93 // Read the document size94 const size = NumberUtils.getInt32LE(buffer, index);95 96 if (size < 5) {97 throw new BSONError(`bson size must be >= 5, is ${size}`);98 }99 100 if (options.allowObjectSmallerThanBufferSize && buffer.length < size) {101 throw new BSONError(`buffer length ${buffer.length} must be >= bson size ${size}`);102 }103 104 if (!options.allowObjectSmallerThanBufferSize && buffer.length !== size) {105 throw new BSONError(`buffer length ${buffer.length} must === bson size ${size}`);106 }107 108 if (size + index > buffer.byteLength) {109 throw new BSONError(110 `(bson size ${size} + options.index ${index} must be <= buffer length ${buffer.byteLength})`111 );112 }113 114 // Illegal end value115 if (buffer[index + size - 1] !== 0) {116 throw new BSONError(117 "One object, sized correctly, with a spot for an EOO, but the EOO isn't 0x00"118 );119 }120 121 // Start deserialization122 return deserializeObject(buffer, index, options, isArray);123}124 125const allowedDBRefKeys = /^\$ref$|^\$id$|^\$db$/;126 127function deserializeObject(128 buffer: Uint8Array,129 index: number,130 options: DeserializeOptions,131 isArray = false132) {133 const fieldsAsRaw = options['fieldsAsRaw'] == null ? null : options['fieldsAsRaw'];134 135 // Return raw bson buffer instead of parsing it136 const raw = options['raw'] == null ? false : options['raw'];137 138 // Return BSONRegExp objects instead of native regular expressions139 const bsonRegExp = typeof options['bsonRegExp'] === 'boolean' ? options['bsonRegExp'] : false;140 141 // Controls the promotion of values vs wrapper classes142 const promoteBuffers = options.promoteBuffers ?? false;143 const promoteLongs = options.promoteLongs ?? true;144 const promoteValues = options.promoteValues ?? true;145 const useBigInt64 = options.useBigInt64 ?? false;146 147 if (useBigInt64 && !promoteValues) {148 throw new BSONError('Must either request bigint or Long for int64 deserialization');149 }150 151 if (useBigInt64 && !promoteLongs) {152 throw new BSONError('Must either request bigint or Long for int64 deserialization');153 }154 155 // Ensures default validation option if none given156 const validation = options.validation == null ? { utf8: true } : options.validation;157 158 // Shows if global utf-8 validation is enabled or disabled159 let globalUTFValidation = true;160 // Reflects utf-8 validation setting regardless of global or specific key validation161 let validationSetting: boolean;162 // Set of keys either to enable or disable validation on163 let utf8KeysSet;164 165 // Check for boolean uniformity and empty validation option166 const utf8ValidatedKeys = validation.utf8;167 if (typeof utf8ValidatedKeys === 'boolean') {168 validationSetting = utf8ValidatedKeys;169 } else {170 globalUTFValidation = false;171 const utf8ValidationValues = Object.keys(utf8ValidatedKeys).map(function (key) {172 return utf8ValidatedKeys[key];173 });174 if (utf8ValidationValues.length === 0) {175 throw new BSONError('UTF-8 validation setting cannot be empty');176 }177 if (typeof utf8ValidationValues[0] !== 'boolean') {178 throw new BSONError('Invalid UTF-8 validation option, must specify boolean values');179 }180 validationSetting = utf8ValidationValues[0];181 // Ensures boolean uniformity in utf-8 validation (all true or all false)182 if (!utf8ValidationValues.every(item => item === validationSetting)) {183 throw new BSONError('Invalid UTF-8 validation option - keys must be all true or all false');184 }185 }186 187 // Add keys to set that will either be validated or not based on validationSetting188 if (!globalUTFValidation) {189 utf8KeysSet = new Set();190 191 for (const key of Object.keys(utf8ValidatedKeys)) {192 utf8KeysSet.add(key);193 }194 }195 196 // Set the start index197 const startIndex = index;198 199 // Validate that we have at least 4 bytes of buffer200 if (buffer.length < 5) throw new BSONError('corrupt bson message < 5 bytes long');201 202 // Read the document size203 const size = NumberUtils.getInt32LE(buffer, index);204 index += 4;205 206 // Ensure buffer is valid size207 if (size < 5 || size > buffer.length) throw new BSONError('corrupt bson message');208 209 // Create holding object210 const object: Document = isArray ? [] : {};211 // Used for arrays to skip having to perform utf8 decoding212 let arrayIndex = 0;213 const done = false;214 215 let isPossibleDBRef = isArray ? false : null;216 217 // While we have more left data left keep parsing218 while (!done) {219 // Read the type220 const elementType = buffer[index++];221 222 // If we get a zero it's the last byte, exit223 if (elementType === 0) break;224 225 // Get the start search index226 let i = index;227 // Locate the end of the c string228 while (buffer[i] !== 0x00 && i < buffer.length) {229 i++;230 }231 232 // If are at the end of the buffer there is a problem with the document233 if (i >= buffer.byteLength) throw new BSONError('Bad BSON Document: illegal CString');234 235 // Represents the key236 const name = isArray ? arrayIndex++ : ByteUtils.toUTF8(buffer, index, i, false);237 238 // shouldValidateKey is true if the key should be validated, false otherwise239 let shouldValidateKey = true;240 if (globalUTFValidation || utf8KeysSet?.has(name)) {241 shouldValidateKey = validationSetting;242 } else {243 shouldValidateKey = !validationSetting;244 }245 246 if (isPossibleDBRef !== false && (name as string)[0] === '$') {247 isPossibleDBRef = allowedDBRefKeys.test(name as string);248 }249 let value;250 251 index = i + 1;252 253 if (elementType === constants.BSON_DATA_STRING) {254 const stringSize = NumberUtils.getInt32LE(buffer, index);255 index += 4;256 if (257 stringSize <= 0 ||258 stringSize > buffer.length - index ||259 buffer[index + stringSize - 1] !== 0260 ) {261 throw new BSONError('bad string length in bson');262 }263 value = ByteUtils.toUTF8(buffer, index, index + stringSize - 1, shouldValidateKey);264 index = index + stringSize;265 } else if (elementType === constants.BSON_DATA_OID) {266 const oid = ByteUtils.allocateUnsafe(12);267 for (let i = 0; i < 12; i++) oid[i] = buffer[index + i];268 value = new ObjectId(oid);269 index = index + 12;270 } else if (elementType === constants.BSON_DATA_INT && promoteValues === false) {271 value = new Int32(NumberUtils.getInt32LE(buffer, index));272 index += 4;273 } else if (elementType === constants.BSON_DATA_INT) {274 value = NumberUtils.getInt32LE(buffer, index);275 index += 4;276 } else if (elementType === constants.BSON_DATA_NUMBER) {277 value = NumberUtils.getFloat64LE(buffer, index);278 index += 8;279 if (promoteValues === false) value = new Double(value);280 } else if (elementType === constants.BSON_DATA_DATE) {281 const lowBits = NumberUtils.getInt32LE(buffer, index);282 const highBits = NumberUtils.getInt32LE(buffer, index + 4);283 index += 8;284 285 value = new Date(new Long(lowBits, highBits).toNumber());286 } else if (elementType === constants.BSON_DATA_BOOLEAN) {287 if (buffer[index] !== 0 && buffer[index] !== 1)288 throw new BSONError('illegal boolean type value');289 value = buffer[index++] === 1;290 } else if (elementType === constants.BSON_DATA_OBJECT) {291 const _index = index;292 const objectSize = NumberUtils.getInt32LE(buffer, index);293 294 if (objectSize <= 0 || objectSize > buffer.length - index)295 throw new BSONError('bad embedded document length in bson');296 297 // We have a raw value298 if (raw) {299 value = buffer.subarray(index, index + objectSize);300 } else {301 let objectOptions = options;302 if (!globalUTFValidation) {303 objectOptions = { ...options, validation: { utf8: shouldValidateKey } };304 }305 value = deserializeObject(buffer, _index, objectOptions, false);306 }307 308 index = index + objectSize;309 } else if (elementType === constants.BSON_DATA_ARRAY) {310 const _index = index;311 const objectSize = NumberUtils.getInt32LE(buffer, index);312 let arrayOptions: DeserializeOptions = options;313 314 // Stop index315 const stopIndex = index + objectSize;316 317 // All elements of array to be returned as raw bson318 if (fieldsAsRaw && fieldsAsRaw[name]) {319 arrayOptions = { ...options, raw: true };320 }321 322 if (!globalUTFValidation) {323 arrayOptions = { ...arrayOptions, validation: { utf8: shouldValidateKey } };324 }325 value = deserializeObject(buffer, _index, arrayOptions, true);326 index = index + objectSize;327 328 if (buffer[index - 1] !== 0) throw new BSONError('invalid array terminator byte');329 if (index !== stopIndex) throw new BSONError('corrupted array bson');330 } else if (elementType === constants.BSON_DATA_UNDEFINED) {331 value = undefined;332 } else if (elementType === constants.BSON_DATA_NULL) {333 value = null;334 } else if (elementType === constants.BSON_DATA_LONG) {335 if (useBigInt64) {336 value = NumberUtils.getBigInt64LE(buffer, index);337 index += 8;338 } else {339 // Unpack the low and high bits340 const lowBits = NumberUtils.getInt32LE(buffer, index);341 const highBits = NumberUtils.getInt32LE(buffer, index + 4);342 index += 8;343 344 const long = new Long(lowBits, highBits);345 // Promote the long if possible346 if (promoteLongs && promoteValues === true) {347 value =348 long.lessThanOrEqual(JS_INT_MAX_LONG) && long.greaterThanOrEqual(JS_INT_MIN_LONG)349 ? long.toNumber()350 : long;351 } else {352 value = long;353 }354 }355 } else if (elementType === constants.BSON_DATA_DECIMAL128) {356 // Buffer to contain the decimal bytes357 const bytes = ByteUtils.allocateUnsafe(16);358 // Copy the next 16 bytes into the bytes buffer359 for (let i = 0; i < 16; i++) bytes[i] = buffer[index + i];360 // Update index361 index = index + 16;362 // Assign the new Decimal128 value363 value = new Decimal128(bytes);364 } else if (elementType === constants.BSON_DATA_BINARY) {365 let binarySize = NumberUtils.getInt32LE(buffer, index);366 index += 4;367 const totalBinarySize = binarySize;368 const subType = buffer[index++];369 370 // Did we have a negative binary size, throw371 if (binarySize < 0) throw new BSONError('Negative binary type element size found');372 373 // Is the length longer than the document374 if (binarySize > buffer.byteLength)375 throw new BSONError('Binary type size larger than document size');376 377 // If we have subtype 2 skip the 4 bytes for the size378 if (subType === Binary.SUBTYPE_BYTE_ARRAY) {379 binarySize = NumberUtils.getInt32LE(buffer, index);380 index += 4;381 if (binarySize < 0)382 throw new BSONError('Negative binary type element size found for subtype 0x02');383 if (binarySize > totalBinarySize - 4)384 throw new BSONError('Binary type with subtype 0x02 contains too long binary size');385 if (binarySize < totalBinarySize - 4)386 throw new BSONError('Binary type with subtype 0x02 contains too short binary size');387 }388 389 if (promoteBuffers && promoteValues) {390 value = ByteUtils.toLocalBufferType(buffer.subarray(index, index + binarySize));391 } else {392 value = new Binary(buffer.subarray(index, index + binarySize), subType);393 if (subType === constants.BSON_BINARY_SUBTYPE_UUID_NEW && UUID.isValid(value)) {394 value = value.toUUID();395 }396 }397 398 // Update the index399 index = index + binarySize;400 } else if (elementType === constants.BSON_DATA_REGEXP && bsonRegExp === false) {401 // Get the start search index402 i = index;403 // Locate the end of the c string404 while (buffer[i] !== 0x00 && i < buffer.length) {405 i++;406 }407 // If are at the end of the buffer there is a problem with the document408 if (i >= buffer.length) throw new BSONError('Bad BSON Document: illegal CString');409 // Return the C string410 const source = ByteUtils.toUTF8(buffer, index, i, false);411 // Create the regexp412 index = i + 1;413 414 // Get the start search index415 i = index;416 // Locate the end of the c string417 while (buffer[i] !== 0x00 && i < buffer.length) {418 i++;419 }420 // If are at the end of the buffer there is a problem with the document421 if (i >= buffer.length) throw new BSONError('Bad BSON Document: illegal CString');422 // Return the C string423 const regExpOptions = ByteUtils.toUTF8(buffer, index, i, false);424 index = i + 1;425 426 // For each option add the corresponding one for javascript427 const optionsArray = new Array(regExpOptions.length);428 429 // Parse options430 for (i = 0; i < regExpOptions.length; i++) {431 switch (regExpOptions[i]) {432 case 'm':433 optionsArray[i] = 'm';434 break;435 case 's':436 optionsArray[i] = 'g';437 break;438 case 'i':439 optionsArray[i] = 'i';440 break;441 }442 }443 444 value = new RegExp(source, optionsArray.join(''));445 } else if (elementType === constants.BSON_DATA_REGEXP && bsonRegExp === true) {446 // Get the start search index447 i = index;448 // Locate the end of the c string449 while (buffer[i] !== 0x00 && i < buffer.length) {450 i++;451 }452 // If are at the end of the buffer there is a problem with the document453 if (i >= buffer.length) throw new BSONError('Bad BSON Document: illegal CString');454 // Return the C string455 const source = ByteUtils.toUTF8(buffer, index, i, false);456 index = i + 1;457 458 // Get the start search index459 i = index;460 // Locate the end of the c string461 while (buffer[i] !== 0x00 && i < buffer.length) {462 i++;463 }464 // If are at the end of the buffer there is a problem with the document465 if (i >= buffer.length) throw new BSONError('Bad BSON Document: illegal CString');466 // Return the C string467 const regExpOptions = ByteUtils.toUTF8(buffer, index, i, false);468 index = i + 1;469 470 // Set the object471 value = new BSONRegExp(source, regExpOptions);472 } else if (elementType === constants.BSON_DATA_SYMBOL) {473 const stringSize = NumberUtils.getInt32LE(buffer, index);474 index += 4;475 if (476 stringSize <= 0 ||477 stringSize > buffer.length - index ||478 buffer[index + stringSize - 1] !== 0479 ) {480 throw new BSONError('bad string length in bson');481 }482 const symbol = ByteUtils.toUTF8(buffer, index, index + stringSize - 1, shouldValidateKey);483 value = promoteValues ? symbol : new BSONSymbol(symbol);484 index = index + stringSize;485 } else if (elementType === constants.BSON_DATA_TIMESTAMP) {486 value = new Timestamp({487 i: NumberUtils.getUint32LE(buffer, index),488 t: NumberUtils.getUint32LE(buffer, index + 4)489 });490 index += 8;491 } else if (elementType === constants.BSON_DATA_MIN_KEY) {492 value = new MinKey();493 } else if (elementType === constants.BSON_DATA_MAX_KEY) {494 value = new MaxKey();495 } else if (elementType === constants.BSON_DATA_CODE) {496 const stringSize = NumberUtils.getInt32LE(buffer, index);497 index += 4;498 if (499 stringSize <= 0 ||500 stringSize > buffer.length - index ||501 buffer[index + stringSize - 1] !== 0502 ) {503 throw new BSONError('bad string length in bson');504 }505 const functionString = ByteUtils.toUTF8(506 buffer,507 index,508 index + stringSize - 1,509 shouldValidateKey510 );511 512 value = new Code(functionString);513 514 // Update parse index position515 index = index + stringSize;516 } else if (elementType === constants.BSON_DATA_CODE_W_SCOPE) {517 const totalSize = NumberUtils.getInt32LE(buffer, index);518 index += 4;519 520 // Element cannot be shorter than totalSize + stringSize + documentSize + terminator521 if (totalSize < 4 + 4 + 4 + 1) {522 throw new BSONError('code_w_scope total size shorter minimum expected length');523 }524 525 // Get the code string size526 const stringSize = NumberUtils.getInt32LE(buffer, index);527 index += 4;528 // Check if we have a valid string529 if (530 stringSize <= 0 ||531 stringSize > buffer.length - index ||532 buffer[index + stringSize - 1] !== 0533 ) {534 throw new BSONError('bad string length in bson');535 }536 537 // Javascript function538 const functionString = ByteUtils.toUTF8(539 buffer,540 index,541 index + stringSize - 1,542 shouldValidateKey543 );544 // Update parse index position545 index = index + stringSize;546 // Parse the element547 const _index = index;548 // Decode the size of the object document549 const objectSize = NumberUtils.getInt32LE(buffer, index);550 // Decode the scope object551 const scopeObject = deserializeObject(buffer, _index, options, false);552 // Adjust the index553 index = index + objectSize;554 555 // Check if field length is too short556 if (totalSize < 4 + 4 + objectSize + stringSize) {557 throw new BSONError('code_w_scope total size is too short, truncating scope');558 }559 560 // Check if totalSize field is too long561 if (totalSize > 4 + 4 + objectSize + stringSize) {562 throw new BSONError('code_w_scope total size is too long, clips outer document');563 }564 565 value = new Code(functionString, scopeObject);566 } else if (elementType === constants.BSON_DATA_DBPOINTER) {567 // Get the code string size568 const stringSize = NumberUtils.getInt32LE(buffer, index);569 index += 4;570 // Check if we have a valid string571 if (572 stringSize <= 0 ||573 stringSize > buffer.length - index ||574 buffer[index + stringSize - 1] !== 0575 )576 throw new BSONError('bad string length in bson');577 // Namespace578 const namespace = ByteUtils.toUTF8(buffer, index, index + stringSize - 1, shouldValidateKey);579 // Update parse index position580 index = index + stringSize;581 582 // Read the oid583 const oidBuffer = ByteUtils.allocateUnsafe(12);584 for (let i = 0; i < 12; i++) oidBuffer[i] = buffer[index + i];585 const oid = new ObjectId(oidBuffer);586 587 // Update the index588 index = index + 12;589 590 // Upgrade to DBRef type591 value = new DBRef(namespace, oid);592 } else {593 throw new BSONError(594 `Detected unknown BSON type ${elementType.toString(16)} for fieldname "${name}"`595 );596 }597 if (name === '__proto__') {598 Object.defineProperty(object, name, {599 value,600 writable: true,601 enumerable: true,602 configurable: true603 });604 } else {605 object[name] = value;606 }607 }608 609 // Check if the deserialization was against a valid array/object610 if (size !== index - startIndex) {611 if (isArray) throw new BSONError('corrupt array bson');612 throw new BSONError('corrupt object bson');613 }614 615 // if we did not find "$ref", "$id", "$db", or found an extraneous $key, don't make a DBRef616 if (!isPossibleDBRef) return object;617 618 if (isDBRefLike(object)) {619 const copy = Object.assign({}, object) as Partial<DBRefLike>;620 delete copy.$ref;621 delete copy.$id;622 delete copy.$db;623 return new DBRef(object.$ref, object.$id, object.$db, copy);624 }625 626 return object;627}628