opusdev/vector-similarity-api

1import {2  type MongoCrypt,3  type MongoCryptConstructor,4  type MongoCryptOptions5} from 'mongodb-client-encryption';6import * as net from 'net';7 8import { deserialize, type Document, serialize } from '../bson';9import { type CommandOptions, type ProxyOptions } from '../cmap/connection';10import { kDecorateResult } from '../constants';11import { getMongoDBClientEncryption } from '../deps';12import { MongoRuntimeError } from '../error';13import { MongoClient, type MongoClientOptions } from '../mongo_client';14import { type Abortable } from '../mongo_types';15import { MongoDBCollectionNamespace } from '../utils';16import { autoSelectSocketOptions } from './client_encryption';17import * as cryptoCallbacks from './crypto_callbacks';18import { MongoCryptInvalidArgumentError } from './errors';19import { MongocryptdManager } from './mongocryptd_manager';20import {21  type CredentialProviders,22  isEmptyCredentials,23  type KMSProviders,24  refreshKMSCredentials25} from './providers';26import { type CSFLEKMSTlsOptions, StateMachine } from './state_machine';27 28/** @public */29export interface AutoEncryptionOptions {30  /** @internal client for metadata lookups */31  metadataClient?: MongoClient;32  /** A `MongoClient` used to fetch keys from a key vault */33  keyVaultClient?: MongoClient;34  /** The namespace where keys are stored in the key vault */35  keyVaultNamespace?: string;36  /** Configuration options that are used by specific KMS providers during key generation, encryption, and decryption. */37  kmsProviders?: KMSProviders;38  /** Configuration options for custom credential providers. */39  credentialProviders?: CredentialProviders;40  /**41   * A map of namespaces to a local JSON schema for encryption42   *43   * **NOTE**: Supplying options.schemaMap provides more security than relying on JSON Schemas obtained from the server.44   * It protects against a malicious server advertising a false JSON Schema, which could trick the client into sending decrypted data that should be encrypted.45   * Schemas supplied in the schemaMap only apply to configuring automatic encryption for Client-Side Field Level Encryption.46   * Other validation rules in the JSON schema will not be enforced by the driver and will result in an error.47   */48  schemaMap?: Document;49  /** Supply a schema for the encrypted fields in the document  */50  encryptedFieldsMap?: Document;51  /** Allows the user to bypass auto encryption, maintaining implicit decryption */52  bypassAutoEncryption?: boolean;53  /** Allows users to bypass query analysis */54  bypassQueryAnalysis?: boolean;55  /**56   * Sets the expiration time for the DEK in the cache in milliseconds. Defaults to 60000.  0 means no timeout.57   */58  keyExpirationMS?: number;59  options?: {60    /** An optional hook to catch logging messages from the underlying encryption engine */61    logger?: (level: AutoEncryptionLoggerLevel, message: string) => void;62  };63  extraOptions?: {64    /**65     * A local process the driver communicates with to determine how to encrypt values in a command.66     * Defaults to "mongodb://%2Fvar%2Fmongocryptd.sock" if domain sockets are available or "mongodb://localhost:27020" otherwise67     */68    mongocryptdURI?: string;69    /** If true, autoEncryption will not attempt to spawn a mongocryptd before connecting  */70    mongocryptdBypassSpawn?: boolean;71    /** The path to the mongocryptd executable on the system */72    mongocryptdSpawnPath?: string;73    /** Command line arguments to use when auto-spawning a mongocryptd */74    mongocryptdSpawnArgs?: string[];75    /**76     * Full path to a MongoDB Crypt shared library to be used (instead of mongocryptd).77     *78     * This needs to be the path to the file itself, not a directory.79     * It can be an absolute or relative path. If the path is relative and80     * its first component is `$ORIGIN`, it will be replaced by the directory81     * containing the mongodb-client-encryption native addon file. Otherwise,82     * the path will be interpreted relative to the current working directory.83     *84     * Currently, loading different MongoDB Crypt shared library files from different85     * MongoClients in the same process is not supported.86     *87     * If this option is provided and no MongoDB Crypt shared library could be loaded88     * from the specified location, creating the MongoClient will fail.89     *90     * If this option is not provided and `cryptSharedLibRequired` is not specified,91     * the AutoEncrypter will attempt to spawn and/or use mongocryptd according92     * to the mongocryptd-specific `extraOptions` options.93     *94     * Specifying a path prevents mongocryptd from being used as a fallback.95     *96     * Requires the MongoDB Crypt shared library, available in MongoDB 6.0 or higher.97     */98    cryptSharedLibPath?: string;99    /**100     * If specified, never use mongocryptd and instead fail when the MongoDB Crypt101     * shared library could not be loaded.102     *103     * This is always true when `cryptSharedLibPath` is specified.104     *105     * Requires the MongoDB Crypt shared library, available in MongoDB 6.0 or higher.106     */107    cryptSharedLibRequired?: boolean;108    /**109     * Search paths for a MongoDB Crypt shared library to be used (instead of mongocryptd)110     * Only for driver testing!111     * @internal112     */113    cryptSharedLibSearchPaths?: string[];114  };115  proxyOptions?: ProxyOptions;116  /** The TLS options to use connecting to the KMS provider */117  tlsOptions?: CSFLEKMSTlsOptions;118}119 120/**121 * @public122 *123 * Extra options related to the mongocryptd process124 * \* _Available in MongoDB 6.0 or higher._125 */126export type AutoEncryptionExtraOptions = NonNullable<AutoEncryptionOptions['extraOptions']>;127 128/** @public */129export const AutoEncryptionLoggerLevel = Object.freeze({130  FatalError: 0,131  Error: 1,132  Warning: 2,133  Info: 3,134  Trace: 4135} as const);136 137/**138 * @public139 * The level of severity of the log message140 *141 * | Value | Level |142 * |-------|-------|143 * | 0 | Fatal Error |144 * | 1 | Error |145 * | 2 | Warning |146 * | 3 | Info |147 * | 4 | Trace |148 */149export type AutoEncryptionLoggerLevel =150  (typeof AutoEncryptionLoggerLevel)[keyof typeof AutoEncryptionLoggerLevel];151 152/**153 * @internal An internal class to be used by the driver for auto encryption154 * **NOTE**: Not meant to be instantiated directly, this is for internal use only.155 */156export class AutoEncrypter {157  _client: MongoClient;158  _bypassEncryption: boolean;159  _keyVaultNamespace: string;160  _keyVaultClient: MongoClient;161  _metaDataClient: MongoClient;162  _proxyOptions: ProxyOptions;163  _tlsOptions: CSFLEKMSTlsOptions;164  _kmsProviders: KMSProviders;165  _bypassMongocryptdAndCryptShared: boolean;166  _contextCounter: number;167  _credentialProviders?: CredentialProviders;168 169  _mongocryptdManager?: MongocryptdManager;170  _mongocryptdClient?: MongoClient;171 172  /** @internal */173  _mongocrypt: MongoCrypt;174 175  /**176   * Used by devtools to enable decorating decryption results.177   *178   * When set and enabled, `decrypt` will automatically recursively179   * traverse a decrypted document and if a field has been decrypted,180   * it will mark it as decrypted.  Compass uses this to determine which181   * fields were decrypted.182   */183  [kDecorateResult] = false;184 185  /** @internal */186  static getMongoCrypt(): MongoCryptConstructor {187    const encryption = getMongoDBClientEncryption();188    if ('kModuleError' in encryption) {189      throw encryption.kModuleError;190    }191    return encryption.MongoCrypt;192  }193 194  /**195   * Create an AutoEncrypter196   *197   * **Note**: Do not instantiate this class directly. Rather, supply the relevant options to a MongoClient198   *199   * **Note**: Supplying `options.schemaMap` provides more security than relying on JSON Schemas obtained from the server.200   * It protects against a malicious server advertising a false JSON Schema, which could trick the client into sending unencrypted data that should be encrypted.201   * Schemas supplied in the schemaMap only apply to configuring automatic encryption for Client-Side Field Level Encryption.202   * Other validation rules in the JSON schema will not be enforced by the driver and will result in an error.203   *204   * @example <caption>Create an AutoEncrypter that makes use of mongocryptd</caption>205   * ```ts206   * // Enabling autoEncryption via a MongoClient using mongocryptd207   * const { MongoClient } = require('mongodb');208   * const client = new MongoClient(URL, {209   *   autoEncryption: {210   *     kmsProviders: {211   *       aws: {212   *         accessKeyId: AWS_ACCESS_KEY,213   *         secretAccessKey: AWS_SECRET_KEY214   *       }215   *     }216   *   }217   * });218   * ```219   *220   * await client.connect();221   * // From here on, the client will be encrypting / decrypting automatically222   * @example <caption>Create an AutoEncrypter that makes use of libmongocrypt's CSFLE shared library</caption>223   * ```ts224   * // Enabling autoEncryption via a MongoClient using CSFLE shared library225   * const { MongoClient } = require('mongodb');226   * const client = new MongoClient(URL, {227   *   autoEncryption: {228   *     kmsProviders: {229   *       aws: {}230   *     },231   *     extraOptions: {232   *       cryptSharedLibPath: '/path/to/local/crypt/shared/lib',233   *       cryptSharedLibRequired: true234   *     }235   *   }236   * });237   * ```238   *239   * await client.connect();240   * // From here on, the client will be encrypting / decrypting automatically241   */242  constructor(client: MongoClient, options: AutoEncryptionOptions) {243    this._client = client;244    this._bypassEncryption = options.bypassAutoEncryption === true;245 246    this._keyVaultNamespace = options.keyVaultNamespace || 'admin.datakeys';247    this._keyVaultClient = options.keyVaultClient || client;248    this._metaDataClient = options.metadataClient || client;249    this._proxyOptions = options.proxyOptions || {};250    this._tlsOptions = options.tlsOptions || {};251    this._kmsProviders = options.kmsProviders || {};252    this._credentialProviders = options.credentialProviders;253 254    if (options.credentialProviders?.aws && !isEmptyCredentials('aws', this._kmsProviders)) {255      throw new MongoCryptInvalidArgumentError(256        'Can only provide a custom AWS credential provider when the state machine is configured for automatic AWS credential fetching'257      );258    }259 260    const mongoCryptOptions: MongoCryptOptions = {261      enableMultipleCollinfo: true,262      cryptoCallbacks263    };264    if (options.schemaMap) {265      mongoCryptOptions.schemaMap = Buffer.isBuffer(options.schemaMap)266        ? options.schemaMap267        : (serialize(options.schemaMap) as Buffer);268    }269 270    if (options.encryptedFieldsMap) {271      mongoCryptOptions.encryptedFieldsMap = Buffer.isBuffer(options.encryptedFieldsMap)272        ? options.encryptedFieldsMap273        : (serialize(options.encryptedFieldsMap) as Buffer);274    }275 276    mongoCryptOptions.kmsProviders = !Buffer.isBuffer(this._kmsProviders)277      ? (serialize(this._kmsProviders) as Buffer)278      : this._kmsProviders;279 280    if (options.options?.logger) {281      mongoCryptOptions.logger = options.options.logger;282    }283 284    if (options.extraOptions && options.extraOptions.cryptSharedLibPath) {285      mongoCryptOptions.cryptSharedLibPath = options.extraOptions.cryptSharedLibPath;286    }287 288    if (options.bypassQueryAnalysis) {289      mongoCryptOptions.bypassQueryAnalysis = options.bypassQueryAnalysis;290    }291 292    if (options.keyExpirationMS != null) {293      mongoCryptOptions.keyExpirationMS = options.keyExpirationMS;294    }295 296    this._bypassMongocryptdAndCryptShared = this._bypassEncryption || !!options.bypassQueryAnalysis;297 298    if (options.extraOptions && options.extraOptions.cryptSharedLibSearchPaths) {299      // Only for driver testing300      mongoCryptOptions.cryptSharedLibSearchPaths = options.extraOptions.cryptSharedLibSearchPaths;301    } else if (!this._bypassMongocryptdAndCryptShared) {302      mongoCryptOptions.cryptSharedLibSearchPaths = ['$SYSTEM'];303    }304 305    const MongoCrypt = AutoEncrypter.getMongoCrypt();306    this._mongocrypt = new MongoCrypt(mongoCryptOptions);307    this._contextCounter = 0;308 309    if (310      options.extraOptions &&311      options.extraOptions.cryptSharedLibRequired &&312      !this.cryptSharedLibVersionInfo313    ) {314      throw new MongoCryptInvalidArgumentError(315        '`cryptSharedLibRequired` set but no crypt_shared library loaded'316      );317    }318 319    // Only instantiate mongocryptd manager/client once we know for sure320    // that we are not using the CSFLE shared library.321    if (!this._bypassMongocryptdAndCryptShared && !this.cryptSharedLibVersionInfo) {322      this._mongocryptdManager = new MongocryptdManager(options.extraOptions);323      const clientOptions: MongoClientOptions = {324        serverSelectionTimeoutMS: 10000325      };326 327      if (328        (options.extraOptions == null || typeof options.extraOptions.mongocryptdURI !== 'string') &&329        !net.getDefaultAutoSelectFamily330      ) {331        // Only set family if autoSelectFamily options are not supported.332        clientOptions.family = 4;333      }334 335      // eslint-disable-next-line @typescript-eslint/ban-ts-comment336      // @ts-ignore: TS complains as this always returns true on versions where it is present.337      if (net.getDefaultAutoSelectFamily) {338        // AutoEncrypter is made inside of MongoClient constructor while options are being parsed,339        // we do not have access to the options that are in progress.340        // TODO(NODE-6449): AutoEncrypter does not use client options for autoSelectFamily341        Object.assign(clientOptions, autoSelectSocketOptions(this._client.s?.options ?? {}));342      }343 344      this._mongocryptdClient = new MongoClient(this._mongocryptdManager.uri, clientOptions);345    }346  }347 348  /**349   * Initializes the auto encrypter by spawning a mongocryptd and connecting to it.350   *351   * This function is a no-op when bypassSpawn is set or the crypt shared library is used.352   */353  async init(): Promise<MongoClient | void> {354    if (this._bypassMongocryptdAndCryptShared || this.cryptSharedLibVersionInfo) {355      return;356    }357    if (!this._mongocryptdManager) {358      throw new MongoRuntimeError(359        'Reached impossible state: mongocryptdManager is undefined when neither bypassSpawn nor the shared lib are specified.'360      );361    }362    if (!this._mongocryptdClient) {363      throw new MongoRuntimeError(364        'Reached impossible state: mongocryptdClient is undefined when neither bypassSpawn nor the shared lib are specified.'365      );366    }367 368    if (!this._mongocryptdManager.bypassSpawn) {369      await this._mongocryptdManager.spawn();370    }371 372    try {373      const client = await this._mongocryptdClient.connect();374      return client;375    } catch (error) {376      throw new MongoRuntimeError(377        'Unable to connect to `mongocryptd`, please make sure it is running or in your PATH for auto-spawn',378        { cause: error }379      );380    }381  }382 383  /**384   * Cleans up the `_mongocryptdClient`, if present.385   */386  async close(): Promise<void> {387    await this._mongocryptdClient?.close();388  }389 390  /**391   * Encrypt a command for a given namespace.392   */393  async encrypt(394    ns: string,395    cmd: Document,396    options: CommandOptions & Abortable = {}397  ): Promise<Document | Uint8Array> {398    options.signal?.throwIfAborted();399 400    if (this._bypassEncryption) {401      // If `bypassAutoEncryption` has been specified, don't encrypt402      return cmd;403    }404 405    const commandBuffer = Buffer.isBuffer(cmd) ? cmd : serialize(cmd, options);406    const context = this._mongocrypt.makeEncryptionContext(407      MongoDBCollectionNamespace.fromString(ns).db,408      commandBuffer409    );410 411    context.id = this._contextCounter++;412    context.ns = ns;413    context.document = cmd;414 415    const stateMachine = new StateMachine({416      promoteValues: false,417      promoteLongs: false,418      proxyOptions: this._proxyOptions,419      tlsOptions: this._tlsOptions,420      socketOptions: autoSelectSocketOptions(this._client.s.options)421    });422 423    return deserialize(await stateMachine.execute(this, context, options), {424      promoteValues: false,425      promoteLongs: false426    });427  }428 429  /**430   * Decrypt a command response431   */432  async decrypt(433    response: Uint8Array,434    options: CommandOptions & Abortable = {}435  ): Promise<Uint8Array> {436    options.signal?.throwIfAborted();437 438    const context = this._mongocrypt.makeDecryptionContext(response);439 440    context.id = this._contextCounter++;441 442    const stateMachine = new StateMachine({443      ...options,444      proxyOptions: this._proxyOptions,445      tlsOptions: this._tlsOptions,446      socketOptions: autoSelectSocketOptions(this._client.s.options)447    });448 449    return await stateMachine.execute(this, context, options);450  }451 452  /**453   * Ask the user for KMS credentials.454   *455   * This returns anything that looks like the kmsProviders original input456   * option. It can be empty, and any provider specified here will override457   * the original ones.458   */459  async askForKMSCredentials(): Promise<KMSProviders> {460    return await refreshKMSCredentials(this._kmsProviders, this._credentialProviders);461  }462 463  /**464   * Return the current libmongocrypt's CSFLE shared library version465   * as `{ version: bigint, versionStr: string }`, or `null` if no CSFLE466   * shared library was loaded.467   */468  get cryptSharedLibVersionInfo(): { version: bigint; versionStr: string } | null {469    return this._mongocrypt.cryptSharedLibVersionInfo;470  }471 472  static get libmongocryptVersion(): string {473    return AutoEncrypter.getMongoCrypt().libmongocryptVersion;474  }475}476