basant307/AI_Governance_Project

1"use strict";2/* eslint no-use-before-define:0 */3var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {4    if (k2 === undefined) k2 = k;5    var desc = Object.getOwnPropertyDescriptor(m, k);6    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {7      desc = { enumerable: true, get: function() { return m[k]; } };8    }9    Object.defineProperty(o, k2, desc);10}) : (function(o, m, k, k2) {11    if (k2 === undefined) k2 = k;12    o[k2] = m[k];13}));14var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {15    Object.defineProperty(o, "default", { enumerable: true, value: v });16}) : function(o, v) {17    o["default"] = v;18});19var __importStar = (this && this.__importStar) || function (mod) {20    if (mod && mod.__esModule) return mod;21    var result = {};22    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);23    __setModuleDefault(result, mod);24    return result;25};26var __importDefault = (this && this.__importDefault) || function (mod) {27    return (mod && mod.__esModule) ? mod : { "default": mod };28};29Object.defineProperty(exports, "__esModule", { value: true });30exports.getEncoding = exports.isBinary = exports.isText = void 0;31const pathUtil = __importStar(require("path"));32const textextensions_1 = __importDefault(require("textextensions"));33const binaryextensions_1 = __importDefault(require("binaryextensions"));34/**35 * Determine if the filename and/or buffer is text.36 * Determined by extension checks first (if filename is available), otherwise if unknown extension or no filename, will perform a slower buffer encoding detection.37 * This order is done, as extension checks are quicker, and also because encoding checks cannot guarantee accuracy for chars between utf8 and utf16.38 * The extension checks are performed using the resources https://github.com/bevry/textextensions and https://github.com/bevry/binaryextensions39 * @param filename The filename for the file/buffer if available40 * @param buffer The buffer for the file if available41 * @returns Will be `null` if neither `filename` nor `buffer` were provided. Otherwise will be a boolean value with the detection result.42 */43function isText(filename, buffer) {44    // Test extensions45    if (filename) {46        // Extract filename47        const parts = pathUtil.basename(filename).split('.').reverse();48        // Cycle extensions49        for (const extension of parts) {50            if (textextensions_1.default.indexOf(extension) !== -1) {51                return true;52            }53            if (binaryextensions_1.default.indexOf(extension) !== -1) {54                return false;55            }56        }57    }58    // Fallback to encoding if extension check was not enough59    if (buffer) {60        return getEncoding(buffer) === 'utf8';61    }62    // No buffer was provided63    return null;64}65exports.isText = isText;66/**67 * Determine if the filename and/or buffer is binary.68 * Determined by extension checks first (if filename is available), otherwise if unknown extension or no filename, will perform a slower buffer encoding detection.69 * This order is done, as extension checks are quicker, and also because encoding checks cannot guarantee accuracy for chars between utf8 and utf16.70 * The extension checks are performed using the resources https://github.com/bevry/textextensions and https://github.com/bevry/binaryextensions71 * @param filename The filename for the file/buffer if available72 * @param buffer The buffer for the file if available73 * @returns Will be `null` if neither `filename` nor `buffer` were provided. Otherwise will be a boolean value with the detection result.74 */75function isBinary(filename, buffer) {76    const text = isText(filename, buffer);77    if (text == null)78        return null;79    return !text;80}81exports.isBinary = isBinary;82/**83 * Get the encoding of a buffer.84 * Checks the start, middle, and end of the buffer for characters that are unrecognized within UTF8 encoding.85 * History has shown that inspection at all three locations is necessary.86 * @returns Will be `null` if `buffer` was not provided. Otherwise will be either `'utf8'` or `'binary'`87 */88function getEncoding(buffer, opts) {89    // Check90    if (!buffer)91        return null;92    // Prepare93    const textEncoding = 'utf8';94    const binaryEncoding = 'binary';95    const chunkLength = opts?.chunkLength ?? 24;96    let chunkBegin = opts?.chunkBegin ?? 0;97    // Discover98    if (opts?.chunkBegin == null) {99        // Start100        let encoding = getEncoding(buffer, { chunkLength, chunkBegin });101        if (encoding === textEncoding) {102            // Middle103            chunkBegin = Math.max(0, Math.floor(buffer.length / 2) - chunkLength);104            encoding = getEncoding(buffer, {105                chunkLength,106                chunkBegin,107            });108            if (encoding === textEncoding) {109                // End110                chunkBegin = Math.max(0, buffer.length - chunkLength);111                encoding = getEncoding(buffer, {112                    chunkLength,113                    chunkBegin,114                });115            }116        }117        // Return118        return encoding;119    }120    else {121        // Extract122        chunkBegin = getChunkBegin(buffer, chunkBegin);123        if (chunkBegin === -1) {124            return binaryEncoding;125        }126        const chunkEnd = getChunkEnd(buffer, Math.min(buffer.length, chunkBegin + chunkLength));127        if (chunkEnd > buffer.length) {128            return binaryEncoding;129        }130        const contentChunkUTF8 = buffer.toString(textEncoding, chunkBegin, chunkEnd);131        // Detect encoding132        for (let i = 0; i < contentChunkUTF8.length; ++i) {133            const charCode = contentChunkUTF8.charCodeAt(i);134            if (charCode === 65533 || charCode <= 8) {135                // 8 and below are control characters (e.g. backspace, null, eof, etc.)136                // 65533 is the unknown character137                // console.log(charCode, contentChunkUTF8[i])138                return binaryEncoding;139            }140        }141        // Return142        return textEncoding;143    }144}145exports.getEncoding = getEncoding;146// ====================================147// The functions below are created to handle multibyte utf8 characters.148// To understand how the encoding works, check this article: https://en.wikipedia.org/wiki/UTF-8#Encoding149// @todo add documentation for these150function getChunkBegin(buf, chunkBegin) {151    // If it's the beginning, just return.152    if (chunkBegin === 0) {153        return 0;154    }155    if (!isLaterByteOfUtf8(buf[chunkBegin])) {156        return chunkBegin;157    }158    let begin = chunkBegin - 3;159    if (begin >= 0) {160        if (isFirstByteOf4ByteChar(buf[begin])) {161            return begin;162        }163    }164    begin = chunkBegin - 2;165    if (begin >= 0) {166        if (isFirstByteOf4ByteChar(buf[begin]) ||167            isFirstByteOf3ByteChar(buf[begin])) {168            return begin;169        }170    }171    begin = chunkBegin - 1;172    if (begin >= 0) {173        // Is it a 4-byte, 3-byte utf8 character?174        if (isFirstByteOf4ByteChar(buf[begin]) ||175            isFirstByteOf3ByteChar(buf[begin]) ||176            isFirstByteOf2ByteChar(buf[begin])) {177            return begin;178        }179    }180    return -1;181}182function getChunkEnd(buf, chunkEnd) {183    // If it's the end, just return.184    if (chunkEnd === buf.length) {185        return chunkEnd;186    }187    let index = chunkEnd - 3;188    if (index >= 0) {189        if (isFirstByteOf4ByteChar(buf[index])) {190            return chunkEnd + 1;191        }192    }193    index = chunkEnd - 2;194    if (index >= 0) {195        if (isFirstByteOf4ByteChar(buf[index])) {196            return chunkEnd + 2;197        }198        if (isFirstByteOf3ByteChar(buf[index])) {199            return chunkEnd + 1;200        }201    }202    index = chunkEnd - 1;203    if (index >= 0) {204        if (isFirstByteOf4ByteChar(buf[index])) {205            return chunkEnd + 3;206        }207        if (isFirstByteOf3ByteChar(buf[index])) {208            return chunkEnd + 2;209        }210        if (isFirstByteOf2ByteChar(buf[index])) {211            return chunkEnd + 1;212        }213    }214    return chunkEnd;215}216function isFirstByteOf4ByteChar(byte) {217    // eslint-disable-next-line no-bitwise218    return byte >> 3 === 30; // 11110xxx?219}220function isFirstByteOf3ByteChar(byte) {221    // eslint-disable-next-line no-bitwise222    return byte >> 4 === 14; // 1110xxxx?223}224function isFirstByteOf2ByteChar(byte) {225    // eslint-disable-next-line no-bitwise226    return byte >> 5 === 6; // 110xxxxx?227}228function isLaterByteOfUtf8(byte) {229    // eslint-disable-next-line no-bitwise230    return byte >> 6 === 2; // 10xxxxxx?231}232