CoolFace
Datasetpublic

basant307/AI_Governance_Project

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face
0likes48downloads
unicode.js109 linesDownload Raw Back to encoding
1"use strict";2var __importDefault = (this && this.__importDefault) || function (mod) {3    return (mod && mod.__esModule) ? mod : { "default": mod };4};5Object.defineProperty(exports, "__esModule", { value: true });6exports.UTF_32LE = exports.UTF_32BE = exports.UTF_16LE = exports.UTF_16BE = void 0;7const match_1 = __importDefault(require("../match"));8class UTF_16BE {9    name() {10        return 'UTF-16BE';11    }12    match(det) {13        const input = det.rawInput;14        if (input.length >= 2 &&15            (input[0] & 0xff) == 0xfe &&16            (input[1] & 0xff) == 0xff) {17            return (0, match_1.default)(det, this, 100);18        }19        return null;20    }21}22exports.UTF_16BE = UTF_16BE;23class UTF_16LE {24    name() {25        return 'UTF-16LE';26    }27    match(det) {28        const input = det.rawInput;29        if (input.length >= 2 &&30            (input[0] & 0xff) == 0xff &&31            (input[1] & 0xff) == 0xfe) {32            if (input.length >= 4 && input[2] == 0x00 && input[3] == 0x00) {33                return null;34            }35            return (0, match_1.default)(det, this, 100);36        }37        return null;38    }39}40exports.UTF_16LE = UTF_16LE;41class UTF_32 {42    name() {43        return 'UTF-32';44    }45    getChar(_input, _index) {46        return -1;47    }48    match(det) {49        let numValid = 0, numInvalid = 0, hasBOM = false, confidence = 0;50        const limit = (det.rawLen / 4) * 4;51        const input = det.rawInput;52        if (limit == 0) {53            return null;54        }55        if (this.getChar(input, 0) == 0x0000feff) {56            hasBOM = true;57        }58        for (let i = 0; i < limit; i += 4) {59            const ch = this.getChar(input, i);60            if (ch < 0 || ch >= 0x10ffff || (ch >= 0xd800 && ch <= 0xdfff)) {61                numInvalid += 1;62            }63            else {64                numValid += 1;65            }66        }67        if (hasBOM && numInvalid == 0) {68            confidence = 100;69        }70        else if (hasBOM && numValid > numInvalid * 10) {71            confidence = 80;72        }73        else if (numValid > 3 && numInvalid == 0) {74            confidence = 100;75        }76        else if (numValid > 0 && numInvalid == 0) {77            confidence = 80;78        }79        else if (numValid > numInvalid * 10) {80            confidence = 25;81        }82        return confidence == 0 ? null : (0, match_1.default)(det, this, confidence);83    }84}85class UTF_32BE extends UTF_32 {86    name() {87        return 'UTF-32BE';88    }89    getChar(input, index) {90        return (((input[index + 0] & 0xff) << 24) |91            ((input[index + 1] & 0xff) << 16) |92            ((input[index + 2] & 0xff) << 8) |93            (input[index + 3] & 0xff));94    }95}96exports.UTF_32BE = UTF_32BE;97class UTF_32LE extends UTF_32 {98    name() {99        return 'UTF-32LE';100    }101    getChar(input, index) {102        return (((input[index + 3] & 0xff) << 24) |103            ((input[index + 2] & 0xff) << 16) |104            ((input[index + 1] & 0xff) << 8) |105            (input[index + 0] & 0xff));106    }107}108exports.UTF_32LE = UTF_32LE;109//# sourceMappingURL=unicode.js.map
basant307/AI_Governance_Project · CoolFace