CoolFace
Datasetpublic

basant307/AI_Governance_Project

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face
0likes48downloads
utf8.js72 linesDownload Raw Back to encoding
1"use strict";2var __importDefault = (this && this.__importDefault) || function (mod) {3    return (mod && mod.__esModule) ? mod : { "default": mod };4};5Object.defineProperty(exports, "__esModule", { value: true });6const match_1 = __importDefault(require("../match"));7class Utf8 {8    name() {9        return 'UTF-8';10    }11    match(det) {12        let hasBOM = false, numValid = 0, numInvalid = 0, trailBytes = 0, confidence;13        const input = det.rawInput;14        if (det.rawLen >= 3 &&15            (input[0] & 0xff) == 0xef &&16            (input[1] & 0xff) == 0xbb &&17            (input[2] & 0xff) == 0xbf) {18            hasBOM = true;19        }20        for (let i = 0; i < det.rawLen; i++) {21            const b = input[i];22            if ((b & 0x80) == 0)23                continue;24            if ((b & 0x0e0) == 0x0c0) {25                trailBytes = 1;26            }27            else if ((b & 0x0f0) == 0x0e0) {28                trailBytes = 2;29            }30            else if ((b & 0x0f8) == 0xf0) {31                trailBytes = 3;32            }33            else {34                numInvalid++;35                if (numInvalid > 5)36                    break;37                trailBytes = 0;38            }39            for (;;) {40                i++;41                if (i >= det.rawLen)42                    break;43                if ((input[i] & 0xc0) != 0x080) {44                    numInvalid++;45                    break;46                }47                if (--trailBytes == 0) {48                    numValid++;49                    break;50                }51            }52        }53        confidence = 0;54        if (hasBOM && numInvalid == 0)55            confidence = 100;56        else if (hasBOM && numValid > numInvalid * 10)57            confidence = 80;58        else if (numValid > 3 && numInvalid == 0)59            confidence = 100;60        else if (numValid > 0 && numInvalid == 0)61            confidence = 80;62        else if (numValid == 0 && numInvalid == 0)63            confidence = 10;64        else if (numValid > numInvalid * 10)65            confidence = 25;66        else67            return null;68        return (0, match_1.default)(det, this, confidence);69    }70}71exports.default = Utf8;72//# sourceMappingURL=utf8.js.map
basant307/AI_Governance_Project · CoolFace