TrinetraLabs/Placebo_AI

1"use strict";2 3var utils = require("./utils");4var support = require("./support");5var nodejsUtils = require("./nodejsUtils");6var GenericWorker = require("./stream/GenericWorker");7 8/**9 * The following functions come from pako, from pako/lib/utils/strings10 * released under the MIT license, see pako https://github.com/nodeca/pako/11 */12 13// Table with utf8 lengths (calculated by first byte of sequence)14// Note, that 5 & 6-byte values and some 4-byte values can not be represented in JS,15// because max possible codepoint is 0x10ffff16var _utf8len = new Array(256);17for (var i=0; i<256; i++) {18    _utf8len[i] = (i >= 252 ? 6 : i >= 248 ? 5 : i >= 240 ? 4 : i >= 224 ? 3 : i >= 192 ? 2 : 1);19}20_utf8len[254]=_utf8len[254]=1; // Invalid sequence start21 22// convert string to array (typed, when possible)23var string2buf = function (str) {24    var buf, c, c2, m_pos, i, str_len = str.length, buf_len = 0;25 26    // count binary size27    for (m_pos = 0; m_pos < str_len; m_pos++) {28        c = str.charCodeAt(m_pos);29        if ((c & 0xfc00) === 0xd800 && (m_pos+1 < str_len)) {30            c2 = str.charCodeAt(m_pos+1);31            if ((c2 & 0xfc00) === 0xdc00) {32                c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00);33                m_pos++;34            }35        }36        buf_len += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4;37    }38 39    // allocate buffer40    if (support.uint8array) {41        buf = new Uint8Array(buf_len);42    } else {43        buf = new Array(buf_len);44    }45 46    // convert47    for (i=0, m_pos = 0; i < buf_len; m_pos++) {48        c = str.charCodeAt(m_pos);49        if ((c & 0xfc00) === 0xd800 && (m_pos+1 < str_len)) {50            c2 = str.charCodeAt(m_pos+1);51            if ((c2 & 0xfc00) === 0xdc00) {52                c = 0x10000 + ((c - 0xd800) << 10) + (c2 - 0xdc00);53                m_pos++;54            }55        }56        if (c < 0x80) {57            /* one byte */58            buf[i++] = c;59        } else if (c < 0x800) {60            /* two bytes */61            buf[i++] = 0xC0 | (c >>> 6);62            buf[i++] = 0x80 | (c & 0x3f);63        } else if (c < 0x10000) {64            /* three bytes */65            buf[i++] = 0xE0 | (c >>> 12);66            buf[i++] = 0x80 | (c >>> 6 & 0x3f);67            buf[i++] = 0x80 | (c & 0x3f);68        } else {69            /* four bytes */70            buf[i++] = 0xf0 | (c >>> 18);71            buf[i++] = 0x80 | (c >>> 12 & 0x3f);72            buf[i++] = 0x80 | (c >>> 6 & 0x3f);73            buf[i++] = 0x80 | (c & 0x3f);74        }75    }76 77    return buf;78};79 80// Calculate max possible position in utf8 buffer,81// that will not break sequence. If that's not possible82// - (very small limits) return max size as is.83//84// buf[] - utf8 bytes array85// max   - length limit (mandatory);86var utf8border = function(buf, max) {87    var pos;88 89    max = max || buf.length;90    if (max > buf.length) { max = buf.length; }91 92    // go back from last position, until start of sequence found93    pos = max-1;94    while (pos >= 0 && (buf[pos] & 0xC0) === 0x80) { pos--; }95 96    // Fuckup - very small and broken sequence,97    // return max, because we should return something anyway.98    if (pos < 0) { return max; }99 100    // If we came to start of buffer - that means vuffer is too small,101    // return max too.102    if (pos === 0) { return max; }103 104    return (pos + _utf8len[buf[pos]] > max) ? pos : max;105};106 107// convert array to string108var buf2string = function (buf) {109    var i, out, c, c_len;110    var len = buf.length;111 112    // Reserve max possible length (2 words per char)113    // NB: by unknown reasons, Array is significantly faster for114    //     String.fromCharCode.apply than Uint16Array.115    var utf16buf = new Array(len*2);116 117    for (out=0, i=0; i<len;) {118        c = buf[i++];119        // quick process ascii120        if (c < 0x80) { utf16buf[out++] = c; continue; }121 122        c_len = _utf8len[c];123        // skip 5 & 6 byte codes124        if (c_len > 4) { utf16buf[out++] = 0xfffd; i += c_len-1; continue; }125 126        // apply mask on first byte127        c &= c_len === 2 ? 0x1f : c_len === 3 ? 0x0f : 0x07;128        // join the rest129        while (c_len > 1 && i < len) {130            c = (c << 6) | (buf[i++] & 0x3f);131            c_len--;132        }133 134        // terminated by end of string?135        if (c_len > 1) { utf16buf[out++] = 0xfffd; continue; }136 137        if (c < 0x10000) {138            utf16buf[out++] = c;139        } else {140            c -= 0x10000;141            utf16buf[out++] = 0xd800 | ((c >> 10) & 0x3ff);142            utf16buf[out++] = 0xdc00 | (c & 0x3ff);143        }144    }145 146    // shrinkBuf(utf16buf, out)147    if (utf16buf.length !== out) {148        if(utf16buf.subarray) {149            utf16buf = utf16buf.subarray(0, out);150        } else {151            utf16buf.length = out;152        }153    }154 155    // return String.fromCharCode.apply(null, utf16buf);156    return utils.applyFromCharCode(utf16buf);157};158 159 160// That's all for the pako functions.161 162 163/**164 * Transform a javascript string into an array (typed if possible) of bytes,165 * UTF-8 encoded.166 * @param {String} str the string to encode167 * @return {Array|Uint8Array|Buffer} the UTF-8 encoded string.168 */169exports.utf8encode = function utf8encode(str) {170    if (support.nodebuffer) {171        return nodejsUtils.newBufferFrom(str, "utf-8");172    }173 174    return string2buf(str);175};176 177 178/**179 * Transform a bytes array (or a representation) representing an UTF-8 encoded180 * string into a javascript string.181 * @param {Array|Uint8Array|Buffer} buf the data de decode182 * @return {String} the decoded string.183 */184exports.utf8decode = function utf8decode(buf) {185    if (support.nodebuffer) {186        return utils.transformTo("nodebuffer", buf).toString("utf-8");187    }188 189    buf = utils.transformTo(support.uint8array ? "uint8array" : "array", buf);190 191    return buf2string(buf);192};193 194/**195 * A worker to decode utf8 encoded binary chunks into string chunks.196 * @constructor197 */198function Utf8DecodeWorker() {199    GenericWorker.call(this, "utf-8 decode");200    // the last bytes if a chunk didn't end with a complete codepoint.201    this.leftOver = null;202}203utils.inherits(Utf8DecodeWorker, GenericWorker);204 205/**206 * @see GenericWorker.processChunk207 */208Utf8DecodeWorker.prototype.processChunk = function (chunk) {209 210    var data = utils.transformTo(support.uint8array ? "uint8array" : "array", chunk.data);211 212    // 1st step, re-use what's left of the previous chunk213    if (this.leftOver && this.leftOver.length) {214        if(support.uint8array) {215            var previousData = data;216            data = new Uint8Array(previousData.length + this.leftOver.length);217            data.set(this.leftOver, 0);218            data.set(previousData, this.leftOver.length);219        } else {220            data = this.leftOver.concat(data);221        }222        this.leftOver = null;223    }224 225    var nextBoundary = utf8border(data);226    var usableData = data;227    if (nextBoundary !== data.length) {228        if (support.uint8array) {229            usableData = data.subarray(0, nextBoundary);230            this.leftOver = data.subarray(nextBoundary, data.length);231        } else {232            usableData = data.slice(0, nextBoundary);233            this.leftOver = data.slice(nextBoundary, data.length);234        }235    }236 237    this.push({238        data : exports.utf8decode(usableData),239        meta : chunk.meta240    });241};242 243/**244 * @see GenericWorker.flush245 */246Utf8DecodeWorker.prototype.flush = function () {247    if(this.leftOver && this.leftOver.length) {248        this.push({249            data : exports.utf8decode(this.leftOver),250            meta : {}251        });252        this.leftOver = null;253    }254};255exports.Utf8DecodeWorker = Utf8DecodeWorker;256 257/**258 * A worker to endcode string chunks into utf8 encoded binary chunks.259 * @constructor260 */261function Utf8EncodeWorker() {262    GenericWorker.call(this, "utf-8 encode");263}264utils.inherits(Utf8EncodeWorker, GenericWorker);265 266/**267 * @see GenericWorker.processChunk268 */269Utf8EncodeWorker.prototype.processChunk = function (chunk) {270    this.push({271        data : exports.utf8encode(chunk.data),272        meta : chunk.meta273    });274};275exports.Utf8EncodeWorker = Utf8EncodeWorker;276