basant307/AI_Governance_Project

1/* eslint no-use-before-define:0 */2 3// Import4import type Buffer from 'node:buffer'5import * as pathUtil from 'node:path'6import textExtensions from 'https://unpkg.com/textextensions@^6.11.0/edition-deno/index.ts'7import binaryExtensions from 'https://unpkg.com/binaryextensions@^6.11.0/edition-deno/index.ts'8 9export interface EncodingOpts {10	/** Defaults to 24 */11	chunkLength?: number12 13	/** If not provided, will check the start, beginning, and end */14	chunkBegin?: number15}16 17/**18 * Determine if the filename and/or buffer is text.19 * Determined by extension checks first (if filename is available), otherwise if unknown extension or no filename, will perform a slower buffer encoding detection.20 * This order is done, as extension checks are quicker, and also because encoding checks cannot guarantee accuracy for chars between utf8 and utf16.21 * The extension checks are performed using the resources https://github.com/bevry/textextensions and https://github.com/bevry/binaryextensions22 * @param filename The filename for the file/buffer if available23 * @param buffer The buffer for the file if available24 * @returns Will be `null` if neither `filename` nor `buffer` were provided. Otherwise will be a boolean value with the detection result.25 */26export function isText(27	filename?: string | null,28	buffer?: Buffer | null29): boolean | null {30	// Test extensions31	if (filename) {32		// Extract filename33		const parts = pathUtil.basename(filename).split('.').reverse()34 35		// Cycle extensions36		for (const extension of parts) {37			if (textExtensions.indexOf(extension) !== -1) {38				return true39			}40			if (binaryExtensions.indexOf(extension) !== -1) {41				return false42			}43		}44	}45 46	// Fallback to encoding if extension check was not enough47	if (buffer) {48		return getEncoding(buffer) === 'utf8'49	}50 51	// No buffer was provided52	return null53}54 55/**56 * Determine if the filename and/or buffer is binary.57 * Determined by extension checks first (if filename is available), otherwise if unknown extension or no filename, will perform a slower buffer encoding detection.58 * This order is done, as extension checks are quicker, and also because encoding checks cannot guarantee accuracy for chars between utf8 and utf16.59 * The extension checks are performed using the resources https://github.com/bevry/textextensions and https://github.com/bevry/binaryextensions60 * @param filename The filename for the file/buffer if available61 * @param buffer The buffer for the file if available62 * @returns Will be `null` if neither `filename` nor `buffer` were provided. Otherwise will be a boolean value with the detection result.63 */64export function isBinary(filename?: string | null, buffer?: Buffer | null) {65	const text = isText(filename, buffer)66	if (text == null) return null67	return !text68}69 70/**71 * Get the encoding of a buffer.72 * Checks the start, middle, and end of the buffer for characters that are unrecognized within UTF8 encoding.73 * History has shown that inspection at all three locations is necessary.74 * @returns Will be `null` if `buffer` was not provided. Otherwise will be either `'utf8'` or `'binary'`75 */76export function getEncoding(77	buffer: Buffer | null,78	opts?: EncodingOpts79): 'utf8' | 'binary' | null {80	// Check81	if (!buffer) return null82 83	// Prepare84	const textEncoding = 'utf8'85	const binaryEncoding = 'binary'86	const chunkLength = opts?.chunkLength ?? 2487	let chunkBegin = opts?.chunkBegin ?? 088 89	// Discover90	if (opts?.chunkBegin == null) {91		// Start92		let encoding = getEncoding(buffer, { chunkLength, chunkBegin })93		if (encoding === textEncoding) {94			// Middle95			chunkBegin = Math.max(0, Math.floor(buffer.length / 2) - chunkLength)96			encoding = getEncoding(buffer, {97				chunkLength,98				chunkBegin,99			})100			if (encoding === textEncoding) {101				// End102				chunkBegin = Math.max(0, buffer.length - chunkLength)103				encoding = getEncoding(buffer, {104					chunkLength,105					chunkBegin,106				})107			}108		}109 110		// Return111		return encoding112	} else {113		// Extract114		chunkBegin = getChunkBegin(buffer, chunkBegin)115		if (chunkBegin === -1) {116			return binaryEncoding117		}118 119		const chunkEnd = getChunkEnd(120			buffer,121			Math.min(buffer.length, chunkBegin + chunkLength)122		)123 124		if (chunkEnd > buffer.length) {125			return binaryEncoding126		}127 128		const contentChunkUTF8 = buffer.toString(textEncoding, chunkBegin, chunkEnd)129 130		// Detect encoding131		for (let i = 0; i < contentChunkUTF8.length; ++i) {132			const charCode = contentChunkUTF8.charCodeAt(i)133			if (charCode === 65533 || charCode <= 8) {134				// 8 and below are control characters (e.g. backspace, null, eof, etc.)135				// 65533 is the unknown character136				// console.log(charCode, contentChunkUTF8[i])137				return binaryEncoding138			}139		}140 141		// Return142		return textEncoding143	}144}145 146// ====================================147// The functions below are created to handle multibyte utf8 characters.148// To understand how the encoding works, check this article: https://en.wikipedia.org/wiki/UTF-8#Encoding149// @todo add documentation for these150 151function getChunkBegin(buf: Buffer, chunkBegin: number) {152	// If it's the beginning, just return.153	if (chunkBegin === 0) {154		return 0155	}156 157	if (!isLaterByteOfUtf8(buf[chunkBegin])) {158		return chunkBegin159	}160 161	let begin = chunkBegin - 3162 163	if (begin >= 0) {164		if (isFirstByteOf4ByteChar(buf[begin])) {165			return begin166		}167	}168 169	begin = chunkBegin - 2170 171	if (begin >= 0) {172		if (173			isFirstByteOf4ByteChar(buf[begin]) ||174			isFirstByteOf3ByteChar(buf[begin])175		) {176			return begin177		}178	}179 180	begin = chunkBegin - 1181 182	if (begin >= 0) {183		// Is it a 4-byte, 3-byte utf8 character?184		if (185			isFirstByteOf4ByteChar(buf[begin]) ||186			isFirstByteOf3ByteChar(buf[begin]) ||187			isFirstByteOf2ByteChar(buf[begin])188		) {189			return begin190		}191	}192 193	return -1194}195 196function getChunkEnd(buf: Buffer, chunkEnd: number) {197	// If it's the end, just return.198	if (chunkEnd === buf.length) {199		return chunkEnd200	}201 202	let index = chunkEnd - 3203 204	if (index >= 0) {205		if (isFirstByteOf4ByteChar(buf[index])) {206			return chunkEnd + 1207		}208	}209 210	index = chunkEnd - 2211 212	if (index >= 0) {213		if (isFirstByteOf4ByteChar(buf[index])) {214			return chunkEnd + 2215		}216 217		if (isFirstByteOf3ByteChar(buf[index])) {218			return chunkEnd + 1219		}220	}221 222	index = chunkEnd - 1223 224	if (index >= 0) {225		if (isFirstByteOf4ByteChar(buf[index])) {226			return chunkEnd + 3227		}228 229		if (isFirstByteOf3ByteChar(buf[index])) {230			return chunkEnd + 2231		}232 233		if (isFirstByteOf2ByteChar(buf[index])) {234			return chunkEnd + 1235		}236	}237 238	return chunkEnd239}240 241function isFirstByteOf4ByteChar(byte: number) {242	// eslint-disable-next-line no-bitwise243	return byte >> 3 === 30 // 11110xxx?244}245 246function isFirstByteOf3ByteChar(byte: number) {247	// eslint-disable-next-line no-bitwise248	return byte >> 4 === 14 // 1110xxxx?249}250 251function isFirstByteOf2ByteChar(byte: number) {252	// eslint-disable-next-line no-bitwise253	return byte >> 5 === 6 // 110xxxxx?254}255 256function isLaterByteOfUtf8(byte: number) {257	// eslint-disable-next-line no-bitwise258	return byte >> 6 === 2 // 10xxxxxx?259}260