23ws-LLMcoder/LLMcoder-GitHub-Python-Mix-Direct
Dataset Card for LLMcoder-GitHub-Python-Mix-Direct Python target autocomplete suggestions in the format of conversations for OpenAI's fine-tuning. Dataset Details Dataset Description Curated by: [More Information Needed] Funded by [optional]: [More Information Needed] Shared by [optional]: [More Information Needed] Language(s) (NLP): [More Information Needed] License: [More Information Needed] Dataset Sources [optional] The data… See the full description on the dataset page: https://huggingface.co/datasets/23ws-LLMcoder/LLMcoder-GitHub-Python-Mix-Direct.
0216
1ent"] = s2 self.app["COM"] = s # compatibility3 self.applist.append(("COM", s))4 5 6def SOF(self, marker):7 #8 # Start of frame marker. Defines the size and mode of the9 # image. JPEG is colour blind, so we use some simple10 # heuristics to map the number of layers to an appropriate11 # mode. Note that this could be made a bit brighter, by12 # looking for JFIF and Adobe APP markers.13 14 n = i16(self.fp.read(2)) - 215 s = ImageFile._safe_read(self.fp, n)16 self._size = i16(s, 3), i16(s, 1)17 18 self.bits = s[0]19 if self.bits != 8:20 msg = f"cannot handle {self.bits}-bit layers"21 raise SyntaxError(msg)22 23 self.layers = s[5]24 if self.layers == 1:25 self._mode = "L"26 elif self.layers == 3:27 self._mode = "RGB"28 elif self.layers == 4:29 self._mode = "CMYK"30 else:31 msg = f"cannot handle {self.layers}-layer images"32 raise SyntaxError(msg)33 34 if marker in [0xFFC2, 0xFFC6, 0xFFCA, 0xFFCE]:35 self.info["progressive"] = self.info["progression"] = 136 37 if self.icclist:38 # fixup icc profile39 self.icclist.sort() # sort by sequence number40 if self.icclist[0][13] == len(self.icclist):41 profile = []42 for p in self.icclist:43 profile.append(p[14:])44 icc_profile = b"".join(profile)45 else:46 icc_profile = None # wrong number of fragments47 self.info["icc_profile"] = icc_profile48 self.icclist = []49 50 for i in range(6, len(s), 3):51 t = s[i : i + 3]52 # 4-tuples: id, vsamp, hsamp, qtable53 self.layer.append((t[0], t[1] // 16, t[1] & 15, t[2]))54 55 56def DQT(self, marker):57 #58 # Define quantization table. Note that there might be more59 # than one table in each marker.60 61 # FIXME: The quantization tables can be used to estimate the62 # compression quality.63 64 n = i16(self.fp.read(2)) - 265 s = ImageFile._safe_read(self.fp, n)66 while len(s):67 v = s[0]68 precision = 1 if (v // 16 == 0) else 2 # in bytes69 qt_length = 1 + precision * 6470 if len(s) < qt_length:71 msg = "bad quantization table marker"72 raise SyntaxError(msg)73 data = array.array("B" if precision == 1 else "H", s[1:qt_length])74 if sys.byteorder == "little" and precision > 1:75 data.byteswap() # the values are always big-endian76 self.quantization[v & 15] = [data[i] for i in zigzag_index]77 s = s[qt_length:]78 79 80#81# JPEG marker table82 83MARKER = {84 0xFFC0: ("SOF0", "Baseline DCT", SOF),85 0xFFC1: ("SOF1", "Extended Sequential DCT", SOF),86 0xFFC2: ("SOF2", "Progressive DCT", SOF),87 0xFFC3: ("SOF3", "Spatial lossless", SOF),88 0xFFC4: ("DHT", "Define Huffman table", Skip),89 0xFFC5: ("SOF5", "Differential sequential DCT", SOF),90 0xFFC6: ("SOF6", "Differential progressive DCT", SOF),91 0xFFC7: ("SOF7", "Differential spatial", SOF),92 0xFFC8: ("JPG", "Extension", None),93 0xFFC9: ("SOF9", "Extended sequential DCT (AC)", SOF),94 0xFFCA: ("SOF10", "Progressive DCT (AC)", SOF),95 0xFFCB: ("SOF11", "Spatial lossless DCT (AC)", SOF),96 0xFFCC: ("DAC", "Define arithmetic coding conditioning", Skip),97 0xFFCD: ("SOF13", "Differential sequential DCT (AC)", SOF),98 0xFFCE: ("SOF14", "Differential progressive DCT (AC)", SOF),99 0xFFCF: ("SOF15", "Differential spatial (AC)", SOF),100 0xFFD0: ("RST0", "Restart 0", None),101 0xFFD1: ("RST1", "Restart 1", None),102 0xFFD2: ("RST2", "Restart 2", None),103 0xFFD3: ("RST3", "Restart 3", None),104 0xFFD4: ("RST4", "Restart 4", None),105 0xFFD5: ("RST5", "Restart 5", None),106 0xFFD6: ("RST6", "Restart 6", None),107 0xFFD7: ("RST7", "Restart 7", None),108 0xFFD8: ("SOI", "Start of image", None),109 0xFFD9: ("EOI", "End of image", None),110 0xFFDA: ("SOS", "Start of scan", Skip),111 0xFFDB: ("DQT", "Define quantization table", DQT),112 0xFFDC: ("DNL", "Define number of lines", Skip),113 0xFFDD: ("DRI", "Define restart interval", Skip),114 0xFFDE: ("DHP", "Define hierarchical progression", SOF),115 0xFFDF: ("EXP", "Expand reference component", Skip),116 0xFFE0: ("APP0", "Application segment 0", APP),117 0xFFE1: ("APP1", "Application segment 1", APP),118 0xFFE2: ("APP2", "Application segment 2", APP),119 0xFFE3: ("APP3", "Application segment 3", APP),120 0xFFE4: ("APP4", "Application segment 4", APP),121 0xFFE5: ("APP5", "Application segment 5", APP),122 0xFFE6: ("APP6", "Application segment 6", APP),123 0xFFE7: ("APP7", "Application segment 7", APP),124 0xFFE8: ("APP8", "Application segment 8", APP),125 0xFFE9: ("APP9", "Application segment 9", APP),126 0xFFEA: ("APP10", "Application segment 10", APP),127 0xFFEB: ("APP11", "Application segment 11", APP),128 0xFFEC: ("APP12", "Application segment 12", APP),129 0xFFED: ("APP13", "Application segment 13", APP),130 0xFFEE: ("APP14", "Application segment 14", APP),131 0xFFEF: ("APP15", "Application segment 15", APP),132 0xFFF0: ("JPG0", "Extension 0", None),133 0xFFF1: ("JPG1", "Extension 1", None),134 0xFFF2: ("JPG2", "Extension 2", None),135 0xFFF3: ("JPG3", "Extension 3", None),136 0xFFF4: ("JPG4", "Extension 4", None),137 0xFFF5: ("JPG5", "Extension 5", None),138 0xFFF6: ("JPG6", "Extension 6", None),139 0xFFF7: ("JPG7", "Extension 7", None),140 0xFFF8: ("JPG8", "Extension 8", None),141 0xFFF9: ("JPG9", "Extension 9", None),142 0xFFFA: ("JPG10", "Extension 10", None),143 0xFFFB: ("JPG11", "Extension 11", None),144 0xFFFC: ("JPG12", "Extension 12", None),145 0xFFFD: ("JPG13", "Extension 13", None),146 0xFFFE: ("COM", "Comment", COM),147}148 149 150def _accept(prefix):151 # Magic number was taken from https://en.wikipedia.org/wiki/JPEG152 return prefix[:3] == b"\xFF\xD8\xFF"153 154 155##156# Image plugin for JPEG and JFIF images.157 158 159class JpegImageFile(ImageFile.ImageFile):160 format = "JPEG"161 format_description = "JPEG (ISO 10918)"162 163 def _open(self):164 s = self.fp.read(3)165 166 if not _accept(s):167 msg = "not a JPEG file"168 raise SyntaxError(msg)169 s = b"\xFF"170 171 # Create attributes172 self.bits = self.layers = 0173 174 # JPEG specifics (internal)175 self.layer = []176 self.huffman_dc = {}177 self.huffman_ac = {}178 self.quantization = {}179 self.app = {} # compatibility180 self.applist = []181 self.icclist = []182 183 while True:184 i = s[0]185 if i == 0xFF:186 s = s + self.fp.read(1)187 i = i16(s)188 else:189 # Skip non-0xFF junk190 s = self.fp.read(1)191 continue192 193 if i in MARKER:194 name, description, handler = MARKER[i]195 if handler is not None:196 handler(self, i)197 if i == 0xFFDA: # start of scan198 rawmode = self.mode199 if self.mode == "CMYK":200 rawmode = "CMYK;I" # assume adobe conventions201 self.tile = [("jpeg", (0, 0) + self.size, 0, (rawmode, ""))]202 # self.__offset = self.fp.tell()203 break204 s = self.fp.read(1)205 elif i == 0 or i == 0xFFFF:206 # padded marker or junk; move on207 s = b"\xff"208 elif i == 0xFF00: # Skip extraneous data (escaped 0xFF)209 s = self.fp.read(1)210 else:211 msg = "no marker found"212 raise SyntaxError(msg)213 214 def load_read(self, read_bytes):215 """216 internal: read more image data217 For premature EOF and LOAD_TRUNCATED_IMAGES adds EOI marker218 so libjpeg can finish decoding219 """220 s = self.fp.read(read_bytes)221 222 if not s and ImageFile.LOAD_TRUNCATED_IMAGES and not hasattr(self, "_ended"):223 # Premature EOF.224 # Pretend file is finished adding EOI marker225 self._ended = True226 return b"\xFF\xD9"227 228 return s229 230 def draft(self, mode, size):231 if len(self.tile) != 1:232 return233 234 # Protect from second call235 if self.decoderconfig:236 return237 238 d, e, o, a = self.tile[0]239 scale = 1240 original_size = self.size241 242 if a[0] == "RGB" and mode in ["L", "YCbCr"]:243 self._mode = mode244 a = mode, ""245 246 if size:247 scale = min(self.size[0] // size[0], self.size[1] // size[1])248 for s in [8, 4, 2, 1]:249 if scale >= s:250 break251 e = (252 e[0],253 e[1],254 (e[2] - e[0] + s - 1) // s + e[0],255 (e[3] - e[1] + s - 1) // s + e[1],256 )257 self._size = ((self.size[0] + s - 1) // s, (self.size[1] + s - 1) // s)258 scale = s259 260 self.tile = [(d, e, o, a)]261 self.decoderconfig = (scale, 0)262 263 b