AnonymousSub/minigpt4.cpp
0
1import os2import sys3import ctypes4import pathlib5from typing import Optional, List6import enum7from pathlib import Path8 9class DataType(enum.IntEnum):10 def __str__(self):11 return str(self.name)12 13 F16 = 014 F32 = 115 I32 = 216 L64 = 317 Q4_0 = 418 Q4_1 = 519 Q5_0 = 620 Q5_1 = 721 Q8_0 = 822 Q8_1 = 923 Q2_K = 1024 Q3_K = 1125 Q4_K = 1226 Q5_K = 1327 Q6_K = 1428 Q8_K = 1529 30class Verbosity(enum.IntEnum):31 SILENT = 032 ERR = 133 INFO = 234 DEBUG = 335 36class ImageFormat(enum.IntEnum):37 UNKNOWN = 038 F32 = 139 U8 = 240 41I32 = ctypes.c_int3242U32 = ctypes.c_uint3243F32 = ctypes.c_float44SIZE_T = ctypes.c_size_t45VOID_PTR = ctypes.c_void_p46CHAR_PTR = ctypes.POINTER(ctypes.c_char)47FLOAT_PTR = ctypes.POINTER(ctypes.c_float)48INT_PTR = ctypes.POINTER(ctypes.c_int32)49CHAR_PTR_PTR = ctypes.POINTER(ctypes.POINTER(ctypes.c_char))50 51MiniGPT4ContextP = VOID_PTR52class MiniGPT4Context:53 def __init__(self, ptr: ctypes.pointer):54 self.ptr = ptr55 56class MiniGPT4Image(ctypes.Structure):57 _fields_ = [58 ('data', VOID_PTR),59 ('width', I32),60 ('height', I32),61 ('channels', I32),62 ('format', I32)63 ]64 65class MiniGPT4Embedding(ctypes.Structure):66 _fields_ = [67 ('data', FLOAT_PTR),68 ('n_embeddings', SIZE_T),69 ]70 71MiniGPT4ImageP = ctypes.POINTER(MiniGPT4Image)72MiniGPT4EmbeddingP = ctypes.POINTER(MiniGPT4Embedding)73 74class MiniGPT4SharedLibrary:75 """76 Python wrapper around minigpt4.cpp shared library.77 """78 79 def __init__(self, shared_library_path: str):80 """81 Loads the shared library from specified file.82 In case of any error, this method will throw an exception.83 84 Parameters85 ----------86 shared_library_path : str87 Path to minigpt4.cpp shared library. On Windows, it would look like 'minigpt4.dll'. On UNIX, 'minigpt4.so'.88 """89 90 self.library = ctypes.cdll.LoadLibrary(shared_library_path)91 92 self.library.minigpt4_model_load.argtypes = [93 CHAR_PTR, # const char *path94 CHAR_PTR, # const char *llm_model95 I32, # int verbosity96 I32, # int seed97 I32, # int n_ctx98 I32, # int n_batch99 I32, # int numa100 ]101 self.library.minigpt4_model_load.restype = MiniGPT4ContextP102 103 self.library.minigpt4_image_load_from_file.argtypes = [104 MiniGPT4ContextP, # struct MiniGPT4Context *ctx105 CHAR_PTR, # const char *path106 MiniGPT4ImageP, # struct MiniGPT4Image *image107 I32, # int flags108 ]109 self.library.minigpt4_image_load_from_file.restype = I32110 111 self.library.minigpt4_encode_image.argtypes = [112 MiniGPT4ContextP, # struct MiniGPT4Context *ctx113 MiniGPT4ImageP, # const struct MiniGPT4Image *image114 MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding115 I32, # size_t n_threads116 ]117 self.library.minigpt4_encode_image.restype = I32118 119 self.library.minigpt4_begin_chat_image.argtypes = [120 MiniGPT4ContextP, # struct MiniGPT4Context *ctx121 MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding122 CHAR_PTR, # const char *s123 I32, # size_t n_threads124 ]125 self.library.minigpt4_begin_chat_image.restype = I32126 127 self.library.minigpt4_end_chat_image.argtypes = [128 MiniGPT4ContextP, # struct MiniGPT4Context *ctx129 CHAR_PTR_PTR, # const char **token130 I32, # size_t n_threads131 F32, # float temp132 I32, # int32_t top_k133 F32, # float top_p134 F32, # float tfs_z135 F32, # float typical_p136 I32, # int32_t repeat_last_n137 F32, # float repeat_penalty138 F32, # float alpha_presence139 F32, # float alpha_frequency140 I32, # int mirostat141 F32, # float mirostat_tau142 F32, # float mirostat_eta143 I32, # int penalize_nl144 ]145 self.library.minigpt4_end_chat_image.restype = I32146 147 self.library.minigpt4_system_prompt.argtypes = [148 MiniGPT4ContextP, # struct MiniGPT4Context *ctx149 I32, # size_t n_threads150 ]151 self.library.minigpt4_system_prompt.restype = I32152 153 self.library.minigpt4_begin_chat.argtypes = [154 MiniGPT4ContextP, # struct MiniGPT4Context *ctx155 CHAR_PTR, # const char *s156 I32, # size_t n_threads157 ]158 self.library.minigpt4_begin_chat.restype = I32159 160 self.library.minigpt4_end_chat.argtypes = [161 MiniGPT4ContextP, # struct MiniGPT4Context *ctx162 CHAR_PTR_PTR, # const char **token163 I32, # size_t n_threads164 F32, # float temp165 I32, # int32_t top_k166 F32, # float top_p167 F32, # float tfs_z168 F32, # float typical_p169 I32, # int32_t repeat_last_n170 F32, # float repeat_penalty171 F32, # float alpha_presence172 F32, # float alpha_frequency173 I32, # int mirostat174 F32, # float mirostat_tau175 F32, # float mirostat_eta176 I32, # int penalize_nl177 ]178 self.library.minigpt4_end_chat.restype = I32179 180 self.library.minigpt4_reset_chat.argtypes = [181 MiniGPT4ContextP, # struct MiniGPT4Context *ctx182 ]183 self.library.minigpt4_reset_chat.restype = I32184 185 self.library.minigpt4_contains_eos_token.argtypes = [186 CHAR_PTR, # const char *s187 ]188 self.library.minigpt4_contains_eos_token.restype = I32189 190 self.library.minigpt4_is_eos.argtypes = [191 CHAR_PTR, # const char *s192 ]193 self.library.minigpt4_is_eos.restype = I32194 195 self.library.minigpt4_free.argtypes = [196 MiniGPT4ContextP, # struct MiniGPT4Context *ctx197 ]198 self.library.minigpt4_free.restype = I32199 200 self.library.minigpt4_free_image.argtypes = [201 MiniGPT4ImageP, # struct MiniGPT4Image *image202 ]203 self.library.minigpt4_free_image.restype = I32204 205 self.library.minigpt4_free_embedding.argtypes = [206 MiniGPT4EmbeddingP, # struct MiniGPT4Embedding *embedding207 ]208 self.library.minigpt4_free_embedding.restype = I32209 210 self.library.minigpt4_error_code_to_string.argtypes = [211 I32, # int error_code212 ]213 self.library.minigpt4_error_code_to_string.restype = CHAR_PTR214 215 self.library.minigpt4_quantize_model.argtypes = [216 CHAR_PTR, # const char *in_path217 CHAR_PTR, # const char *out_path218 I32, # int data_type219 ]220 self.library.minigpt4_quantize_model.restype = I32221 222 self.library.minigpt4_set_verbosity.argtypes = [223 I32, # int verbosity224 ]225 self.library.minigpt4_set_verbosity.restype = None226 227 def panic_if_error(self, error_code: int) -> None:228 """229 Raises an exception if the error code is not 0.230 231 Parameters232 ----------233 error_code : int234 Error code to check.235 """236 237 if error_code != 0:238 raise RuntimeError(self.library.minigpt4_error_code_to_string(I32(error_code)))239 240 def minigpt4_model_load(self, model_path: str, llm_model_path: str, verbosity: int = 1, seed: int = 1337, n_ctx: int = 2048, n_batch: int = 512, numa: int = 0) -> MiniGPT4Context:241 """242 Loads a model from a file.243 244 Args:245 model_path (str): Path to model file.246 llm_model_path (str): Path to LLM model file.247 verbosity (int): Verbosity level: 0 = silent, 1 = error, 2 = info, 3 = debug. Defaults to 0.248 n_ctx (int): Size of context for llm model. Defaults to 2048.249 seed (int): Seed for llm model. Defaults to 1337.250 numa (int): NUMA node to use (0 = NUMA disabled, 1 = NUMA enabled). Defaults to 0.251 252 Returns:253 MiniGPT4Context: Context.254 """255 256 ptr = self.library.minigpt4_model_load(257 model_path.encode('utf-8'),258 llm_model_path.encode('utf-8'),259 I32(verbosity),260 I32(seed),261 I32(n_ctx),262 I32(n_batch),263 I32(numa),264 )265 266 assert ptr is not None, 'minigpt4_model_load failed'267 268 return MiniGPT4Context(ptr)269 270 def minigpt4_image_load_from_file(self, ctx: MiniGPT4Context, path: str, flags: int) -> MiniGPT4Image:271 """272 Loads an image from a file273 274 Args:275 ctx (MiniGPT4Context): context276 path (str): path277 flags (int): flags278 279 Returns:280 MiniGPT4Image: image281 """282 283 image = MiniGPT4Image()284 self.panic_if_error(self.library.minigpt4_image_load_from_file(ctx.ptr, path.encode('utf-8'), ctypes.pointer(image), I32(flags)))285 return image286 287 def minigpt4_preprocess_image(self, ctx: MiniGPT4Context, image: MiniGPT4Image, flags: int = 0) -> MiniGPT4Image:288 """289 Preprocesses an image290 291 Args:292 ctx (MiniGPT4Context): Context293 image (MiniGPT4Image): Image294 flags (int): Flags. Defaults to 0.295 296 Returns:297 MiniGPT4Image: Preprocessed image298 """299 300 preprocessed_image = MiniGPT4Image()301 self.panic_if_error(self.library.minigpt4_preprocess_image(ctx.ptr, ctypes.pointer(image), ctypes.pointer(preprocessed_image), I32(flags)))302 return preprocessed_image303 304 def minigpt4_encode_image(self, ctx: MiniGPT4Context, image: MiniGPT4Image, n_threads: int = 0) -> MiniGPT4Embedding:305 """306 Encodes an image into embedding307 308 Args:309 ctx (MiniGPT4Context): Context.310 image (MiniGPT4Image): Image.311 n_threads (int): Number of threads to use, if 0, uses all available. Defaults to 0.312 313 Returns:314 embedding (MiniGPT4Embedding): Output embedding.315 """316 317 embedding = MiniGPT4Embedding()318 self.panic_if_error(self.library.minigpt4_encode_image(ctx.ptr, ctypes.pointer(image), ctypes.pointer(embedding), n_threads))319 return embedding320 321 def minigpt4_begin_chat_image(self, ctx: MiniGPT4Context, image_embedding: MiniGPT4Embedding, s: str, n_threads: int = 0):322 """323 Begins a chat with an image.324 325 Args:326 ctx (MiniGPT4Context): Context.327 image_embedding (MiniGPT4Embedding): Image embedding.328 s (str): Question to ask about the image.329 n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.330 331 Returns:332 None333 """334 335 self.panic_if_error(self.library.minigpt4_begin_chat_image(ctx.ptr, ctypes.pointer(image_embedding), s.encode('utf-8'), n_threads))336 337 def minigpt4_end_chat_image(self, ctx: MiniGPT4Context, n_threads: int = 0, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1) -> str:338 """339 Ends a chat with an image.340 341 Args:342 ctx (MiniGPT4Context): Context.343 n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.344 temp (float, optional): Temperature. Defaults to 0.8.345 top_k (int, optional): Top K. Defaults to 40.346 top_p (float, optional): Top P. Defaults to 0.9.347 tfs_z (float, optional): Tfs Z. Defaults to 1.0.348 typical_p (float, optional): Typical P. Defaults to 1.0.349 repeat_last_n (int, optional): Repeat last N. Defaults to 64.350 repeat_penalty (float, optional): Repeat penality. Defaults to 1.1.351 alpha_presence (float, optional): Alpha presence. Defaults to 1.0.352 alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.353 mirostat (int, optional): Mirostat. Defaults to 0.354 mirostat_tau (float, optional): Mirostat Tau. Defaults to 5.0.355 mirostat_eta (float, optional): Mirostat Eta. Defaults to 1.0.356 penalize_nl (int, optional): Penalize NL. Defaults to 1.357 358 Returns:359 str: Token generated.360 """361 362 token = CHAR_PTR()363 self.panic_if_error(self.library.minigpt4_end_chat_image(ctx.ptr, ctypes.pointer(token), n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl))364 return ctypes.cast(token, ctypes.c_char_p).value.decode('utf-8')365 366 def minigpt4_system_prompt(self, ctx: MiniGPT4Context, n_threads: int = 0):367 """368 Generates a system prompt.369 370 Args:371 ctx (MiniGPT4Context): Context.372 n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.373 """374 375 self.panic_if_error(self.library.minigpt4_system_prompt(ctx.ptr, n_threads))376 377 def minigpt4_begin_chat(self, ctx: MiniGPT4Context, s: str, n_threads: int = 0):378 """379 Begins a chat continuing after minigpt4_begin_chat_image380 381 Args:382 ctx (MiniGPT4Context): Context.383 s (str): Question to ask about the image.384 n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.385 386 Returns:387 None388 """389 self.panic_if_error(self.library.minigpt4_begin_chat(ctx.ptr, s.encode('utf-8'), n_threads))390 391 def minigpt4_end_chat(self, ctx: MiniGPT4Context, n_threads: int = 0, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1) -> str:392 """393 Ends a chat.394 395 Args:396 ctx (MiniGPT4Context): Context.397 n_threads (int, optional): Number of threads to use, if 0, uses all available. Defaults to 0.398 temp (float, optional): Temperature. Defaults to 0.8.399 top_k (int, optional): Top K. Defaults to 40.400 top_p (float, optional): Top P. Defaults to 0.9.401 tfs_z (float, optional): Tfs Z. Defaults to 1.0.402 typical_p (float, optional): Typical P. Defaults to 1.0.403 repeat_last_n (int, optional): Repeat last N. Defaults to 64.404 repeat_penalty (float, optional): Repeat penality. Defaults to 1.1.405 alpha_presence (float, optional): Alpha presence. Defaults to 1.0.406 alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.407 mirostat (int, optional): Mirostat. Defaults to 0.408 mirostat_tau (float, optional): Mirostat Tau. Defaults to 5.0.409 mirostat_eta (float, optional): Mirostat Eta. Defaults to 1.0.410 penalize_nl (int, optional): Penalize NL. Defaults to 1.411 412 Returns:413 str: Token generated.414 """415 416 token = CHAR_PTR()417 self.panic_if_error(self.library.minigpt4_end_chat(ctx.ptr, ctypes.pointer(token), n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl))418 return ctypes.cast(token, ctypes.c_char_p).value.decode('utf-8')419 420 def minigpt4_reset_chat(self, ctx: MiniGPT4Context):421 """422 Resets the chat.423 424 Args:425 ctx (MiniGPT4Context): Context.426 """427 self.panic_if_error(self.library.minigpt4_reset_chat(ctx.ptr))428 429 def minigpt4_contains_eos_token(self, s: str) -> bool:430 431 """432 Checks if a string contains an EOS token.433 434 Args:435 s (str): String to check.436 437 Returns:438 bool: True if the string contains an EOS token, False otherwise.439 """440 441 return self.library.minigpt4_contains_eos_token(s.encode('utf-8'))442 443 def minigpt4_is_eos(self, s: str) -> bool:444 445 """446 Checks if a string is EOS.447 448 Args:449 s (str): String to check.450 451 Returns:452 bool: True if the string contains an EOS, False otherwise.453 """454 455 return self.library.minigpt4_is_eos(s.encode('utf-8'))456 457 458 def minigpt4_free(self, ctx: MiniGPT4Context) -> None:459 """460 Frees a context.461 462 Args:463 ctx (MiniGPT4Context): Context.464 """465 466 self.panic_if_error(self.library.minigpt4_free(ctx.ptr))467 468 def minigpt4_free_image(self, image: MiniGPT4Image) -> None:469 """470 Frees an image.471 472 Args:473 image (MiniGPT4Image): Image.474 """475 476 self.panic_if_error(self.library.minigpt4_free_image(ctypes.pointer(image)))477 478 def minigpt4_free_embedding(self, embedding: MiniGPT4Embedding) -> None:479 """480 Frees an embedding.481 482 Args:483 embedding (MiniGPT4Embedding): Embedding.484 """485 486 self.panic_if_error(self.library.minigpt4_free_embedding(ctypes.pointer(embedding)))487 488 def minigpt4_error_code_to_string(self, error_code: int) -> str:489 """490 Converts an error code to a string.491 492 Args:493 error_code (int): Error code.494 495 Returns:496 str: Error string.497 """498 499 return self.library.minigpt4_error_code_to_string(error_code).decode('utf-8')500 501 def minigpt4_quantize_model(self, in_path: str, out_path: str, data_type: DataType):502 """503 Quantizes a model file.504 505 Args:506 in_path (str): Path to input model file.507 out_path (str): Path to write output model file.508 data_type (DataType): Must be one DataType enum values.509 """510 511 self.panic_if_error(self.library.minigpt4_quantize_model(in_path.encode('utf-8'), out_path.encode('utf-8'), data_type))512 513 def minigpt4_set_verbosity(self, verbosity: Verbosity):514 """515 Sets verbosity.516 517 Args:518 verbosity (int): Verbosity.519 """520 521 self.library.minigpt4_set_verbosity(I32(verbosity))522 523def load_library() -> MiniGPT4SharedLibrary:524 """525 Attempts to find minigpt4.cpp shared library and load it.526 """527 528 file_name: str529 530 if 'win32' in sys.platform or 'cygwin' in sys.platform:531 file_name = 'minigpt4.dll'532 elif 'darwin' in sys.platform:533 file_name = 'libminigpt4.dylib'534 else:535 file_name = 'libminigpt4.so'536 537 cwd = pathlib.Path(os.getcwd())538 repo_root_dir: pathlib.Path = pathlib.Path(os.path.abspath(__file__)).parent.parent539 540 paths = [541 # If we are in "minigpt4" directory542 f'../bin/Release/{file_name}',543 # If we are in repo root directory544 f'bin/Release/{file_name}',545 # If we compiled in build directory546 f'build/bin/Release/{file_name}',547 # If we compiled in build directory548 f'build/{file_name}',549 f'../build/{file_name}',550 # Search relative to this file551 str(repo_root_dir / 'bin' / 'Release' / file_name),552 # Fallback553 str(repo_root_dir / file_name),554 str(cwd / file_name)555 ]556 557 for path in paths:558 if os.path.isfile(path):559 return MiniGPT4SharedLibrary(path)560 561 return MiniGPT4SharedLibrary(paths[-1])562 563class MiniGPT4ChatBot:564 def __init__(self, model_path: str, llm_model_path: str, verbosity: Verbosity = Verbosity.SILENT, n_threads: int = 0):565 """566 Creates a new MiniGPT4ChatBot instance.567 568 Args:569 model_path (str): Path to model file.570 llm_model_path (str): Path to language model model file.571 verbosity (Verbosity, optional): Verbosity. Defaults to Verbosity.SILENT.572 n_threads (int, optional): Number of threads to use. Defaults to 0.573 """574 575 self.library = load_library()576 self.ctx = self.library.minigpt4_model_load(model_path, llm_model_path, verbosity)577 self.n_threads = n_threads578 579 from PIL import Image580 from torchvision import transforms581 from torchvision.transforms.functional import InterpolationMode582 self.image_size = 224583 584 mean = (0.48145466, 0.4578275, 0.40821073)585 std = (0.26862954, 0.26130258, 0.27577711)586 self.transform = transforms.Compose(587 [588 transforms.RandomResizedCrop(589 self.image_size,590 interpolation=InterpolationMode.BICUBIC,591 ),592 transforms.ToTensor(),593 transforms.Normalize(mean, std)594 ]595 )596 self.embedding: Optional[MiniGPT4Embedding] = None597 self.is_image_chat = False598 self.chat_history = []599 600 def free(self):601 if self.ctx:602 self.library.minigpt4_free(self.ctx)603 604 def generate(self, message: str, limit: int = 1024, temp: float = 0.8, top_k: int = 40, top_p: float = 0.9, tfs_z: float = 1.0, typical_p: float = 1.0, repeat_last_n: int = 64, repeat_penalty: float = 1.1, alpha_presence: float = 1.0, alpha_frequency: float = 1.0, mirostat: int = 0, mirostat_tau: float = 5.0, mirostat_eta: float = 1.0, penalize_nl: int = 1):605 """606 Generates a chat response.607 608 Args:609 message (str): Message.610 limit (int, optional): Limit. Defaults to 1024.611 temp (float, optional): Temperature. Defaults to 0.8.612 top_k (int, optional): Top K. Defaults to 40.613 top_p (float, optional): Top P. Defaults to 0.9.614 tfs_z (float, optional): TFS Z. Defaults to 1.0.615 typical_p (float, optional): Typical P. Defaults to 1.0.616 repeat_last_n (int, optional): Repeat last N. Defaults to 64.617 repeat_penalty (float, optional): Repeat penalty. Defaults to 1.1.618 alpha_presence (float, optional): Alpha presence. Defaults to 1.0.619 alpha_frequency (float, optional): Alpha frequency. Defaults to 1.0.620 mirostat (int, optional): Mirostat. Defaults to 0.621 mirostat_tau (float, optional): Mirostat tau. Defaults to 5.0.622 mirostat_eta (float, optional): Mirostat eta. Defaults to 1.0.623 penalize_nl (int, optional): Penalize NL. Defaults to 1.624 """625 if self.is_image_chat:626 self.is_image_chat = False627 self.library.minigpt4_begin_chat_image(self.ctx, self.embedding, message, self.n_threads)628 chat = ''629 for _ in range(limit):630 token = self.library.minigpt4_end_chat_image(self.ctx, self.n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl)631 chat += token632 if self.library.minigpt4_contains_eos_token(token):633 continue634 if self.library.minigpt4_is_eos(chat):635 break636 yield token637 else:638 self.library.minigpt4_begin_chat(self.ctx, message, self.n_threads)639 chat = ''640 for _ in range(limit):641 token = self.library.minigpt4_end_chat(self.ctx, self.n_threads, temp, top_k, top_p, tfs_z, typical_p, repeat_last_n, repeat_penalty, alpha_presence, alpha_frequency, mirostat, mirostat_tau, mirostat_eta, penalize_nl)642 chat += token643 if self.library.minigpt4_contains_eos_token(token):644 continue645 if self.library.minigpt4_is_eos(chat):646 break647 yield token648 649 def reset_chat(self):650 """651 Resets the chat.652 """653 654 self.is_image_chat = False655 if self.embedding:656 self.library.minigpt4_free_embedding(self.embedding)657 self.embedding = None658 659 self.library.minigpt4_reset_chat(self.ctx)660 self.library.minigpt4_system_prompt(self.ctx, self.n_threads)661 662 def upload_image(self, image):663 """664 Uploads an image.665 666 Args:667 image (Image): Image.668 """669 670 self.reset_chat()671 672 image = self.transform(image)673 image = image.unsqueeze(0)674 image = image.numpy()675 image = image.ctypes.data_as(ctypes.c_void_p)676 minigpt4_image = MiniGPT4Image(image, self.image_size, self.image_size, 3, ImageFormat.F32)677 self.embedding = self.library.minigpt4_encode_image(self.ctx, minigpt4_image, self.n_threads)678 679 self.is_image_chat = True680 681 682if __name__ == "__main__":683 import argparse684 parser = argparse.ArgumentParser(description='Test loading minigpt4')685 parser.add_argument('model_path', help='Path to model file')686 parser.add_argument('llm_model_path', help='Path to llm model file')687 parser.add_argument('-i', '--image_path', help='Image to test', default='images/llama.png')688 parser.add_argument('-p', '--prompts', help='Text to test', default='what is the text in the picture?,what is the color of it?')689 args = parser.parse_args()690 691 model_path = args.model_path692 llm_model_path = args.llm_model_path693 image_path = args.image_path694 prompts = args.prompts695 696 if not Path(model_path).exists():697 print(f'Model does not exist: {model_path}')698 exit(1) 699 700 if not Path(llm_model_path).exists():701 print(f'LLM Model does not exist: {llm_model_path}')702 exit(1)703 704 prompts = prompts.split(',')705 706 print('Loading minigpt4 shared library...')707 library = load_library()708 print(f'Loaded library {library}')709 ctx = library.minigpt4_model_load(model_path, llm_model_path, Verbosity.DEBUG)710 image = library.minigpt4_image_load_from_file(ctx, image_path, 0)711 preprocessed_image = library.minigpt4_preprocess_image(ctx, image, 0)712 713 question = prompts[0]714 n_threads = 0715 embedding = library.minigpt4_encode_image(ctx, preprocessed_image, n_threads)716 library.minigpt4_system_prompt(ctx, n_threads)717 library.minigpt4_begin_chat_image(ctx, embedding, question, n_threads)718 chat = ''719 while True:720 token = library.minigpt4_end_chat_image(ctx, n_threads)721 chat += token722 if library.minigpt4_contains_eos_token(token):723 continue724 if library.minigpt4_is_eos(chat):725 break726 print(token, end='')727 728 for i in range(1, len(prompts)):729 prompt = prompts[i]730 library.minigpt4_begin_chat(ctx, prompt, n_threads)731 chat = ''732 while True:733 token = library.minigpt4_end_chat(ctx, n_threads)734 chat += token735 if library.minigpt4_contains_eos_token(token):736 continue737 if library.minigpt4_is_eos(chat):738 break739 print(token, end='')740 741 library.minigpt4_free_image(image)742 library.minigpt4_free_image(preprocessed_image)743 library.minigpt4_free(ctx)744