atlury/vision-test

1import * as webllm from "https://esm.run/@mlc-ai/web-llm";
2
3// Global variables and configuration
4let selectedModel = "Phi-3.5-vision-instruct-q4f16_1-MLC";
5let uploadedBase64Image = "";
6let modelInitialized = false;
7
8// Callback function to update initialization progress
9function initProgressCallback(report) {
10  document.getElementById("download-status").textContent = report.text;
11  document.getElementById("download-status").classList.remove("hidden");
12}
13
14// Function to append messages to the chat box
15function appendMessage(message, role = "user") {
16  const chatBox = document.getElementById("chat-box");
17  const container = document.createElement("div");
18  container.classList.add("message-container", role);
19
20  const newMessage = document.createElement("div");
21  newMessage.classList.add("message");
22  newMessage.textContent = message;
23
24  container.appendChild(newMessage);
25  chatBox.appendChild(container);
26  chatBox.scrollTop = chatBox.scrollHeight; // Scroll to the latest message
27}
28
29// Function to update the last message
30function updateLastMessage(content) {
31  const messageDoms = document.querySelectorAll(".message");
32  const lastMessageDom = messageDoms[messageDoms.length - 1];
33  lastMessageDom.textContent = content;
34}
35
36// Function to check if both image and model are ready and enable the Send button
37function checkIfReadyToSend() {
38  if (uploadedBase64Image && modelInitialized) {
39    document.getElementById("send").disabled = false;
40  }
41}
42
43// Main function to initialize the engine and process images
44async function main() {
45  if (!uploadedBase64Image) {
46    alert("Please upload an image first!");
47    return;
48  }
49
50  // Initialize the engine configuration
51  const engineConfig = {
52    initProgressCallback: initProgressCallback,
53    logLevel: "INFO",
54  };
55
56  const chatOpts = {
57    context_window_size: 6144,
58  };
59
60  // Create the engine
61  const engine = await webllm.CreateMLCEngine(selectedModel, engineConfig, chatOpts);
62
63  // Indicate that the model is initialized
64  modelInitialized = true;
65  checkIfReadyToSend(); // Check if we can enable the Send button now
66
67  // Construct chat messages with the uploaded image
68  const messages = [
69    {
70      role: "user",
71      content: [
72        { type: "text", text: "Describe the uploaded image." },
73        { type: "image_url", image_url: { url: uploadedBase64Image } },
74      ],
75    },
76  ];
77
78  // Send the chat request
79  const request = { stream: false, messages: messages };
80  const reply = await engine.chat.completions.create(request);
81
82  // Get the reply and display it
83  const replyMessage = await engine.getMessage();
84  appendMessage(replyMessage, "assistant");
85  document.getElementById("send").disabled = false;
86  console.log(reply);
87}
88
89// Handle file uploads
90document.getElementById("image-input").addEventListener("change", async function(event) {
91  const file = event.target.files[0];
92  if (file) {
93    uploadedBase64Image = await imageFileToBase64(file);
94    console.log("Image uploaded and converted to base64");
95    checkIfReadyToSend(); // Check if we can enable the Send button now
96  }
97});
98
99// Set up UI bindings and event listeners
100document.getElementById("download").addEventListener("click", async function () {
101  selectedModel = document.getElementById("model-selection").value;
102  await main(); // Initialize and run the model
103});
104
105document.getElementById("send").addEventListener("click", function () {
106  const input = document.getElementById("user-input").value.trim();
107  if (input.length === 0) return;
108
109  appendMessage(input, "user");
110  document.getElementById("user-input").value = "";
111  document.getElementById("user-input").setAttribute("placeholder", "Generating...");
112
113  // Additional logic for new user questions can be added here
114});
115
116// Populate model selection dropdown
117const availableModels = ["Phi-3.5-vision-instruct-q4f16_1-MLC", "Phi-3.5-vision-instruct-q4f32_1-MLC"];
118availableModels.forEach((modelId) => {
119  const option = document.createElement("option");
120  option.value = modelId;
121  option.textContent = modelId;
122  document.getElementById("model-selection").appendChild(option);
123});
124document.getElementById("model-selection").value = selectedModel;
125