Lyon28/Caca-Chatbot-V2

User Query
    ↓
Preprocessing (lowercase, clean, normalize)
    ↓
Language Detection (ID/EN auto-detect)
    ↓
┌──────────────────────────────────┐
│  Parallel Execution (10 Techniques)    │
├──────────────────────────────────┤
│ 1. BM25 Scoring                        │
│ 2. TF-IDF Cosine                       │
│ 3. SBERT MiniLM (FAISS)                │
│ 4. SBERT MPNet (FAISS)                 │
│ 5. USE Similarity                      │
│ 6. Fuzzy Matching (Top 100)            │
│ 7. Jaccard Similarity (Top 100)        │
│ 8. N-gram Overlap (Top 100)            │
│ 9. Pattern Detection                   │
│ 10. Keyword Boosting                   │
│ BONUS: Context History (if enabled)    │
└──────────────────────────────────┘
    ↓
Weighted Ensemble (Sum all scores)
    ↓
Top-K Selection
    ↓
Best Response + Confidence Score

{
  "messages": [
    {"role": "user", "content": "Halo CACA, siapa kamu?"},
    {"role": "assistant", "content": "Halo! Aku CACA, chatbot pintar yang siap membantu!"}
  ]
}

pip install -r requirements.txt

datasets
huggingface_hub
pandas
numpy
scikit-learn
rank-bm25
python-Levenshtein
fuzzywuzzy
sentence-transformers
faiss-cpu
nltk
langdetect
tensorflow
tensorflow-hub

from huggingface_hub import hf_hub_download
import pickle
import json
import faiss
import numpy as np

repo_id = "Lyon28/Caca-Chatbot-V2"

# Download all files
files = [
    "bm25_index.pkl",
    "tfidf_vectorizer.pkl",
    "tfidf_matrix.pkl",
    "faiss_mini_index.bin",
    "faiss_mpnet_index.bin",
    "sbert_mini_embeddings.npy",
    "sbert_mpnet_embeddings.npy",
    "use_embeddings.npy",
    "queries.json",
    "responses.json",
    "query_patterns.json",
    "config.json",
    "patterns.json",
    "keywords.json"
]

print("📥 Downloading CACA models...")
for file in files:
    hf_hub_download(repo_id, file, local_dir="./caca_models")

print("✅ All models downloaded!")

from sentence_transformers import SentenceTransformer
import tensorflow_hub as hub
from sklearn.metrics.pairwise import cosine_similarity
from fuzzywuzzy import fuzz
from langdetect import detect
from rank_bm25 import BM25Okapi
import re

# Load all models
print("Loading CACA models...")

with open('caca_models/bm25_index.pkl', 'rb') as f:
    bm25 = pickle.load(f)

with open('caca_models/tfidf_vectorizer.pkl', 'rb') as f:
    tfidf_vectorizer = pickle.load(f)

with open('caca_models/tfidf_matrix.pkl', 'rb') as f:
    tfidf_matrix = pickle.load(f)

faiss_mini = faiss.read_index('caca_models/faiss_mini_index.bin')
faiss_mpnet = faiss.read_index('caca_models/faiss_mpnet_index.bin')

sbert_mini_embeddings = np.load('caca_models/sbert_mini_embeddings.npy')
sbert_mpnet_embeddings = np.load('caca_models/sbert_mpnet_embeddings.npy')
use_embeddings = np.load('caca_models/use_embeddings.npy')

with open('caca_models/queries.json', 'r', encoding='utf-8') as f:
    queries = json.load(f)

with open('caca_models/responses.json', 'r', encoding='utf-8') as f:
    responses = json.load(f)

with open('caca_models/query_patterns.json', 'r', encoding='utf-8') as f:
    query_patterns = json.load(f)

with open('caca_models/config.json', 'r', encoding='utf-8') as f:
    config = json.load(f)

with open('caca_models/patterns.json', 'r', encoding='utf-8') as f:
    PATTERNS = json.load(f)

with open('caca_models/keywords.json', 'r', encoding='utf-8') as f:
    IMPORTANT_KEYWORDS = json.load(f)

# Load transformer models
sbert_mini = SentenceTransformer('all-MiniLM-L6-v2')
sbert_mpnet = SentenceTransformer('paraphrase-mpnet-base-v2')
use_model = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")

print("✅ All models loaded!")

# Helper functions
def preprocess_text(text):
    text = text.lower()
    text = re.sub(r'[^\w\s]', ' ', text)
    text = re.sub(r'\s+', ' ', text).strip()
    return text

def ngram_similarity(text1, text2, n=3):
    ngrams1 = set([text1[i:i+n] for i in range(len(text1)-n+1)])
    ngrams2 = set([text2[i:i+n] for i in range(len(text2)-n+1)])
    if not ngrams1 or not ngrams2:
        return 0.0
    return len(ngrams1 & ngrams2) / len(ngrams1 | ngrams2)

def jaccard_similarity(text1, text2):
    set1, set2 = set(text1.split()), set(text2.split())
    if not set1 or not set2:
        return 0.0
    return len(set1 & set2) / len(set1 | set2)

def detect_pattern(query):
    for pattern, tag in PATTERNS.items():
        if re.search(pattern, query, re.IGNORECASE):
            return tag
    return None

def detect_language(text):
    try:
        return detect(text)
    except:
        return 'id'

# Main chat function
def chat(query, verbose=False):
    """Chat with CACA"""
    query_clean = preprocess_text(query)
    lang = detect_language(query_clean)
    
    scores = np.zeros(len(queries))
    weights = config['techniques']
    
    # 1. BM25
    bm25_scores = bm25.get_scores(query_clean.split())
    bm25_scores = (bm25_scores - bm25_scores.min()) / (bm25_scores.max() - bm25_scores.min() + 1e-10)
    scores += weights['bm25'] * bm25_scores
    
    # 2. TF-IDF
    query_tfidf = tfidf_vectorizer.transform([query_clean])
    tfidf_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
    scores += weights['tfidf'] * tfidf_scores
    
    # 3. SBERT MiniLM
    query_mini = sbert_mini.encode([query_clean])
    faiss.normalize_L2(query_mini)
    D_mini, I_mini = faiss_mini.search(query_mini, len(queries))
    sbert_mini_scores = np.zeros(len(queries))
    sbert_mini_scores[I_mini[0]] = D_mini[0]
    sbert_mini_scores = (sbert_mini_scores - sbert_mini_scores.min()) / (sbert_mini_scores.max() - sbert_mini_scores.min() + 1e-10)
    scores += weights['sbert_mini'] * sbert_mini_scores
    
    # 4. SBERT MPNet
    query_mpnet = sbert_mpnet.encode([query_clean])
    faiss.normalize_L2(query_mpnet)
    D_mpnet, I_mpnet = faiss_mpnet.search(query_mpnet, len(queries))
    sbert_mpnet_scores = np.zeros(len(queries))
    sbert_mpnet_scores[I_mpnet[0]] = D_mpnet[0]
    sbert_mpnet_scores = (sbert_mpnet_scores - sbert_mpnet_scores.min()) / (sbert_mpnet_scores.max() - sbert_mpnet_scores.min() + 1e-10)
    scores += weights['sbert_mpnet'] * sbert_mpnet_scores
    
    # 5. USE
    query_use = use_model([query_clean]).numpy()
    use_scores = cosine_similarity(query_use, use_embeddings).flatten()
    use_scores = (use_scores - use_scores.min()) / (use_scores.max() - use_scores.min() + 1e-10)
    scores += weights['use'] * use_scores
    
    # 6-8. Fuzzy, Jaccard, N-gram (Top 100)
    top_100_idx = np.argsort(scores)[-100:]
    
    fuzzy_scores = np.zeros(len(queries))
    jaccard_scores = np.zeros(len(queries))
    ngram_scores = np.zeros(len(queries))
    
    for idx in top_100_idx:
        fuzzy_scores[idx] = fuzz.ratio(query_clean, queries[idx]) / 100.0
        jaccard_scores[idx] = jaccard_similarity(query_clean, queries[idx])
        ngram_scores[idx] = ngram_similarity(query_clean, queries[idx])
    
    scores += weights['fuzzy'] * fuzzy_scores
    scores += weights['jaccard'] * jaccard_scores
    scores += weights['ngram'] * ngram_scores
    
    # 9. Pattern Matching
    pattern_tag = detect_pattern(query_clean)
    pattern_scores = np.zeros(len(queries))
    if pattern_tag:
        for i, tag in enumerate(query_patterns):
            if tag == pattern_tag:
                pattern_scores[i] = 1.0
    scores += weights['pattern'] * pattern_scores
    
    # 10. Keyword Boost
    keyword_scores = np.zeros(len(queries))
    query_words = query_clean.split()
    for i, q in enumerate(queries):
        boost = sum(1 for kw in IMPORTANT_KEYWORDS if kw in q and kw in query_words)
        keyword_scores[i] = boost / len(IMPORTANT_KEYWORDS) if IMPORTANT_KEYWORDS else 0
    scores += weights['keyword_boost'] * keyword_scores
    
    # Get best match
    top_idx = np.argmax(scores)
    
    result = {
        'response': responses[top_idx],
        'score': float(scores[top_idx]),
        'matched_query': queries[top_idx],
        'detected_language': lang,
        'pattern': pattern_tag
    }
    
    if verbose:
        result['technique_scores'] = {
            'bm25': float(bm25_scores[top_idx]),
            'tfidf': float(tfidf_scores[top_idx]),
            'sbert_mini': float(sbert_mini_scores[top_idx]),
            'sbert_mpnet': float(sbert_mpnet_scores[top_idx]),
            'use': float(use_scores[top_idx]),
            'fuzzy': float(fuzzy_scores[top_idx]),
            'jaccard': float(jaccard_scores[top_idx]),
            'ngram': float(ngram_scores[top_idx]),
            'pattern': float(pattern_scores[top_idx]),
            'keyword': float(keyword_scores[top_idx])
        }
    
    return result

# Test CACA
print("\n🤖 Testing CACA...")
result = chat("Halo CACA, apa kabar?", verbose=True)
print(f"User: Halo CACA, apa kabar?")
print(f"CACA: {result['response']}")
print(f"Score: {result['score']:.4f}")
print(f"Language: {result['detected_language']}")
print(f"Pattern: {result['pattern']}")

if 'technique_scores' in result:
    print("\nTechnique Scores:")
    for tech, score in sorted(result['technique_scores'].items(), key=lambda x: x[1], reverse=True):
        print(f"  {tech}: {score:.4f}")

# Quick chat
response = chat("Siapa kamu?")
print(response['response'])

# With details
response = chat("What is AI?", verbose=True)
print(f"Response: {response['response']}")
print(f"Confidence: {response['score']:.2%}")
print(f"Language: {response['detected_language']}")

import gradio as gr

def chat_interface(message, history):
    result = chat(message)
    return result['response']

demo = gr.ChatInterface(
    chat_interface,
    title="🤖 CACA - Contextual Adaptive Conversational AI",
    description="Ultimate hybrid chatbot dengan 10+ teknik retrieval | Support ID & EN",
    examples=[
        "Halo CACA, siapa kamu?",
        "Apa itu kecerdasan buatan?",
        "Bagaimana cara belajar coding?",
        "What is machine learning?",
        "Terima kasih banyak!"
    ],
    theme="soft",
    chatbot=gr.Chatbot(height=500)
)

demo.launch(share=True)

# Re-build CACA
python build_caca.py

# Upload to HF Hub
python upload_to_hub.py

# Clone repository
git clone https://huggingface.co/Lyon28/Caca-Chatbot-V2
cd Caca-Chatbot

# Install dependencies
pip install -r requirements.txt

# Run tests
python test_caca.py

# Start Flask API
python app_flask.py

# Or start Gradio
python app_gradio.py

FROM python:3.9-slim

WORKDIR /app

COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

COPY . .

EXPOSE 7860

CMD ["python", "app_gradio.py"]