ank52/logic_stream

1import os2import sqlite33import pandas as pd4from datasets import Dataset5from transformers import (6    AutoTokenizer, 7    AutoModelForSequenceClassification, 8    TrainingArguments, 9    Trainer10)11 12def run_continuous_learning_pipeline():13    print("🔁 Starting Nightly Continuous Learning Pipeline...")14 15    # 1. Connect to Django's SQLite Database16    db_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "db.sqlite3")17    print(f"🔌 Connecting to Database: {db_path}")18    19    conn = sqlite3.connect(db_path)20    # In a real enterprise system, we would filter by `WHERE is_verified=1` to only train on human-approved labels.21    # For this demo, we fetch all logged interactions.22    query = "SELECT ticket_text as text, predicted_id as label FROM customer_portal_aitraininglog;"23    24    try:25        df = pd.read_sql_query(query, conn)26    except Exception as e:27        print("Error reading from database. Have you submitted any tickets yet?")28        return29        30    if df.empty:31        print("💤 No new tickets found in the database. Going back to sleep.")32        return33        34    print(f"📈 Found {len(df)} new verified ticket(s)! Initiating Incremental Fine-Tuning.")35    36    # 2. Convert to HuggingFace Dataset37    dataset = Dataset.from_pandas(df)38    39    # We duplicate the dataset slightly so the batch isn't too small for the Trainer API40    if len(dataset) < 8:41        print("Small batch detected. Augmenting data for mathematical stability...")42        dataset = Dataset.from_pandas(pd.concat([df]*10, ignore_index=True))43 44    # 3. Load the pre-existing, already fine-tuned model (not from scratch!)45    MODEL_PATH = "../my_fine_tuned_bert"46    if not os.path.exists(MODEL_PATH):47        print(f"❌ Could not find {MODEL_PATH}. You must have the base model available.")48        return49 50    print(f"🧠 Loading existing neural weights from {MODEL_PATH}...")51    tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)52    model = AutoModelForSequenceClassification.from_pretrained(MODEL_PATH)53 54    # 4. Tokenization55    def tokenize_function(examples):56        return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=128)57 58    tokenized_dataset = dataset.map(tokenize_function, batched=True)59 60    # 5. Incremental Training Configuration61    # We use a very low learning rate (e.g., 1e-5) because we only want to *tweak* the weights, not destroy them (Catastrophic Forgetting)62    training_args = TrainingArguments(63        output_dir="./results_continuous",64        learning_rate=1e-5,65        per_device_train_batch_size=4,66        num_train_epochs=1, # Just 1 pass over the new data!67        weight_decay=0.01,68        save_strategy="no"69    )70 71    trainer = Trainer(72        model=model,73        args=training_args,74        train_dataset=tokenized_dataset,75    )76 77    # 6. Execute Incremental Fine-Tuning78    print("🔥 Executing Active Learning adjustments...")79    trainer.train()80 81    # 7. Overwrite the Model with the newly improved weights82    print(f"💾 Saving smarter, self-improved model back to {MODEL_PATH}...")83    model.save_pretrained(MODEL_PATH)84    tokenizer.save_pretrained(MODEL_PATH)85    86    # 8. Clean up (e.g., mark rows as 'consumed' in the database)87    cursor = conn.cursor()88    cursor.execute("DELETE FROM customer_portal_aitraininglog;")89    conn.commit()90    conn.close()91    92    print("✨ Continuous Learning Cycle Complete! The AI is now smarter!")93 94if __name__ == "__main__":95    run_continuous_learning_pipeline()96