imseldrith/ChatGPT-Detection

1import gradio as gr2import spacy3import re4import numpy as np5import torch6from transformers import AutoTokenizer, AutoModelForCausalLM7 8# Define some constants for error messages9MAX_TEXT_LENGTH = 204810MIN_GEN_LENGTH = 111MAX_GEN_LENGTH = 204812MIN_AI_PERCENTAGE = 5013 14# Download the Spacy model for English15spacy.cli.download("en_core_web_sm")16nlp = spacy.load("en_core_web_sm")17 18# Define a function to detect AI-generated content and calculate the perplexity score,19# burstiness score, and average perplexity score for a given text input20def detect_ai_content(text, max_gen_length, temperature, model_name, model_size, num_return_sequences, min_ai_percentage):21    # Clean the text by removing extra spaces, line breaks, and special characters22    cleaned_text = re.sub(r'\s+', ' ', text).strip()23    cleaned_text = re.sub(r'[^\w\s]', '', cleaned_text)24 25    # If the cleaned text is empty or contains only one sentence, return an error message26    doc = nlp(cleaned_text)27    if not re.search('\S', cleaned_text) or len(list(doc.sents)) < 2:28        return {"error": "Input text must contain at least two sentences."}29 30    # Check if the cleaned text is longer than the maximum allowed by the GPT model31    if len(cleaned_text) > MAX_TEXT_LENGTH:32        return {"error": f"Input text must be no longer than {MAX_TEXT_LENGTH} characters."}33 34    # Check if the minimum threshold for the percentage of AI-generated content is within the allowed range35    if not (0 <= min_ai_percentage <= 100):36        return {"error": "Minimum threshold for AI percentage must be between 0 and 100."}37 38    # Load the specified GPT model and tokenizer39    model_name = f"{model_name}-{model_size}"40    tokenizer = AutoTokenizer.from_pretrained(model_name)41    model = AutoModelForCausalLM.from_pretrained(model_name)42 43    # Set the device to run the model on (either "cuda" or "cpu")44    device = "cuda" if torch.cuda.is_available() else "cpu"45    model.to(device)46 47    # Set the end of sequence token ID for text generation48    eos_token_id = tokenizer.eos_token_id49 50    # Generate multiple sequences using the pre-trained GPT model51    input_ids = tokenizer.encode(cleaned_text, add_special_tokens=True, return_tensors='pt').to(device)52 53    output_sequences = []54    for i in range(num_return_sequences):55        output_sequence = model.generate(56            input_ids=input_ids,57            max_length=max_gen_length + len(input_ids[0]),58            temperature=temperature,59            top_k=50,60            top_p=0.95,61            repetition_penalty=1.5,62            do_sample=True,63            num_return_sequences=164        )[0]65        output_sequences.append(output_sequence)66    67    # Decode the generated sequences using the GPT tokenizer68    generated_texts = []69    perplexities = []70    ai_percentages = []71 72    for i, output_sequence in enumerate(output_sequences):73        generated_text = tokenizer.decode(output_sequence.tolist()[len(input_ids[0]):], skip_special_tokens=True)74 75        # Calculate the percentage of AI-generated content in the generated text76        ai_percentage = round(len(generated_text) / len(cleaned_text) * 100, 2)77        ai_percentages.append(ai_percentage)78 79        # Check if the AI percentage and perplexity score are above their respective thresholds80        generated_input_ids = tokenizer.encode(generated_text, add_special_tokens=False, return_tensors='pt').to(device)81        with torch.no_grad():82            loss = model(generated_input_ids, labels=generated_input_ids).loss.item()83        perplexity = np.exp(loss)84        perplexities.append(perplexity)85        generated_texts.append(generated_text)86 87    # Remove the generated sequences that are identical or highly similar to the cleaned text88    clean_doc = nlp(cleaned_text)89    unique_generated_texts = []90 91    for generated_text in generated_texts:92        gen_doc = nlp(generated_text)93        similarity = clean_doc.similarity(gen_doc)94 95        if similarity < 0.8:96            is_unique = True97 98            for unique_text in unique_generated_texts:99                unique_doc = nlp(unique_text)100                unique_similarity = unique_doc.similarity(gen_doc)101 102                if unique_similarity >= 0.8:103                    is_unique = False104                    break105 106            if is_unique:107                unique_generated_texts.append(generated_text)108 109    # Calculate the burstiness score for the input text, which measures the diversity of vocabulary in the input text110    doc = nlp(cleaned_text)111    # Extract the tokens from the input text's sentences112    all_tokens = []113    for sent in doc.sents:114        tokens = [token.text.lower() for token in sent if not token.is_punct and not token.is_stop]115        all_tokens += tokens116 117    # Calculate the burstiness score for the input text118    unique_tokens = set(all_tokens)119    num_unique_tokens = len(unique_tokens)120    total_tokens = len(all_tokens)121    burstiness_score = (num_unique_tokens * num_unique_tokens) / (total_tokens * total_tokens)122 123    # Calculate the average perplexity score and AI percentage for the generated texts124    avg_perplexity = np.mean(perplexities)125    avg_ai_percentage = round(np.mean(ai_percentages), 2)126 127    # Check if the AI percentage and perplexity score are above their respective thresholds128    if avg_ai_percentage < min_ai_percentage:129        return {"error": f"The generated text has an AI percentage of {avg_ai_percentage}%, which is below the minimum threshold of {min_ai_percentage}%."}130    if avg_perplexity < MIN_GEN_LENGTH:131        return {"error": f"The generated text has a perplexity score of {avg_perplexity}, which is below the minimum threshold of {MIN_GEN_LENGTH}."}132    if avg_perplexity > MAX_GEN_LENGTH:133        return {"error": f"The generated text has a perplexity score of {avg_perplexity}, which is above the maximum threshold of {MAX_GEN_LENGTH}."}134 135    # Return the unique generated texts, burstiness score, and average AI percentage and perplexity score136    return {137        "generated_texts": unique_generated_texts,138        "burstiness_score": round(burstiness_score, 2),139        "avg_ai_percentage": avg_ai_percentage,140        "avg_perplexity": round(avg_perplexity, 2)141    }142 143# Define the Gradio interface for the BAI Chat function144def bai_chat(text, max_gen_length=256, temperature=0.7, model_name="gpt2", model_size="medium", num_return_sequences=5, min_ai_percentage=50):145    result = detect_ai_content(text, max_gen_length, temperature, model_name, model_size, num_return_sequences, min_ai_percentage)146 147    if "error" in result:148        return result["error"]149    else:150        generated_texts = "\n\n".join(result["generated_texts"])151        return f"Burstiness Score: {result['burstiness_score']}\nAverage AI Percentage: {result['avg_ai_percentage']}%\nAverage Perplexity Score: {result['avg_perplexity']}\n\n{generated_texts}"152 153gr.Interface(fn=bai_chat, 154            inputs=[gr.inputs.Textbox("Enter your text here...")], 155            outputs=[gr.outputs.Textbox(label="Generated Texts")]).launch()