antitheft159/HelpDeskMultiLanguage

1!pip install datasets2import pandas as pd3import plotly.express as px4import os5import plotly.graph_objects as go6from plotly.subplots import make_subplots7from sklearn.model_selection import train_test_split8from sklearn.metrics import classification_report, confusion_matrix9from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer10from datasets import Dataset11import torch12import numpy as np13 14!pip install wandb15 16import wandb17wandb.login(key='eb4c4a1fa7eec1ffbabc36420ba1166f797d4ac5')18 19data_path = "/content/ticket_helpdesk_labeled_multi_languages_english_spain_french_german.csv"20df = pd.read_csv(data_path)21 22print("First few rows of the dataset:")23print(df.head())24 25print("\nEDA and Visualization")26print("\nSummary statistics:")27print(df.describe(include='all'))28 29fig_queue = px.histogram(df, x='queue', title='Distribution of Queue Categories', color='queue')30fig_queue.show()31 32fig_priority = px.histogram(df, x='priority', title='Distribution of Priority Levels', color='priority')33fig_priority.show()34 35fig_language = px.histogram(df, x='language', title='Distribution of Languages', color='language')36fig_language.show()37 38fig_software = px.histogram(df, x='software_used', title='Distribution of Software Used', color='software_used')39fig_software.show()40 41fig_hardware = px.histogram(df, x='hardware_used', title='Distribution of Hardware Used', color='hardware_used')42fig_hardware.show()43 44fig_accounting = px.histogram(df, x='accounting_category', title='Distribution of Accounting Categories', color='accounting_category')45fig_accounting.show()46 47fig = make_subplots(rows=3, cols=1, subplot_titles=('Priority Distribution', 'Language Distribution', 'Queue Distribution'))48 49fig.add_trace(go.Histogram(x=df['priority'], name='Priority'), row=1, col=1)50fig.add_trace(go.Histogram(x=df['language'], name='Language'), row=2, col=1)51fig.add_trace(go.Histogram(x=df['queue'], name='Queue'), row=3, col=1)52 53fig.update_layout(title_text='Distributions of Priority, Language, and Queue', showlegend=False)54fig.show()55 56fig_scatter = px.scatter(df, x='priority', y='queue', color='priority', title='Scatter Plot of Priority vs. Queue')57fig_scatter.show()58 59df = df.dropna(subset=['text'])60df['text'] = df['text'].astype(str)61 62df['queue_encoded'] = df['queue'].astype('category').cat.codes63queue_mapping = dict(enumerate(df['queue'].astype('category').cat.categories))64 65X_train, X_test, y_train, y_test = train_test_split(df['text'], df['queue_encoded'], test_size=0.2, random_state=42)66 67train_data = Dataset.from_dict({'text': X_train.tolist(), 'label': y_train.tolist()})68test_data = Dataset.from_dict({'text': X_test.tolist(), 'label': y_test.tolist()})69 70model_name = "xlm-roberta-base"71tokenizer = AutoTokenizer.from_pretrained(model_name)72model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=df['queue_encoded'].nunique())73 74def preprocess_function(examples):75  return tokenizer(examples['text'], truncation=True, padding=True)76 77train_data = train_data.map(preprocess_function, batched=True)78test_data = test_data.map(preprocess_function, batched=True)