JeeKay/Malaria-classification

1#!/usr/bin/env python2# coding: utf-83 4# # Visualize data5 6# In[1]:7 8 9import os10import numpy as np11import matplotlib.pyplot as plt12import cv213from pathlib import Path14from collections import defaultdict15 16 17# In[2]:18 19 20data_dir = "malaria_data/cell_images"21parasitized_dir = os.path.join(data_dir, 'Parasitized')22uninfected_dir = os.path.join(data_dir, 'Uninfected')23 24parasitized_files = list(Path(parasitized_dir).glob('*.png'))25uninfected_files = list(Path(uninfected_dir).glob('*.png'))26 27print(f"Parasitized Images: {len(parasitized_files)}")28print(f"Uninfected Images: {len(uninfected_files)}")29 30 31# In[3]:32 33 34labels = ['Parasitized', 'Uninfected']35counts = [len(parasitized_files), len(uninfected_files)]36 37plt.figure(figsize=(6, 4))38plt.bar(labels, counts, color=['#ff7f0e', '#1f77b4'])39plt.title("Class Distribution")40plt.ylabel("Number of Images")41plt.show()42 43 44# In[4]:45 46 47def plot_samples(image_files, title, num_samples=5):48    plt.figure(figsize=(15, 3))49    for i in range(num_samples):50        img = cv2.imread(str(image_files[i]))51        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)52        plt.subplot(1, num_samples, i+1)53        plt.imshow(img)54        plt.axis("off")55    plt.suptitle(title)56    plt.show()57 58plot_samples(parasitized_files, "Parasitized Cells")59plot_samples(uninfected_files, "Uninfected Cells")60 61 62# In[5]:63 64 65def get_image_sizes(file_list):66    sizes = []67    for f in file_list:68        img = cv2.imread(str(f))69        sizes.append(img.shape[:2])  # height, width70    return sizes71 72parasitized_sizes = get_image_sizes(parasitized_files)73uninfected_sizes = get_image_sizes(uninfected_files)74 75all_sizes = parasitized_sizes + uninfected_sizes76unique_sizes = set(all_sizes)77 78print("Unique image sizes found:")79print(unique_sizes)80 81 82# In[6]:83 84 85total_images = len(parasitized_files) + len(uninfected_files)86avg_height = np.mean([size[0] for size in all_sizes])87avg_width = np.mean([size[1] for size in all_sizes])88 89print(f"\nTotal Images: {total_images}")90print(f"Average Image Size: {avg_width:.0f}x{avg_height:.0f}")91print(f"Min/Max Height: {min(s[0] for s in all_sizes)} / {max(s[0] for s in all_sizes)}")92print(f"Min/Max Width: {min(s[1] for s in all_sizes)} / {max(s[1] for s in all_sizes)}")93 94 95# In[7]:96 97 98sample_img = cv2.imread(str(parasitized_files[5]))99print("Image shape:", sample_img.shape)100 101 102# # Data preprocessing103 104# In[8]:105 106 107import matplotlib.pyplot as plt108import numpy as np109 110# Assuming you have your image data in a numpy array called 'image_data'111# For a single image:112plt.figure(figsize=(10, 6))113plt.hist(sample_img.ravel(), bins=256, range=(0, 256), color='blue', alpha=0.7)114plt.title('Pixel Value Distribution')115plt.xlabel('Pixel Intensity')116plt.ylabel('Frequency')117plt.grid(True, linestyle='--', alpha=0.5)118plt.show()119 120 121# # Data Splitting122 123# In[20]:124 125 126import os127import shutil128from pathlib import Path129import random130from sklearn.model_selection import train_test_split131import numpy as np132import matplotlib.pyplot as plt133import cv2134import torch135from torchvision import datasets, transforms136from torch.utils.data import DataLoader137 138 139# In[21]:140 141 142RAW_DATA_DIR = 'malaria_data/cell_images'143OUTPUT_DIR = 'malaria_ds/split_dataset'144 145PARASITIZED_DIR = os.path.join(RAW_DATA_DIR, 'Parasitized')146UNINFECTED_DIR = os.path.join(RAW_DATA_DIR, 'Uninfected')147 148# Output directories149TRAIN_DIR = os.path.join(OUTPUT_DIR, 'train')150VAL_DIR = os.path.join(OUTPUT_DIR, 'validation')151TEST_DIR = os.path.join(OUTPUT_DIR, 'test')152 153# Ensure output directories exist154os.makedirs(PARASITIZED_DIR, exist_ok=True)155os.makedirs(UNINFECTED_DIR, exist_ok=True)156 157print("Paths defined.")158 159 160# In[22]:161 162 163def split_class_files(class_dir, train_dir, val_dir, test_dir):164    all_files = list(Path(class_dir).glob('*.*'))165    train_files, test_files = train_test_split(all_files, test_size=0.1, random_state=42)166    train_files, val_files = train_test_split(train_files, test_size=0.1 / (1 - 0.1), random_state=42)167 168    for f in train_files:169        shutil.copy(f, train_dir)170    for f in val_files:171        shutil.copy(f, val_dir)172    for f in test_files:173        shutil.copy(f, test_dir)174 175    return len(all_files)176 177def create_split_folders():178    class_names = ['Parasitized', 'Uninfected']179    for folder in ['train', 'validation', 'test']:180        for cls in class_names:181            os.makedirs(os.path.join(OUTPUT_DIR, folder, cls), exist_ok=True)182 183    print("Splitting Parasitized Images:")184    total_parasitized = split_class_files(185        os.path.join(RAW_DATA_DIR, 'Parasitized'),186        os.path.join(OUTPUT_DIR, 'train', 'Parasitized'),187        os.path.join(OUTPUT_DIR, 'validation', 'Parasitized'),188        os.path.join(OUTPUT_DIR, 'test', 'Parasitized')189    )190 191    print("\nSplitting Uninfected Images:")192    total_uninfected = split_class_files(193        os.path.join(RAW_DATA_DIR, 'Uninfected'),194        os.path.join(OUTPUT_DIR, 'train', 'Uninfected'),195        os.path.join(OUTPUT_DIR, 'validation', 'Uninfected'),196        os.path.join(OUTPUT_DIR, 'test', 'Uninfected')197    )198 199    print(f"\nTotal Parasitized: {total_parasitized}, Uninfected: {total_uninfected}")200    print("Dataset split completed.")201 202 203# ## Data Aug and transforms204 205# In[23]:206 207 208IMG_SIZE = (128, 128)209BATCH_SIZE = 32210 211# Custom class_to_idx mapping to fix label order212class_to_idx = {'Uninfected': 0, 'Parasitized': 1}213idx_to_class = {v: k for k, v in class_to_idx.items()}214 215# Define transforms216train_transforms = transforms.Compose([217    transforms.Resize(IMG_SIZE),218    transforms.ToTensor(),219    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),220    transforms.RandomRotation(20),221    transforms.RandomHorizontalFlip(),222])223 224val_test_transforms = transforms.Compose([225    transforms.Resize(IMG_SIZE),226    transforms.ToTensor(),227    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),228])229 230# Custom Dataset Class to enforce class_to_idx231class CustomImageFolder(datasets.ImageFolder):232    def __init__(self, root, transform, class_to_idx_override=None):233        super().__init__(root=root, transform=transform)234        if class_to_idx_override:235            self.class_to_idx = class_to_idx_override236            self.samples = [237                (path, class_to_idx[cls]) 238                for path, cls_idx in self.samples 239                for cls in [self.classes[cls_idx]] 240                if cls in class_to_idx_override241            ]242            self.classes = list(class_to_idx_override.keys())243 244 245 246# In[24]:247 248 249def get_dataloaders():250    # Create datasets251    train_dataset = CustomImageFolder(root=os.path.join(OUTPUT_DIR, 'train'), transform=train_transforms, class_to_idx_override=class_to_idx)252    val_dataset = CustomImageFolder(root=os.path.join(OUTPUT_DIR, 'validation'), transform=val_test_transforms, class_to_idx_override=class_to_idx)253    test_dataset = CustomImageFolder(root=os.path.join(OUTPUT_DIR, 'test'), transform=val_test_transforms, class_to_idx_override=class_to_idx)254 255    # Create data loaders256    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)257    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)258    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)259 260    print(f"Train: {len(train_dataset)}, Val: {len(val_dataset)}, Test: {len(test_dataset)}")261    print("Class Mapping:", train_dataset.class_to_idx)262 263    return train_loader, val_loader, test_loader, train_dataset, val_dataset, test_dataset264 265 266# In[26]:267 268 269def show_batch_sample(loader, dataset):270    images, labels = next(iter(loader))271    plt.figure(figsize=(12, 6))272    for i in range(min(6, BATCH_SIZE)):273        img = images[i].numpy().transpose((1, 2, 0))274        img = np.clip(img * np.array([0.229, 0.224, 0.225]) + np.array([0.485, 0.456, 0.406]), 0, 1)275        plt.subplot(2, 3, i+1)276        plt.imshow(img)277        plt.title(idx_to_class[labels[i].item()])278        plt.axis("off")279    plt.suptitle("Sample Batch from DataLoader")280    plt.show()281 282 283# In[32]:284 285 286create_split_folders()287train_loader, val_loader, test_loader, train_dataset, val_dataset, test_dataset = get_dataloaders()288show_batch_sample(train_loader, train_dataset)289 290 291# In[34]:292 293 294print(train_dataset)295 296