xkas2001/uzbek-language-dataset

uzbek-dataset/
├── community-oscar-uzbek/          # 1.1GB - OSCAR community data
│   ├── all_metadata_text.txt       # 666MB - To'liq matn to'plami  
│   ├── top5_metadata_text.txt      # 167MB - Top 5 sifatli matnlar
│   ├── uzbek_words_community.txt   # 5.6MB - Community so'zlar
│   ├── uzbek_words_frequency_community.txt  # 9.8MB - Chastota
│   ├── uzbek_words_unique_clean.txt         # 4.5MB - Unique clean words
│   ├── uzbek_top1000_community.txt          # 7KB - Top 1000 so'zlar
│   └── data/                       # Raw OSCAR data (2014-2023)
│       ├── 2014-42/ to 2023-50/   # Yillik snapshot'lar
│       └── */uz_meta/              # Uzbek metadata
├── custom-uzbek/                   # 2.1GB - Custom matnlar
│   ├── parsed_txt.txt              # 546MB - Parsed matnlar
│   ├── parsed_with_imlo.txt        # 561MB - Imlo bilan qayta ishlangan
│   ├── parsed_with_imlo_without_emoji.txt  # 546MB - Emoji'siz
│   └── word_lists/                 # So'z ro'yxatlari
│       ├── words_list.txt          # Barcha so'zlar
│       ├── frequency_list.txt      # Chastota ro'yxati
│       └── unique_words.txt        # Unique so'zlar
├── oscar-uzbek/                    # 38MB - OSCAR 2301 Uzbek
│   ├── uzbek_words_unique.txt      # 2MB - Unique so'zlar
│   ├── uzbek_words_frequency.txt   # 3MB - Chastota jadval
│   ├── uzbek_top1000_words.txt     # 7KB - Top 1000
│   ├── uzbek_words_filtered.txt    # 1.6MB - Filtrlangan
│   ├── uzbek_proper_nouns.txt      # 376KB - Proper nouns
│   └── uz_meta/                    # Metadata
└── merge/                          # 122MB - Merge tools va natijalar
    ├── frequency_list.txt          # 31MB - Birlashtirilgan frequency
    ├── unique_words.txt            # 22MB - Birlashtirilgan unique
    ├── merged_data/                # Merged datasets
    └── *.py                        # Merge script'lari

# Repository'ni clone qilish
git clone https://huggingface.co/datasets/xkas2001/uzbek-language-dataset
cd uzbek-language-dataset

# Yoki to'g'ridan-to'g'ri fayllarni yuklash
wget https://huggingface.co/datasets/xkas2001/uzbek-language-dataset/resolve/main/community-oscar-uzbek/all_metadata_text.txt

from datasets import load_dataset

# Hugging Face datasets orqali yuklash
dataset = load_dataset("xkas2001/uzbek-language-dataset")

# Yoki local fayllarni o'qish
def load_uzbek_text(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        return f.readlines()

# Barcha OSCAR matnlarini yuklash
oscar_texts = load_uzbek_text('community-oscar-uzbek/all_metadata_text.txt')
print(f"OSCAR lines: {len(oscar_texts)}")

# Custom matnlarni yuklash  
custom_texts = load_uzbek_text('custom-uzbek/parsed_with_imlo.txt')
print(f"Custom lines: {len(custom_texts)}")

import json
from collections import Counter

def load_frequency_list(file_path):
    """Frequency list'ni yuklash"""
    word_freq = {}
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            if '\t' in line:
                word, freq = line.strip().split('\t', 1)
                word_freq[word] = int(freq)
    return word_freq

# Frequency data'ni yuklash
freq_data = load_frequency_list('merge/frequency_list.txt')
print(f"Total unique words: {len(freq_data)}")
print(f"Most frequent words: {list(freq_data.items())[:10]}")

# Top N so'zlarni olish
top_words = sorted(freq_data.items(), key=lambda x: x[1], reverse=True)[:1000]
print("Top 10 so'zlar:")
for word, count in top_words[:10]:
    print(f"{word}: {count}")

import re
from typing import List

def clean_uzbek_text(text: str) -> str:
    """O'zbek matnini tozalash"""
    # Unicode normalizatsiya
    import unicodedata
    text = unicodedata.normalize('NFC', text)
    
    # Keraksiz bo'sh joylarni olib tashlash
    text = re.sub(r'\s+', ' ', text).strip()
    
    # URL va email'larni olib tashlash
    text = re.sub(r'http[s]?://\S+', '', text)
    text = re.sub(r'\S+@\S+\.\S+', '', text)
    
    return text

def tokenize_uzbek(text: str) -> List[str]:
    """O'zbek matnini tokenizatsiya qilish"""
    # Apostrophe va tire'larni saqlab qolish
    tokens = re.findall(r"[a-zA-ZÀ-ÿŞşĞğÇçıİöüÖÜ''-]+", text, re.UNICODE)
    return [token.lower() for token in tokens if len(token) > 1]

# Misol ishlatish
sample_text = "O'zbekiston Respublikasi mustaqil davlat hisoblanadi."
cleaned = clean_uzbek_text(sample_text)
tokens = tokenize_uzbek(cleaned)
print(f"Cleaned: {cleaned}")
print(f"Tokens: {tokens}")

def create_word_frequency(text_files: List[str], output_file: str):
    """Bir nechta fayl asosida frequency list yaratish"""
    word_counts = Counter()
    
    for file_path in text_files:
        print(f"Processing: {file_path}")
        with open(file_path, 'r', encoding='utf-8') as f:
            for line_num, line in enumerate(f, 1):
                if line_num % 100000 == 0:
                    print(f"  Processed {line_num} lines")
                
                cleaned_line = clean_uzbek_text(line)
                tokens = tokenize_uzbek(cleaned_line)
                word_counts.update(tokens)
    
    # Natijalarni saqlash
    with open(output_file, 'w', encoding='utf-8') as f:
        for word, count in word_counts.most_common():
            f.write(f"{word}\t{count}\n")
    
    print(f"Created frequency list with {len(word_counts)} unique words")
    return word_counts

# Dataset'larni birlashtirish
text_files = [
    'community-oscar-uzbek/all_metadata_text.txt',
    'custom-uzbek/parsed_with_imlo.txt'
]
word_freq = create_word_frequency(text_files, 'combined_frequency.txt')

def filter_high_quality_text(input_file: str, output_file: str, min_length=50):
    """Yuqori sifatli matnlarni filtrlash"""
    quality_lines = []
    
    with open(input_file, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            
            # Minimal uzunlik
            if len(line) < min_length:
                continue
            
            # Latin harflar foizi
            latin_chars = sum(1 for c in line if c.isalpha() and ord(c) < 128)
            latin_ratio = latin_chars / len(line) if line else 0
            
            # O'zbek harflar foizi (apostrophe bilan)
            uzbek_chars = sum(1 for c in line if c in "aeiouAEIOUoʻ'ğüşçıÖÜŞÇĞIi")
            
            # Quality criteria
            if (latin_ratio > 0.7 and  # Mostly Latin alphabet
                uzbek_chars > 5 and    # Has Uzbek-specific chars
                line.count('.') > 0):  # Has sentences
                quality_lines.append(line)
    
    # Save high-quality lines
    with open(output_file, 'w', encoding='utf-8') as f:
        f.writelines(line + '\n' for line in quality_lines)
    
    print(f"Filtered {len(quality_lines)} high-quality lines")

# High-quality text filtrlash
filter_high_quality_text(
    'community-oscar-uzbek/all_metadata_text.txt',
    'community-oscar-uzbek/high_quality_text.txt'
)

# GPT-style model uchun data preparation
def prepare_for_language_modeling(input_files, output_file, max_length=512):
    """Language modeling uchun matnlarni tayyorlash"""
    import random
    
    all_texts = []
    for file_path in input_files:
        with open(file_path, 'r', encoding='utf-8') as f:
            texts = [clean_uzbek_text(line.strip()) for line in f 
                    if len(line.strip()) > 50]
            all_texts.extend(texts)
    
    # Shuffle va chunk qilish
    random.shuffle(all_texts)
    
    with open(output_file, 'w', encoding='utf-8') as f:
        current_chunk = ""
        for text in all_texts:
            if len(current_chunk + text) < max_length:
                current_chunk += text + " "
            else:
                f.write(current_chunk.strip() + '\n')
                current_chunk = text + " "
        
        if current_chunk.strip():
            f.write(current_chunk.strip() + '\n')

# Language modeling data tayyorlash
prepare_for_language_modeling([
    'community-oscar-uzbek/all_metadata_text.txt',
    'custom-uzbek/parsed_with_imlo.txt'
], 'uzbek_language_modeling_data.txt')

# Word2Vec/FastText uchun corpus tayyorlash
def prepare_for_embeddings(input_files, output_file):
    """Word embeddings uchun corpus yaratish"""
    with open(output_file, 'w', encoding='utf-8') as out_f:
        for file_path in input_files:
            with open(file_path, 'r', encoding='utf-8') as in_f:
                for line in in_f:
                    cleaned = clean_uzbek_text(line.strip())
                    if len(cleaned) > 20:  # Minimal length
                        tokens = tokenize_uzbek(cleaned)
                        if len(tokens) > 3:  # Minimal token count
                            out_f.write(' '.join(tokens) + '\n')

# FastText training example
"""
from gensim.models import FastText

# Corpus tayyorlash
prepare_for_embeddings([
    'community-oscar-uzbek/all_metadata_text.txt',
    'custom-uzbek/parsed_with_imlo.txt'
], 'uzbek_embeddings_corpus.txt')

# FastText model o'qitish
sentences = []
with open('uzbek_embeddings_corpus.txt', 'r') as f:
    sentences = [line.strip().split() for line in f]

model = FastText(
    sentences=sentences,
    vector_size=300,
    window=5,
    min_count=5,
    workers=4,
    sg=1  # Skip-gram
)

model.save('uzbek_fasttext.model')
"""

def create_classification_dataset(texts, labels, output_file):
    """Text classification uchun dataset yaratish"""
    import pandas as pd
    
    data = []
    for text, label in zip(texts, labels):
        cleaned_text = clean_uzbek_text(text)
        if len(cleaned_text) > 30:
            data.append({
                'text': cleaned_text,
                'label': label,
                'length': len(cleaned_text),
                'word_count': len(tokenize_uzbek(cleaned_text))
            })
    
    df = pd.DataFrame(data)
    df.to_csv(output_file, index=False, encoding='utf-8')
    
    print(f"Created classification dataset with {len(df)} samples")
    print(f"Label distribution:\n{df['label'].value_counts()}")

# Topic classification uchun misol
# Bu yerda siz o'z topic labeling logikangizni yozasiz

import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import numpy as np

def analyze_dataset_quality(file_path):
    """Dataset sifatini tahlil qilish"""
    
    line_lengths = []
    word_counts = []
    char_distribution = Counter()
    total_lines = 0
    
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            total_lines += 1
            line = line.strip()
            
            line_lengths.append(len(line))
            words = tokenize_uzbek(line)
            word_counts.append(len(words))
            
            # Character distribution
            char_distribution.update(line.lower())
            
            if total_lines % 100000 == 0:
                print(f"Analyzed {total_lines} lines")
    
    # Statistics
    stats = {
        'total_lines': total_lines,
        'avg_line_length': np.mean(line_lengths),
        'median_line_length': np.median(line_lengths),
        'avg_word_count': np.mean(word_counts),
        'median_word_count': np.median(word_counts),
        'most_common_chars': char_distribution.most_common(20)
    }
    
    return stats, line_lengths, word_counts

# Analysis running
stats, lengths, word_counts = analyze_dataset_quality(
    'community-oscar-uzbek/all_metadata_text.txt'
)

print("Dataset Statistics:")
for key, value in stats.items():
    if key != 'most_common_chars':
        print(f"{key}: {value}")

print("\nMost common characters:")
for char, count in stats['most_common_chars']:
    print(f"'{char}': {count}")

def visualize_dataset_stats(lengths, word_counts, output_dir='plots'):
    """Dataset statistikalarini vizualizatsiya qilish"""
    import os
    os.makedirs(output_dir, exist_ok=True)
    
    plt.style.use('seaborn-v0_8')
    
    # Line length distribution
    plt.figure(figsize=(12, 5))
    
    plt.subplot(1, 2, 1)
    plt.hist(lengths, bins=50, alpha=0.7, color='skyblue', edgecolor='black')
    plt.title('Line Length Distribution')
    plt.xlabel('Characters')
    plt.ylabel('Frequency')
    plt.axvline(np.mean(lengths), color='red', linestyle='--', 
                label=f'Mean: {np.mean(lengths):.1f}')
    plt.legend()
    
    # Word count distribution  
    plt.subplot(1, 2, 2)
    plt.hist(word_counts, bins=50, alpha=0.7, color='lightgreen', edgecolor='black')
    plt.title('Word Count Distribution')
    plt.xlabel('Words')
    plt.ylabel('Frequency')
    plt.axvline(np.mean(word_counts), color='red', linestyle='--',
                label=f'Mean: {np.mean(word_counts):.1f}')
    plt.legend()
    
    plt.tight_layout()
    plt.savefig(f'{output_dir}/dataset_distribution.png', dpi=300, bbox_inches='tight')
    plt.show()

# Visualization
visualize_dataset_stats(lengths, word_counts)

# So'z ro'yxatlarini birlashtirish
cd merge/
python merge_wordlists.py

# Frequency list'larni birlashtirish  
python merge_frequency_lists.py

# custom-uzbek/process_words.py
# Maxsus so'z qayta ishlash script'i
python custom-uzbek/process_words.py

# Repository'ni fork qiling
git fork https://github.com/xkas2001/uzbek-dataset

# Feature branch yarating
git checkout -b feature/new-improvement

# O'zgarishlarni commit qiling
git commit -m "Add: new data cleaning improvements"

# Pull request yarating
git push origin feature/new-improvement