MMo4/csit-ned-chatbot

1import re2import yaml3import json4from typing import Dict, Any, List, Optional5from pathlib import Path6import logging7 8def setup_logging(level: str = "INFO"):9    """Setup logging configuration"""10    logging.basicConfig(11        level=getattr(logging, level.upper()),12        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',13        datefmt='%Y-%m-%d %H:%M:%S'14    )15    return logging.getLogger(__name__)16 17def extract_yaml_metadata(content: str) -> tuple[Dict[str, Any], str]:18    """Extract YAML frontmatter and content from markdown"""19    if content.startswith('---'):20        try:21            # Find the end of YAML frontmatter22            end_marker = content.find('\n---\n', 4)23            if end_marker == -1:24                end_marker = content.find('\n---', 4)25                if end_marker == -1:26                    return {}, content27            28            yaml_content = content[4:end_marker]29            markdown_content = content[end_marker + 4:].strip()30            31            metadata = yaml.safe_load(yaml_content)32            return metadata or {}, markdown_content33        except yaml.YAMLError as e:34            logging.warning(f"Failed to parse YAML metadata: {e}")35            return {}, content36    return {}, content37 38def clean_text(text: str) -> str:39    """Clean and normalize text content"""40    # Remove excessive whitespace41    text = re.sub(r'\s+', ' ', text)42    # Remove markdown formatting for embedding43    text = re.sub(r'[#*`_\[\]()]', '', text)44    # Clean up multiple spaces45    text = ' '.join(text.split())46    return text.strip()47 48def chunk_text(text: str, chunk_size: int = 500, overlap: int = 50) -> List[str]:49    """Split text into overlapping chunks"""50    words = text.split()51    chunks = []52    53    for i in range(0, len(words), chunk_size - overlap):54        chunk = ' '.join(words[i:i + chunk_size])55        if chunk.strip():56            chunks.append(chunk)57    58    return chunks59 60def validate_metadata(metadata: Dict[str, Any]) -> bool:61    """Validate metadata against required schema"""62    required_fields = [63        'chunk_id', 'title', 'category', 'subcategory', 64        'departments', 'topics', 'data_freshness'65    ]66    67    for field in required_fields:68        if field not in metadata:69            return False70    71    # Validate category values72    valid_categories = [73        'comparisons', 'departments', 'career_outcomes', 74        'admissions', 'misconceptions', 'student_concerns'75    ]76    77    if metadata.get('category') not in valid_categories:78        return False79    80    return True81 82def get_file_paths(directory: Path, pattern: str = "*.md") -> List[Path]:83    """Get all files matching pattern in directory recursively"""84    return list(directory.rglob(pattern))85 86def sanitize_filename(text: str) -> str:87    """Convert text to safe filename"""88    # Remove special characters and replace with underscores89    safe_text = re.sub(r'[^\w\s-]', '', text)90    # Replace spaces and multiple underscores with single underscore91    safe_text = re.sub(r'[-\s_]+', '_', safe_text)92    return safe_text.lower().strip('_')93 94def calculate_text_stats(text: str) -> Dict[str, int]:95    """Calculate basic text statistics"""96    words = text.split()97    return {98        'word_count': len(words),99        'character_count': len(text),100        'sentence_count': len(re.findall(r'[.!?]+', text)),101        'paragraph_count': len([p for p in text.split('\n\n') if p.strip()])102    }103 104def load_config(config_path: str) -> Dict[str, Any]:105    """Load configuration from JSON or YAML file."""106    try:107        config_file = Path(config_path)108        109        if not config_file.exists():110            return {}111        112        with open(config_file, 'r', encoding='utf-8') as f:113            if config_path.endswith('.json'):114                return json.load(f)115            elif config_path.endswith('.yaml') or config_path.endswith('.yml'):116                return yaml.safe_load(f) or {}117            else:118                # Try JSON first, then YAML119                try:120                    f.seek(0)121                    return json.load(f)122                except json.JSONDecodeError:123                    f.seek(0)124                    return yaml.safe_load(f) or {}125                    126    except Exception as e:127        logging.warning(f"Error loading config from {config_path}: {e}")128        return {}