ndwdgda/replit2

1import pandas as pd2import os3from datetime import datetime4import logging5 6logger = logging.getLogger(__name__)7 8class DataProductManager:9    def __init__(self, data_dir=None):10        self.data_dir = data_dir or os.getenv("DATA_DIR", "data")11        # Create directory structure12        self.dirs = {13            'bundles': os.path.join(self.data_dir, 'bundles'),14            'yearly': os.path.join(self.data_dir, 'yearly'),15            'quarterly': os.path.join(self.data_dir, 'quarterly'),16            'monthly': os.path.join(self.data_dir, 'monthly')17        }18        for d in self.dirs.values():19            os.makedirs(d, exist_ok=True)20    21    def calculate_price(self, file_type, row_count):22        """Calculate optimal pricing based on data volume"""23        pricing_model = {24            'monthly': {'base': 99, 'per_10k': 5, 'cap': 299},25            'quarterly': {'base': 249, 'per_10k': 10, 'cap': 699},26            'yearly': {'base': 899, 'per_10k': 20, 'cap': 1999},27            'bundle': {'base': 2999, 'per_10k': 50, 'cap': 4999}28        }29        30        model = pricing_model.get(file_type, pricing_model['monthly'])31        price = model['base'] + ((row_count // 10000) * model['per_10k'])32        return min(price, model['cap'])33    34    def smart_split_csv(self, master_file, product_type):35        """36        Intelligently split master CSV into marketable products37        """38        if not os.path.exists(master_file):39            logger.warning(f"Master file not found: {master_file}")40            return {}41 42        try:43            df = pd.read_csv(master_file)44            45            # Normalize date column46            if 'date' in df.columns:47                df['date'] = pd.to_datetime(df['date'])48            elif 'scraped_date' in df.columns:49                df['date'] = pd.to_datetime(df['scraped_date'])50            else:51                # Fallback if no date column52                logger.warning(f"No date column found in {master_file}")53                return {}54            55            created_files = {}56            57            # 1. Create Bundle (Master File)58            bundle_path = os.path.join(self.dirs['bundles'], f'{product_type}_FULL.csv')59            df.to_csv(bundle_path, index=False)60            created_files[bundle_path] = {61                'type': 'bundle',62                'period': 'All Time',63                'rows': len(df),64                'size_mb': os.path.getsize(bundle_path) / (1024*1024),65                'price': self.calculate_price('bundle', len(df)),66                'description': f'Complete Historical Bundle'67            }68            69            # 2. Split by Year70            for year, year_data in df.groupby(df['date'].dt.year):71                yearly_path = os.path.join(self.dirs['yearly'], f'{product_type}_{year}.csv')72                year_data.to_csv(yearly_path, index=False)73                74                created_files[yearly_path] = {75                    'type': 'yearly',76                    'period': str(year),77                    'rows': len(year_data),78                    'size_mb': os.path.getsize(yearly_path) / (1024*1024),79                    'price': self.calculate_price('yearly', len(year_data)),80                    'description': f'{year} Full Year Dataset'81                }82                83                # 3. Split by Quarter84                for quarter in range(1, 5):85                    q_data = year_data[year_data['date'].dt.quarter == quarter]86                    if len(q_data) > 0:87                        q_path = os.path.join(self.dirs['quarterly'], f'{product_type}_{year}_Q{quarter}.csv')88                        q_data.to_csv(q_path, index=False)89                        90                        created_files[q_path] = {91                            'type': 'quarterly',92                            'period': f'{year} Q{quarter}',93                            'rows': len(q_data),94                            'size_mb': os.path.getsize(q_path) / (1024*1024),95                            'price': self.calculate_price('quarterly', len(q_data)),96                            'description': f'{year} Q{quarter} Dataset'97                        }98                        99                        # 4. Split by Month (only if we have quarterly data)100                        # Optimization: Only do this if requested, but for now we do it.101                        # Actually, let's stick to Q/Y/Bundle to avoid file explosion for this demo102                        # unless the user explicitly wants monthly. The prompt said "Tier 3: Monthly".103                        # Okay, let's do monthly.104                        # 4. Split by Month (DISABLED per user request)105                        # for month in range((quarter-1)*3 + 1, quarter*3 + 1):106                        #     m_data = q_data[q_data['date'].dt.month == month]107                        #     if len(m_data) > 0:108                        #         m_path = os.path.join(self.dirs['monthly'], f'{product_type}_{year}_{month:02d}.csv')109                        #         m_data.to_csv(m_path, index=False)110                        #         created_files[m_path] = {111                        #             'type': 'monthly',112                        #             'period': f'{year}-{month:02d}',113                        #             'rows': len(m_data),114                        #             'size_mb': os.path.getsize(m_path) / (1024*1024),115                        #             'price': self.calculate_price('monthly', len(m_data)),116                        #             'description': f'{year}-{month:02d} Dataset'117                        #         }118 119            return created_files120            121        except Exception as e:122            logger.error(f"Error processing {master_file}: {e}")123            return {}124 125    def generate_catalog(self, all_products):126        """Generate a list of products for the UI."""127        catalog = []128        for filepath, info in all_products.items():129            catalog.append({130                'filename': os.path.basename(filepath),131                'path': filepath, # Internal use132                'type': info['type'],133                'period': info['period'],134                'rows': info['rows'],135                'size_mb': f"{info['size_mb']:.2f}",136                'price': info['price'],137                'description': info['description'],138                'download_url': f"/download/{os.path.basename(filepath)}"139            })140        # Sort by type (Bundle -> Yearly -> Quarterly -> Monthly)141        order = {'bundle': 0, 'yearly': 1, 'quarterly': 2, 'monthly': 3}142        catalog.sort(key=lambda x: (order.get(x['type'], 99), x['period']))143        return catalog144