Alpha108/GenerativeEngineOptimization
0
1"""2Content Optimization Module3Enhances content for better AI/LLM performance and GEO scores4"""5 6import json7import re8from typing import Dict, Any, List, Optional9from langchain.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate10 11 12class ContentOptimizer:13 """Main class for optimizing content for AI search engines"""14 15 def __init__(self, llm):16 self.llm = llm17 self.setup_prompts()18 19 def setup_prompts(self):20 """Initialize optimization prompts"""21 22 # Main content enhancement prompt23 self.enhancement_prompt = (24 "You are an AI Content Enhancement Specialist. Your purpose is to optimize user-provided text to maximize its effectiveness for large language models (LLMs) in search, question-answering, and conversational AI systems.\n\n"25 "Evaluate the input text based on the following criteria, assigning a score from 1-10 for each:\n"26 "- Clarity: How easily can the content be understood?\n"27 "- Structuredness: How well-organized and coherent is the content?\n"28 "- LLM Answerability: How easily can an LLM extract precise answers from the content?\n\n"29 "Identify the most salient keywords.\n\n"30 "Rewrite the text to improve:\n"31 "- Clarity and precision\n"32 "- Logical structure and flow\n"33 "- Suitability for LLM-based information retrieval\n\n"34 "Present your analysis and optimized text in the following JSON format:\n"35 "```json\n"36 "{{\n"37 " \"scores\": {{\n"38 " \"clarity\": 8.5,\n"39 " \"structuredness\": 7.0,\n"40 " \"answerability\": 9.0\n"41 " }},\n"42 " \"keywords\": [\"example\", \"installation\", \"setup\"],\n"43 " \"optimized_text\": \"...\"\n"44 "}}\n"45 "```"46 )47 48 # SEO-style optimization prompt49 self.seo_style_prompt = (50 "You are an AI-first SEO specialist. Optimize this content for AI search engines and LLM systems. "51 "Focus on:\n"52 "1. Semantic keyword optimization\n"53 "2. Question-answer format enhancement\n"54 "3. Factual accuracy and authority signals\n"55 "4. Conversational readiness\n"56 "5. Citation-worthy structure\n"57 "Provide analysis and optimization in JSON:\n"58 "```json\n"59 "{{\n"60 " \"seo_analysis\": {{\n"61 " \"keyword_density\": \"analysis of current keywords\",\n"62 " \"semantic_gaps\": [\"missing semantic terms\"],\n"63 " \"readability_score\": 8.5,\n"64 " \"authority_signals\": [\"credentials\", \"citations\"]\n"65 " }},\n"66 " \"optimized_content\": {{\n"67 " \"title_suggestions\": [\"optimized title 1\", \"optimized title 2\"],\n"68 " \"meta_description\": \"AI-optimized meta description\",\n"69 " \"enhanced_content\": \"full optimized content...\",\n"70 " \"structured_data_suggestions\": [\"schema markup recommendations\"]\n"71 " }},\n"72 " \"improvement_summary\": {{\n"73 " \"changes_made\": [\"change 1\", \"change 2\"],\n"74 " \"expected_impact\": \"description of expected improvements\"\n"75 " }}\n"76 "}}\n"77 "```"78 )79 80 # Competitive content analysis prompt81 self.competitive_analysis_prompt = (82 "Compare this content against best practices for AI search optimization. Identify gaps and opportunities.\n"83 "Original Content: {content}\n"84 "Analyze against these AI search factors:\n"85 "- Entity recognition and linking\n"86 "- Question coverage completeness\n"87 "- Factual statement clarity\n"88 "- Conversational flow\n"89 "- Semantic relationship mapping\n\n"90 "Provide competitive analysis in JSON format with specific recommendations:\n"91 "{{\n"92 " \"competitive_analysis\": {{\n"93 " \"entity_gaps\": [\"gap1\", \"gap2\"],\n"94 " \"question_coverage\": \"summary of coverage\",\n"95 " \"factual_clarity\": \"assessment\",\n"96 " \"conversational_flow\": \"assessment\",\n"97 " \"semantic_relationships\": [\"relationship1\", \"relationship2\"]\n"98 " }},\n"99 " \"recommendations\": [\"recommendation 1\", \"recommendation 2\"]\n"100 "}}\n"101 )102 103 def optimize_content(self, content: str, analyze_only: bool = False, 104 include_keywords: bool = True, optimization_type: str = "standard") -> Dict[str, Any]:105 """106 Main content optimization function107 Args:108 content (str): Content to optimize109 analyze_only (bool): If True, only analyze without rewriting110 include_keywords (bool): Whether to include keyword analysis111 optimization_type (str): Type of optimization ("standard", "seo", "competitive") 112 Returns:113 Dict: Optimization results with scores and enhanced content114 """115 try:116 # Choose optimization approach117 if optimization_type == "seo":118 return self._seo_style_optimization(content, analyze_only)119 elif optimization_type == "competitive":120 return self._competitive_optimization(content)121 else:122 return self._standard_optimization(content, analyze_only, include_keywords)123 124 except Exception as e:125 return {'error': f"Optimization failed: {str(e)}"}126 127 def _standard_optimization(self, content: str, analyze_only: bool, include_keywords: bool) -> Dict[str, Any]:128 """Standard content optimization using enhancement prompt"""129 try:130 # Modify prompt based on options131 prompt_text = self.enhancement_prompt132 133 if analyze_only:134 prompt_text = prompt_text.replace(135 "Rewrite the text to improve:",136 "Analyze the text for potential improvements in:"137 ).replace(138 '"optimized_text": "..."',139 '"optimization_suggestions": ["suggestion 1", "suggestion 2"]'140 )141 142 if not include_keywords:143 prompt_text = prompt_text.replace(144 '"keywords": ["example", "installation", "setup"],',145 ''146 )147 148 # Create and run chain149 prompt_template = ChatPromptTemplate.from_messages([150 SystemMessagePromptTemplate.from_template(prompt_text),151 HumanMessagePromptTemplate.from_template(content[:6000]) # Limit content length152 ])153 # ("system", prompt_text),154 # ("user", content[:6000]) # Limit content length155 156 chain = prompt_template | self.llm157 result = chain.invoke({})158 159 # Parse result160 result_content = result.content if hasattr(result, 'content') else str(result)161 parsed_result = self._parse_optimization_result(result_content)162 163 # Add metadata164 parsed_result.update({165 'optimization_type': 'standard',166 'analyze_only': analyze_only,167 'original_length': len(content),168 'original_word_count': len(content.split())169 })170 171 return parsed_result172 173 except Exception as e:174 return {'error': f"Standard optimization failed: {str(e)}"}175 176 def _seo_style_optimization(self, content: str, analyze_only: bool) -> Dict[str, Any]:177 """SEO-focused optimization for AI search engines"""178 try:179 prompt_template = ChatPromptTemplate.from_messages([180 ("system", self.seo_style_prompt),181 ("user", f"Optimize this content for AI search engines:\n\n{content[:6000]}")182 ])183 184 chain = prompt_template | self.llm185 result = chain.invoke({})186 187 result_content = result.content if hasattr(result, 'content') else str(result)188 parsed_result = self._parse_optimization_result(result_content)189 190 # Add SEO-specific metadata191 parsed_result.update({192 'optimization_type': 'seo',193 'analyze_only': analyze_only,194 'seo_focused': True195 })196 197 return parsed_result198 199 except Exception as e:200 return {'error': f"SEO optimization failed: {str(e)}"}201 202 def _competitive_optimization(self, content: str) -> Dict[str, Any]:203 """Competitive analysis-based optimization"""204 try:205 formatted_prompt = self.competitive_analysis_prompt.format(content=content[:5000])206 207 prompt_template = ChatPromptTemplate.from_messages([208 ("system", formatted_prompt),209 ("user", "Perform the competitive analysis and provide optimization recommendations.")210 ])211 212 chain = prompt_template | self.llm213 result = chain.invoke({})214 215 result_content = result.content if hasattr(result, 'content') else str(result)216 parsed_result = self._parse_optimization_result(result_content)217 218 parsed_result.update({219 'optimization_type': 'competitive',220 'competitive_analysis': True221 })222 223 return parsed_result224 225 except Exception as e:226 return {'error': f"Competitive optimization failed: {str(e)}"}227 228 def batch_optimize_content(self, content_list: List[str], optimization_type: str = "standard") -> List[Dict[str, Any]]:229 """230 Optimize multiple pieces of content in batch231 232 Args:233 content_list (List[str]): List of content pieces to optimize234 optimization_type (str): Type of optimization to apply235 236 Returns:237 List[Dict]: List of optimization results238 """239 results = []240 241 for i, content in enumerate(content_list):242 try:243 result = self.optimize_content(244 content, 245 optimization_type=optimization_type246 )247 result['batch_index'] = i248 results.append(result)249 250 except Exception as e:251 results.append({252 'batch_index': i,253 'error': f"Batch optimization failed: {str(e)}"254 })255 256 return results257 258 def generate_content_variations(self, content: str, num_variations: int = 3) -> List[Dict[str, Any]]:259 """260 Generate multiple optimized variations of the same content261 262 Args:263 content (str): Original content264 num_variations (int): Number of variations to generate265 266 Returns:267 List[Dict]: List of content variations with analysis268 """269 variations = []270 271 variation_prompts = [272 "Create a more conversational version optimized for AI chat responses",273 "Create a more authoritative version optimized for citations",274 "Create a more structured version optimized for question-answering"275 ]276 277 for i in range(min(num_variations, len(variation_prompts))):278 try:279 custom_prompt = f"""You are optimizing content for AI systems. {variation_prompts[i]}.280 281Original content: {content[:4000]}282 283Provide the optimized variation in JSON format:284```json285{{286"variation_type": "conversational/authoritative/structured",287"optimized_content": "the rewritten content...",288"key_changes": ["change 1", "change 2"],289"target_use_case": "description of ideal use case"290}}291```"""292 293 prompt_template = ChatPromptTemplate.from_messages([294 ("system", custom_prompt),295 ("user", "Generate the variation.")296 ])297 298 chain = prompt_template | self.llm299 result = chain.invoke({})300 301 result_content = result.content if hasattr(result, 'content') else str(result)302 parsed_result = self._parse_optimization_result(result_content)303 304 parsed_result.update({305 'variation_index': i,306 'variation_prompt': variation_prompts[i]307 })308 309 variations.append(parsed_result)310 311 except Exception as e:312 variations.append({313 'variation_index': i,314 'error': f"Variation generation failed: {str(e)}"315 })316 317 return variations318 319 def analyze_content_readability(self, content: str) -> Dict[str, Any]:320 """321 Analyze content readability for AI systems322 323 Args:324 content (str): Content to analyze325 326 Returns:327 Dict: Readability analysis results328 """329 try:330 # Basic readability metrics331 words = content.split()332 sentences = re.split(r'[.!?]+', content)333 sentences = [s.strip() for s in sentences if s.strip()]334 335 paragraphs = [p.strip() for p in content.split('\n\n') if p.strip()]336 337 # Calculate metrics338 avg_words_per_sentence = len(words) / len(sentences) if sentences else 0339 avg_sentences_per_paragraph = len(sentences) / len(paragraphs) if paragraphs else 0340 341 # Character-based metrics342 avg_word_length = sum(len(word) for word in words) / len(words) if words else 0343 344 # Complexity indicators345 long_sentences = [s for s in sentences if len(s.split()) > 20]346 complex_words = [w for w in words if len(w) > 6]347 348 return {349 'basic_metrics': {350 'total_words': len(words),351 'total_sentences': len(sentences),352 'total_paragraphs': len(paragraphs),353 'avg_words_per_sentence': avg_words_per_sentence,354 'avg_sentences_per_paragraph': avg_sentences_per_paragraph,355 'avg_word_length': avg_word_length356 },357 'complexity_indicators': {358 'long_sentences_count': len(long_sentences),359 'long_sentences_percentage': len(long_sentences) / len(sentences) * 100 if sentences else 0,360 'complex_words_count': len(complex_words),361 'complex_words_percentage': len(complex_words) / len(words) * 100 if words else 0362 },363 'ai_readability_score': self._calculate_ai_readability_score({364 'avg_words_per_sentence': avg_words_per_sentence,365 'avg_word_length': avg_word_length,366 'complex_words_percentage': len(complex_words) / len(words) * 100 if words else 0367 }),368 'recommendations': self._generate_readability_recommendations({369 'avg_words_per_sentence': avg_words_per_sentence,370 'long_sentences_percentage': len(long_sentences) / len(sentences) * 100 if sentences else 0,371 'complex_words_percentage': len(complex_words) / len(words) * 100 if words else 0372 })373 }374 375 except Exception as e:376 return {'error': f"Readability analysis failed: {str(e)}"}377 378 def extract_key_entities(self, content: str) -> Dict[str, Any]:379 """380 Extract key entities and topics for optimization381 382 Args:383 content (str): Content to analyze384 385 Returns:386 Dict: Extracted entities and topics387 """388 try:389 entity_prompt = """Extract key entities, topics, and concepts from this content for AI optimization.390 391Content: {content}392 393Identify:3941. Named entities (people, places, organizations)3952. Key concepts and topics3963. Technical terms and jargon3974. Potential semantic keywords3985. Question-answer opportunities399 400Format as JSON:401```json402{{403"named_entities": ["entity1", "entity2"],404"key_topics": ["topic1", "topic2"],405"technical_terms": ["term1", "term2"],406"semantic_keywords": ["keyword1", "keyword2"],407"question_opportunities": ["What is...", "How does..."],408"entity_relationships": ["relationship descriptions"]409}}410```"""411 412 prompt_template = ChatPromptTemplate.from_messages([413 ("system", entity_prompt.format(content=content[:5000])),414 ("user", "Extract the entities and topics.")415 ])416 417 chain = prompt_template | self.llm418 result = chain.invoke({})419 420 result_content = result.content if hasattr(result, 'content') else str(result)421 return self._parse_optimization_result(result_content)422 423 except Exception as e:424 return {'error': f"Entity extraction failed: {str(e)}"}425 426 def optimize_for_voice_search(self, content: str) -> Dict[str, Any]:427 """428 Optimize content specifically for voice search and conversational AI429 430 Args:431 content (str): Content to optimize432 433 Returns:434 Dict: Voice search optimization results435 """436 try:437 voice_prompt = """Optimize this content for voice search and conversational AI systems.438 439Focus on:4401. Natural language patterns4412. Question-based structure4423. Conversational tone4434. Clear, direct answers4445. Featured snippet optimization445 446Original content: {content}447 448Provide optimization in JSON:449```json450{{451"voice_optimized_content": "conversational version...",452"question_answer_pairs": [453 {{"question": "What is...", "answer": "Direct answer..."}},454 {{"question": "How does...", "answer": "Step by step..."}}455],456"featured_snippet_candidates": ["snippet 1", "snippet 2"],457"natural_language_improvements": ["improvement 1", "improvement 2"],458"conversational_score": 8.5459}}460```"""461 462 prompt_template = ChatPromptTemplate.from_messages([463 ("system", voice_prompt.format(content=content[:4000])),464 ("user", "Optimize for voice search.")465 ])466 467 chain = prompt_template | self.llm468 result = chain.invoke({})469 470 result_content = result.content if hasattr(result, 'content') else str(result)471 parsed_result = self._parse_optimization_result(result_content)472 473 parsed_result.update({474 'optimization_type': 'voice_search',475 'voice_optimized': True476 })477 478 return parsed_result479 480 except Exception as e:481 return {'error': f"Voice search optimization failed: {str(e)}"}482 483 def _parse_optimization_result(self, response_text: str) -> Dict[str, Any]:484 """Parse LLM response and extract structured results"""485 try:486 # Find JSON content in the response487 json_start = response_text.find('{')488 json_end = response_text.rfind('}') + 1489 490 if json_start != -1 and json_end != -1:491 json_str = response_text[json_start:json_end]492 parsed = json.loads(json_str)493 494 # Ensure consistent structure495 if 'scores' not in parsed and 'score' in parsed:496 parsed['scores'] = parsed['score']497 498 return parsed499 else:500 # If no JSON found, return raw response with error flag501 return {502 'raw_response': response_text,503 'parsing_error': 'No JSON structure found in response',504 'scores': {'clarity': 0, 'structuredness': 0, 'answerability': 0}505 }506 507 except json.JSONDecodeError as e:508 return {509 'raw_response': response_text,510 'parsing_error': f'JSON decode error: {str(e)}',511 'scores': {'clarity': 0, 'structuredness': 0, 'answerability': 0}512 }513 except Exception as e:514 return {515 'raw_response': response_text,516 'parsing_error': f'Unexpected parsing error: {str(e)}',517 'scores': {'clarity': 0, 'structuredness': 0, 'answerability': 0}518 }519 520 def _calculate_ai_readability_score(self, metrics: Dict[str, float]) -> float:521 """Calculate AI-specific readability score"""522 try:523 # Optimal ranges for AI consumption524 optimal_words_per_sentence = 15 # Sweet spot for AI processing525 optimal_word_length = 5 # Balance of complexity and clarity526 optimal_complex_words_percentage = 15 # Some complexity is good for authority527 528 # Calculate deviations from optimal529 sentence_score = max(0, 10 - abs(metrics['avg_words_per_sentence'] - optimal_words_per_sentence) * 0.5)530 word_length_score = max(0, 10 - abs(metrics['avg_word_length'] - optimal_word_length) * 2)531 complexity_score = max(0, 10 - abs(metrics['complex_words_percentage'] - optimal_complex_words_percentage) * 0.3)532 533 # Weighted average534 overall_score = (sentence_score * 0.4 + word_length_score * 0.3 + complexity_score * 0.3)535 536 return round(overall_score, 1)537 538 except Exception:539 return 5.0 # Default neutral score540 541 def _generate_readability_recommendations(self, metrics: Dict[str, float]) -> List[str]:542 """Generate specific readability improvement recommendations"""543 recommendations = []544 545 try:546 if metrics['avg_words_per_sentence'] > 20:547 recommendations.append("Break down long sentences for better AI processing")548 elif metrics['avg_words_per_sentence'] < 8:549 recommendations.append("Consider combining very short sentences for better context")550 551 if metrics['long_sentences_percentage'] > 30:552 recommendations.append("Reduce the number of complex sentences (>20 words)")553 554 if metrics['complex_words_percentage'] > 25:555 recommendations.append("Simplify vocabulary where possible for broader accessibility")556 elif metrics['complex_words_percentage'] < 5:557 recommendations.append("Add more specific terminology to establish authority")558 559 return recommendations560 561 except Exception:562 return ["Unable to generate specific recommendations"]