salmareda999988/R_D
0
1import streamlit as st2import requests3import json4import os5import sys6import subprocess7import asyncio8import aiohttp9import logging10from datetime import datetime11from langdetect import detect12import speech_recognition as sr13from gtts import gTTS14import tempfile15import pandas as pd16import matplotlib.pyplot as plt17import base6418from transformers import AutoTokenizer, AutoModel19import torch20import numpy as np21from typing import List, Dict, Any22from dotenv import load_dotenv23 24# Import custom modules25from construction_news_fetcher import ConstructionNewsFetcher26from job_market_analyzer import JobMarketAnalyzer27 28# Set up logging29logging.basicConfig(level=logging.INFO)30 31# Load environment variables32load_dotenv()33 34# --- Gemini API Setup ---35GEMINI_AVAILABLE = False36try:37 import google.generativeai as genai38 GEMINI_AVAILABLE = True39except ImportError:40 try:41 subprocess.check_call([sys.executable, "-m", "pip", "install", "google-generativeai"])42 import google.generativeai as genai43 GEMINI_AVAILABLE = True44 except Exception as e:45 st.error("Gemini API is not available. Please install google-generativeai.")46 47GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "")48if GEMINI_AVAILABLE and GEMINI_API_KEY:49 genai.configure(api_key=GEMINI_API_KEY)50 51# Other API keys52HUGGINGFACE_API_KEY = os.getenv("HUGGINGFACE_API_KEY", "")53GNEWS_API_KEY = os.getenv("GNEWS_API_KEY", "")54 55# Set Streamlit page configuration56st.set_page_config(page_title="Construction Industry HR Trends Chatbot", page_icon="👷♂️", layout="wide")57 58# --- Caching for Expensive Operations ---59 60@st.cache_data(show_spinner=False)61def get_cached_embeddings(texts):62 try:63 tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base", use_auth_token=HUGGINGFACE_API_KEY)64 model = AutoModel.from_pretrained("xlm-roberta-base", use_auth_token=HUGGINGFACE_API_KEY)65 device = "cuda" if torch.cuda.is_available() else "cpu"66 model = model.to(device)67 embeddings = []68 for text in texts:69 inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)70 inputs = {k: v.to(device) for k, v in inputs.items()}71 with torch.no_grad():72 outputs = model(**inputs)73 embedding = outputs.last_hidden_state[:, 0, :].cpu().numpy()74 embeddings.append(embedding[0])75 return embeddings76 except Exception as e:77 logging.error("Error in get_cached_embeddings: %s", e)78 return [np.random.rand(768) for _ in texts]79 80@st.cache_data(show_spinner=False)81def translate_text_cached(text, source_lang, target_lang):82 if source_lang == "en" and target_lang == "ar":83 model_name = "Helsinki-NLP/opus-mt-en-ar"84 elif source_lang == "ar" and target_lang == "en":85 model_name = "Helsinki-NLP/opus-mt-ar-en"86 else:87 return text88 API_URL = f"https://api-inference.huggingface.co/models/{model_name}"89 headers = {"Authorization": f"Bearer {HUGGINGFACE_API_KEY}"}90 payload = {"inputs": text}91 try:92 response = requests.post(API_URL, headers=headers, json=payload, timeout=10)93 result = response.json()94 if isinstance(result, list) and len(result) > 0:95 return result[0]["translation_text"]96 return text97 except Exception as e:98 logging.error("Translation error: %s", e)99 return text100 101# --- Asynchronous Helper for News (if needed) ---102async def fetch_news_async(url, headers):103 async with aiohttp.ClientSession() as session:104 async with session.get(url, headers=headers, timeout=10) as response:105 return await response.json()106 107# --- Construction Market Research Class ---108class ConstructionMarketResearch:109 """Handles construction market data and report generation."""110 111 def __init__(self, api_key=None):112 self.api_key = api_key113 self.session = requests.Session()114 self.session.headers.update({'User-Agent': 'Mozilla/5.0'})115 self.load_market_data()116 117 def load_market_data(self):118 self.market_growth = {119 'GCC': 5.2,120 'Middle East (other)': 3.8,121 'North Africa': 2.9,122 'Europe': 1.7,123 'Asia-Pacific': 4.5,124 'North America': 2.2125 }126 self.sector_performance = {127 'Residential': 3.7,128 'Commercial': 2.9,129 'Infrastructure': 5.4,130 'Industrial': 3.2,131 'Energy': 4.1132 }133 self.workforce_challenges = {134 'Skilled labor shortage': 8.7,135 'Safety compliance': 7.5,136 'Workforce retention': 7.2,137 'Training and certification': 6.9,138 'Remote site management': 7.8,139 'Multilingual workforce': 8.3,140 'Heat stress management': 8.6,141 'Competitive compensation': 6.5142 }143 self.salary_data = {144 'Project Manager': 85,145 'Construction Manager': 78,146 'Site Engineer': 52,147 'Safety Manager': 68,148 'Foreman': 48,149 'Skilled Tradesperson': 42,150 'HR Manager (Construction)': 72,151 'Construction Recruiter': 55152 }153 154 def analyze_keywords(self, keywords):155 results = {}156 keywords_lower = [k.lower() for k in keywords]157 regions = {158 'gcc': 'GCC', 159 'middle east': 'Middle East (other)',160 'north africa': 'North Africa', 161 'europe': 'Europe',162 'asia': 'Asia-Pacific', 163 'america': 'North America'164 }165 for key, region in regions.items():166 if any(key in kw for kw in keywords_lower):167 results['market_growth'] = {region: self.market_growth[region]}168 sectors = {169 'residential': 'Residential',170 'commercial': 'Commercial',171 'infrastructure': 'Infrastructure',172 'industrial': 'Industrial',173 'energy': 'Energy'174 }175 for key, sector in sectors.items():176 if any(key in kw for kw in keywords_lower):177 results['sector_performance'] = {sector: self.sector_performance[sector]}178 workforce_keys = {179 'skill': 'Skilled labor shortage',180 'safety': 'Safety compliance',181 'retention': 'Workforce retention',182 'training': 'Training and certification',183 'remote': 'Remote site management',184 'language': 'Multilingual workforce',185 'multilingual': 'Multilingual workforce',186 'heat': 'Heat stress management',187 'compensation': 'Competitive compensation'188 }189 for key, challenge in workforce_keys.items():190 if any(key in kw for kw in keywords_lower):191 if 'workforce_challenges' not in results:192 results['workforce_challenges'] = {}193 results['workforce_challenges'][challenge] = self.workforce_challenges[challenge]194 if any(kw in ['salary', 'compensation', 'pay', 'wage'] for kw in keywords_lower):195 results['salary_data'] = self.salary_data196 roles = {197 'manager': ['Project Manager', 'Construction Manager', 'Safety Manager', 'HR Manager (Construction)'],198 'engineer': ['Site Engineer'],199 'safety': ['Safety Manager'],200 'hr': ['HR Manager (Construction)', 'Construction Recruiter'],201 'recruit': ['Construction Recruiter'],202 'trade': ['Skilled Tradesperson'],203 'foreman': ['Foreman']204 }205 for key, role_list in roles.items():206 if any(key in kw for kw in keywords_lower):207 results['salary_data'] = {role: self.salary_data[role] for role in role_list}208 return results209 210 def generate_market_report(self, query):211 keywords = query.lower().split()212 analysis = self.analyze_keywords(keywords)213 report = "Construction Market Research Report\n" + "=" * 40 + "\n\n"214 if 'market_growth' in analysis:215 report += "Market Growth:\n"216 for region, growth in analysis['market_growth'].items():217 report += f"- {region}: {growth}% year-over-year growth\n"218 report += "\n"219 if 'sector_performance' in analysis:220 report += "Sector Performance:\n"221 for sector, performance in analysis['sector_performance'].items():222 report += f"- {sector}: {performance}% growth\n"223 report += "\n"224 if 'workforce_challenges' in analysis:225 report += "Workforce Challenges (scale 1-10):\n"226 for challenge, rating in analysis['workforce_challenges'].items():227 report += f"- {challenge}: {rating}/10\n"228 report += "\n"229 if 'salary_data' in analysis:230 report += "Salary Data (thousands USD):\n"231 for role, salary in analysis['salary_data'].items():232 report += f"- {role}: ${salary}k per year\n"233 report += "\n"234 if not analysis:235 report += "Overall Construction Market Overview:\n"236 report += f"- Average market growth: {sum(self.market_growth.values())/len(self.market_growth):.1f}%\n"237 report += f"- Top performing sector: {max(self.sector_performance.items(), key=lambda x: x[1])[0]}\n"238 report += f"- Most significant workforce challenge: {max(self.workforce_challenges.items(), key=lambda x: x[1])[0]}\n"239 report += f"- Average construction management salary: ${sum(self.salary_data.values())/len(self.salary_data):.0f}k\n"240 return report241 242# --- Gemini Query Function ---243def query_with_gemini(question, lang="en"):244 if not GEMINI_AVAILABLE or not GEMINI_API_KEY:245 return None246 try:247 language_prompt = "in English" if lang == "en" else "in Arabic"248 prompt = f"""You are an expert HR advisor specializing in construction industry HR trends, safety regulations, workforce management for construction projects, and skill development in the building trades. Answer the following construction HR-related question {language_prompt}. Be concise but informative. Include relevant statistics or best practices specific to the construction industry if appropriate.249 250Question: {question}251"""252 model_instance = genai.GenerativeModel('gemini-1.0-pro')253 generation_config = {254 "temperature": 0.7,255 "top_p": 0.95,256 "top_k": 40,257 "max_output_tokens": 1024,258 }259 response = model_instance.generate_content(prompt, generation_config=generation_config)260 if response and hasattr(response, 'text'):261 return response.text262 else:263 return None264 except Exception as e:265 logging.error("Gemini API error: %s", e)266 st.error(f"Gemini API error: {e}")267 return None268 269# --- HR Knowledge Base & Query Processing ---270def query_hr_knowledge_base(query, lang="en"):271 gemini_response = query_with_gemini(query, lang) if GEMINI_AVAILABLE else None272 if gemini_response:273 return gemini_response274 hr_knowledge = [275 {276 "en": "Construction safety management involves implementing safety regulations, conducting regular site inspections, and enforcing PPE compliance to protect workers on construction sites.",277 "ar": "تتضمن إدارة السلامة في البناء تطبيق لوائح السلامة، وإجراء عمليات تفتيش منتظمة للموقع، وفرض الامتثال لمعدات الحماية الشخصية لحماية العمال في مواقع البناء."278 },279 {280 "en": "Skilled trades recruitment in construction focuses on finding qualified electricians, plumbers, carpenters, welders, and equipment operators who have the necessary certifications and experience.",281 "ar": "يركز توظيف الحرفيين المهرة في قطاع البناء على العثور على كهربائيين ومُركّبي أنابيب ونجارين ولحامين ومشغلي معدات مؤهلين يمتلكون الشهادات والخبرة اللازمة."282 },283 {284 "en": "Project-based staffing in construction requires flexible HR strategies to manage crews that move between sites and adjust workforce levels based on project phases.",285 "ar": "يتطلب التوظيف على أساس المشروع في البناء استراتيجيات مرنة للموارد البشرية لإدارة الفرق التي تتنقل بين المواقع وتعديل مستويات القوى العاملة بناءً على مراحل المشروع."286 },287 {288 "en": "Construction labor compliance includes managing certified payroll, prevailing wage requirements, and adherence to local labor laws for construction projects.",289 "ar": "يشمل الامتثال لقوانين العمل في البناء إدارة كشوف المرتبات المعتمدة، ومتطلبات الأجور السائدة، والالتزام بقوانين العمل المحلية لمشاريع البناء."290 },291 {292 "en": "Safety training in construction is mandatory and includes certification, fall protection, hazard communication, and equipment-specific training to prevent workplace accidents.",293 "ar": "التدريب على السلامة في البناء إلزامي ويشمل الشهادات، والحماية من السقوط، والتوعية بالمخاطر، والتدريب الخاص بالمعدات لمنع حوادث العمل."294 },295 {296 "en": "Construction HR departments often manage multicultural workforces and must develop strategies for clear communication across language barriers to ensure safety and productivity.",297 "ar": "غالبًا ما تدير إدارات الموارد البشرية في البناء قوى عاملة متعددة الثقافات ويجب أن تطور استراتيجيات للتواصل الواضح عبر حواجز اللغة لضمان السلامة والإنتاجية."298 },299 {300 "en": "Heat stress management is a critical HR function in construction, especially in Middle Eastern countries, requiring proper hydration protocols, rest schedules, and monitoring systems.",301 "ar": "تعتبر إدارة الإجهاد الحراري وظيفة حيوية للموارد البشرية في البناء، خاصة في دول الشرق الأوسط، مما يتطلب بروتوكولات ترطيب مناسبة وجداول راحة وأنظمة مراقبة."302 },303 {304 "en": "Construction site access control systems are increasingly using biometric technology to verify worker identity, track hours accurately, and ensure only qualified personnel access restricted areas.",305 "ar": "تستخدم أنظمة التحكم في الوصول إلى مواقع البناء بشكل متزايد تقنية القياسات الحيوية للتحقق من هوية العامل، وتتبع الساعات بدقة، وضمان وصول الموظفين المؤهلين فقط إلى المناطق المقيدة."306 }307 ]308 try:309 query_embedding = get_cached_embeddings([query])[0]310 knowledge_texts = [entry[lang] for entry in hr_knowledge]311 knowledge_embeddings = get_cached_embeddings(knowledge_texts)312 similarities = []313 for emb in knowledge_embeddings:314 similarity = np.dot(query_embedding, emb) / (np.linalg.norm(query_embedding) * np.linalg.norm(emb))315 similarities.append(similarity)316 best_match_idx = np.argmax(similarities)317 best_match = hr_knowledge[best_match_idx][lang]318 except Exception as e:319 logging.error("Error in HR knowledge base matching: %s", e)320 query_terms = query.lower().split()321 best_match_idx = 0322 best_match_score = 0323 for idx, entry in enumerate(hr_knowledge):324 entry_text = entry[lang].lower()325 score = sum(1 for term in query_terms if term in entry_text)326 if score > best_match_score:327 best_match_score = score328 best_match_idx = idx329 best_match = hr_knowledge[best_match_idx][lang]330 return best_match331 332def process_query(query, lang="en"):333 base_response = query_hr_knowledge_base(query, lang)334 if lang == "en":335 response = "# Construction HR Analysis Report\n\n"336 response += "## Expert Overview\n"337 response += base_response + "\n\n"338 else:339 response = "# تقرير تحليل الموارد البشرية في قطاع البناء\n\n"340 response += "## نظرة خبير عامة\n"341 response += base_response + "\n\n"342 if "safety" in query.lower() or "سلامة" in query:343 if lang == "en":344 response += "## Safety Compliance Insights\n"345 response += "Construction industries globally are seeing increased focus on safety protocols, with:\n"346 response += "* 67% of companies implementing advanced PPE monitoring systems\n"347 response += "* 82% increase in safety training hours per worker annually\n"348 response += "* 42% reduction in incidents at sites using AI-powered safety monitoring\n\n"349 else:350 response += "## رؤى الامتثال للسلامة\n"351 response += "تشهد صناعات البناء عالميًا تركيزًا متزايدًا على بروتوكولات السلامة، مع:\n"352 response += "* 67% من الشركات تنفذ أنظمة متقدمة لمراقبة معدات الحماية الشخصية\n"353 response += "* زيادة بنسبة 82% في ساعات تدريب السلامة لكل عامل سنويًا\n"354 response += "* انخفاض بنسبة 42% في الحوادث في المواقع التي تستخدم أنظمة مراقبة السلامة المدعومة بالذكاء الاصطناعي\n\n"355 if "recruit" in query.lower() or "توظيف" in query or "talent" in query or "مواهب" in query:356 if lang == "en":357 response += "## Recruitment Strategy Best Practices\n"358 response += "Leading construction firms are revolutionizing their hiring approaches with:\n"359 response += "* Trade-specific assessment tools reducing mismatched hires by 34%\n"360 response += "* Partnerships with technical schools increasing qualified candidate pools by 58%\n"361 response += "* Apprenticeship programs showing 72% retention after 3 years vs. 41% for traditional hires\n\n"362 else:363 response += "## أفضل ممارسات استراتيجية التوظيف\n"364 response += "تقوم شركات البناء الرائدة بثورة في نهجها للتوظيف من خلال:\n"365 response += "* أدوات تقييم خاصة بالمهن تقلل من التعيينات غير المتطابقة بنسبة 34%\n"366 response += "* شراكات مع المدارس الفنية تزيد من مجموعات المرشحين المؤهلين بنسبة 58%\n"367 response += "* برامج التدريب المهني تُظهر احتفاظًا بنسبة 72% بعد 3 سنوات مقابل 41% للتوظيف التقليدي\n\n"368 # Append industry news using ConstructionNewsFetcher369 news_fetcher = ConstructionNewsFetcher(gnews_api_key=GNEWS_API_KEY)370 news_articles = news_fetcher.fetch_hr_construction_news(query=query, language=lang, max_results=3)371 news_text = ""372 if news_articles:373 if lang == "en":374 news_text = "\n## Latest Industry Developments\n"375 else:376 news_text = "\n## أحدث التطورات في الصناعة\n"377 for i, article in enumerate(news_articles, 1):378 news_text += f"{i}. **{article['title']}** - {article['description']}\n"379 response += news_text380 if lang == "en":381 response += "\n## Strategic Recommendations\n"382 response += "Based on current industry trends and your specific query, we recommend:\n"383 response += "1. **Implement digital competency training** for field supervisors\n"384 response += "2. **Develop multilingual safety protocols** to address diverse workforce needs\n"385 response += "3. **Establish structured career pathways** for skilled trades to improve retention\n"386 response += "4. **Adopt mobile-first HR tools** for better field workforce management\n"387 else:388 response += "\n## توصيات استراتيجية\n"389 response += "بناءً على اتجاهات الصناعة الحالية واستفسارك المحدد، نوصي بما يلي:\n"390 response += "1. **تنفيذ تدريب الكفاءة الرقمية** للمشرفين الميدانيين\n"391 response += "2. **تطوير بروتوكولات سلامة متعددة اللغات** لتلبية احتياجات القوى العاملة المتنوعة\n"392 response += "3. **إنشاء مسارات وظيفية منظمة** للحرف الماهرة لتحسين الاحتفاظ\n"393 response += "4. **اعتماد أدوات موارد بشرية تعتمد على الأجهزة المحمولة** لإدارة أفضل للقوى العاملة الميدانية\n"394 return response395 396# --- Voice and Text-to-Speech Functions ---397def voice_input():398 r = sr.Recognizer()399 with sr.Microphone() as source:400 st.write("Listening...")401 audio = r.listen(source)402 try:403 text = r.recognize_google(audio)404 lang = detect(text)405 if lang == "ar":406 text = r.recognize_google(audio, language="ar-AR")407 else:408 text = r.recognize_google(audio, language="en-US")409 return text, lang410 except Exception as e:411 logging.error("Voice input error: %s", e)412 st.error(f"Could not recognize speech: {e}")413 return None, "en"414 415def text_to_speech(text, lang):416 try:417 tts = gTTS(text=text, lang=lang[:2])418 with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as fp:419 tts.save(fp.name)420 return fp.name421 except Exception as e:422 logging.error("Text to speech error: %s", e)423 st.error(f"Text to speech error: {e}")424 return None425 426# --- Chat Interaction and UI Elements ---427def chat_interaction(user_input=None, is_voice=False):428 if user_input:429 lang = detect(user_input)430 st.session_state.messages.append({"role": "user", "content": user_input, "language": lang})431 response = process_query(user_input, lang)432 st.session_state.messages.append({"role": "assistant", "content": response, "language": lang})433 st.session_state.history.append({"query": user_input, "response": response, "language": lang})434 if is_voice:435 audio_file = text_to_speech(response, lang)436 if audio_file:437 st.audio(audio_file)438 st.markdown(response)439 440# --- Sidebar Options and Interactive Widgets ---441with st.sidebar:442 st.subheader("Options")443 preferred_lang = st.radio("Preferred Language:", ["English", "Arabic"])444 use_voice = st.checkbox("Enable Voice Interaction")445 if use_voice and st.button("🎤 Speak"):446 user_input, detected_lang = voice_input()447 if user_input:448 chat_interaction(user_input, is_voice=True)449 st.subheader("Data Sources")450 use_news = st.checkbox("Include News Data", value=True)451 use_job_market = st.checkbox("Include Job Market Insights", value=True)452 use_gemini = st.checkbox("Use Gemini AI", 453 value=GEMINI_AVAILABLE and GEMINI_API_KEY,454 disabled=not (GEMINI_AVAILABLE and GEMINI_API_KEY))455 if not GEMINI_AVAILABLE:456 st.warning("Gemini API is not available. Run 'pip install google-generativeai' to enable it.")457 elif not GEMINI_API_KEY and use_gemini:458 st.info("Gemini integration requires an API key. Please add it in your environment variables.")459 if use_job_market:460 st.subheader("Job Market Insights")461 job_market_search = st.text_input("Search for HR trends in:")462 if job_market_search and st.button("Analyze Job Market"):463 with st.spinner("Analyzing job market data..."):464 job_market_analyzer = JobMarketAnalyzer()465 job_market_data = job_market_analyzer.analyze_hr_trends(job_market_search)466 st.write("**Top Skills in Demand:**")467 for skill, count in job_market_data['top_skills'][:5]:468 st.write(f"- {skill}: {count} mentions")469 st.write("**Top Hiring Companies:**")470 for company, count in job_market_data['top_companies'][:3]:471 st.write(f"- {company}: {count} jobs")472 st.write("**Top Locations:**")473 for location, count in job_market_data['top_locations'][:3]:474 st.write(f"- {location}: {count} jobs")475 st.subheader("Market Research")476 market_query = st.text_input("Research construction market:")477 if market_query and st.button("Generate Market Report"):478 with st.spinner("Generating market research report..."):479 market_research = ConstructionMarketResearch()480 report = market_research.generate_market_report(market_query)481 st.text_area("Market Research Report", report, height=300)482 st.subheader("About")483 st.markdown("""484 This specialized Construction HR chatbot combines multiple AI technologies:485 486 - XLM-RoBERTa for multilingual understanding487 - Gemini AI for advanced construction HR knowledge (when available)488 - Construction Job Market Analysis for real-world insights489 - News APIs and RSS feeds for the latest construction industry trends490 - Market Research data for construction sector analysis491 492 It supports both English and Arabic and provides voice interaction, focusing on construction-specific 493 HR challenges like safety compliance, skilled trades recruitment, project staffing, and field workforce management.494 """)495 496# --- Main Chat Interface ---497if "messages" not in st.session_state:498 st.session_state.messages = []499if "history" not in st.session_state:500 st.session_state.history = []501 502st.title("🏗️ Construction Industry HR Trends Chatbot")503st.subheader("Ask questions about construction HR trends in English or Arabic")504 505user_input = st.chat_input("Ask something about construction HR trends...")506if user_input:507 chat_interaction(user_input)508 509# --- Visualization Section: Interactive Plotly Chart ---510if st.session_state.messages and len(st.session_state.messages) > 2:511 st.subheader("Interactive HR Trends Visualization")512 data = {513 'Trend': ['Safety Technology', 'Mobile Workforce Management', 'Skilled Trades Training', 'Compliance Automation', 'Multilingual Safety Programs'],514 'Adoption Rate': [82, 71, 65, 58, 49]515 }516 df = pd.DataFrame(data)517 import plotly.express as px518 fig = px.bar(df, x='Trend', y='Adoption Rate', title="Construction HR Trends 2023-2024")519 st.plotly_chart(fig)520 521st.markdown("---")522st.markdown("Specialized for Construction Industry HR")523