samdo20/website-contact

1# main.py2 3import asyncio4from concurrent.futures import ThreadPoolExecutor5from flask import Flask, request, jsonify6from bs4 import BeautifulSoup7import uuid8import logging9import os10from urllib.parse import urlparse11from utils.email_extractor import extract_emails_html, extract_emails_jsonld12from utils.phone_extractor import extract_phones_html, extract_phones_jsonld, validate_phones13from utils.social_links import extract_social_links_jsonld14from utils.link_scraper import link_scraper, extract_links, is_valid_url15from utils.user_agent import get_user_agent_headers  # Ensure this import is present16from utils.link_analyzer import analyze_links17import requests18from email_validator import validate_email, EmailNotValidError19 20app = Flask(__name__)21SCRIPT_VERSION = "V 1.5 // commit"22 23logging.basicConfig(level=logging.INFO)24logger = logging.getLogger(__name__)25 26def analyze_links_parallel(links, headers):27    valid_links = [link for link in links if is_valid_url(link)]28    with ThreadPoolExecutor() as executor:29        loop = asyncio.new_event_loop()30        asyncio.set_event_loop(loop)31        tasks = [32            loop.run_in_executor(executor, analyze_links, link, headers)33            for link in valid_links34        ]35        results = loop.run_until_complete(asyncio.gather(*tasks))36    return results37 38@app.route('/scrape', methods=['GET'])39def scrape():40    url = request.args.get('url')41    headers = get_user_agent_headers()42 43    include_emails = request.args.get('include_emails', 'true').lower() == 'true'44    include_phones = request.args.get('include_phones', 'true').lower() == 'true'45    include_social_links = request.args.get('include_social_links', 'true').lower() == 'true'46    include_unique_links = request.args.get('include_unique_links', 'true').lower() == 'true'47 48    links, error = link_scraper(url, headers)49    if error:50        return jsonify({'error': error}), 50051 52    domain = urlparse(url).netloc53 54    emails, phones, visited_links = {}, {}, set()55    if include_emails or include_phones or include_unique_links:56        results = analyze_links_parallel(links, headers)57        for result in results:58            emails.update(result[0])59            phones.update(result[1])60            visited_links.update(result[2])61 62    social_links = {}63    if include_social_links:64        response = requests.get(url, headers=headers)65        if response.status_code == 200:66            soup = BeautifulSoup(response.text, 'html.parser')67            social_links = extract_social_links_jsonld(soup)68        else:69            return jsonify({'error': 'Failed to fetch the URL'}), 50070 71    result = {72        "request_id": str(uuid.uuid4()),73        "domain": url.split("//")[-1].split("/")[0],74        "query": url,75        "status": "OK",76        "data": [77            {78                "emails": [{"value": email, "sources": sources} for email, sources in emails.items() if validate_email_address(email)] if include_emails else [],79                "phone_numbers": [{"value": phone, "sources": sources} for phone, sources in phones.items()] if include_phones else [],80                "social_links": social_links if include_social_links else {},81                "unique_links": sorted(list(visited_links)) if include_unique_links else []82            }83        ]84    }85 86    logger.info(f"Processed URL: {url}")87    return jsonify(result)88 89def validate_email_address(email):90    try:91        validate_email(email)92        return True93    except EmailNotValidError:94        return False95 96@app.route('/', methods=['GET'])97def read_root():98    return jsonify({"message": "API is live. Use the /predict endpoint."}), 20099 100@app.route('/health', methods=['GET'])101def health_check():102    return jsonify({"status": "healthy"}), 200103