lyzq/malicious-url-app

1from __future__ import annotations2 3import math4import re5from collections import Counter6from typing import Iterable7from urllib.parse import urlsplit8 9import numpy as np10import pandas as pd11from sklearn.base import BaseEstimator, TransformerMixin12 13 14SUSPICIOUS_TOKENS = (15    "login",16    "verify",17    "update",18    "account",19    "secure",20    "signin",21    "banking",22    "confirm",23    "password",24    "paypal",25    "invoice",26    "wallet",27    "bonus",28    "gift",29    "free",30)31 32SHORTENERS = {33    "bit.ly",34    "tinyurl.com",35    "goo.gl",36    "t.co",37    "ow.ly",38    "is.gd",39    "buff.ly",40    "cutt.ly",41    "rebrand.ly",42    "lnkd.in",43}44 45TLD_PATTERN = re.compile(r"\.([a-z]{2,24})$")46IPV4_PATTERN = re.compile(r"(?:\d{1,3}\.){3}\d{1,3}")47 48 49def standardize_url(url: object) -> str:50    if url is None:51        return ""52    text = str(url).strip().strip('"').strip("'")53    if not text:54        return ""55    return text.replace("\\", "/").lower()56 57 58def ensure_parsable_url(url: str) -> str:59    if not url:60        return ""61    return url if "://" in url else f"http://{url}"62 63 64def shannon_entropy(text: str) -> float:65    if not text:66        return 0.067    counts = Counter(text)68    length = len(text)69    return -sum((count / length) * math.log2(count / length) for count in counts.values())70 71 72def safe_ratio(numerator: float, denominator: float) -> float:73    return float(numerator) / float(denominator) if denominator else 0.074 75 76def lexical_signals(url: str) -> dict[str, float]:77    clean_url = standardize_url(url)78    parsed = urlsplit(ensure_parsable_url(clean_url))79 80    hostname = parsed.netloc.split("@")[-1]81    path = parsed.path or ""82    query = parsed.query or ""83    fragment = parsed.fragment or ""84    full_path = f"{path}?{query}" if query else path85 86    digit_count = sum(char.isdigit() for char in clean_url)87    alpha_count = sum(char.isalpha() for char in clean_url)88    special_count = sum(not char.isalnum() for char in clean_url)89    at_count = clean_url.count("@")90 91    host_parts = [part for part in hostname.split(".") if part]92    subdomain_count = max(len(host_parts) - 2, 0)93    tld_match = TLD_PATTERN.search(hostname)94    tld_length = len(tld_match.group(1)) if tld_match else 095    path_depth = len([part for part in path.split("/") if part])96 97    suspicious_token_count = sum(token in clean_url for token in SUSPICIOUS_TOKENS)98    brand_count = sum(token in clean_url for token in ("paypal", "microsoft", "apple", "google", "amazon"))99 100    return {101        "url_length": len(clean_url),102        "hostname_length": len(hostname),103        "path_length": len(path),104        "query_length": len(query),105        "fragment_length": len(fragment),106        "path_depth": path_depth,107        "subdomain_count": subdomain_count,108        "dot_count": clean_url.count("."),109        "digit_count": digit_count,110        "alpha_count": alpha_count,111        "special_count": special_count,112        "hyphen_count": clean_url.count("-"),113        "underscore_count": clean_url.count("_"),114        "slash_count": clean_url.count("/"),115        "ampersand_count": clean_url.count("&"),116        "eq_count": clean_url.count("="),117        "percent_count": clean_url.count("%"),118        "at_count": at_count,119        "suspicious_token_count": suspicious_token_count,120        "brand_token_count": brand_count,121        "contains_ip": float(bool(IPV4_PATTERN.search(hostname))),122        "contains_punycode": float("xn--" in hostname),123        "contains_https_token": float("https" in clean_url),124        "uses_https_scheme": float(parsed.scheme == "https"),125        "has_query": float(bool(query)),126        "has_fragment": float(bool(fragment)),127        "has_at_symbol": float(at_count > 0),128        "has_double_slash_in_path": float("//" in full_path),129        "is_shortener": float(hostname in SHORTENERS),130        "tld_length": tld_length,131        "digit_ratio": safe_ratio(digit_count, len(clean_url)),132        "special_ratio": safe_ratio(special_count, len(clean_url)),133        "entropy": shannon_entropy(clean_url),134    }135 136 137def extract_lexical_features(urls: Iterable[object]) -> pd.DataFrame:138    rows = [lexical_signals(standardize_url(url)) for url in urls]139    return pd.DataFrame(rows).fillna(0.0)140 141 142def summarize_url_signals(url: str) -> list[str]:143    features = lexical_signals(url)144    messages: list[str] = []145    if features["contains_ip"]:146        messages.append("Host appears to use a raw IP address.")147    if features["suspicious_token_count"] >= 2:148        messages.append("URL contains multiple high-risk lure words like login, verify, or secure.")149    if features["url_length"] >= 75:150        messages.append("URL is unusually long, which often hides redirect or phishing paths.")151    if features["subdomain_count"] >= 3:152        messages.append("URL uses several subdomains, a common obfuscation trick.")153    if features["has_at_symbol"]:154        messages.append("URL contains '@', which can hide the true destination.")155    if features["uses_https_scheme"] == 0 and features["contains_https_token"]:156        messages.append("The text contains 'https' but the actual scheme is not HTTPS.")157    if features["is_shortener"]:158        messages.append("URL uses a shortening service that can conceal the final destination.")159    if not messages:160        messages.append("No obvious lexical red flags were triggered by the heuristic layer.")161    return messages162 163 164class LexicalFeatureExtractor(BaseEstimator, TransformerMixin):165    def fit(self, X: Iterable[object], y: object = None) -> "LexicalFeatureExtractor":166        return self167 168    def transform(self, X: Iterable[object]) -> np.ndarray:169        frame = extract_lexical_features(X)170        return frame.to_numpy(dtype=float)171 172