Aluode/PerceptionLabPortable

1"""Handwritten parser of dependency specifiers.2 3The docstring for each __parse_* function contains EBNF-inspired grammar representing4the implementation.5"""6 7from __future__ import annotations8 9import ast10from typing import NamedTuple, Sequence, Tuple, Union11 12from ._tokenizer import DEFAULT_RULES, Tokenizer13 14 15class Node:16    def __init__(self, value: str) -> None:17        self.value = value18 19    def __str__(self) -> str:20        return self.value21 22    def __repr__(self) -> str:23        return f"<{self.__class__.__name__}('{self}')>"24 25    def serialize(self) -> str:26        raise NotImplementedError27 28 29class Variable(Node):30    def serialize(self) -> str:31        return str(self)32 33 34class Value(Node):35    def serialize(self) -> str:36        return f'"{self}"'37 38 39class Op(Node):40    def serialize(self) -> str:41        return str(self)42 43 44MarkerVar = Union[Variable, Value]45MarkerItem = Tuple[MarkerVar, Op, MarkerVar]46MarkerAtom = Union[MarkerItem, Sequence["MarkerAtom"]]47MarkerList = Sequence[Union["MarkerList", MarkerAtom, str]]48 49 50class ParsedRequirement(NamedTuple):51    name: str52    url: str53    extras: list[str]54    specifier: str55    marker: MarkerList | None56 57 58# --------------------------------------------------------------------------------------59# Recursive descent parser for dependency specifier60# --------------------------------------------------------------------------------------61def parse_requirement(source: str) -> ParsedRequirement:62    return _parse_requirement(Tokenizer(source, rules=DEFAULT_RULES))63 64 65def _parse_requirement(tokenizer: Tokenizer) -> ParsedRequirement:66    """67    requirement = WS? IDENTIFIER WS? extras WS? requirement_details68    """69    tokenizer.consume("WS")70 71    name_token = tokenizer.expect(72        "IDENTIFIER", expected="package name at the start of dependency specifier"73    )74    name = name_token.text75    tokenizer.consume("WS")76 77    extras = _parse_extras(tokenizer)78    tokenizer.consume("WS")79 80    url, specifier, marker = _parse_requirement_details(tokenizer)81    tokenizer.expect("END", expected="end of dependency specifier")82 83    return ParsedRequirement(name, url, extras, specifier, marker)84 85 86def _parse_requirement_details(87    tokenizer: Tokenizer,88) -> tuple[str, str, MarkerList | None]:89    """90    requirement_details = AT URL (WS requirement_marker?)?91                        | specifier WS? (requirement_marker)?92    """93 94    specifier = ""95    url = ""96    marker = None97 98    if tokenizer.check("AT"):99        tokenizer.read()100        tokenizer.consume("WS")101 102        url_start = tokenizer.position103        url = tokenizer.expect("URL", expected="URL after @").text104        if tokenizer.check("END", peek=True):105            return (url, specifier, marker)106 107        tokenizer.expect("WS", expected="whitespace after URL")108 109        # The input might end after whitespace.110        if tokenizer.check("END", peek=True):111            return (url, specifier, marker)112 113        marker = _parse_requirement_marker(114            tokenizer, span_start=url_start, after="URL and whitespace"115        )116    else:117        specifier_start = tokenizer.position118        specifier = _parse_specifier(tokenizer)119        tokenizer.consume("WS")120 121        if tokenizer.check("END", peek=True):122            return (url, specifier, marker)123 124        marker = _parse_requirement_marker(125            tokenizer,126            span_start=specifier_start,127            after=(128                "version specifier"129                if specifier130                else "name and no valid version specifier"131            ),132        )133 134    return (url, specifier, marker)135 136 137def _parse_requirement_marker(138    tokenizer: Tokenizer, *, span_start: int, after: str139) -> MarkerList:140    """141    requirement_marker = SEMICOLON marker WS?142    """143 144    if not tokenizer.check("SEMICOLON"):145        tokenizer.raise_syntax_error(146            f"Expected end or semicolon (after {after})",147            span_start=span_start,148        )149    tokenizer.read()150 151    marker = _parse_marker(tokenizer)152    tokenizer.consume("WS")153 154    return marker155 156 157def _parse_extras(tokenizer: Tokenizer) -> list[str]:158    """159    extras = (LEFT_BRACKET wsp* extras_list? wsp* RIGHT_BRACKET)?160    """161    if not tokenizer.check("LEFT_BRACKET", peek=True):162        return []163 164    with tokenizer.enclosing_tokens(165        "LEFT_BRACKET",166        "RIGHT_BRACKET",167        around="extras",168    ):169        tokenizer.consume("WS")170        extras = _parse_extras_list(tokenizer)171        tokenizer.consume("WS")172 173    return extras174 175 176def _parse_extras_list(tokenizer: Tokenizer) -> list[str]:177    """178    extras_list = identifier (wsp* ',' wsp* identifier)*179    """180    extras: list[str] = []181 182    if not tokenizer.check("IDENTIFIER"):183        return extras184 185    extras.append(tokenizer.read().text)186 187    while True:188        tokenizer.consume("WS")189        if tokenizer.check("IDENTIFIER", peek=True):190            tokenizer.raise_syntax_error("Expected comma between extra names")191        elif not tokenizer.check("COMMA"):192            break193 194        tokenizer.read()195        tokenizer.consume("WS")196 197        extra_token = tokenizer.expect("IDENTIFIER", expected="extra name after comma")198        extras.append(extra_token.text)199 200    return extras201 202 203def _parse_specifier(tokenizer: Tokenizer) -> str:204    """205    specifier = LEFT_PARENTHESIS WS? version_many WS? RIGHT_PARENTHESIS206              | WS? version_many WS?207    """208    with tokenizer.enclosing_tokens(209        "LEFT_PARENTHESIS",210        "RIGHT_PARENTHESIS",211        around="version specifier",212    ):213        tokenizer.consume("WS")214        parsed_specifiers = _parse_version_many(tokenizer)215        tokenizer.consume("WS")216 217    return parsed_specifiers218 219 220def _parse_version_many(tokenizer: Tokenizer) -> str:221    """222    version_many = (SPECIFIER (WS? COMMA WS? SPECIFIER)*)?223    """224    parsed_specifiers = ""225    while tokenizer.check("SPECIFIER"):226        span_start = tokenizer.position227        parsed_specifiers += tokenizer.read().text228        if tokenizer.check("VERSION_PREFIX_TRAIL", peek=True):229            tokenizer.raise_syntax_error(230                ".* suffix can only be used with `==` or `!=` operators",231                span_start=span_start,232                span_end=tokenizer.position + 1,233            )234        if tokenizer.check("VERSION_LOCAL_LABEL_TRAIL", peek=True):235            tokenizer.raise_syntax_error(236                "Local version label can only be used with `==` or `!=` operators",237                span_start=span_start,238                span_end=tokenizer.position,239            )240        tokenizer.consume("WS")241        if not tokenizer.check("COMMA"):242            break243        parsed_specifiers += tokenizer.read().text244        tokenizer.consume("WS")245 246    return parsed_specifiers247 248 249# --------------------------------------------------------------------------------------250# Recursive descent parser for marker expression251# --------------------------------------------------------------------------------------252def parse_marker(source: str) -> MarkerList:253    return _parse_full_marker(Tokenizer(source, rules=DEFAULT_RULES))254 255 256def _parse_full_marker(tokenizer: Tokenizer) -> MarkerList:257    retval = _parse_marker(tokenizer)258    tokenizer.expect("END", expected="end of marker expression")259    return retval260 261 262def _parse_marker(tokenizer: Tokenizer) -> MarkerList:263    """264    marker = marker_atom (BOOLOP marker_atom)+265    """266    expression = [_parse_marker_atom(tokenizer)]267    while tokenizer.check("BOOLOP"):268        token = tokenizer.read()269        expr_right = _parse_marker_atom(tokenizer)270        expression.extend((token.text, expr_right))271    return expression272 273 274def _parse_marker_atom(tokenizer: Tokenizer) -> MarkerAtom:275    """276    marker_atom = WS? LEFT_PARENTHESIS WS? marker WS? RIGHT_PARENTHESIS WS?277                | WS? marker_item WS?278    """279 280    tokenizer.consume("WS")281    if tokenizer.check("LEFT_PARENTHESIS", peek=True):282        with tokenizer.enclosing_tokens(283            "LEFT_PARENTHESIS",284            "RIGHT_PARENTHESIS",285            around="marker expression",286        ):287            tokenizer.consume("WS")288            marker: MarkerAtom = _parse_marker(tokenizer)289            tokenizer.consume("WS")290    else:291        marker = _parse_marker_item(tokenizer)292    tokenizer.consume("WS")293    return marker294 295 296def _parse_marker_item(tokenizer: Tokenizer) -> MarkerItem:297    """298    marker_item = WS? marker_var WS? marker_op WS? marker_var WS?299    """300    tokenizer.consume("WS")301    marker_var_left = _parse_marker_var(tokenizer)302    tokenizer.consume("WS")303    marker_op = _parse_marker_op(tokenizer)304    tokenizer.consume("WS")305    marker_var_right = _parse_marker_var(tokenizer)306    tokenizer.consume("WS")307    return (marker_var_left, marker_op, marker_var_right)308 309 310def _parse_marker_var(tokenizer: Tokenizer) -> MarkerVar:311    """312    marker_var = VARIABLE | QUOTED_STRING313    """314    if tokenizer.check("VARIABLE"):315        return process_env_var(tokenizer.read().text.replace(".", "_"))316    elif tokenizer.check("QUOTED_STRING"):317        return process_python_str(tokenizer.read().text)318    else:319        tokenizer.raise_syntax_error(320            message="Expected a marker variable or quoted string"321        )322 323 324def process_env_var(env_var: str) -> Variable:325    if env_var in ("platform_python_implementation", "python_implementation"):326        return Variable("platform_python_implementation")327    else:328        return Variable(env_var)329 330 331def process_python_str(python_str: str) -> Value:332    value = ast.literal_eval(python_str)333    return Value(str(value))334 335 336def _parse_marker_op(tokenizer: Tokenizer) -> Op:337    """338    marker_op = IN | NOT IN | OP339    """340    if tokenizer.check("IN"):341        tokenizer.read()342        return Op("in")343    elif tokenizer.check("NOT"):344        tokenizer.read()345        tokenizer.expect("WS", expected="whitespace after 'not'")346        tokenizer.expect("IN", expected="'in' after 'not'")347        return Op("not in")348    elif tokenizer.check("OP"):349        return Op(tokenizer.read().text)350    else:351        return tokenizer.raise_syntax_error(352            "Expected marker operator, one of <=, <, !=, ==, >=, >, ~=, ===, in, not in"353        )354