Aluode/PerceptionLabPortable

1"""distutils.filelist2 3Provides the FileList class, used for poking about the filesystem4and building lists of files.5"""6 7from __future__ import annotations8 9import fnmatch10import functools11import os12import re13from collections.abc import Iterable14from typing import Literal, overload15 16from ._log import log17from .errors import DistutilsInternalError, DistutilsTemplateError18from .util import convert_path19 20 21class FileList:22    """A list of files built by on exploring the filesystem and filtered by23    applying various patterns to what we find there.24 25    Instance attributes:26      dir27        directory from which files will be taken -- only used if28        'allfiles' not supplied to constructor29      files30        list of filenames currently being built/filtered/manipulated31      allfiles32        complete list of files under consideration (ie. without any33        filtering applied)34    """35 36    def __init__(self, warn: object = None, debug_print: object = None) -> None:37        # ignore argument to FileList, but keep them for backwards38        # compatibility39        self.allfiles: Iterable[str] | None = None40        self.files: list[str] = []41 42    def set_allfiles(self, allfiles: Iterable[str]) -> None:43        self.allfiles = allfiles44 45    def findall(self, dir: str | os.PathLike[str] = os.curdir) -> None:46        self.allfiles = findall(dir)47 48    def debug_print(self, msg: object) -> None:49        """Print 'msg' to stdout if the global DEBUG (taken from the50        DISTUTILS_DEBUG environment variable) flag is true.51        """52        from distutils.debug import DEBUG53 54        if DEBUG:55            print(msg)56 57    # Collection methods58 59    def append(self, item: str) -> None:60        self.files.append(item)61 62    def extend(self, items: Iterable[str]) -> None:63        self.files.extend(items)64 65    def sort(self) -> None:66        # Not a strict lexical sort!67        sortable_files = sorted(map(os.path.split, self.files))68        self.files = []69        for sort_tuple in sortable_files:70            self.files.append(os.path.join(*sort_tuple))71 72    # Other miscellaneous utility methods73 74    def remove_duplicates(self) -> None:75        # Assumes list has been sorted!76        for i in range(len(self.files) - 1, 0, -1):77            if self.files[i] == self.files[i - 1]:78                del self.files[i]79 80    # "File template" methods81 82    def _parse_template_line(self, line):83        words = line.split()84        action = words[0]85 86        patterns = dir = dir_pattern = None87 88        if action in ('include', 'exclude', 'global-include', 'global-exclude'):89            if len(words) < 2:90                raise DistutilsTemplateError(91                    f"'{action}' expects <pattern1> <pattern2> ..."92                )93            patterns = [convert_path(w) for w in words[1:]]94        elif action in ('recursive-include', 'recursive-exclude'):95            if len(words) < 3:96                raise DistutilsTemplateError(97                    f"'{action}' expects <dir> <pattern1> <pattern2> ..."98                )99            dir = convert_path(words[1])100            patterns = [convert_path(w) for w in words[2:]]101        elif action in ('graft', 'prune'):102            if len(words) != 2:103                raise DistutilsTemplateError(104                    f"'{action}' expects a single <dir_pattern>"105                )106            dir_pattern = convert_path(words[1])107        else:108            raise DistutilsTemplateError(f"unknown action '{action}'")109 110        return (action, patterns, dir, dir_pattern)111 112    def process_template_line(self, line: str) -> None:  # noqa: C901113        # Parse the line: split it up, make sure the right number of words114        # is there, and return the relevant words.  'action' is always115        # defined: it's the first word of the line.  Which of the other116        # three are defined depends on the action; it'll be either117        # patterns, (dir and patterns), or (dir_pattern).118        (action, patterns, dir, dir_pattern) = self._parse_template_line(line)119 120        # OK, now we know that the action is valid and we have the121        # right number of words on the line for that action -- so we122        # can proceed with minimal error-checking.123        if action == 'include':124            self.debug_print("include " + ' '.join(patterns))125            for pattern in patterns:126                if not self.include_pattern(pattern, anchor=True):127                    log.warning("warning: no files found matching '%s'", pattern)128 129        elif action == 'exclude':130            self.debug_print("exclude " + ' '.join(patterns))131            for pattern in patterns:132                if not self.exclude_pattern(pattern, anchor=True):133                    log.warning(134                        "warning: no previously-included files found matching '%s'",135                        pattern,136                    )137 138        elif action == 'global-include':139            self.debug_print("global-include " + ' '.join(patterns))140            for pattern in patterns:141                if not self.include_pattern(pattern, anchor=False):142                    log.warning(143                        (144                            "warning: no files found matching '%s' "145                            "anywhere in distribution"146                        ),147                        pattern,148                    )149 150        elif action == 'global-exclude':151            self.debug_print("global-exclude " + ' '.join(patterns))152            for pattern in patterns:153                if not self.exclude_pattern(pattern, anchor=False):154                    log.warning(155                        (156                            "warning: no previously-included files matching "157                            "'%s' found anywhere in distribution"158                        ),159                        pattern,160                    )161 162        elif action == 'recursive-include':163            self.debug_print("recursive-include {} {}".format(dir, ' '.join(patterns)))164            for pattern in patterns:165                if not self.include_pattern(pattern, prefix=dir):166                    msg = "warning: no files found matching '%s' under directory '%s'"167                    log.warning(msg, pattern, dir)168 169        elif action == 'recursive-exclude':170            self.debug_print("recursive-exclude {} {}".format(dir, ' '.join(patterns)))171            for pattern in patterns:172                if not self.exclude_pattern(pattern, prefix=dir):173                    log.warning(174                        (175                            "warning: no previously-included files matching "176                            "'%s' found under directory '%s'"177                        ),178                        pattern,179                        dir,180                    )181 182        elif action == 'graft':183            self.debug_print("graft " + dir_pattern)184            if not self.include_pattern(None, prefix=dir_pattern):185                log.warning("warning: no directories found matching '%s'", dir_pattern)186 187        elif action == 'prune':188            self.debug_print("prune " + dir_pattern)189            if not self.exclude_pattern(None, prefix=dir_pattern):190                log.warning(191                    ("no previously-included directories found matching '%s'"),192                    dir_pattern,193                )194        else:195            raise DistutilsInternalError(196                f"this cannot happen: invalid action '{action}'"197            )198 199    # Filtering/selection methods200    @overload201    def include_pattern(202        self,203        pattern: str,204        anchor: bool = True,205        prefix: str | None = None,206        is_regex: Literal[False] = False,207    ) -> bool: ...208    @overload209    def include_pattern(210        self,211        pattern: str | re.Pattern[str],212        anchor: bool = True,213        prefix: str | None = None,214        *,215        is_regex: Literal[True],216    ) -> bool: ...217    @overload218    def include_pattern(219        self,220        pattern: str | re.Pattern[str],221        anchor: bool,222        prefix: str | None,223        is_regex: Literal[True],224    ) -> bool: ...225    def include_pattern(226        self,227        pattern: str | re.Pattern,228        anchor: bool = True,229        prefix: str | None = None,230        is_regex: bool = False,231    ) -> bool:232        """Select strings (presumably filenames) from 'self.files' that233        match 'pattern', a Unix-style wildcard (glob) pattern.  Patterns234        are not quite the same as implemented by the 'fnmatch' module: '*'235        and '?'  match non-special characters, where "special" is platform-236        dependent: slash on Unix; colon, slash, and backslash on237        DOS/Windows; and colon on Mac OS.238 239        If 'anchor' is true (the default), then the pattern match is more240        stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If241        'anchor' is false, both of these will match.242 243        If 'prefix' is supplied, then only filenames starting with 'prefix'244        (itself a pattern) and ending with 'pattern', with anything in between245        them, will match.  'anchor' is ignored in this case.246 247        If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and248        'pattern' is assumed to be either a string containing a regex or a249        regex object -- no translation is done, the regex is just compiled250        and used as-is.251 252        Selected strings will be added to self.files.253 254        Return True if files are found, False otherwise.255        """256        # XXX docstring lying about what the special chars are?257        files_found = False258        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)259        self.debug_print(f"include_pattern: applying regex r'{pattern_re.pattern}'")260 261        # delayed loading of allfiles list262        if self.allfiles is None:263            self.findall()264 265        for name in self.allfiles:266            if pattern_re.search(name):267                self.debug_print(" adding " + name)268                self.files.append(name)269                files_found = True270        return files_found271 272    @overload273    def exclude_pattern(274        self,275        pattern: str,276        anchor: bool = True,277        prefix: str | None = None,278        is_regex: Literal[False] = False,279    ) -> bool: ...280    @overload281    def exclude_pattern(282        self,283        pattern: str | re.Pattern[str],284        anchor: bool = True,285        prefix: str | None = None,286        *,287        is_regex: Literal[True],288    ) -> bool: ...289    @overload290    def exclude_pattern(291        self,292        pattern: str | re.Pattern[str],293        anchor: bool,294        prefix: str | None,295        is_regex: Literal[True],296    ) -> bool: ...297    def exclude_pattern(298        self,299        pattern: str | re.Pattern,300        anchor: bool = True,301        prefix: str | None = None,302        is_regex: bool = False,303    ) -> bool:304        """Remove strings (presumably filenames) from 'files' that match305        'pattern'.  Other parameters are the same as for306        'include_pattern()', above.307        The list 'self.files' is modified in place.308        Return True if files are found, False otherwise.309        """310        files_found = False311        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)312        self.debug_print(f"exclude_pattern: applying regex r'{pattern_re.pattern}'")313        for i in range(len(self.files) - 1, -1, -1):314            if pattern_re.search(self.files[i]):315                self.debug_print(" removing " + self.files[i])316                del self.files[i]317                files_found = True318        return files_found319 320 321# Utility functions322 323 324def _find_all_simple(path):325    """326    Find all files under 'path'327    """328    all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True))329    results = (330        os.path.join(base, file) for base, dirs, files in all_unique for file in files331    )332    return filter(os.path.isfile, results)333 334 335class _UniqueDirs(set):336    """337    Exclude previously-seen dirs from walk results,338    avoiding infinite recursion.339    Ref https://bugs.python.org/issue44497.340    """341 342    def __call__(self, walk_item):343        """344        Given an item from an os.walk result, determine345        if the item represents a unique dir for this instance346        and if not, prevent further traversal.347        """348        base, dirs, files = walk_item349        stat = os.stat(base)350        candidate = stat.st_dev, stat.st_ino351        found = candidate in self352        if found:353            del dirs[:]354        self.add(candidate)355        return not found356 357    @classmethod358    def filter(cls, items):359        return filter(cls(), items)360 361 362def findall(dir: str | os.PathLike[str] = os.curdir):363    """364    Find all files under 'dir' and return the list of full filenames.365    Unless dir is '.', return full filenames with dir prepended.366    """367    files = _find_all_simple(dir)368    if dir == os.curdir:369        make_rel = functools.partial(os.path.relpath, start=dir)370        files = map(make_rel, files)371    return list(files)372 373 374def glob_to_re(pattern):375    """Translate a shell-like glob pattern to a regular expression; return376    a string containing the regex.  Differs from 'fnmatch.translate()' in377    that '*' does not match "special characters" (which are378    platform-specific).379    """380    pattern_re = fnmatch.translate(pattern)381 382    # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which383    # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,384    # and by extension they shouldn't match such "special characters" under385    # any OS.  So change all non-escaped dots in the RE to match any386    # character except the special characters (currently: just os.sep).387    sep = os.sep388    if os.sep == '\\':389        # we're using a regex to manipulate a regex, so we need390        # to escape the backslash twice391        sep = r'\\\\'392    escaped = rf'\1[^{sep}]'393    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)394    return pattern_re395 396 397def translate_pattern(pattern, anchor=True, prefix=None, is_regex=False):398    """Translate a shell-like wildcard pattern to a compiled regular399    expression.  Return the compiled regex.  If 'is_regex' true,400    then 'pattern' is directly compiled to a regex (if it's a string)401    or just returned as-is (assumes it's a regex object).402    """403    if is_regex:404        if isinstance(pattern, str):405            return re.compile(pattern)406        else:407            return pattern408 409    # ditch start and end characters410    start, _, end = glob_to_re('_').partition('_')411 412    if pattern:413        pattern_re = glob_to_re(pattern)414        assert pattern_re.startswith(start) and pattern_re.endswith(end)415    else:416        pattern_re = ''417 418    if prefix is not None:419        prefix_re = glob_to_re(prefix)420        assert prefix_re.startswith(start) and prefix_re.endswith(end)421        prefix_re = prefix_re[len(start) : len(prefix_re) - len(end)]422        sep = os.sep423        if os.sep == '\\':424            sep = r'\\'425        pattern_re = pattern_re[len(start) : len(pattern_re) - len(end)]426        pattern_re = rf'{start}\A{prefix_re}{sep}.*{pattern_re}{end}'427    else:  # no prefix -- respect anchor flag428        if anchor:429            pattern_re = rf'{start}\A{pattern_re[len(start) :]}'430 431    return re.compile(pattern_re)432