Aluode/PerceptionLabPortable

1import base642import re3 4import requests5 6from ..spec import AbstractFileSystem7from ..utils import infer_storage_options8from .memory import MemoryFile9 10 11class GithubFileSystem(AbstractFileSystem):12    """Interface to files in github13 14    An instance of this class provides the files residing within a remote github15    repository. You may specify a point in the repos history, by SHA, branch16    or tag (default is current master).17 18    For files less than 1 MB in size, file content is returned directly in a19    MemoryFile. For larger files, or for files tracked by git-lfs, file content20    is returned as an HTTPFile wrapping the ``download_url`` provided by the21    GitHub API.22 23    When using fsspec.open, allows URIs of the form:24 25    - "github://path/file", in which case you must specify org, repo and26      may specify sha in the extra args27    - 'github://org:repo@/precip/catalog.yml', where the org and repo are28      part of the URI29    - 'github://org:repo@sha/precip/catalog.yml', where the sha is also included30 31    ``sha`` can be the full or abbreviated hex of the commit you want to fetch32    from, or a branch or tag name (so long as it doesn't contain special characters33    like "/", "?", which would have to be HTTP-encoded).34 35    For authorised access, you must provide username and token, which can be made36    at https://github.com/settings/tokens37    """38 39    url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}"40    content_url = "https://api.github.com/repos/{org}/{repo}/contents/{path}?ref={sha}"41    protocol = "github"42    timeout = (60, 60)  # connect, read timeouts43 44    def __init__(45        self, org, repo, sha=None, username=None, token=None, timeout=None, **kwargs46    ):47        super().__init__(**kwargs)48        self.org = org49        self.repo = repo50        if (username is None) ^ (token is None):51            raise ValueError("Auth required both username and token")52        self.username = username53        self.token = token54        if timeout is not None:55            self.timeout = timeout56        if sha is None:57            # look up default branch (not necessarily "master")58            u = "https://api.github.com/repos/{org}/{repo}"59            r = requests.get(60                u.format(org=org, repo=repo), timeout=self.timeout, **self.kw61            )62            r.raise_for_status()63            sha = r.json()["default_branch"]64 65        self.root = sha66        self.ls("")67        try:68            from .http import HTTPFileSystem69 70            self.http_fs = HTTPFileSystem(**kwargs)71        except ImportError:72            self.http_fs = None73 74    @property75    def kw(self):76        if self.username:77            return {"auth": (self.username, self.token)}78        return {}79 80    @classmethod81    def repos(cls, org_or_user, is_org=True):82        """List repo names for given org or user83 84        This may become the top level of the FS85 86        Parameters87        ----------88        org_or_user: str89            Name of the github org or user to query90        is_org: bool (default True)91            Whether the name is an organisation (True) or user (False)92 93        Returns94        -------95        List of string96        """97        r = requests.get(98            f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos",99            timeout=cls.timeout,100        )101        r.raise_for_status()102        return [repo["name"] for repo in r.json()]103 104    @property105    def tags(self):106        """Names of tags in the repo"""107        r = requests.get(108            f"https://api.github.com/repos/{self.org}/{self.repo}/tags",109            timeout=self.timeout,110            **self.kw,111        )112        r.raise_for_status()113        return [t["name"] for t in r.json()]114 115    @property116    def branches(self):117        """Names of branches in the repo"""118        r = requests.get(119            f"https://api.github.com/repos/{self.org}/{self.repo}/branches",120            timeout=self.timeout,121            **self.kw,122        )123        r.raise_for_status()124        return [t["name"] for t in r.json()]125 126    @property127    def refs(self):128        """Named references, tags and branches"""129        return {"tags": self.tags, "branches": self.branches}130 131    def ls(self, path, detail=False, sha=None, _sha=None, **kwargs):132        """List files at given path133 134        Parameters135        ----------136        path: str137            Location to list, relative to repo root138        detail: bool139            If True, returns list of dicts, one per file; if False, returns140            list of full filenames only141        sha: str (optional)142            List at the given point in the repo history, branch or tag name or commit143            SHA144        _sha: str (optional)145            List this specific tree object (used internally to descend into trees)146        """147        path = self._strip_protocol(path)148        if path == "":149            _sha = sha or self.root150        if _sha is None:151            parts = path.rstrip("/").split("/")152            so_far = ""153            _sha = sha or self.root154            for part in parts:155                out = self.ls(so_far, True, sha=sha, _sha=_sha)156                so_far += "/" + part if so_far else part157                out = [o for o in out if o["name"] == so_far]158                if not out:159                    raise FileNotFoundError(path)160                out = out[0]161                if out["type"] == "file":162                    if detail:163                        return [out]164                    else:165                        return path166                _sha = out["sha"]167        if path not in self.dircache or sha not in [self.root, None]:168            r = requests.get(169                self.url.format(org=self.org, repo=self.repo, sha=_sha),170                timeout=self.timeout,171                **self.kw,172            )173            if r.status_code == 404:174                raise FileNotFoundError(path)175            r.raise_for_status()176            types = {"blob": "file", "tree": "directory"}177            out = [178                {179                    "name": path + "/" + f["path"] if path else f["path"],180                    "mode": f["mode"],181                    "type": types[f["type"]],182                    "size": f.get("size", 0),183                    "sha": f["sha"],184                }185                for f in r.json()["tree"]186                if f["type"] in types187            ]188            if sha in [self.root, None]:189                self.dircache[path] = out190        else:191            out = self.dircache[path]192        if detail:193            return out194        else:195            return sorted([f["name"] for f in out])196 197    def invalidate_cache(self, path=None):198        self.dircache.clear()199 200    @classmethod201    def _strip_protocol(cls, path):202        opts = infer_storage_options(path)203        if "username" not in opts:204            return super()._strip_protocol(path)205        return opts["path"].lstrip("/")206 207    @staticmethod208    def _get_kwargs_from_urls(path):209        opts = infer_storage_options(path)210        if "username" not in opts:211            return {}212        out = {"org": opts["username"], "repo": opts["password"]}213        if opts["host"]:214            out["sha"] = opts["host"]215        return out216 217    def _open(218        self,219        path,220        mode="rb",221        block_size=None,222        cache_options=None,223        sha=None,224        **kwargs,225    ):226        if mode != "rb":227            raise NotImplementedError228 229        # construct a url to hit the GitHub API's repo contents API230        url = self.content_url.format(231            org=self.org, repo=self.repo, path=path, sha=sha or self.root232        )233 234        # make a request to this API, and parse the response as JSON235        r = requests.get(url, timeout=self.timeout, **self.kw)236        if r.status_code == 404:237            raise FileNotFoundError(path)238        r.raise_for_status()239        content_json = r.json()240 241        # if the response's content key is not empty, try to parse it as base64242        if content_json["content"]:243            content = base64.b64decode(content_json["content"])244 245            # as long as the content does not start with the string246            # "version https://git-lfs.github.com/"247            # then it is probably not a git-lfs pointer and we can just return248            # the content directly249            if not content.startswith(b"version https://git-lfs.github.com/"):250                return MemoryFile(None, None, content)251 252        # we land here if the content was not present in the first response253        # (regular file over 1MB or git-lfs tracked file)254        # in this case, we get let the HTTPFileSystem handle the download255        if self.http_fs is None:256            raise ImportError(257                "Please install fsspec[http] to access github files >1 MB "258                "or git-lfs tracked files."259            )260        return self.http_fs.open(261            content_json["download_url"],262            mode=mode,263            block_size=block_size,264            cache_options=cache_options,265            **kwargs,266        )267 268    def rm(self, path, recursive=False, maxdepth=None, message=None):269        path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth)270        for p in reversed(path):271            self.rm_file(p, message=message)272 273    def rm_file(self, path, message=None, **kwargs):274        """275        Remove a file from a specified branch using a given commit message.276 277        Since Github DELETE operation requires a branch name, and we can't reliably278        determine whether the provided SHA refers to a branch, tag, or commit, we279        assume it's a branch. If it's not, the user will encounter an error when280        attempting to retrieve the file SHA or delete the file.281 282        Parameters283        ----------284        path: str285            The file's location relative to the repository root.286        message: str, optional287            The commit message for the deletion.288        """289 290        if not self.username:291            raise ValueError("Authentication required")292 293        path = self._strip_protocol(path)294 295        # Attempt to get SHA from cache or Github API296        sha = self._get_sha_from_cache(path)297        if not sha:298            url = self.content_url.format(299                org=self.org, repo=self.repo, path=path.lstrip("/"), sha=self.root300            )301            r = requests.get(url, timeout=self.timeout, **self.kw)302            if r.status_code == 404:303                raise FileNotFoundError(path)304            r.raise_for_status()305            sha = r.json()["sha"]306 307        # Delete the file308        delete_url = self.content_url.format(309            org=self.org, repo=self.repo, path=path, sha=self.root310        )311        branch = self.root312        data = {313            "message": message or f"Delete {path}",314            "sha": sha,315            **({"branch": branch} if branch else {}),316        }317 318        r = requests.delete(delete_url, json=data, timeout=self.timeout, **self.kw)319        error_message = r.json().get("message", "")320        if re.search(r"Branch .+ not found", error_message):321            error = "Remove only works when the filesystem is initialised from a branch or default (None)"322            raise ValueError(error)323        r.raise_for_status()324 325        self.invalidate_cache(path)326 327    def _get_sha_from_cache(self, path):328        for entries in self.dircache.values():329            for entry in entries:330                entry_path = entry.get("name")331                if entry_path and entry_path == path and "sha" in entry:332                    return entry["sha"]333        return None334