Aluode/PerceptionLabPortable
0
1"""This file is largely copied from http.py"""2 3import io4import logging5import re6import urllib.error7import urllib.parse8from copy import copy9from json import dumps, loads10from urllib.parse import urlparse11 12try:13 import yarl14except (ImportError, ModuleNotFoundError, OSError):15 yarl = False16 17from fsspec.callbacks import _DEFAULT_CALLBACK18from fsspec.registry import register_implementation19from fsspec.spec import AbstractBufferedFile, AbstractFileSystem20from fsspec.utils import DEFAULT_BLOCK_SIZE, isfilelike, nullcontext, tokenize21 22from ..caching import AllBytes23 24# https://stackoverflow.com/a/15926317/382115425ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P<url>[^"']+)""")26ex2 = re.compile(r"""(?P<url>http[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""")27logger = logging.getLogger("fsspec.http")28 29 30class JsHttpException(urllib.error.HTTPError): ...31 32 33class StreamIO(io.BytesIO):34 # fake class, so you can set attributes on it35 # will eventually actually stream36 ...37 38 39class ResponseProxy:40 """Looks like a requests response"""41 42 def __init__(self, req, stream=False):43 self.request = req44 self.stream = stream45 self._data = None46 self._headers = None47 48 @property49 def raw(self):50 if self._data is None:51 b = self.request.response.to_bytes()52 if self.stream:53 self._data = StreamIO(b)54 else:55 self._data = b56 return self._data57 58 def close(self):59 if hasattr(self, "_data"):60 del self._data61 62 @property63 def headers(self):64 if self._headers is None:65 self._headers = dict(66 [67 _.split(": ")68 for _ in self.request.getAllResponseHeaders().strip().split("\r\n")69 ]70 )71 return self._headers72 73 @property74 def status_code(self):75 return int(self.request.status)76 77 def raise_for_status(self):78 if not self.ok:79 raise JsHttpException(80 self.url, self.status_code, self.reason, self.headers, None81 )82 83 def iter_content(self, chunksize, *_, **__):84 while True:85 out = self.raw.read(chunksize)86 if out:87 yield out88 else:89 break90 91 @property92 def reason(self):93 return self.request.statusText94 95 @property96 def ok(self):97 return self.status_code < 40098 99 @property100 def url(self):101 return self.request.response.responseURL102 103 @property104 def text(self):105 # TODO: encoding from headers106 return self.content.decode()107 108 @property109 def content(self):110 self.stream = False111 return self.raw112 113 def json(self):114 return loads(self.text)115 116 117class RequestsSessionShim:118 def __init__(self):119 self.headers = {}120 121 def request(122 self,123 method,124 url,125 params=None,126 data=None,127 headers=None,128 cookies=None,129 files=None,130 auth=None,131 timeout=None,132 allow_redirects=None,133 proxies=None,134 hooks=None,135 stream=None,136 verify=None,137 cert=None,138 json=None,139 ):140 from js import Blob, XMLHttpRequest141 142 logger.debug("JS request: %s %s", method, url)143 144 if cert or verify or proxies or files or cookies or hooks:145 raise NotImplementedError146 if data and json:147 raise ValueError("Use json= or data=, not both")148 req = XMLHttpRequest.new()149 extra = auth if auth else ()150 if params:151 url = f"{url}?{urllib.parse.urlencode(params)}"152 req.open(method, url, False, *extra)153 if timeout:154 req.timeout = timeout155 if headers:156 for k, v in headers.items():157 req.setRequestHeader(k, v)158 159 req.setRequestHeader("Accept", "application/octet-stream")160 req.responseType = "arraybuffer"161 if json:162 blob = Blob.new([dumps(data)], {type: "application/json"})163 req.send(blob)164 elif data:165 if isinstance(data, io.IOBase):166 data = data.read()167 blob = Blob.new([data], {type: "application/octet-stream"})168 req.send(blob)169 else:170 req.send(None)171 return ResponseProxy(req, stream=stream)172 173 def get(self, url, **kwargs):174 return self.request("GET", url, **kwargs)175 176 def head(self, url, **kwargs):177 return self.request("HEAD", url, **kwargs)178 179 def post(self, url, **kwargs):180 return self.request("POST}", url, **kwargs)181 182 def put(self, url, **kwargs):183 return self.request("PUT", url, **kwargs)184 185 def patch(self, url, **kwargs):186 return self.request("PATCH", url, **kwargs)187 188 def delete(self, url, **kwargs):189 return self.request("DELETE", url, **kwargs)190 191 192class HTTPFileSystem(AbstractFileSystem):193 """194 Simple File-System for fetching data via HTTP(S)195 196 This is the BLOCKING version of the normal HTTPFileSystem. It uses197 requests in normal python and the JS runtime in pyodide.198 199 ***This implementation is extremely experimental, do not use unless200 you are testing pyodide/pyscript integration***201 """202 203 protocol = ("http", "https", "sync-http", "sync-https")204 sep = "/"205 206 def __init__(207 self,208 simple_links=True,209 block_size=None,210 same_scheme=True,211 cache_type="readahead",212 cache_options=None,213 client_kwargs=None,214 encoded=False,215 **storage_options,216 ):217 """218 219 Parameters220 ----------221 block_size: int222 Blocks to read bytes; if 0, will default to raw requests file-like223 objects instead of HTTPFile instances224 simple_links: bool225 If True, will consider both HTML <a> tags and anything that looks226 like a URL; if False, will consider only the former.227 same_scheme: True228 When doing ls/glob, if this is True, only consider paths that have229 http/https matching the input URLs.230 size_policy: this argument is deprecated231 client_kwargs: dict232 Passed to aiohttp.ClientSession, see233 https://docs.aiohttp.org/en/stable/client_reference.html234 For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}``235 storage_options: key-value236 Any other parameters passed on to requests237 cache_type, cache_options: defaults used in open238 """239 super().__init__(self, **storage_options)240 self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE241 self.simple_links = simple_links242 self.same_schema = same_scheme243 self.cache_type = cache_type244 self.cache_options = cache_options245 self.client_kwargs = client_kwargs or {}246 self.encoded = encoded247 self.kwargs = storage_options248 249 try:250 import js # noqa: F401251 252 logger.debug("Starting JS session")253 self.session = RequestsSessionShim()254 self.js = True255 except Exception as e:256 import requests257 258 logger.debug("Starting cpython session because of: %s", e)259 self.session = requests.Session(**(client_kwargs or {}))260 self.js = False261 262 request_options = copy(storage_options)263 self.use_listings_cache = request_options.pop("use_listings_cache", False)264 request_options.pop("listings_expiry_time", None)265 request_options.pop("max_paths", None)266 request_options.pop("skip_instance_cache", None)267 self.kwargs = request_options268 269 @property270 def fsid(self):271 return "sync-http"272 273 def encode_url(self, url):274 if yarl:275 return yarl.URL(url, encoded=self.encoded)276 return url277 278 @classmethod279 def _strip_protocol(cls, path: str) -> str:280 """For HTTP, we always want to keep the full URL"""281 path = path.replace("sync-http://", "http://").replace(282 "sync-https://", "https://"283 )284 return path285 286 @classmethod287 def _parent(cls, path):288 # override, since _strip_protocol is different for URLs289 par = super()._parent(path)290 if len(par) > 7: # "http://..."291 return par292 return ""293 294 def _ls_real(self, url, detail=True, **kwargs):295 # ignoring URL-encoded arguments296 kw = self.kwargs.copy()297 kw.update(kwargs)298 logger.debug(url)299 r = self.session.get(self.encode_url(url), **self.kwargs)300 self._raise_not_found_for_status(r, url)301 text = r.text302 if self.simple_links:303 links = ex2.findall(text) + [u[2] for u in ex.findall(text)]304 else:305 links = [u[2] for u in ex.findall(text)]306 out = set()307 parts = urlparse(url)308 for l in links:309 if isinstance(l, tuple):310 l = l[1]311 if l.startswith("/") and len(l) > 1:312 # absolute URL on this server313 l = parts.scheme + "://" + parts.netloc + l314 if l.startswith("http"):315 if self.same_schema and l.startswith(url.rstrip("/") + "/"):316 out.add(l)317 elif l.replace("https", "http").startswith(318 url.replace("https", "http").rstrip("/") + "/"319 ):320 # allowed to cross http <-> https321 out.add(l)322 else:323 if l not in ["..", "../"]:324 # Ignore FTP-like "parent"325 out.add("/".join([url.rstrip("/"), l.lstrip("/")]))326 if not out and url.endswith("/"):327 out = self._ls_real(url.rstrip("/"), detail=False)328 if detail:329 return [330 {331 "name": u,332 "size": None,333 "type": "directory" if u.endswith("/") else "file",334 }335 for u in out336 ]337 else:338 return sorted(out)339 340 def ls(self, url, detail=True, **kwargs):341 if self.use_listings_cache and url in self.dircache:342 out = self.dircache[url]343 else:344 out = self._ls_real(url, detail=detail, **kwargs)345 self.dircache[url] = out346 return out347 348 def _raise_not_found_for_status(self, response, url):349 """350 Raises FileNotFoundError for 404s, otherwise uses raise_for_status.351 """352 if response.status_code == 404:353 raise FileNotFoundError(url)354 response.raise_for_status()355 356 def cat_file(self, url, start=None, end=None, **kwargs):357 kw = self.kwargs.copy()358 kw.update(kwargs)359 logger.debug(url)360 361 if start is not None or end is not None:362 if start == end:363 return b""364 headers = kw.pop("headers", {}).copy()365 366 headers["Range"] = self._process_limits(url, start, end)367 kw["headers"] = headers368 r = self.session.get(self.encode_url(url), **kw)369 self._raise_not_found_for_status(r, url)370 return r.content371 372 def get_file(373 self, rpath, lpath, chunk_size=5 * 2**20, callback=_DEFAULT_CALLBACK, **kwargs374 ):375 kw = self.kwargs.copy()376 kw.update(kwargs)377 logger.debug(rpath)378 r = self.session.get(self.encode_url(rpath), **kw)379 try:380 size = int(381 r.headers.get("content-length", None)382 or r.headers.get("Content-Length", None)383 )384 except (ValueError, KeyError, TypeError):385 size = None386 387 callback.set_size(size)388 self._raise_not_found_for_status(r, rpath)389 if not isfilelike(lpath):390 lpath = open(lpath, "wb")391 for chunk in r.iter_content(chunk_size, decode_unicode=False):392 lpath.write(chunk)393 callback.relative_update(len(chunk))394 395 def put_file(396 self,397 lpath,398 rpath,399 chunk_size=5 * 2**20,400 callback=_DEFAULT_CALLBACK,401 method="post",402 **kwargs,403 ):404 def gen_chunks():405 # Support passing arbitrary file-like objects406 # and use them instead of streams.407 if isinstance(lpath, io.IOBase):408 context = nullcontext(lpath)409 use_seek = False # might not support seeking410 else:411 context = open(lpath, "rb")412 use_seek = True413 414 with context as f:415 if use_seek:416 callback.set_size(f.seek(0, 2))417 f.seek(0)418 else:419 callback.set_size(getattr(f, "size", None))420 421 chunk = f.read(chunk_size)422 while chunk:423 yield chunk424 callback.relative_update(len(chunk))425 chunk = f.read(chunk_size)426 427 kw = self.kwargs.copy()428 kw.update(kwargs)429 430 method = method.lower()431 if method not in ("post", "put"):432 raise ValueError(433 f"method has to be either 'post' or 'put', not: {method!r}"434 )435 436 meth = getattr(self.session, method)437 resp = meth(rpath, data=gen_chunks(), **kw)438 self._raise_not_found_for_status(resp, rpath)439 440 def _process_limits(self, url, start, end):441 """Helper for "Range"-based _cat_file"""442 size = None443 suff = False444 if start is not None and start < 0:445 # if start is negative and end None, end is the "suffix length"446 if end is None:447 end = -start448 start = ""449 suff = True450 else:451 size = size or self.info(url)["size"]452 start = size + start453 elif start is None:454 start = 0455 if not suff:456 if end is not None and end < 0:457 if start is not None:458 size = size or self.info(url)["size"]459 end = size + end460 elif end is None:461 end = ""462 if isinstance(end, int):463 end -= 1 # bytes range is inclusive464 return f"bytes={start}-{end}"465 466 def exists(self, path, strict=False, **kwargs):467 kw = self.kwargs.copy()468 kw.update(kwargs)469 try:470 logger.debug(path)471 r = self.session.get(self.encode_url(path), **kw)472 if strict:473 self._raise_not_found_for_status(r, path)474 return r.status_code < 400475 except FileNotFoundError:476 return False477 except Exception:478 if strict:479 raise480 return False481 482 def isfile(self, path, **kwargs):483 return self.exists(path, **kwargs)484 485 def _open(486 self,487 path,488 mode="rb",489 block_size=None,490 autocommit=None, # XXX: This differs from the base class.491 cache_type=None,492 cache_options=None,493 size=None,494 **kwargs,495 ):496 """Make a file-like object497 498 Parameters499 ----------500 path: str501 Full URL with protocol502 mode: string503 must be "rb"504 block_size: int or None505 Bytes to download in one request; use instance value if None. If506 zero, will return a streaming Requests file-like instance.507 kwargs: key-value508 Any other parameters, passed to requests calls509 """510 if mode != "rb":511 raise NotImplementedError512 block_size = block_size if block_size is not None else self.block_size513 kw = self.kwargs.copy()514 kw.update(kwargs)515 size = size or self.info(path, **kwargs)["size"]516 if block_size and size:517 return HTTPFile(518 self,519 path,520 session=self.session,521 block_size=block_size,522 mode=mode,523 size=size,524 cache_type=cache_type or self.cache_type,525 cache_options=cache_options or self.cache_options,526 **kw,527 )528 else:529 return HTTPStreamFile(530 self,531 path,532 mode=mode,533 session=self.session,534 **kw,535 )536 537 def ukey(self, url):538 """Unique identifier; assume HTTP files are static, unchanging"""539 return tokenize(url, self.kwargs, self.protocol)540 541 def info(self, url, **kwargs):542 """Get info of URL543 544 Tries to access location via HEAD, and then GET methods, but does545 not fetch the data.546 547 It is possible that the server does not supply any size information, in548 which case size will be given as None (and certain operations on the549 corresponding file will not work).550 """551 info = {}552 for policy in ["head", "get"]:553 try:554 info.update(555 _file_info(556 self.encode_url(url),557 size_policy=policy,558 session=self.session,559 **self.kwargs,560 **kwargs,561 )562 )563 if info.get("size") is not None:564 break565 except Exception as exc:566 if policy == "get":567 # If get failed, then raise a FileNotFoundError568 raise FileNotFoundError(url) from exc569 logger.debug(str(exc))570 571 return {"name": url, "size": None, **info, "type": "file"}572 573 def glob(self, path, maxdepth=None, **kwargs):574 """575 Find files by glob-matching.576 577 This implementation is idntical to the one in AbstractFileSystem,578 but "?" is not considered as a character for globbing, because it is579 so common in URLs, often identifying the "query" part.580 """581 import re582 583 ends = path.endswith("/")584 path = self._strip_protocol(path)585 indstar = path.find("*") if path.find("*") >= 0 else len(path)586 indbrace = path.find("[") if path.find("[") >= 0 else len(path)587 588 ind = min(indstar, indbrace)589 590 detail = kwargs.pop("detail", False)591 592 if not has_magic(path):593 root = path594 depth = 1595 if ends:596 path += "/*"597 elif self.exists(path):598 if not detail:599 return [path]600 else:601 return {path: self.info(path)}602 else:603 if not detail:604 return [] # glob of non-existent returns empty605 else:606 return {}607 elif "/" in path[:ind]:608 ind2 = path[:ind].rindex("/")609 root = path[: ind2 + 1]610 depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1611 else:612 root = ""613 depth = None if "**" in path else path[ind + 1 :].count("/") + 1614 615 allpaths = self.find(616 root, maxdepth=maxdepth or depth, withdirs=True, detail=True, **kwargs617 )618 # Escape characters special to python regex, leaving our supported619 # special characters in place.620 # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html621 # for shell globbing details.622 pattern = (623 "^"624 + (625 path.replace("\\", r"\\")626 .replace(".", r"\.")627 .replace("+", r"\+")628 .replace("//", "/")629 .replace("(", r"\(")630 .replace(")", r"\)")631 .replace("|", r"\|")632 .replace("^", r"\^")633 .replace("$", r"\$")634 .replace("{", r"\{")635 .replace("}", r"\}")636 .rstrip("/")637 )638 + "$"639 )640 pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern)641 pattern = re.sub("[*]", "[^/]*", pattern)642 pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*"))643 out = {644 p: allpaths[p]645 for p in sorted(allpaths)646 if pattern.match(p.replace("//", "/").rstrip("/"))647 }648 if detail:649 return out650 else:651 return list(out)652 653 def isdir(self, path):654 # override, since all URLs are (also) files655 try:656 return bool(self.ls(path))657 except (FileNotFoundError, ValueError):658 return False659 660 661class HTTPFile(AbstractBufferedFile):662 """663 A file-like object pointing to a remove HTTP(S) resource664 665 Supports only reading, with read-ahead of a predermined block-size.666 667 In the case that the server does not supply the filesize, only reading of668 the complete file in one go is supported.669 670 Parameters671 ----------672 url: str673 Full URL of the remote resource, including the protocol674 session: requests.Session or None675 All calls will be made within this session, to avoid restarting676 connections where the server allows this677 block_size: int or None678 The amount of read-ahead to do, in bytes. Default is 5MB, or the value679 configured for the FileSystem creating this file680 size: None or int681 If given, this is the size of the file in bytes, and we don't attempt682 to call the server to find the value.683 kwargs: all other key-values are passed to requests calls.684 """685 686 def __init__(687 self,688 fs,689 url,690 session=None,691 block_size=None,692 mode="rb",693 cache_type="bytes",694 cache_options=None,695 size=None,696 **kwargs,697 ):698 if mode != "rb":699 raise NotImplementedError("File mode not supported")700 self.url = url701 self.session = session702 self.details = {"name": url, "size": size, "type": "file"}703 super().__init__(704 fs=fs,705 path=url,706 mode=mode,707 block_size=block_size,708 cache_type=cache_type,709 cache_options=cache_options,710 **kwargs,711 )712 713 def read(self, length=-1):714 """Read bytes from file715 716 Parameters717 ----------718 length: int719 Read up to this many bytes. If negative, read all content to end of720 file. If the server has not supplied the filesize, attempting to721 read only part of the data will raise a ValueError.722 """723 if (724 (length < 0 and self.loc == 0) # explicit read all725 # but not when the size is known and fits into a block anyways726 and not (self.size is not None and self.size <= self.blocksize)727 ):728 self._fetch_all()729 if self.size is None:730 if length < 0:731 self._fetch_all()732 else:733 length = min(self.size - self.loc, length)734 return super().read(length)735 736 def _fetch_all(self):737 """Read whole file in one shot, without caching738 739 This is only called when position is still at zero,740 and read() is called without a byte-count.741 """742 logger.debug(f"Fetch all for {self}")743 if not isinstance(self.cache, AllBytes):744 r = self.session.get(self.fs.encode_url(self.url), **self.kwargs)745 r.raise_for_status()746 out = r.content747 self.cache = AllBytes(size=len(out), fetcher=None, blocksize=None, data=out)748 self.size = len(out)749 750 def _parse_content_range(self, headers):751 """Parse the Content-Range header"""752 s = headers.get("Content-Range", "")753 m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s)754 if not m:755 return None, None, None756 757 if m[1] == "*":758 start = end = None759 else:760 start, end = [int(x) for x in m[1].split("-")]761 total = None if m[2] == "*" else int(m[2])762 return start, end, total763 764 def _fetch_range(self, start, end):765 """Download a block of data766 767 The expectation is that the server returns only the requested bytes,768 with HTTP code 206. If this is not the case, we first check the headers,769 and then stream the output - if the data size is bigger than we770 requested, an exception is raised.771 """772 logger.debug(f"Fetch range for {self}: {start}-{end}")773 kwargs = self.kwargs.copy()774 headers = kwargs.pop("headers", {}).copy()775 headers["Range"] = f"bytes={start}-{end - 1}"776 logger.debug("%s : %s", self.url, headers["Range"])777 r = self.session.get(self.fs.encode_url(self.url), headers=headers, **kwargs)778 if r.status_code == 416:779 # range request outside file780 return b""781 r.raise_for_status()782 783 # If the server has handled the range request, it should reply784 # with status 206 (partial content). But we'll guess that a suitable785 # Content-Range header or a Content-Length no more than the786 # requested range also mean we have got the desired range.787 cl = r.headers.get("Content-Length", r.headers.get("content-length", end + 1))788 response_is_range = (789 r.status_code == 206790 or self._parse_content_range(r.headers)[0] == start791 or int(cl) <= end - start792 )793 794 if response_is_range:795 # partial content, as expected796 out = r.content797 elif start > 0:798 raise ValueError(799 "The HTTP server doesn't appear to support range requests. "800 "Only reading this file from the beginning is supported. "801 "Open with block_size=0 for a streaming file interface."802 )803 else:804 # Response is not a range, but we want the start of the file,805 # so we can read the required amount anyway.806 cl = 0807 out = []808 for chunk in r.iter_content(2**20, False):809 out.append(chunk)810 cl += len(chunk)811 out = b"".join(out)[: end - start]812 return out813 814 815magic_check = re.compile("([*[])")816 817 818def has_magic(s):819 match = magic_check.search(s)820 return match is not None821 822 823class HTTPStreamFile(AbstractBufferedFile):824 def __init__(self, fs, url, mode="rb", session=None, **kwargs):825 self.url = url826 self.session = session827 if mode != "rb":828 raise ValueError829 self.details = {"name": url, "size": None}830 super().__init__(fs=fs, path=url, mode=mode, cache_type="readahead", **kwargs)831 832 r = self.session.get(self.fs.encode_url(url), stream=True, **kwargs)833 self.fs._raise_not_found_for_status(r, url)834 self.it = r.iter_content(1024, False)835 self.leftover = b""836 837 self.r = r838 839 def seek(self, *args, **kwargs):840 raise ValueError("Cannot seek streaming HTTP file")841 842 def read(self, num=-1):843 bufs = [self.leftover]844 leng = len(self.leftover)845 while leng < num or num < 0:846 try:847 out = self.it.__next__()848 except StopIteration:849 break850 if out:851 bufs.append(out)852 else:853 break854 leng += len(out)855 out = b"".join(bufs)856 if num >= 0:857 self.leftover = out[num:]858 out = out[:num]859 else:860 self.leftover = b""861 self.loc += len(out)862 return out863 864 def close(self):865 self.r.close()866 self.closed = True867 868 869def get_range(session, url, start, end, **kwargs):870 # explicit get a range when we know it must be safe871 kwargs = kwargs.copy()872 headers = kwargs.pop("headers", {}).copy()873 headers["Range"] = f"bytes={start}-{end - 1}"874 r = session.get(url, headers=headers, **kwargs)875 r.raise_for_status()876 return r.content877 878 879def _file_info(url, session, size_policy="head", **kwargs):880 """Call HEAD on the server to get details about the file (size/checksum etc.)881 882 Default operation is to explicitly allow redirects and use encoding883 'identity' (no compression) to get the true size of the target.884 """885 logger.debug("Retrieve file size for %s", url)886 kwargs = kwargs.copy()887 ar = kwargs.pop("allow_redirects", True)888 head = kwargs.get("headers", {}).copy()889 # TODO: not allowed in JS890 # head["Accept-Encoding"] = "identity"891 kwargs["headers"] = head892 893 info = {}894 if size_policy == "head":895 r = session.head(url, allow_redirects=ar, **kwargs)896 elif size_policy == "get":897 r = session.get(url, allow_redirects=ar, **kwargs)898 else:899 raise TypeError(f'size_policy must be "head" or "get", got {size_policy}')900 r.raise_for_status()901 902 # TODO:903 # recognise lack of 'Accept-Ranges',904 # or 'Accept-Ranges': 'none' (not 'bytes')905 # to mean streaming only, no random access => return None906 if "Content-Length" in r.headers:907 info["size"] = int(r.headers["Content-Length"])908 elif "Content-Range" in r.headers:909 info["size"] = int(r.headers["Content-Range"].split("/")[1])910 elif "content-length" in r.headers:911 info["size"] = int(r.headers["content-length"])912 elif "content-range" in r.headers:913 info["size"] = int(r.headers["content-range"].split("/")[1])914 915 for checksum_field in ["ETag", "Content-MD5", "Digest"]:916 if r.headers.get(checksum_field):917 info[checksum_field] = r.headers[checksum_field]918 919 return info920 921 922# importing this is enough to register it923def register():924 register_implementation("http", HTTPFileSystem, clobber=True)925 register_implementation("https", HTTPFileSystem, clobber=True)926 register_implementation("sync-http", HTTPFileSystem, clobber=True)927 register_implementation("sync-https", HTTPFileSystem, clobber=True)928 929 930register()931 932 933def unregister():934 from fsspec.implementations.http import HTTPFileSystem935 936 register_implementation("http", HTTPFileSystem, clobber=True)937 register_implementation("https", HTTPFileSystem, clobber=True)938