CoolFace
Apppublic

Aluode/PerceptionLabPortable

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
rootpython_embedLibsite-packagesfsspecimplementations__pycache__http.cpython-310.pyc
http.cpython-310.pyc309 linesDownload Raw Back to __pycache__
1o

2��Yi�w�@sVddlZddlZddlZddlZddlZddlmZddlmZddlZddl	Z	ddl3mZmZm
Z
mZddlmZddlmZddlmZddlmZmZmZmZmZd	d4lmZe�d�Ze�d�Ze� d
�Z!dd�Z"Gdd�de�Z#Gdd�de�Z$e�d�Z%dd�Z&Gdd�de�Z'Gdd�de�Z(d"dd�Z)d#dd�Z*d"d d!�Z+ee+�Z,dS)$�N)�copy)�urlparse)�AbstractAsyncStreamedFile�AsyncFileSystem�sync�sync_wrapper)�DEFAULT_CALLBACK)�FSTimeoutError)�AbstractBufferedFile)�DEFAULT_BLOCK_SIZE�glob_translate�5isfilelike�nullcontext�tokenize�)�AllBytesz6<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P<url>[^"']+)z-(?P<url>http[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)zfsspec.httpc�s�tjdi|��S)N�)�aiohttpZ
ClientSession)�kwargsrr��E:\DocsHouse\542 percep lab latest\PerceptionLab\PerceptionLab_Portable\python_embed\Lib\site-packages\fsspec/implementations/http.py�6get_client s�rcs0eZdZdZdZdZdddddddddedf�fdd	�	Zed7d��Z	dd
�Z8edd��Zdd�Z
edd��Ze�fdd��Zd8dd�Zd8dd�Zee�Zdd�Zd9dd�Zdefdd �Zded!d"fd#d$�Zd:d%d&�Zd'd(�Z	)					d;d*d+�Zd<d,d-�Zd.d/�Zd0d1�Zd=d2d3�Z d4d5�Z!d>d6d7�Z"�Z#S)?�HTTPFileSystema29    Simple File-System for fetching data via HTTP(S)10 11    ``ls()`` is implemented by loading the parent page and doing a regex12    match on the result. If simple_link=True, anything of the form13    "http(s)://server.com/stuff?thing=other"; otherwise only links within14    HTML href tags will be used.15    )�http�https�/TN�bytesFcs�t�j|f||d�|��|dur|nt|_||_||_||_||_|	p%i|_|16|_	||_17||_d|_t
|�}
|
�dd�|_|
�dd�|
�dd�|
�dd�|
|_dS)a�18        NB: if this is called async, you must await set_client19 20        Parameters21        ----------22        block_size: int23            Blocks to read bytes; if 0, will default to raw requests file-like24            objects instead of HTTPFile instances25        simple_links: bool26            If True, will consider both HTML <a> tags and anything that looks27            like a URL; if False, will consider only the former.28        same_scheme: True29            When doing ls/glob, if this is True, only consider paths that have30            http/https matching the input URLs.31        size_policy: this argument is deprecated32        client_kwargs: dict33            Passed to aiohttp.ClientSession, see34            https://docs.aiohttp.org/en/stable/client_reference.html35            For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}``36        get_client: Callable[..., aiohttp.ClientSession]37            A callable, which takes keyword arguments and constructs38            an aiohttp.ClientSession. Its state will be managed by39            the HTTPFileSystem class.40        storage_options: key-value41            Any other parameters passed on to requests42        cache_type, cache_options: defaults used in open()43        )�asynchronous�loopN�use_listings_cacheFZlistings_expiry_timeZ	max_pathsZskip_instance_cache)�super�__init__r�44block_size�simple_links�same_schema�45cache_type�
cache_options�
client_kwargsr�encodedr�_sessionr�popr)�selfr"r!Zsame_scheme�size_policyr$r%rrr&rr'Zstorage_optionsZrequest_options��	__class__rrr 1s"*4647zHTTPFileSystem.__init__cCsdS)Nrr�r*rrr�fsidrszHTTPFileSystem.fsidcCstj||jd�S)N)r')�yarl�URLr'�r*�urlrrr�48encode_urlvszHTTPFileSystem.encode_urlc49Csd|dur |��r zt||jdd�WdStttfyYnwt|dd�}|dur0|��dSdS)Ng�������?)�timeoutZ50_connector)Z51is_runningr�close�TimeoutErrorr	�NotImplementedError�getattr�_close)r�sessionZ	connectorrrr�
close_sessionys��zHTTPFileSystem.close_sessionc�sN�|jdur$|jdd|ji|j��IdH|_|js$t�||j|j|j�|jS)Nrr)r(rrr&r�weakref�finalizer<r.rrr�set_session�s�52 zHTTPFileSystem.set_sessioncCs|S)z-For HTTP, we always want to keep the full URLr)�cls�pathrrr�_strip_protocol�szHTTPFileSystem._strip_protocolcs t��|�}t|�dkr|SdS)N��)r�_parent�len)r@rAZparr,rrrE�szHTTPFileSystem._parentc
	�sF�|j��}|�|�t�|�|��IdH}|j|�|�fi|j��4IdH�]}|�||�d|j	vr?|j	d�53d�d}nd}|dvryz(|jdd�IdH}|jrbt
�|�dd�t�|�D�}	n54d	d�t�|�D�}	Wn
tyxg}	Ynwg}	Wd�IdHn1IdHs�wYt�}55t|�}|	D]e}t|t�r�|d56}|�d�r�t|�d57kr�|j�d|j�|��}|�d
�r�|jr�|�|�d�d�r�|58�|�q�|�dd
��|�dd
��d�d�r�|59�|�q�|dvr�|60�d�|�d�|�d�g��q�|61�s|�d��r|j |�d�dd�IdH}62|�rdd�|63D�St!|64�S)N�Content-Type�;r)z	text/htmlN�ignore)�errorscS�g|]}|d�qS�rr��.0�urrr�65<listcomp>��z+HTTPFileSystem._ls_real.<locals>.<listcomp>cSrKrLrrMrrrrP�rQ�rz://rr)z..z../F)�detailcSs&g|]}|d|�d�r
dndd��qS)Nr�	directory�file��name�size�type)�endswithrMrrrrP�s���)"rr�update�logger�debugr?�getr4�_raise_not_found_for_status�headers�	partition�textr"�ex2�findall�ex�UnicodeDecodeError�setr�66isinstance�tuple�67startswithrF�scheme�netlocr#�rstrip�add�replace�join�lstriprZ�_ls_real�sorted)
r*r3rSr�kwr;�r�mimetyperb�links�out�parts�lrrrrr�s^�686970$71 ��(�7273�74� ��	zHTTPFileSystem._ls_realc�sJ�|jr||jvr|j|}|S|j|fd|i|��IdH}||j|<|S)NrS)rZdircacherr)r*r3rSrrxrrr�_ls�s�75�76zHTTPFileSystem._lscCs|jdkr	t|��|��dS)zU77        Raises FileNotFoundError for 404s, otherwise uses raise_for_status.78        i�N)�status�FileNotFoundError�raise_for_status)r*�responser3rrrr_�s79z*HTTPFileSystem._raise_not_found_for_statusc80	�s��|j��}|�|�t�|�|dus|dur6||krdS|�di���}|�|||�IdH|d<||d<|��IdH}|j|�	|�fi|��4IdH�}|�81�IdH}	|�||�Wd�IdH|	S1IdHslwY|	S)N�r`�Range)rrr[r\r]r)Z_process_limitsr?r^r4�readr_)82r*r3�start�endrrtr`r;rurxrrr�	_cat_file�s$�838485"��zHTTPFileSystem._cat_fileiPc86�s<�|j��}|�|�t�|�|��IdH}|j|�|�fi|��4IdH�j}z	t|j	d�}	Wn
t87tfy>d}	Ynw|�|	�|�
||�t|�rQ|}88nt|d�}89z%d}|rr|j�|�IdH}|90�|�|�t|��|s[Wt|�s{|91��n92t|�s�|93��wwWd�IdHdS1IdHs�wYdS)Nzcontent-length�wbT)rrr[r\r]r?r^r4�intr`�94ValueError�KeyError�set_sizer_r
�open�contentr��write�relative_updaterFr6)r*�rpath�lpath�95chunk_size�callbackrrtr;rurX�outfile�chunkrrr�	_get_file�s@�969798"�99100101����102��.�zHTTPFileSystem._get_file�post�	overwritec
	�s��|dkr	td�����fdd�}|j��}	|	�|�|��IdH}103|��}|dvr1td|����t|104|�}||�|�fd|�i|	��4IdH�}|�	||�Wd�IdHdS1IdHs`wYdS)Nr�zExclusive writecs��t�tj�rt��}d}nt�d�}d}|�C}|r)��|�dd��|�d�n	��t|dd��|���}|rR|V��	t105|��|���}|s9Wd�dSWd�dS1s]wYdS)NF�rbTrrrX)rh�io�IOBaserr�r��seekr9r�r�rF)�contextZuse_seek�fr��r�r�r�rr�106gen_chunks%s(�107108109��"�z,HTTPFileSystem._put_file.<locals>.gen_chunks)r��putz.method has to be either 'post' or 'put', not: �data)110r8rrr[r?�lowerr�r9r4r_)
r*r�r�r�r��method�moderr�rtr;�meth�resprr�r�	_put_files �111112113�114&.�zHTTPFileSystem._put_filec	�s��|j��}|�|�zHt�|�|��IdH}|j|�|�fi|��IdH}|4IdH�|r6|�||�|j	dkWd�IdHWS1IdHsLwYWdSt115y]YdStjyj|rg�YdSw)Ni�F)
rrr[r\r]r?r^r4r_r|r}rZClientError)r*rA�strictrrtr;rurrr�_existsJs&�1161171184��zHTTPFileSystem._existsc�s�|j|fi|��IdHS�N)r�)r*rArrrr�_isfile\s�zHTTPFileSystem._isfiler�c119Ks�|dkrt�|dur|n|j}|j��}	|j|	d<|	�|�i}120|p1|121�|j|fi|���p1|122d}t|j|j	�}|r[|r[|123�124dd�r[t||f|||||pO|j|pS|j
|jd�|	��St||f||j|d�|	��S)	a�Make a file-like object125 126        Parameters127        ----------128        path: str129            Full URL with protocol130        mode: string131            must be "rb"132        block_size: int or None133            Bytes to download in one request; use instance value if None. If134            zero, will return a streaming Requests file-like instance.135        kwargs: key-value136            Any other parameters, passed to requests calls137        r�NrrX�partialT)r;r!r�rXr$r%r)r�rr;)r8r!rrrr[�inforrr?r^�HTTPFiler$r%�HTTPStreamFile)r*rAr�r!Z138autocommitr$r%rXrrtr�r;rrr�_open_sD139140141$��142�
���zHTTPFileSystem._openc�sh�|��IdH}|dur&z|j|fi|��IdHd}Wn	ty%Ynwt||f|j||d�|��S)NrX)rr;rX)r?�_infor}�AsyncStreamFiler)r*rAr�rXrr;rrr�143open_async�s$� ����zHTTPFileSystem.open_asynccCst||j|j�S)z;Unique identifier; assume HTTP files are static, unchanging)rr�protocolr2rrr�ukey�szHTTPFileSystem.ukeyc�s��i}|��IdH}dD]G}z#|�t|�|�f||d�|j�|��IdH�|�d�dur0Wn$WqtyS}z|dkrBt|�|�tj	d|d�WYd}~qd}~ww|dd�|�d	d144i�S)aHGet info of URL145 146        Tries to access location via HEAD, and then GET methods, but does147        not fetch the data.148 149        It is possible that the server does not supply any size information, in150        which case size will be given as None (and certain operations on the151        corresponding file will not work).152        N)�headr^)r+r;rXr^rD)�exc_info�rWrXrYrU)153r?r[�154_file_infor4rr^�	Exceptionr}r\r])r*r3rr�r;�policy�excrrrr��s6�155���156��	�157��zHTTPFileSystem._infoc�s�|dur
|dkr
td��ddl}|�d�}|�|�}|p!|�d��|�d�dkr.|�d�nt|�}|�d�dkr>|�d�nt|�}t||�}|�d	d158�}	t|�sv|j	|fi|��IdHrp|	sb|gS||j159|fi|��IdHiS|	stgSiSd|d|�vr�|d|��d�}|d|d�}160||dd��d�d}nd}161||dd��d�d}d|vr�|dur�|�d�}||d��d�d}
||
|}nd}|j
|162f|d
d
d�|��IdH}t||r�dnd��|������fdd�t|���D�}|	r�|St|�S)a163        Find files by glob-matching.164 165        This implementation is idntical to the one in AbstractFileSystem,166        but "?" is not considered as a character for globbing, because it is167        so common in URLs, often identifying the "query" part.168        NrRzmaxdepth must be at least 1rr)z/**z/*�*�[rSFrDz**T)�maxdepthZwithdirsrScsJi|]!\}}��|�d��r�s |ddkr |�d�r |�d�n||�qS)rrYrT)�matchrmrZ)rN�pr��Zappend_slash_to_dirname�patternrr�169<dictcomp>s���170��z(HTTPFileSystem._glob.<locals>.<dictcomp>)r��rerZrB�findrF�minr)�	has_magicr�r��rindex�count�_findr�compilers�items�list)r*rAr�rr�Zends_with_slashZidx_starZ	idx_brace�min_idxrS�root�depthZidx_double_starsZdepth_double_starsZallpathsrxrr�r�_glob�s\�171172  173174���175176�zHTTPFileSystem._globc	�s2�zt|�|�IdH�WSttfyYdSw)NF)�boolr{r}r�)r*rArrr�_isdirs��zHTTPFileSystem._isdirc		�s��|�|�}|�di�}tt|��|d<|��IdH}|j|f||d�|��4IdH�}|��Wd�IdHdS1IdHsAwYdS)a�177        Write bytes to a remote file over HTTP.178 179        Parameters180        ----------181        path : str182            Target URL where the data should be written183        value : bytes184            Data to be written185        mode : str186            How to write to the file - 'overwrite' or 'append'187        **kwargs : dict188            Additional parameters to pass to the HTTP request189        r`�Content-LengthN)r�r`)rBr)�strrFr?r�r~)	r*rA�valuer�rr3r`r;rurrr�190_pipe_files�191"192.�zHTTPFileSystem._pipe_file)T)NN)F)r�NNNNN)r�Nr�)r�)$�__name__�193__module__�__qualname__�__doc__r��seprr �propertyr/r4�staticmethodr<r?�classmethodrBrErrr{rZlsr_r�rr�r�r�r�r�r�r�r�r�r�r��
__classcell__rrr,rr$sj	�A194195196197198<199200�"201�2022203�2049205"IrcsheZdZdZ								d�fdd�	Zd�fd	d206�	Zdd�Zee�Zd
d�Z	dd�Z207ee208�Z�ZS)r�a�209    A file-like object pointing to a remote HTTP(S) resource210 211    Supports only reading, with read-ahead of a predetermined block-size.212 213    In the case that the server does not supply the filesize, only reading of214    the complete file in one go is supported.215 216    Parameters217    ----------218    url: str219        Full URL of the remote resource, including the protocol220    session: aiohttp.ClientSession or None221        All calls will be made within this session, to avoid restarting222        connections where the server allows this223    block_size: int or None224        The amount of read-ahead to do, in bytes. Default is 5MB, or the value225        configured for the FileSystem creating this file226    size: None or int227        If given, this is the size of the file in bytes, and we don't attempt228        to call the server to find the value.229    kwargs: all other key-values are passed to requests calls.230    Nr�rFc	sZ|dkrtd��|231|_|	|_||_||_||dd�|_t�jd||||||d�|��dS)Nr�zFile mode not supportedrUrV)�fsrAr�r!r$r%r)r8rrr3r;�detailsrr )r*r�r3r;r!r�r$r%rXrrrr,rrr Ps"�232�zHTTPFile.__init__�����csj|dkr|jdkr|jdur|j|jks|��|jdur&|dkr%|��n	t|j|j|�}t��|�S)a5Read bytes from file233 234        Parameters235        ----------236        length: int237            Read up to this many bytes. If negative, read all content to end of238            file. If the server has not supplied the filesize, attempting to239            read only part of the data will raise a ValueError.240        rN)�locrX�	blocksize�241_fetch_allr�rr�)r*�lengthr,rrr�os242�z
HTTPFile.readc	�s��t�d|���t|jt�s\|jj|j�|j	�fi|j243��IdH}|4IdH�'|��|��IdH}tt
|�dd|d�|_t
|�|_Wd�IdHdS1IdHsUwYdSdS)z�Read whole file in one shot, without caching244 245        This is only called when position is still at zero,246        and read() is called without a byte-count.247        zFetch all for N)rXZfetcherr�r�)r\r]rh�cacherr;r^r�r4r3rr~r�rFrX)r*rurxrrr�async_fetch_all�s�&�.��zHTTPFile.async_fetch_allcCsx|�dd�}t�d|�}|sdS|ddkrd}}ndd	�|d�d248�D�\}}|ddkr1dnt|d�}|||fS)zParse the Content-Range header�
Content-RangerDzbytes (\d+-\d+|\*)/(\d+|\*))NNNrRr�NcSsg|]}t|��qSr)r�)rN�xrrrrP�rQz1HTTPFile._parse_content_range.<locals>.<listcomp>�-r)r^r�r��splitr�)r*r`�s�mr�r��totalrrr�_parse_content_range�s249250zHTTPFile._parse_content_rangec251	�s��t�d|�d|�d|���|j��}|�di���}d|�d|d��|d<t�|j�d|d���|jj|j�	|j�fd|i|��Id	H}|4Id	H��|j252d253kra	Wd	�Id	HdS|��|j254dkp�|�|j
�d
|kp�t|j
�d|d��||k}|r�|��Id	H}n8|d
kr�td��d
}g}	|j�d�Id	H}	|	r�|�|	�|t|	�7}|||kr�nnnq�d�|�d	||�}|Wd	�Id	HS1Id	Hs�wYd	S)a3Download a block of data255 256        The expectation is that the server returns only the requested bytes,257        with HTTP code 206. If this is not the case, we first check the headers,258        and then stream the output - if the data size is bigger than we259        requested, an exception is raised.260        zFetch range for z: r�r`�bytes=rRr�z : Ni�r���rr�z�The HTTP server doesn't appear to support range requests. Only reading this file from the beginning is supported. Open with block_size=0 for a streaming file interface.Ti)r\r]rrr)r3r;r^r�r4r|r~r�r`r�r�r�r��appendrFrp)261r*r�r�rr`ruZresponse_is_rangerx�clr�rrr�async_fetch_range�sX�262���263�264���265��2660�zHTTPFile.async_fetch_range)NNr�rNNNF�r�)
r�r�r�r�r r�r�rr�r�r�Z_fetch_ranger�rrr,rr�7s"�;r�z([*[])cCst�|�}|duSr�)�magic_check�search)r�r�rrrr��s267r�csNeZdZd�fdd�	Zddd�Zdd	d268�Zee�Zdd�Z�fd
d�Z	�Z269S)r�r�Ncs~��dd��_��_|�_|�_|dkrt��dd��_t�jd	|�|dd�������fdd�}t	�j|��_270|j�_dS)271NrFr�r��none�r�rAr�r$c�s:��jj�j���fi�����IdH}�j�|��|Sr�)r;r^r�r4�272__aenter__r_)ru�rr*r3rr�cor�s�&z$HTTPStreamFile.__init__.<locals>.corr)r)rr3rr;r�r�rr rru)r*r�r3r�rr;rr�r,r�rr �szHTTPStreamFile.__init__rcCs2|dkr273|dkr274dS||jkr|dkrdStd��)NrrRzCannot seek streaming HTTP file)r�r�)r*r��whencerrrr�s275zHTTPStreamFile.seekr�c�s,�|jj�|�IdH}|jt|�7_|Sr�)rur�r�r�rF)r*�numrxrrr�_reads�zHTTPStreamFile._readc�s�|j��dSr�)rur6r.rrrr:s�zHTTPStreamFile._closecs t�|��|j�t���dSr�)�asyncioZrun_coroutine_threadsafer:rrr6r.r,rrr6szHTTPStreamFile.close)r�NN)rr�)r�r�r�r r�rrr�r:r6r�rrr,rr��s276277r�cs6eZdZ	d278�fdd�	Zddd�Z�fdd	�Z�ZS)r�r�NcsN||_||_d|_|dkrt�|dd�|_||_t�j|||dd�||_dS)Nr�r�r�r�)	r3r;rur�r�rrr rX)r*r�r3r�rr;rXrr,rrr s279zAsyncStreamFile.__init__r�c�sv�|jdur&|jj|j�|j�fi|j����IdH}|j�||j�||_|jj	�280|�IdH}|jt|�7_|Sr�)
rur;r^r�r4r3rr�r_r�r�r�rF)r*rrurxrrrr�$s�281���zAsyncStreamFile.readc�s0�|jdur|j��d|_t���IdHdSr�)rur6rr.r,rrr6/s282�283284zAsyncStreamFile.close)r�NNNr�)r�r�r�r r�r6r�rrr,rr�s285�286
r�c287	�s��|��}|�di���}d|�d|d��|d<|j|fd|i|��IdH}|��|4IdH�|��IdH}Wd�IdHn1IdHsIwY|rrt|d��}	|	�|�|	�|�Wd�dS1skwYdS|S)Nr`r�r�rRr�zr+b)rr)r^r~r�r�r�r�)288r;r3r�r�rUrr`rurxr�rrr�	get_range6s�(�289"�rr�c		�s��t�d|�|��}|�dd�}|�di���}d|d<||d<i}|dkr6|j|fd|i|��IdH}n|d	krI|j|fd|i|��IdH}ntd290|����|4IdH�t|��d|jvrvd|jvsl|jdd
vrut	|jd�|d<nd|jvr�t	|jd�291d�d�|d<d|jvr�|jd�d�d|d<|j�d�dkr�d|d<t|j
�|d<dD]}|j�|�r�|j|||<q�Wd�IdH|S1IdHs�wY|S)z�Call HEAD on the server to get details about the file (size/checksum etc.)292 293    Default operation is to explicitly allow redirects and use encoding294    'identity' (no compression) to get the true size of the target.295    zRetrieve file size for %s�allow_redirectsTr`�identityzAccept-Encodingr�Nr^z)size_policy must be "head" or "get", got r�zContent-Encoding)rrDrXr�rrRrGrHrrvz
Accept-Rangesr�Fr�r3)�ETagzContent-MD5�Digestz
Last-Modified)r\r]rr)r^r��	TypeErrorr~r`r�r�rar�r3)	r3r;r+r�arr�r�ruZchecksum_fieldrrrr�GsF�296�297298����r�c�s@�|durt�IdH}t|g|�Rd|i|��IdH}|�d�S)Nr;rX)rr�r^)r3r;�argsrr�rrr�299_file_sizezs300� 301rr�)r�)-rr��loggingr�r=r�urllib.parserrr0Zfsspec.asynrrrrZfsspec.callbacksrZfsspec.exceptionsr	Zfsspec.specr302Zfsspec.utilsrrr
rrZcachingrr�rerc�	getLoggerr\rrr�r�r�r�r�rr�r�	file_sizerrrr�<module>sD303304305306/)307 3083093
Aluode/PerceptionLabPortable · CoolFace