Apppublic
Aluode/PerceptionLabPortable

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
rootpython_embedLibsite-packagesjoblibtest__pycache__test_memmapping.cpython-310.pyc
test_memmapping.cpython-310.pyc346 linesDownload Raw Back to __pycache__
1o

2��YiӪ�
@s�ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl3mZddlZddl
mZddl
mZmZmZmZmZmZddlmZddlmZddlmZmZddlmZddl m!Z!m"Z"m#Z#m$Z$m%Z%dd	l&m'Z'm(Z(m)Z)d4d�Z*dd
�Z+dd�Z,dd�Z-dd�Z.e%e$dd���Z/e%e$e)ejdkdd�dd����Z0e%e$dd���Z1e%dd��Z2e%e$e'deej3gd d!gd"�d#d$����Z4e%e$e'deej3gd d!gd"�d%d&����Z5e%e$e'd'd d!g�d(d)����Z6e%e$e'd'd d!g�d*d+����Z7e%e$e'd'd d!g�d,d-����Z8e%e$e'd'd d!g�d.d/����Z9e%e$d0d1���Z:e%e$d2d3���Z;e%e$e'd'd d!g�d4d5����Z<e%e$e'd'd d!g�d6d7����Z=e%e$e'd'd d!g�d8d9����Z>e%e$e'deej3gd d!gd"�d:d;����Z?e%e$e'd'ej@d ejAjBd<d�d=�d!g�d>d?����ZCe%e$e'deej3gd d!gd"�d@dA����ZDe%e$e#e'deej3gd d!gd"�dBdC�����ZEe%e$e#e'deej3gd d!gd"�dDdE�����ZFe%e$e'deej3gd d!gd"�dFdG����ZGdHdI�ZHe%e$e'deej3gd d!gd"�dJdK����ZIdLdM�ZJe%e$e'dNeKe�Leej3geMdO���dPdQ�e�Ld d!geNeOeMdO���D�d"�dRdS����ZPdTdU�ZQdVdW�ZRe%e)ejdkdXd�e'dYdZd[g�d\d]����ZSe%d^d_��ZTd`da�ZUe%e$dbdc���ZVe%e$ddde���ZWdS)f�N)�sleep)�ArrayMemmapForwardReducer�_get_backing_memmap�
_get_temp_dir�_strided_from_memmap�_WeakArrayKeyMap�has_shareable_memory)�make_memmap)�_TestingMemmappingExecutor)�Parallel�delayed)�MemmappingPool)�IS_GIL_DISABLED�np�with_dev_shm�with_multiprocessing�5with_numpy)�parametrize�raises�skipifcCstjddd�dS)Ni,T)�timeout�exit)�faulthandlerZdump_traceback_later�rr��E:\DocsHouse\542 percep lab latest\PerceptionLab\PerceptionLab_Portable\python_embed\Lib\site-packages\joblib/test/test_memmapping.py�setup_module&srcCst��dS�N)rZcancel_dump_traceback_laterrrrr�teardown_module*srcCst|�dusJ�|Sr)r��arrayrrr�check_memmap_and_send_back.sr cCs |\}}}tj�|||�dS)z�Dummy helper function to be executed in subprocesses6 7    Check that the provided array has the expected values in the provided8    range.9 10    N�r�testing�assert_array_equal��args�data�position�expectedrrr�check_array3s11r)cCsD|\}}}|||ks
J�||d9<tj�||d|�dS)z�Dummy helper function to be executed in subprocesses12 13 14    Check that the input array has the right values in the provided range15    and perform an inplace modification to double the values in the range by16    two.17 18    �Nr!r$rrr�inplace_double>s19	r+cs.tjj}|�d�j}tj|tjddd�}dtj|jd|j	d�|dd�<|�20�tj|tjd	d21ddd
�}t�d��|j�|dd�<|dd�dd�dd�f}|�dd�}t�|�}|j
}td|jdd���fdd�}	|	|�}22t|23�suJ�t|24tj�s}J�||25|�|	|�}t|�s�J�|||�|	|�}t|�s�J�|||�|	|�}
t|
tj�r�J�t|
�s�J�||
|�|	|�}t|tj�r�J�t|�s�J�|||�|d}t|�r�J�|	|�}t|�r�J�t|tj�r�J�|||d�t�|�}t|�r�J�|	|�}t|tj��s	J�t|��rJ�|||�dS)z9Check that it is possible to reduce a memmap backed array�	test.mmapi��w+��dtype�shape�modeg�r�r/N)����r+�Fr5)r/r0r1�order�offset�<������r*�26�2�cTc��|�\}}||�Srr��xZconsr%�Zreducerrr�reconstruct_array_or_memmaps�zEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapr3)rr"r#�join�strpath�memmap�float64�aranger0r/�flush�reshape�asarray�Trr�27isinstance�ndarray)�tmpdirr#�filename�buffer�a�b�b2r?�drD�a_reconstructed�b_reconstructedZb2_reconstructed�c_reconstructed�d_reconstructed�a3Za3_reconstructedZb3Zb3_reconstructedrrCr� test_memmap_based_array_reducingMsZ"�28293031323334r]�win32z2PermissionError only easily triggerable on Windows)�reasoncCs�|�d�j}dj|d�}tjtjd|gtjtjd�}|��|�	�\}}|j35dks/J|����|dks5J�d�|�}||��vsBJ�dS)	Nr,a�if 1:36    import os37    import numpy as np38    import time39    from joblib.externals.loky.backend import resource_tracker40    resource_tracker.VERBOSE = 141 42    # Start the resource tracker43    resource_tracker.ensure_running()44    time.sleep(1)45 46    # Create a file containing numpy data47    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')48    memmap[:] = np.arange(10).astype(np.int8).data49    memmap.flush()50    assert os.path.exists(r"{filename}")51    del memmap52 53    # Create a np.memmap backed by this file54    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')55    resource_tracker.register(r"{filename}", "file")56 57    # Ask the resource_tracker to delete the file backing the np.memmap , this58    # should raise PermissionError that the resource_tracker will log.59    resource_tracker.maybe_unlink(r"{filename}", "file")60 61    # Wait for the resource_tracker to process the maybe_unlink before cleaning62    # up the memmap63    time.sleep(2)64    )rR�-c��stderr�stdoutr�z'tried to unlink {}, got PermissionError)rFrG�format�65subprocess�Popen�sys�66executable�PIPE�wait�communicate�67returncode�decode)rQrR�cmd�p�out�err�msgrrr�2test_resource_tracker_retries_when_permissionerror�s��68rtcsRtjj}|�d�j}tj|tjddd�}t�d��|j	�|dd�<|dd�}|dd�dd�f}|dd�dd�dd�df}|d	d69d�}t70d|jdd
���fdd�}||�}	t|	�s^J�t|	tj�sfJ�||	|�||�}71t|72�suJ�||73|�||�}t|�s�J�|||�||�}t|�s�J�|||�||�}
t|
�s�J�||
|�dS)Nr,)�d�rvr3r-r.i�rr=r4r;r3r5r?Tcr@rrrArCrrrD�rEzNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap)
rr"r#rFrGrHrIrJrLr0rrrO)rQr#rRrTrUr?rW�erDrXrYrZr[Ze_reconstructedrrCr�)test_high_dimension_memmap_array_reducing�s474757677rxcCs�|�d�j}dtj}tjd}tj|d||d�}t|dd|d|ddd	d78�	}t|tj�s/J�|j|ks6J�t|dd|d|dfd|d	d79�	}t	|�j|ksOJ�dS)
Nr,r4r;r-)r1r0�uint8�r�CF)r/r1r9r8r0�stridesZtotal_buffer_lenZunlink_on_gc_collectr*)r*)80rFrG�mmap�ALLOCATIONGRANULARITYrrHrrOr9r)rQ�fname�sizer9Z81memmap_objZmemmap_backed_objrrr�test__strided_from_memmaps:8283��r��factory�multiprocessing�loky)Zidscs�tjj}|�d�j}|dd|d�}z�|�d�j}tj|tjddd����d	�|�	t84�fd85d�t�jd�D��|�dt�
�j��tj|tjd
dd��|�	t86�fdd�t�jd�D��t�|�gksiJ�|�dt�
�j��|�dt�
�j��tj|tjdddd��tt��|�	t�fdd�t�jd�D��Wd�n1s�wYtttf��|�	t87�fdd�t�jd�D��Wd�n1s�wYW|��~dSW|��~dS|��~w)z@Check that subprocess can access and update shared memory memmap�poolr=r*��88max_nbytes�temp_folderr,�r3r4r-r.��?c�.g|]}t�jd�D]	}�||fdf�qqS�r;r���ranger0��.0�i�j)rTrr�89<listcomp>=�.z)test_pool_with_memmap.<locals>.<listcomp>r)r4r3r?cr�)r;�@r�r��rUrrr�Gr��r=rz�)r/r0r1r9c�g|]}�|df�qS)g@r�r�r��r?rrr�V�Ncr�)r�rr�r�rrr�[r�)rr"r#�mkdirrGrFrH�float32�fill�mapr+r�r0�ones�os�listdirr�AssertionErrorr)�RuntimeError�90ValueError�	terminate�r�rQr#�pool_temp_folderrprRr)rTrUr?r�test_pool_with_memmap(sD	91��92&�&���r�cs�tjj}|�d�j}|dd|d�}zb|�d�j}tj|tjddd����d	�t�	���t93�tj�r4J�t��s:J�|�t
��fd94d�t�jd�D��|�dt��j��|�dt��j��t�|�gkslJ�W|��~d
S|��~w)z?Check that subprocess can access and update shared memory arrayr�r=r*r�r,r�r-r.r�cs.g|]}t�jd�D]	}�||fdf�qqSr�r�r��rTZa_viewrrr�|s���z4test_pool_with_memmap_array_view.<locals>.<listcomp>rN)rr"r#r�rGrFrHr�r�rMrOrr�r+r�r0r�r�r�r�r�rr�r� test_pool_with_memmap_array_viewbs.	9596��97�r��backendcCsbdj|d�}tjtjd|gtjtjd�}|��|��\}}|jdks/J|�	�d|�	���dS)N�if 1:98        import numpy as np99        from joblib import Parallel, delayed100 101 102        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))103 104        # Build a complex cyclic reference that is likely to delay garbage105        # collection of the memmapped array in the worker processes.106        first_list = current_list = [data]107        for i in range(10):108            current_list = [current_list]109        first_list.append(current_list)110 111        if __name__ == "__main__":112            results = Parallel(n_jobs=2, backend="{b}")(113                delayed(len)(current_list) for i in range(10))114            assert results == [1] * 10115    r�r`rar�116 117)118rerfrgrhrirjrkrlrmrn�r�rorprqrrrrr�-test_permission_error_windows_reference_cycle�s	��&r�cCs�dj|d�}td�D]=}tj��}tj�t�|d<tj	t119jd|gtjtj|d�}|�
�|��\}}|jdks;J|��|dksAJ�d	|vsGJ�q120dS)121Nakif 1:122        import os123        import time124 125        import numpy as np126 127        from joblib import Parallel, delayed128        from testutils import return_slice_of_data129 130        data = np.ones(int(2e6))131 132        if __name__ == '__main__':133            # warm-up call to launch the workers and start the resource_tracker134            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(135                delayed(id)(i) for i in range(20))136 137            time.sleep(0.5)138 139            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(140                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))141    r�r3�142PYTHONPATHr`�rbrc�envrrd�resource_tracker)rer�r��environ�copy�path�dirname�__file__rfrgrhrirjrkrlrm)r�ro�_r�rprqrrrrr�3test_permission_error_windows_memmap_sent_to_parent�s$143�144145��r�cszt�td���td|dd��fdd�td�D��\}td|dd��fdd�td�D��\}tj�|�tj�|�ks;J�dS)	N�Y@r*r=��n_jobsr�r�c3��|]146}tt��d�VqdS�rRN�r�getattr�r�r�rrr�	<genexpr>���147�z6test_parallel_isolated_temp_folders.<locals>.<genexpr>r;c3r�r�r�r�rrrr��r��rrJ�intrr�r�r�r�)r��148filename_1�149filename_2rrr�#test_parallel_isolated_temp_folders�s150�151� r�cs�t�td���td|dd��$}|�fdd�td�D��\}|�fdd�td�D��\}Wd�n1s5wYtj�|�tj�|�ksHJ�dS)	Nr�r*r=r�c3r�r�r�r�rrrr����z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>r;c3r�r�r�r�rrrr��r�r�)r�rpr�r�rrr�&test_managed_backend_reuse_temp_folder�s� r�cCs�t�td��}t�}t�}dd�}tj|||fd�}tj|||fd�}|��|��|��|��t|�dks;J�t|�dksCJ�||ksIJ�dS)Nr�csptdddd��'}td�D]}|�fdd�td�D��\}|�tj�|��qWd�dS1s1wYdS)Nr�r*r=)r�r�r�c3r�r�r�r�rrrr�r�z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>r;)rr��addr�r�r�)rZ	temp_dirsrpr�rRrrr�concurrent_get_filename
s�"�zJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename)�targetr%r;)	rrJr��set�	threading�Thread�startrF�len)rZtemp_dirs_thread_1Ztemp_dirs_thread_2r��t1�t2rrr�)test_memmapping_temp_folder_thread_safetys"��r�cCs�d}gd�}|D]7\}}}tjtjd|j||d�gtjtjd�}|��|��\}}|j|ks5J|�	���d|vs?J|�	���qdS)Na>if 1:152        import os153        import numpy as np154        from joblib import Parallel, delayed155        from joblib.externals.loky.backend import resource_tracker156        from concurrent.futures import ThreadPoolExecutor, wait157 158        resource_tracker.VERBOSE = 0159 160        array = np.arange(int(1e2))161 162        temp_dirs_thread_1 = set()163        temp_dirs_thread_2 = set()164 165 166        def raise_error(array):167            raise ValueError168 169 170        def parallel_get_filename(array, temp_dirs):171            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:172                for i in range(10):173                    [filename] = p(174                        delayed(getattr)(array, "filename") for _ in range(1)175                    )176                    temp_dirs.add(os.path.dirname(filename))177 178 179        def parallel_raise(array, temp_dirs):180            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:181                for i in range(10):182                    [filename] = p(183                        delayed(raise_error)(array) for _ in range(1)184                    )185                    temp_dirs.add(os.path.dirname(filename))186 187 188        executor = ThreadPoolExecutor(max_workers=2)189 190        # both function calls will use the same loky executor, but with a191        # different Parallel object.192        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)193        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)194 195        # Wait for both threads to terminate their backend196        wait([future_1, future_2])197 198        future_1.result()199        future_2.result()200    ))�parallel_get_filenamer�r)r��parallel_raiser;)r�r�r;r`)�f1�f2rar�)201rfrgrhrirerjrkrlrmrn)roZfunctions_and_returncodesr�r�rmrpr�rrrrr�?test_multithreaded_parallel_termination_resource_tracker_silent&s2��r�cCs�dj|d�}tj��}tj�t�|d<tjt	j202d|gtjtj|d�}|��|�
�\}}|jdks7J|����|dksAJ|����d|vsGJ�dS)	Na�if 1:203        import os204        import time205 206        import numpy as np207 208        from joblib import Parallel, delayed209        from testutils import return_slice_of_data210 211        data = np.ones(100)212 213        if __name__ == '__main__':214            for i in range(5):215                slice_of_data = Parallel(216                    n_jobs=2, max_nbytes=1, backend='{b}')(217                        delayed(return_slice_of_data)(data, 0, 20)218                        for _ in range(10)219                    )220    r�r�r`r�rrdr�)rer�r�r�r�r�r�rfrgrhrirjrkrlrmrn)r�ror�rprqrrrrr�'test_many_parallel_calls_on_same_objectqs 	�221222�r�csHt�td���td|dd��fdd�td�D��\}t|�dus"J�dS)Ng@�@r*rur�c3s�|]	}tt���VqdSr)rr r��r&rrr��s�223�z8test_memmap_returned_as_regular_array.<locals>.<genexpr>r;)rr�r�rr�r)r��resultrr�r�%test_memmap_returned_as_regular_array�s224225�r�cCs�|dkrtj�d�rt�d�dj|d�}tjtjd|gtj	tj	d�}|�226�|��\}}|��}|��}|j
dksBJ|d	|��d227|vsJJ|��dS)Nr��winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler�r�r`rarr�Zresource_tracker)rh�platform�228startswith�pytest�xfailrerfrgrirjrkrlrnrmr�rrr�2test_resource_tracker_silent_when_reference_cycles�s ���r�c229s�t�|j�gks230J�|dd|jdd�}z�t�|j�gksJ�tj�|j�r'J�tjdtjd���j	dks6J�|�231t�fdd	�t�j
d232�D��t�|j�gksRJ�tjdtjd���j	dksaJ�|�233t�fd
d	�t�j
d234�D��tj�|j�s|J�t�|j�}t|�dks�J�tjdgddd�}|�235t|g�}|d236r�J�W|��td�D]}td�tj�|j�s�~dSq�td�|j���|��td�D]}td�tj�|j�s�~wq�td�|j���)z0Check that large arrays are not copied in memoryr3�(r*)r�r��verboser4r2r�cr��r�rr�)�smallrrr��r�z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>rru� cr�r�rr���largerrr��r�r;�abc�objectr=皙�����?z#temporary folder {} was not deletedN)r�r�rGr��exists�_temp_folderrr�r��nbytesr�r)r�r0rI�isdirr�rrr�rr�re)r�rQrpZdumped_filenames�objects�resultsr�r)r�r�r�%test_memmapping_pool_for_large_arrays�sL$$�237���238�r�z,https://github.com/joblib/joblib/issues/1086)ZmarkscCs�dj|d�}tj��}tj�t�|d<tjt	j239d|gtjtj|d�}|��|�
�\}}|��|��}}|�d�d}|jdksGJ|pE|��|dksMJ�tj�|�rUJ�dS)	Na�if 1:240        import os241        from pathlib import Path242        from time import sleep243 244        import numpy as np245        from joblib import Parallel, delayed246        from testutils import print_filename_and_raise247 248        data = np.random.rand(1000)249 250        def get_temp_folder(parallel_obj, backend):251            if "{b}" == "loky":252                return Path(parallel_obj._backend._workers._temp_folder)253            else:254                return Path(parallel_obj._backend._pool._temp_folder)255 256 257        if __name__ == "__main__":258            try:259                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:260                    temp_folder = get_temp_folder(p, "{b}")261                    p(delayed(print_filename_and_raise)(data)262                              for i in range(1))263            except ValueError as e:264                # the temporary folder should be deleted by the end of this265                # call but apparently on some file systems, this takes266                # some time to be visible.267                #268                # We attempt to write into the temporary folder to test for269                # its existence and we wait for a maximum of 10 seconds.270                for i in range(100):271                    try:272                        with open(temp_folder / "some_file.txt", "w") as f:273                            f.write("some content")274                    except FileNotFoundError:275                        # temp_folder has been deleted, all is fine276                        break277 278                    # ... else, wait a bit and try again279                    sleep(.1)280                else:281                    raise AssertionError(282                        str(temp_folder) + " was not deleted"283                    ) from e284    r�r�r`r��285r�)rer�r�r�r�r�r�rfrgrhrirjrkrlrn�splitrmr�)r�ror�rprqrrrRrrr�&test_child_raises_parent_exits_cleanlys$-�286.287�r�cs�|dd|jd�}z=t�|j�gksJ�tjdtjd���jdks"J�|�t�fdd�t	�j288d	�D��t�|j�gks>J�W|��~dS|��~w)289z2Check that large arrays memmapping can be disabledr3Nr�rur2r�cr�r�rr�r�rrr��r�zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>r)rGr�r�rr�rIr�r�r)r�r0r�)r�rQrprr�r�.test_memmapping_pool_for_large_arrays_disabledws290$�r�cCs4tj}z�td�t_|ddd�}zc|j}d}|�|�sJ�tj�|�s$J�tj	dtj291d�}|jdks3J�|�t
|gd�tt�|��d	ksGJ�tj	dtj292d�d293}|jdksXJ�|�t
|gd�tt�|��d294kslJ�W|��~n|��~wtd�D]}tj�|�s�n	td�q}td��W|t_d
S|t_w)z1Check that memmapping uses /dev/shm when possibleg��~Ar3r=�r�z"/dev/shm/joblib_memmapping_folder_rur2r�r;r*r�z(temporary folder of pool was not deletedN)�jmr�SYSTEM_SHARED_MEM_FS_MIN_SIZEr�r�r�r�r�r�rr�rIr�r��idr�r�r�r�rr�)r��	orig_sizerpr�Z
folder_prefixrTrUr�rrr�'test_memmapping_on_large_enough_dev_shm�s8295296�297�rcCsrtj}z1td�t_|ddd�}z|j}|�d�rJ�W|��~n|��~wtj�|�r/J�W|t_dS|t_w)Ng@����6Dr3r=r�z/dev/shm)	r�rr�r�r�r�r�r�r�)r�rrpr�rrr�$test_memmapping_on_too_small_dev_shm�s	298�rcCsjtjj}|dd|jd�}z"|jtjdd�}|��}t|�rJ�||t�d��W|��~dS|��~w)z:Check that large arrays are not copied in memory in returnr3r=r�)���r%rN)	rr"r#rG�apply_asyncr��getrr�)r�rQr#rp�resr�rrr�/test_memmapping_pool_for_large_arrays_in_return�s		�r299cCst|�sJ�||S)z4Multiplication function to be executed by subprocess)r)rTZn_timesrrr�_worker_multiply300srcCs�tjj}|dd|jd�}z8tjt�d��d�dd�dd�dd	�dd�f}|jt|dfd301��	�}t302|�r6J�||d|�W|��~dS|��~w)z�Check that memmaps with a bad buffer are returned as regular arrays303 304    Unary operations and ufuncs on memmap instances return a new memmap305    instance with an in-memory buffer (probably a numpy bug).306    r3r=r�ip)rr*r3r7)r8Nr;r)rr"r#rGrMrJrLrrrrr�)r�rQr#rprTrUrrr�6test_workaround_against_bad_memmap_with_copied_bufferss
0�rcCs|Srr)�argrrr�identity/srzfactory,retry_nor3cCsg|]307\}}d�||��qS)z{}, {})re)r�rB�yrrrr�<s308��r�c	Cs�|�d�j}dtj}tjd}t|d|d|d�}|d|jd�}|jt|fd	���}t|t	j309�s2J�|j|ks9J�t	j�
||�|��dS)310Nr,r4r;r-ry)r1r0r/r9r*�r�r)rFrGr}r~r	rrrrOrrHr9r"r#r�)	r�Zretry_norQrr�r9�objrpr�rrr� test_pool_memmap_with_big_offset3s311312rcCsnd}t||j�\}}|dusJ�||�d�jksJ�t|dd�\}}tj�d�r.|dus.J�|�|�s5J�dS)N�test.tmpdirFrr�)rrGrFrhr�r��endswith)rQ�pool_folder_name�pool_folder�313shared_memrrr�test_pool_get_temp_dirSsrcCsdd}ddl}t|jjd�r|�|jjd�t|dd�\}}tj�d�r)|dus)J�|�	|�s0J�dS)z`Check that _get_temp_dir works when os.statvfs is not defined314 315    Regression test for #902316    rrNZstatvfsrr�F)317�joblib._memmapping_reducer�hasattr�_memmapping_reducerr��delattrrrhr�r�r)rQZmonkeypatchrZjoblibrrrrr�!test_pool_get_temp_dir_no_statvfs_srz1This test fails with a PermissionError on Windows�	mmap_moder6r-csbdd��dd�td�D�}t|ddd��fd	d318�t|�D��}t|�D]\}}tj�||�q#dS)NcSs||dd�<|Srr)�arr�valuerrr�funcwsz4test_numpy_arrays_use_different_memory.<locals>.funccSsg|]	}tjddd��qS))r=r=rIr2)rZzerosr�rrrr�{sz:test_numpy_arrays_use_different_memory.<locals>.<listcomp>r=rr*)rr�r�c3s"�|]\}}t��||�VqdSr�r)r�r�r�r!rrr�}s�319�z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>)r�r�	enumeraterr"r#)rZarraysr�r�rrr#r�&test_numpy_arrays_use_different_memoryqs��r%csxddd�}t�d�}t����|d���|�dksJ�|}��|�dks'J���|d���|�dks6J�~t��t�j�dksDJ���|�dksMJ�~|�j�t�d�}��|d�t�j�dksgJ���|�dkspJ�t	t320����t�d��Wd�n1s�wY~|�j�d	d321��t��fdd�td
�D��}t�
�dkr�tr�dnd}t|�|ks�J�dSdS)NrucSsDt|�D]}t|�dkrdSt��td�qt|�dks J�dS)Nrr�)r�r��gc�collectr)�	container�retriesr�rrr�assert_empty_after_gc_collect�s322z>test_weak_array_key_map.<locals>.assert_empty_after_gc_collect�*rTrUr;r?cSsbt�d�}tt��
|�|�Wd�n1swY|�||�|�|�|ks-J�t|�S)Nr+)rr�r�KeyErrorrr�r)�mr�rTrrr�get_set_get_collect�s323324�z4test_weak_array_key_map.<locals>.get_set_get_collectcsg|]}��|��qSrrr��r.r-rrr��r�z+test_weak_array_key_map.<locals>.<listcomp>r�CPythoni�)ru)rr�rr�rr&r'r��_datarr,r�r��python_implementationr)r*rTrUr?Z325unique_idsZmax_len_unique_idsrr/r�test_weak_array_key_map�s>326327328329330�331�r3cCs@t�}ttj��t�|�Wd�dS1swYdSr)rr�pickle�
PicklingError�dumps)r-rrr�#test_weak_array_key_map_no_pickling�s"�r7cs�t|�d���tjddd�}|����fdd��dd����}td	d332��fdd�|fD��}td	d333��fd
d�|fD��}tj�||�td	d334��fdd�td�D��}tj�|d|�dS)Nzarr.datr=ryr2csTt���}tj|��dtjdd�}Wd�n1swYtjdtj|dd�S)Nr)�accessr9r�)r/rSr9)�openr}�fileno�ACCESS_READrrPry)�fd�mm��testfilerr�_read_array�s335�z%test_direct_mmap.<locals>._read_arraycS�|dS�Nr*r�rBrrrr!��ztest_direct_mmap.<locals>.funcr*�r�c3��|]	}t��|�VqdSrr"�r�rBr#rrr����z#test_direct_mmap.<locals>.<genexpr>c3rFrr"rGr#rrr��rHc3��|]}t���VqdSrr"r��r@rrr����r;r)	�strrFrrJ�tofilerr"r#r��rQrTr�refr�r�r@r!r?r�test_direct_mmap�s336 rQcs�t|�d���tjddd��dd�}|����fdd��d	d337����}tdd��fdd
�|fD��}tdd��fdd
�|fD��}|djdrKJ�tj�	||�tdd��fdd
�t338d�D��}tj�	|d|�dS)Nzarr2.datr=ryr2r4r*cst���}|�dd�S)Nr4r*)rrHrL)r=r>rrr@�s339z=test_parallel_memmap2d_as_memmap_1d_base.<locals>._read_arraycSrArBrrCrrrr!�rDz6test_parallel_memmap2d_as_memmap_1d_base.<locals>.funcrEc3rFrr"rGr#rrr��rHz;test_parallel_memmap2d_as_memmap_1d_base.<locals>.<genexpr>c3rFrr"rGr#rrr��rHrZF_CONTIGUOUSc3rIrr"r�rJrrr��rKr;)rLrFrrJrLrMr�flagsr"r#r�rNrrPr�(test_parallel_memmap2d_as_memmap_1d_base�s340 rS)Xrr&�	itertoolsr}r�r4r�rfrhr��timerr�rrr�rrrrrrZjoblib.backportsr	Zjoblib.executorr341ZTestExecutorZjoblib.parallelrrZjoblib.poolr
Zjoblib.test.commonrrrrrZjoblib.testingrrrrrr r)r+r]rtrxr�Zget_memmapping_executorr�r�r�r�r�r�r�r�r�r�r�r��param�markr�r�r�rrr342rrr�list�productr�r�rLrrrr%r3r7rQrSrrrr�<module>st V343�-,344"�3�'!-
	$I(	5�4����N��1�������345�346;