CoolFace
Apppublic

Aluode/PerceptionLabPortable

sourceHugging Faceupdated 9mo agoView on Hugging Face
0likes
test_parallel.cpython-310.pyc409 linesDownload Raw Back to __pycache__
1o

2��Yi1�@s�
dZddlZddlZddlZddlZddlZddlZddlZddlZddl	m3Z4ddlmZddl
mZddlmZddlmZddlmZddlZddlZdd	lmZmZmZdd5lmZddlmZmZmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%edur�dd
l&m'Z'ddl(m)Z)zddl*Z*Wne+y�dZ*Ynwzddl,m-Z-Wne+y�dZ-Ynwzddl.Z.Wne+y�dZ.Ynwddl/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;m<Z<m=Z=e6�>�Z?e?�@dd�dgeAe6�B��ZCeCdd�e6D�7ZCedu�rgZDnddgZDeDdgZEeFed��reC�Ge�Hd��dd�ZIdd�ZJdd�ZKd d!�ZLGd"d#�d#eM�ZN�dTd%d&�ZOd'd(�ZP�dUd)d*�ZQd+d,�ZRd-d.�ZSd/d0�ZTd1d2�ZUe"d3e<e;g�ejVj"d4d5d6e:d6d7�fd8ggd9�d:�ed;d<����ZWe"d=eC�e"d>gd?��e"d@gdA��dBdC����ZXe"d=eC�e"d>ddDg�dEdF���ZYe"d=eC�dGdH��ZZdIdJ�Z[ee"dKgdL��dMdN���Z\ee"d=gdO��dPdQ���Z]dRdS�Z^e"dTe6�e"dUe6�dVdW���Z_dXdY�Z`edZd[��Zad\d]�Zbe"d>gd^��d_d`��Zce"d=eE�dadb��Zdedcdd��Zeeee"degdf��e"dgddhg�didj�����Zfe"d=eE�dkdl��Zgee"d=eE�dmdn���Zhee"d=eie?�doh�e"dpdqdrg�dsdt����Zjee"d=eD�dudv���Zkee"d=e6�dwdx���Zldydz�Zme"d=e6�e"d{dgd|�fd}gd~�fg�dd����Znee"d=eE�d�d����Zod�d��Zpee"d=eD�d�d����Zqd�d��Zrd�d��Zsd�d��Ztee"d=eE�ejVjud�d��d�d�����ZvGd�d��d�e3�Zwd�d��Zxe"d=eC�d�d���Zyee"d=eE�e"d>gd���d�d�����Zzd�d��Z{d�d��Z|e$edud�d��d�d���Z}d�d��Z~eEdd�Ze��d�d�e�d��D��ee"d=e�e"d3e;e<g�d�d�����Z�Gd�d��d�e3�Z�e"d3e<e;g�d�d���Z�e"d3e<e;g�d�d���Z�d�d��Z�d�d��Z�Gd�d��d�ej�j0�Z�e=d�e��ee"d=gd���e"d3e<e;g�d�d�����Z�ee"d>gd���e"d=eE�e"d3e<e;g�d�d������Z�d�d��Z�ee"d�eE�e"d�eE�e"d3e<e;g�d�d������Z�ee"d3e<e;g�e"d�d�d$g�d�dÄ���Z�e"d�gdŢ�d�dDŽ�Z�e"d�gdɢ�d�d˄�Z�ed�d̈́�Z�eee"d=eD�d�dτ���Z�d�Z�ee"d=eD�d�d҄��Z�d�Z�d�Z�d�Z�d�Z�ee"d=eDedu�r�gndg�e"d�e�e�e�g�e"d�gd٢�d�dۄ����Z�d�j�ej���ej���ej���dݍZ�ed�d߄�Z�d�j�ej���ej���ej���dݍZ�ed�d��Z�d�d�Z�d�d�Z�eee"d=eD�d�d����Z�d�d�Z�eed�d���Z�d�d�Z�d�d��Z�ejV�"d>gd��d�d��Z�d�d��Z�d�d��Z�ejV�"d>dDd}g�ee"d=eie?�doh�d�d�����Z�e"d=eC�e"d>gd���d�d����Z�d�d��Z�d��d�Z�ee"d=e?�e"dpdqdrg�e"d>gd����d�d�����Z�e"d=e?�e"dpdqdrg�e"d>gd����d�d����Z�e"d=e?�e"dpdqdrg�e"d>gd����d�d����Z�e"d=e?�e"�ddqdrg�e"�ddqdrg�e"d>gd����d	�d6�����Z�e"�dg�d��e"�ddqdrg�e"�ddqdrg��d
�d����Z�eee"d=eD��d�d����Z�e"d=edu�rBddgng�d���d�d��Z�ee"d=eD��d�d���Z�ee"d3e<e;g��d�d���Z�e"d>ddDg�e"�dg�d���d�d���Z�e"d3e<e;g��d�d��Z��d�d�Z��dV�d �d!�Z�ee"d=ddg�e"d3e<e;g��d"�d#����Z��dW�d$�d%�Z�ejVj�e"d3e<e;g�e"d=edu�r�dgnddg��d&�d'����Z��d(�d)�Z�e"d=edu�r�ddgndg�e$e-du�d*d���d+�d,���Z�e"d3e<e;g�e$e.du�d-d���d.�d/���Z�e"d3e<e;g��d0�d1��ZĐd2�d3�ZŐd4�d5�ZƐd6�d7�Zǐd8�d9�Z�eee"d>g�d:���d;�d<����Z�eee"�d=g�d>��e"d>dDd6g�e"d3e<e;g��d?�d@������Z�ee"d>dDd6g�e"�dAg�dB��e"d3e<e;g��dC�dD�����Z�ee"d>g�dE���dF�dG���Z̐dH�dI�Z͐dT�dJ�dK�Z�ee"d>dDd}g�e"d=eD�e"d3e<e;g��dL�dM�����Z�ee"d>dDd}g�e"d=eD��dN�dO����Z�eejV�"d>dDd}g��dP�dQ���Z�eejV�"d>dDd}g��dR�dS���Z�dS(Xz7Test the parallel module.8�N)�nullcontext��sqrt��TimeoutError)�
PicklingError)�sleep)�format_exception)�dump�load�parallel)�mp)�IS_GIL_DISABLED�np�with_multiprocessing�9with_numpy)�check_subprocess_call�parametrize�raises�skipif�warns)�get_reusable_executor)�Queue�)�parallel_sum)�LokyBackend�MultiprocessingBackend�ParallelBackendBase�SequentialBackend�ThreadingBackend)	�BACKENDS�Parallel�	cpu_count�delayed�effective_n_jobsr
�parallel_backend�parallel_config�register_parallel_backend�multiprocessingcCsg|]}t|��qS�)r )�.0Zbackend_strr)r)��E:\DocsHouse\542 percep lab latest\PerceptionLab\PerceptionLab_Portable\python_embed\Lib\site-packages\joblib/test/test_parallel.py�10<listcomp>Rsr,�loky�	threading�get_context�spawncCs11ttjS�N)r r�DEFAULT_BACKENDr)r)r)r+�get_default_backend_instance^s12r3cCst|dt|dd��S)N�_poolZ_workers)�getattr��backendr)r)r+�get_workersfsr8cCs||Sr1r))�x�yr)r)r+�divisionj�r;cC�|dS�N�r)�r9r)r)r+�squarenr<rAc@seZdZdZdd�ZdS)�MyExceptionWithFinickyInitz,An exception class with non trivial __init__cC�dSr1r))�self�a�b�c�dr)r)r+�__init__u�z#MyExceptionWithFinickyInit.__init__N��__name__�13__module__�__qualname__�__doc__rIr)r)r)r+rBr�rBFcCs"|dkr|r
tdddd��t�|S)N�rErFrGrH)rB�14ValueError)r9�custom_exceptionr)r)r+�exception_raiserys���rTcCst�d�t�)Ng�������?)�timer�KeyboardInterruptr@r)r)r+�interrupt_raiser�s15rWcCs|d||S)zNA module-level function so that it can be spawn with16    multiprocessing.17    r?r))r9r:�zr)r)r+�f�srYcCstt��d�S�Nr)�typer�get_active_backendr)r)r)r+�_active_backend_type��r]cCst||d�dd�td�D��S)N��n_jobsr7cs��|]	}tt�|�VqdSr1�r#rA�r*�ir)r)r+�	<genexpr>���18�z parallel_func.<locals>.<genexpr>��r!�range)�inner_n_jobsr7r)r)r+�
parallel_func�s�rkcC�t�dksJ�dSrZ)r"r)r)r)r+�test_cpu_count��rmcCrlrZ�r$r)r)r)r+�test_effective_n_jobs�rnrp�contextzbackend_n_jobs, expected_n_jobs)rgrg������r`�Nr)zpositive-intznegative-int�None)ZidscCsT|d|d��tdd�|ksJ�Wd�n1swYtdd�dks(J�dS)Nr.rsrro)rqZbackend_n_jobsZexpected_n_jobsr)r)r+�test_effective_n_jobs_None�s19�rvr7r`)rr?rr������verbose)r?��dcCs:dd�td�D�t|||d�dd�td�D��ksJ�dS)NcS�g|]}t|��qSr)�rA�r*r9r)r)r+r,��z(test_simple_parallel.<locals>.<listcomp>��r`r7rxcsrar1rbr}r)r)r+re���z'test_simple_parallel.<locals>.<genexpr>�rir!)r7r`rxr)r)r+�test_simple_parallel�s20��r�r?cCs�d}t�d�}Gdd�dt�}|||dd�}|dd�t|�D��t�}|jD]}|�|�}r=|��\}	}21|�|22|	�q't	|�d	ksFJ�dS)23Nrzz(Done\s+\d+ out of \d+ \|)c@seZdZgZdd�ZdS)z/test_parallel_pretty_print.<locals>.ParallelLogcSs|j�|�dSr1)�messages�append)rD�msgr)r)r+�_print�r^z6test_parallel_pretty_print.<locals>.ParallelLog._printN)rLrMrNr�r�r)r)r)r+�ParallelLog�rPr�i'r�cS�g|]}tt�|��qSr)�r#rYrcr)r)r+r,��z.test_parallel_pretty_print.<locals>.<listcomp>r)24�re�compiler!ri�setr��search�span�add�len)r7r`�n_tasks�patternr��executorZlens�message�srErFr)r)r+�test_parallel_pretty_print�s2526�r�cs�|jt��ddd�tjdd��}td|d�dd	�td27�D��}|gd�ks(J�Wd�n1s2wYdd
�|D�}|dvsGt|t�rRd��fdd
�|D�}t	|�dksZJ�dS)N�nameZ!some_new_name_for_the_main_thread)�targetr��valueT��recordr?r_csrar1rbr}r)r)r+re�rfz6test_main_thread_renamed_no_warning.<locals>.<genexpr>rg)rr�cSs*g|]}dt|j�vrt|jt�s|�qS)zworker timeout)�strr��28isinstance�DeprecationWarning�r*�wr)r)r+r,�s29��z7test_main_thread_renamed_no_warning.<locals>.<listcomp>)Nr(z3multi-threaded, use of fork() may lead to deadlockscsg|]}�t|j�vr|�qSr))r�r�r���message_partr)r+r,��r)30�setattrr.�current_thread�warnings�catch_warningsr!rir�rr�)r7�monkeypatch�warninfo�resultsr)r�r+�#test_main_thread_renamed_no_warning�s(���	�31 32�r�cCs�tjdd��}t�d�t||d�Wd�n1swYdd�|D�}|rI|rGtdd�|D��}tr=t|�d	knt|�d	k}|oF|Sd33S|rMJ�dS)NTr��always)r7rjcSsg|]}|j�qSr))r�r�r)r)r+r,sz*_assert_warning_nested.<locals>.<listcomp>css�|]34}d|jdvVqdS)zbacked parallel loops cannotrN)�args)r*Zeachr)r)r+re35��36�z)_assert_warning_nested.<locals>.<genexpr>rF)r�r��simplefilterrk�allrr�)r7rj�expectedr�Zwarnings_are_correctZwarnings_have_the_right_lengthr)r)r+�_assert_warning_nesteds 37���r�z%parent_backend,child_backend,expected))r-r(T)r-r-F)r(r(T)r(r-T)r.r(T)r.r-Tcsntd|d��fdd�td�D��td|d���fdd�td�D��}|dkr/t|�s-J�dSt|�s5J�dS)Nr?r_c3s"�|]}tt��ddd�VqdS)rF�r7rjr�N�r#r��r*�_��
child_backendr)r+re(����38�z0test_nested_parallel_warnings.<locals>.<genexpr>rc3s"�|]}tt��d�d�VqdS)r?r�Nr�r��r�r�r)r+re0r�r.)r!ri�anyr�)�parent_backendr�r��resr)r�r+�test_nested_parallel_warningss��	r�)r-r(r.cCs>dg}dd�}tj||fd�}|��|��|dsJ�dS)NFcSshtjdd��}tdd�dd�td�D��Wd�n1swYtt|��t|�dk|d<dS)	NTr�r?rscs��|]	}tt�d�VqdS)皙�����?N�r#rr�r)r)r+reFr�zPtest_background_thread_parallelism.<locals>.background_thread.<locals>.<genexpr>r�r)r�r�r!ri�printr�)�is_run_parallelr�r)r)r+�background_threadDs39�z=test_background_thread_parallelism.<locals>.background_thread)r�r�r)r.�Thread�start�join)r7r�r��tr)r)r+�"test_background_thread_parallelism?sr�cCs"td|d�dd�td�D��dS)Nr?r_csr���{�G�z�?Nrbr�r)r)r+reQr�znested_loop.<locals>.<genexpr>rhr6r)r)r+�nested_loopPs"r�r�r�cs&td|d��fdd�td�D��dS)Nr?r_c3��|]	}tt���VqdSr1)r#r�r�r�r)r+reWrfz#test_nested_loop.<locals>.<genexpr>rh)r�r�r)r�r+�test_nested_loopTs�r�cCst�r1)rRr6r)r)r+�raise_exception\rJr�c	Cs�tt��5tddd��}|tt�d�tt�d�g�Wd�n1s$wYWd�dSWd�dS1s<wYdS)Nr?r-r_)rrRr!r#r�r�)rr)r)r+�)test_nested_loop_with_exception_with_loky`s40��"�r�cs<tdd��tddd��fdd�td�D�����sJ�dS)	z1Input is mutable when using the threading backendr)�maxsizer?r.r_c3s�|]41}t�j�d�VqdS�rN)r#�putr���qr)r+rej��z1test_mutate_input_with_threads.<locals>.<genexpr>N)rr!ri�fullr)r)r�r+�test_mutate_input_with_threadsgs42"r�)rr?rgcCs6td�}dd�|D�t|d�dd�|D��ksJ�dS)z.Check the keyword argument processing of pmap.�43cS�g|]}t|dd��qS�r�r:�rYr}r)r)r+r,rr�z(test_parallel_kwargs.<locals>.<listcomp>rscs� �|]}tt�|dd�VqdS�rr�Nr�r}r)r)r+rer��44�z'test_parallel_kwargs.<locals>.<genexpr>Nr�)r`�lstr)r)r+�test_parallel_kwargsns�r�cCs�td�}dd�|D�}td|d��4}|j}||dd�|D��ks"J�||dd�|D��ks/J�tdur>t|�t|j�us>J�Wd�n1sHwYtdurZt|j�dusZJ�||d	d�|D��ksgJ�tdurtt|j�dusvJ�dSdS)45Nr�cSr�r�r�r}r)r)r+r,zr�z4test_parallel_as_context_manager.<locals>.<listcomp>r�r_csr�r�r�r}r)r)r+re���z3test_parallel_as_context_manager.<locals>.<genexpr>csr�r�r�r}r)r)r+re�r�csr�r�r�r}r)r)r+re�r�)rir!�_backendr
r8)r7r�r��pZmanaged_backendr)r)r+� test_parallel_as_context_managerws"���r�csdGdd�dt��ttdd��tddd��fdd	�td46�D��Wd�dS1s+wYdS)z`Check that pmap captures the errors when it is passed an object47    that cannot be pickled.48    c@seZdZdd�ZdS)z1test_parallel_pickling.<locals>.UnpicklableObjectcS�td��)NZ123)�RuntimeError�rDr)r)r+�49__reduce__�r<z<test_parallel_pickling.<locals>.UnpicklableObject.__reduce__N)rLrMrNr�r)r)r)r+�UnpicklableObject�sr�zthe task to send��matchr?r-r_c3s�|]50}tt����VqdSr1�r#�idr��r�r)r+re�r�z)test_parallel_pickling.<locals>.<genexpr>r�N)�objectrrr!rir)r)r�r+�test_parallel_pickling�s51�"�r��	byteorder)�<�>�=�52max_nbytesZ1Mcs�dd��t�d��d��|�d����jj}tdd|d���fd	d53�td�D��}|D]\}}||ks5J�||jjks=J�tj�	�|�q+dS)NcSs||jjfSr1)�dtyper�r@r)r)r+�inspect_byteorder��z=test_parallel_byteorder_corruption.<locals>.inspect_byteorder�)r?rg�i4r?r-)r`r7r�c3��|]	}t����VqdSr1�r#r��r�r9r)r+re�rfz5test_parallel_byteorder_corruption.<locals>.<genexpr>rg)54rZarangeZreshape�viewr�r�r!ri�testing�assert_array_equal)r�r�Zinitial_np_byteorder�resultZ55x_returnedZbyteorder_in_workerr)rr+�"test_parallel_byteorder_corruption�s��rcCs0ttd|dd�dd�td�D���dksJ�dS)Nr?��r`r7�timeoutcsr�)g����MbP?Nr�r}r)r)r+re�rfz0test_parallel_timeout_success.<locals>.<genexpr>r�)r�r!rir6r)r)r+�test_parallel_timeout_success�s���rcCsNtt��td|dd�dd�td�D��Wd�dS1s wYdS)Nr?r�r	csr��r�Nr�r}r)r)r+re�rfz-test_parallel_timeout_fail.<locals>.<genexpr>r�)rrr!rir6r)r)r+�test_parallel_timeout_fail�s565758�"�r
�59sequential�	return_as�	generator�generator_unorderedcCsxtt��ttd||dd�dd�td�D���Wd�n1s"wYttd||dd�dd�td�D���dS)Nr?r�)r`r7rr60csr�rr�r}r)r)r+re�rfz<test_parallel_timeout_fail_with_generator.<locals>.<genexpr>r�csr�r�r�r}r)r)r+re�rf)rr�listr!ri�r7rr)r)r+�)test_parallel_timeout_fail_with_generator�s61�����rc	Cstdu�rtt��td|d�dd�tdd�D��Wd�n1s$wYtt��td|d�dd�dD��Wd�n1sEwYtd|d���}t|j�dusZJ�t|j�}tt��|dd�tdd�D��Wd�n1szwYt|j�dus�J�t|j�|us�J�d	d�td62�D�|dd�td63�D��ks�J�t|j�}tt��|d
d�dD��Wd�n1s�wYt|j�dus�J�t|j�|us�J�dd�td64�D�|dd�td65�D��ks�J|j	|j66|j|jf��Wd�n	1�swYt|j�du�sJ�n!tt��tdd�dd�dD��Wd�n	1�s2wYtt��tdd�dd�tdd�D��Wd�n	1�sVwYtt
��tddd�dd�td�D��Wd�dS1�s{wYdS)Nr?r_cS�g|]\}}tt�||��qSr)�r#r;�r*r9r:r)r)r+r,�r�z&test_error_capture.<locals>.<listcomp>)rr)rrcSr�r)�r#rWr}r)r)r+r,�r�cSrr)rrr)r)r+r,�r�cSr�r�r�r}r)r)r+r,r�r�csr�r�r�r}r)r)r+rer�z%test_error_capture.<locals>.<genexpr>cSr�r)rr}r)r)r+r,r�cSr�r�r�r}r)r)r+r,r�csr�r�r�r}r)r)r+rer�rscSr�r)rr}r)r)r+r,"r�cSrr)rrr)r)r+r,'r�r)r`rxcsr�)T)rSN�r#rTrcr)r)r+re+r�r)r
r�ZeroDivisionErrorr!�ziprVr8r�ri�67_iteratingZn_completed_tasksZn_dispatched_tasks�	_abortingrB)r7rZoriginal_workersr)r)r+�test_error_capture�sd686970��7172��7374��7576�����'77�78 �7980�$�rc	Csddd�}td|d��w}ttdd��|dd	�|dd81�D��Wd�n1s)wYttdd��|dd	�|dd82�D��Wd�n1sKwYttdd��|d
d	�|dd83�D��Wd�n1smwYWd�dSWd�dS1s�wYdS)Nrcss*�td�D]
}||krtd��|VqdS)N��Iterator Raising Error)rirR)�raise_atrdr)r)r+�my_generator2s��z1test_error_in_task_iterator.<locals>.my_generatorr?r_r r�csrar1rbrcr)r)r+re;r�z.test_error_in_task_iterator.<locals>.<genexpr>)r!csrar1rbrcr)r)r+re@r�rcsrar1rbrcr)r)r+reDr���r)r!rrR)r7r"r�r)r)r+�test_error_in_task_iterator/s84����"�r%cCs|�d|�dS)NzConsumed %s)r�)�queue�itemr)r)r+�consumerGrnr(zbatch_size, expected_queue)�85Produced 0�86Consumed 0�87Produced 1�88Consumed 1�89Produced 2�90Consumed 2�91Produced 3�92Consumed 3�93Produced 4�94Consumed 4�95Produced 5�96Consumed 5r�)r)r+r-r/r*r,r.r0r1r3r2r4csTt���fdd�}td||d��fdd�|�D���|ks J�t��dks(J�dS)	z=Test that with only one job, Parallel does act as a iterator.c3�(�td�D]}��d|�|VqdS�Nr�zProduced %i�rir��rd�r&r)r+�producerx�97��z'test_dispatch_one_job.<locals>.producerr�r`�98batch_sizer7c3s�|]99}tt��|�VqdSr1�r#r(r}r9r)r+re}r�z(test_dispatch_one_job.<locals>.<genexpr>�N)rr!r�)r7r=Zexpected_queuer:r)r9r+�test_dispatch_one_jobKs+�r@cs�t��}|����fdd�}tddd|d��fdd�|�D��t��}|d	d100ks,J�|dd��d
�}|dks;J�|�d�}||ksFJ�t��dksNJ�dS)zRCheck that using pre_dispatch Parallel does indeed dispatch items101    lazily.102    c3r5r6r7r8r9r)r+r:�r;z/test_dispatch_multiprocessing.<locals>.producerr?rrg)r`r=�pre_dispatchr7c3s�|]103}tt��d�VqdS)r�Nr>r�r9r)r+re�r�z0test_dispatch_multiprocessing.<locals>.<genexpr>rr)Nr�zConsumed anyrrr/r?)r
�Managerrr!�indexr�)r7�managerr:Zqueue_contentsZfirst_consumption_indexZproduced_3_indexr)r9r+�test_dispatch_multiprocessing�s�104rEcCs\tdddd��}|dd�td�D��|j��dksJ�Wd�dS1s'wYdS)	Nr?�autor.r<csrar1r�rcr)r)r+re�r�z/test_batching_auto_threading.<locals>.<genexpr>�r�r!rir�Zcompute_batch_size)r�r)r)r+�test_batching_auto_threading�s"�rIcCs\tdd|d��}|dd�td�D��|j��dksJ�Wd�dS1s'wYdS)Nr?rFr<csrar1r�rcr)r)r+re�r�z2test_batching_auto_subprocesses.<locals>.<genexpr>rGrrH)r7r�r)r)r+�test_batching_auto_subprocesses�s"�rJcCsNtt��tdddd�dd�td�D��Wd�dS1s wYdS)	zCMake sure that exception raised during dispatch are indeed capturedr?�r)r`rArxcsrar1rrcr)r)r+re�rfz*test_exception_dispatch.<locals>.<genexpr>rN)rrRr!rir)r)r)r+�test_exception_dispatch�s105106107�"�rLcC� tdd�dd�td�D��dS)Nr?rscsrar1r�r*�jr)r)r+re�r�z(nested_function_inner.<locals>.<genexpr>rrhr8r)r)r+�nested_function_inner�� rPcCrM)Nr?rscsrar1)r#rPrNr)r)r+re�r�z(nested_function_outer.<locals>.<genexpr>rrhr8r)r)r+�nested_function_outer�rQrRz'https://github.com/joblib/loky/pull/255��reasoncCs�tt��}td|d�dd�td�D��Wd�n1swYt|j|j|j�}d�|�}d|vs7J�d	|vs=J�d108|vsCJ�t|j�tusLJ�dS)z�Ensure errors for nested joblib cases gets propagated109 110    We rely on the Python 3 built-in __cause__ system that already111    report this kind of information to the user.112    r?r_csrar1)r#rRrcr)r)r+re�rfz1test_nested_exception_dispatch.<locals>.<genexpr>rN�rRrPrT)	rrRr!rir	r[r��tbr�)r7�excinfoZreport_lines�reportr)r)r+�test_nested_exception_dispatch�s113	114��115rYc@s$eZdZdZddd�Zd	dd�ZdS)116�FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|�|�|_||_|Sr1)r$r`r)rDr`rZbackend_argsr)r)r+�	configure�szFakeParallelBackend.configurecCs"|dkrtt��d|d�}|S)Nrr)�maxr
r"�rDr`r)r)r+r$�sz$FakeParallelBackend.effective_n_jobsr�)r)rLrMrNrOr[r$r)r)r)r+rZ�s117rZc	Cs�ttdd��
tdd�Wd�n1swYttdd��tdd��Wd�n1s2wYWd�n1sAwYttdd��&tdd��Wd�n1s]wYWd�dSWd�dS1suwYdS)NzInvalid backend:r�zunit-testingr6)rrRr!r&r)r)r)r+�test_invalid_backend�s ������"�r^cCs,tt��}td|d���Wd�n1swYdt|j�vs%J�tt��}td|d���Wd�n1s<wYdt|j�vsJJ�tt��}td|d���Wd�n1sawYdt|j�vsoJ�tt��}td|d���Wd�n1s�wYdt|j�vs�J�dS)Nrr_z&n_jobs == 0 in Parallel has no meaning��?z2.3z$n_jobs could not be converted to intZinvalid_str)rrRr!Z_initialize_backendr�r�)r7rWr)r)r+�test_invalid_njobs�s 118�119�120�121�r`)�2gffffff@r?cCsPt||d�}|��dksJ�|dd�td�D��}tdd�t|�D��s&J�dS)Nr_r?csrar1rbrcr)r)r+rer�z.test_njobs_converted_to_int.<locals>.<genexpr>r�css �|]\}}|t|�kVqdSr1r|)r*rd�rr)r)r+rer�)r!�_effective_n_jobsrir��	enumerate)r7r`r�r�r)r)r+�test_njobs_converted_to_intsrecCs<ztdt�dtvsJ�tdtksJ�Wtd=dStd=w)NZtest_backend)r'rZr r)r)r)r+�test_register_parallel_backends122123rfcCs^tj}t�t�ksJ�ztdtddd�t�tksJ�W|t_n|t_wt�t�ks-J�dS)Nr.T)Zmake_default)rr2r]r3r'r r)Zdefault_backend_origr)r)r+�test_overwrite_default_backend%srgzOnly without multiprocessingcCs�ttdd��tdd�dd�td�D��Wd�n1swYtdd��t�dd�td�D��Wd�dS1sAwYdS)	Nz)joblib backend '.*' is not available on.*r�r-r6csrar1rbrcr)r)r+re4r�z2test_backend_no_multiprocessing.<locals>.<genexpr>rgcsrar1rbrcr)r)r+re8r�)r�UserWarningr!rir&r)r)r)r+�test_backend_no_multiprocessing1s�"�ricCs^||dd���t��\}}|dksJ�td�dksJ�t�}|jdks%J�|dkr;t|�tus1J�t|j�tus:J�nB|dkrQt|�tusGJ�t|j�tusPJ�n4|dkrgt|�t	us]J�t|j�t	usfJ�n.|�124d�r�t|�tustJ�t|j�tus�J�Wd�dSWd�dSWd�dSWd�dSWd�dS1s�wYdS)Nrgrsr(r-r.Ztest_)rr\r$r!r`r[rr�rr�125startswithrZ)rq�backend_name�active_backend�
active_n_jobsr�r)r)r+�check_backend_context_manager;s8126�����"�rncCsg|]}d|�qS)ztest_backend_%dr)rcr)r)r+r,Qr~rgcsr|tvr|�t|t�t�t�ksJ�t�|�t�t�ks J�tddd��fdd�tD��t�t�ks7J�dS)Nr?r.r_c3s"�|]}|stt��|�VqdSr1)r#rn)r*rF�rqr)r+recs��127�z/test_backend_context_manager.<locals>.<genexpr>)r �setitemrZr]r3rnr!� all_backends_for_context_manager)r�r7rqr)ror+�test_backend_context_managerTs128�rrc@seZdZdZddd�ZdS)�ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd��||_dS)Nzparam should not be None)rR�param)rDrtr)r)r+rIps129z%ParameterizedParallelBackend.__init__r1rKr)r)r)r+rsmsrscCs�|�tdt�t�t�ksJ�|dddd��?t��\}}t|�tus%J�|jdks,J�|dks2J�t	�}|j130dks<J�|j|usCJ�|dd�td�D��}Wd�n1sXwY|dd	�td�D�ksjJ�t�t�ksrJ�dS)131NZ
param_backend�*rg)rtr`csrar1�r#rrcr)r)r+re�r�z=test_parameterized_backend_context_manager.<locals>.<genexpr>rcSr{r)rrcr)r)r+r,�r~z>test_parameterized_backend_context_manager.<locals>.<listcomp>)
rpr rsr]r3rr\r[rtr!r`r�ri)r�rqrlrmr�r�r)r)r+�*test_parameterized_backend_context_managervs�	rwcCs�t�t�ksJ�|tdd�dd��?t��\}}t|�tus J�|jdks'J�|dks-J�t�}|jdks7J�|j	|us>J�|dd�t132d�D��}Wd�n1sSwY|dd�t133d�D�kseJ�t�t�ksmJ�dS)	N�+)rtrrscsrar1rvrcr)r)r+re�r�zFtest_directly_parameterized_backend_context_manager.<locals>.<genexpr>cSr{r)rrcr)r)r+r,�r~zGtest_directly_parameterized_backend_context_manager.<locals>.<listcomp>)r]r3rsrr\r[rtr!r`r�ri)rqrlrmr�r�r)r)r+�3test_directly_parameterized_backend_context_manager�s�	rycCstd�t��S)Nr�)r�os�getpidr)r)r)r+�sleep_and_return_pid�sr|cCs<t�tksJ�t���dksJ�tdd�dd�td�D��S)Nrr?rscs��|]}tt��VqdSr1)r#r|r�r)r)r+re���z"get_nested_pids.<locals>.<genexpr>)r]rr!rcrir)r)r)r+�get_nested_pids�srcs eZdZdZ�fdd�Z�ZS)�	MyBackendz:Backend to test backward compatibility with older backendscstt|���dSrZ)�superr��get_nested_backendr���	__class__r)r+r��szMyBackend.get_nested_backend)rLrMrNrOr��
__classcell__r)r)r�r+r��sr��back_compat_backend)r.r-r(r�cCsh||��&tdd�dd�td�D��}|D]}tt|��dks!J�qWd�dS1s-wYdS)Nr?rscsr}r1)r#rr�r)r)r+re�r~z6test_nested_backend_context_manager.<locals>.<genexpr>r�r)r!rir�r�)rqr7Z134pid_groupsZ	pid_groupr)r)r+�#test_nested_backend_context_manager�s135�"�r�)r?rrNcs~dd��tdd��fdd�td�D��|��d��tdd����fdd�td�D��Wd�dS1s8wYdS)	NcSs0t�t|ks	J�t|�}t���|ksJ�dSr1)r]r r$r!rc)Zexpected_backend_typeZexpected_n_jobr)r)r+�check_nested_backend�sz?test_nested_backend_in_sequential.<locals>.check_nested_backendrrsc3s �|]}t��tjd�VqdSr�)r#rr2r�)r�r)r+re�r�z4test_nested_backend_in_sequential.<locals>.<genexpr>r�c3s�|]136}t�����VqdSr1rr��r7r�r`r)r+re�r�rh)r7r`rqr)r�r+�!test_nested_backend_in_sequential�s137�138�"�r�cCs\||�� }|tur
|d}|tur|d}|j|ksJ�Wd�dS1s'wYdS)Nr7r)r&r%�
nesting_level)rq�
inner_backendZexpected_level�ctxr7r)r)r+�check_nesting_level�s139"�r��
outer_backendr�cs�t�|d�td|d���fdd�td�D����dd��t���fdd�td�D��Wd�dS1s9wYdS)	Nrr?r_c3� �|]}tt���d�VqdSr��r#r�r��rqr�r)r+re�r�z-test_backend_nesting_level.<locals>.<genexpr>r�rsc3r�r�r�r�r�r)r+re�r�)r�r!ri)rqr�r�r)r�r+�test_backend_nesting_level�s�140�"�r��with_retrieve_callbackTcs�ddl�G��fdd�dt�}td|�dd��|d��;}tdd��fd	d141�td�D��|tur:|djd
ks:J�|turG|djd
ksOJ�Wd�dSWd�dS1sZwYdS)Nrcs"eZdZdZ�Z�jdd��ZdS)z)test_retrieval_context.<locals>.MyBackendrcss�|jd7_dVdSrtr8r�r)r)r+�retrieval_context	s�142z;test_retrieval_context.<locals>.MyBackend.retrieval_contextN)rLrMrNrdZsupports_retrieve_callback�contextmanagerr�r))�143contextlibr�r)r+r�s144r�Z	retrievalcSstdd�dd�t|�D��S)Nr?rscsrar1r�rcr)r)r+rer�z>test_retrieval_context.<locals>.nested_call.<locals>.<genexpr>rh)�nr)r)r+�nested_callsz+test_retrieval_context.<locals>.nested_callr?rsc3s�|]	}t��|�VqdSr1rrc)r�r)r+rer�z)test_retrieval_context.<locals>.<genexpr>rr7r)r�rr'r!rir&rdr%)rqr�r�r�r))r�r�r�r+�test_retrieval_context�s145	146 ��"�r�r=)rrrg���Q��?cCs8tt��t|d�Wd�dS1swYdS)N�r=)rrRr!r�r)r)r+�test_invalid_batch_sizes147"�r�z)n_tasks, n_jobs, pre_dispatch, batch_size))r?r?r�rF)r?r?r`rF�r�r?r`rF)ir?r`rFr�)r�r�r`rF)��r?r`rF)�r?�1482 * n_jobsr)��r?r�r)r�r?r�rQ)r�r?r�rFcCsN|||d�}dd�t|�D�}tdi|��dd�t|�D��}||ks%J�dS)N)r`rAr=cSr{r)r|rcr)r)r+r,:r~z0test_dispatch_race_condition.<locals>.<listcomp>csrar1rbrcr)r)r+re;r�z/test_dispatch_race_condition.<locals>.<genexpr>r)r�)r�r`rAr=�paramsr�r�r)r)r+�test_dispatch_race_condition%s r�cCs8t��}tddd�}|j�d�}|��}||ksJ�dS)Nr?r(r_rq)r
�get_start_methodr!Z_backend_kwargs�get)Zmp_start_methodr�rqZstart_methodr)r)r+�test_default_mp_context?s149r�cs^|dkr	t�d�}tj�d�}|�dd��t���j�td|d��fdd�t	d�D��dS)	Nr(r0rui�r?r_c3s"�|]}ttj���j�VqdSr1)r#r�dot�Trc�rEr)r+re^s� zAtest_no_blas_crash_or_freeze_with_subprocesses.<locals>.<genexpr>)150r
r/r�randomZRandomState�randnr�r�r!ri)r7�rngr)r�r+�.test_no_blas_crash_or_freeze_with_subprocessesHs151&r�afrom joblib import Parallel, delayed152 153def square(x):154    return x ** 2155 156backend = "{}"157if backend == "spawn":158    from multiprocessing import get_context159    backend = get_context(backend)160 161print(Parallel(n_jobs=2, backend=backend)(162      delayed(square)(i) for i in range(5)))163cCsB|dkrt��dkrt�d�t�|�}ttjd|gddd�dS)Nr(ZforkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr��\[0, 1, 4, 9, 16\]�r164�stdout_regex)	r
r��pytest�skip�,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAIN�formatr�sys�165executable)r7�coder)r)r+�2test_parallel_with_interactively_defined_functionsqs�166167�r�a�import sys168# Make sure that joblib is importable in the subprocess launching this169# script. This is needed in case we run the tests from the joblib root170# folder without having installed joblib171sys.path.insert(0, {joblib_root_folder!r})172 173from joblib import Parallel, delayed174 175def run(f, x):176    return f(x)177 178{define_func}179 180if __name__ == "__main__":181    backend = "{backend}"182    if backend == "spawn":183        from multiprocessing import get_context184        backend = get_context(backend)185 186    callable_position = "{callable_position}"187    if callable_position == "delayed":188        print(Parallel(n_jobs=2, backend=backend)(189                delayed(square)(i) for i in range(5)))190    elif callable_position == "args":191        print(Parallel(n_jobs=2, backend=backend)(192                delayed(run)(square, i) for i in range(5)))193    else:194        print(Parallel(n_jobs=2, backend=backend)(195                delayed(run)(f=square, x=i) for i in range(5)))196z!def square(x):197    return x ** 2198zcdef gen_square():199    def square(x):200        return x ** 2201    return square202square = gen_square()203zsquare = lambda x: x ** 2204�define_func�callable_position)r#r��kwargsc	Csv|dvr|tks
tjdkrt�d�tj|||tj�	tj�	t205j��d�}|�d�}|�
|�ttj|jgddd�dS)	N)r(r0�win32zNot picklable with pickle)r�r7r��joblib_root_folderzunpicklable_func_script.pyr�r�r�)�SQUARE_MAINr��platformr�r��)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINr�rz�path�dirname�joblib�__file__r��writerr��strpath)r7r�r��tmpdirr�Z	code_filer)r)r+�0test_parallel_with_unpicklable_functions_in_args�s 206�207208209210�r�a�import sys211import faulthandler212# Make sure that joblib is importable in the subprocess launching this213# script. This is needed in case we run the tests from the joblib root214# folder without having installed joblib215sys.path.insert(0, {joblib_root_folder!r})216 217from joblib import Parallel, delayed218from functools import partial219 220class MyClass:221    '''Class defined in the __main__ namespace'''222    def __init__(self, value):223        self.value = value224 225 226def square(x, ignored=None, ignored2=None):227    '''Function defined in the __main__ namespace'''228    return x.value ** 2229 230 231square2 = partial(square, ignored2='something')232 233# Here, we do not need the `if __name__ == "__main__":` safeguard when234# using the default `loky` backend (even on Windows).235 236# To make debugging easier237faulthandler.dump_traceback_later(30, exit=True)238 239# The following baroque function call is meant to check that joblib240# introspection rightfully uses cloudpickle instead of the (faster) pickle241# module of the standard library when necessary. In particular cloudpickle is242# necessary for functions and instances of classes interactively defined in the243# __main__ module.244 245print(Parallel(backend="loky", n_jobs=2)(246    delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))])247    for i in range(5)248))249)r�cCs.|�d�}|�t�ttj|jgddd�dS)Nz(joblib_interactively_defined_function.pyr�)r�r250)r�r��5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrr�r�r��r��scriptr)r)r+�7test_parallel_with_interactively_defined_functions_loky�s251252253254�r�a�import sys255# Make sure that joblib is importable in the subprocess launching this256# script. This is needed in case we run the tests from the joblib root257# folder without having installed joblib258sys.path.insert(0, {joblib_root_folder!r})259 260from joblib import Parallel, delayed, hash261import multiprocessing as mp262mp.util.log_to_stderr(5)263 264class MyList(list):265    '''MyList is interactively defined by MyList.append is a built-in'''266    def __hash__(self):267        # XXX: workaround limitation in cloudpickle268        return hash(self).__hash__()269 270l = MyList()271 272print(Parallel(backend="loky", n_jobs=2)(273    delayed(l.append)(i) for i in range(3)274))275cCs0|�d�}|�t�ttj|jgdddd�dS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]ZLokyProcess�)r�Zstderr_regexr276)r�r��9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrr�r�r�r�r)r)r+�:test_parallel_with_interactively_defined_bound_method_lokys277278279280�r�cCs"tg�}tdd�|�gksJ�dS)Nr?rs)�iterr!)Zexhausted_iteratorr)r)r+�%test_parallel_with_exhausted_iterator)sr�cCs"t|tj�s
tdt|���|��S)Nz#Expected np.memmap instance, got %r)r�r�memmap�	TypeErrorr[�copyr�r)r)r+�check_memmap.sr�cCs�dd�}tdd|d�dd�|d�D��}t||t|���D]\}}tj�||�qtd	d|d�d281d�|d�D��}t||t|���D]\}}tj�||�qBdS)Ncss*�t|�D]
}tjdtjd�|VqdS)Nr��r�)rir�onesZfloat32)r�rdr)r)r+�generate_arrays<s��zBtest_auto_memmap_on_arrays_from_generator.<locals>.generate_arraysr?r�r`r�r7csrar1�r#r��r*rEr)r)r+reBrfz<test_auto_memmap_on_arrays_from_generator.<locals>.<genexpr>rzr�csrar1r�r�r)r)r+reKrf)r!rr�rrr)r7r�r�rr�r)r)r+�)test_auto_memmap_on_arrays_from_generator4s���r�cCs|Sr1r))�argr)r)r+�identityRrJr�cs�|�d�j}tj}tj|dd�tj|dd�g}t||�t|dd��t	dd��fdd	�d282D��\}t283�dtj�s;J��dj|ksDJ�tj
�||�dS)Nz	test.mmapZuint8r�rb)Z	mmap_moder?rsc3r�r1)r#r�r��r�r)r+re^r�z.test_memmap_with_big_offset.<locals>.<genexpr>r$r)r�r��mmap�ALLOCATIONGRANULARITYrZzerosr�r284rr!r�r��offsetrr)r��fname�size�objrr)r�r+�test_memmap_with_big_offsetVs285r�cCs�tjdd��}tddd�dd�td�D��Wd�n1s wYt|�dks-J�|d}t|jt�s9J�t|j�d	ksBJ�dS)286NTr�r)r`r287csrar1rbrcr)r)r+refr�zFtest_warning_about_timeout_not_supported_by_backend.<locals>.<genexpr>�2rz�The backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.)	r�r�r!rir�r�r�rhr�)r�r�r)r)r+�3test_warning_about_timeout_not_supported_by_backendds ��r�cCs|||<|Sr1r))�288input_listrCr�r)r)r+�set_list_valueqsr�)rr?r�csVdgd�t|ddd��fdd�td�D��}t|�}tdd�t�|�D��s)J�dS)	Nrrrr.�r`rr7c3s �|]}tt��||�VqdSr1)r#r�rc�r�r)r+re|r�zPtest_parallel_return_order_with_return_as_generator_parameter.<locals>.<genexpr>cs��|]	\}}||kVqdSr1r)�r*�vrbr)r)r+re�r�)r!rirr�r)r`rr)r�r+�=test_parallel_return_order_with_return_as_generator_parametervs289� r�cCs|rtd�t|�S)Nr)rr)�e�delayr)r)r+�_sqrt_with_delay�sr�cspt|d|d�dd�td�D���t�fdd�td�D��}dgttd	d��}td290d�t||�D��s5J��dS)Nrr�css&�|]}tt�|d|dk�VqdS)r?rN)r#r�rcr)r)r+re�s�291�zK_test_parallel_unordered_generator_returns_fastest_first.<locals>.<genexpr>r�c3s�|]}t��VqdSr1)�nextr��rr)r+re�s��	rr?csr�r1r)r�r)r)r+re�r�)r!ri�sortedrr�r)r7r`Zquickly_returnedZexpected_quickly_returnedr)r�r+�8_test_parallel_unordered_generator_returns_fastest_first�s�r�cCst||�dSr1)r��r7r`r)r)r+�7test_parallel_unordered_generator_returns_fastest_first�sr�)rr?rwrrcCsvdgdgd}tt��t��}t||d�dd�|D��Wd�n1s(wYt��|}|dks9J�dS)NrEr�rzr_css�|]292}ttj�|�VqdSr1�r#rUrrcr)r)r+re�r�z%test_abort_backend.<locals>.<genexpr>r)rr�rUr!)r`r7Zdelays�t_start�dtr)r)r+�test_abort_backend�s293�r�cCstjtd�td�}d|d<|S)Ng��Ar�Fr)rr��int�bool)r�rr)r)r+�get_large_object�srcCs\t|||d��}|dd�td�D��}t|�t|�~Wd�dS1s'wYdS)N)r`r7rcsrar1)r#rrcr)r)r+re�r�z0_test_deadlock_with_generator.<locals>.<genexpr>r�)r!rir�)r7rr`rrr)r)r+�_test_deadlock_with_generator�s"�rcCst|||�dSr1)r)r7rr`r)r)r+�test_deadlock_with_generator�srcCs�ttdd��)t|||d�}|dd�td�D��}t��}|dd�td�D��}Wd�n1s2wYt��|d	ksCJd294��~dS)N�)This Parallel instance is already runningr�rcsr�r�r�r�r)r)r+re�r�z/test_multiple_generator_call.<locals>.<genexpr>r�csrar1r�rcr)r)r+re�r�rzr?�[The error should be raised immediately when submitting a new task but it took more than 2s.)rr�r!rirU)r7rr`r�gr�Zgen2r)r)r+�test_multiple_generator_call�s��rc	Cs�t|||d��F}|dd�td�D��}t��}ttdd��|dd�td�D��}Wd�n1s3wYt��|d	ksDJd295��Wd�~dS1sPwY~dS)Nrcsr�rr�r�r)r)r+re�r�z7test_multiple_generator_call_managed.<locals>.<genexpr>r�rr�csrar1r�rcr)r)r+re�r�rzr?r)r!rirUrr�)r7rr`rrr��g2r)r)r+�$test_multiple_generator_call_managed�s��296��r	�return_as_1�return_as_2cCs�t|||d�dd�td�D��}t|||d�dd�tdd�D��}|dkr)t|�}|dkr1t|�}tdd�t|td��D��sAJ�td	d�t|tdd��D��sRJ�dS)297Nrcs� �|]}tt�|d�VqdS�r?Nrvrcr)r)r+rer�z9test_multiple_generator_call_separated.<locals>.<genexpr>r�csrr
rvrcr)r)r+rer�rrcsr�r1r)�r*r�rdr)r)r+rer�csr�r1r)rr)r)r+rer�)r!rir�r�r)r7r298rr`rrr)r)r+�&test_multiple_generator_call_separated�s�� &rzbackend, error))r-T)r.F)rFc299Cs\|dkr
tdur
t�d�td||d�}|dd�td�D��}t�|dd	��}|r/ttd300d�nt	�}|�6t301302�}td||d�dd�tdd
�D��}|dkrRt|�}tdd�t
|tdd
��D��scJ�Wd�n1smwYt303304�|dks|J�d}	|jr�|	dkr�|	d7}	t305�d�|jr�|	dks�t306307�|dks�J�|��dkr�|js�J�dSdS)Nr-zRequires multiprocessingr?rcsr�rr�rcr)r)r+re&r�z<test_multiple_generator_call_separated_gc.<locals>.<genexpr>r�cSstd�S)NzGenerator collected)r�r)r)r)r+�<lambda>'�z;test_multiple_generator_call_separated_gc.<locals>.<lambda>z The executor underlying Parallelr�csrr
rvrcr)r)r+re3r�rrcsr�r1r)rr)r)r+re:r�rrrgrr_)r
r�r�r!ri�weakref�finalizerr�rrUr�r�r�aliverrcr)308r7r309r�errorrrZg_wrr�r��retryr)r)r+�)test_multiple_generator_call_separated_gcs:310���$�311�312�rcCs�|j}tdd||d��#}|dd�tj�d�gdD��tt�|��dks'J�Wd�n1s1wYtd�D]
}t�|�sCn	td	�q:t	d313��tdd|d�}|dd�tj�d�gdD��td�D]}t�|�sqdStd	�qgt	d314��)
Nr?r)r`r�r7Ztemp_foldercsrar1r�r�r)r)r+reVr�z(test_memmapping_leaks.<locals>.<genexpr>r�rrzr�z/temporary directory of Parallel was not removedr�csrar1r�r�r)r)r+refr�)315r�r!rr�r�rz�listdirrir�AssertionError)r7r�r�r�r)r)r+�test_memmapping_leaksKs" �316317 318319r)Nr-r.cCs<td|d�dd�td�D��}|dd�td�D�ksJ�dS)Nr?r_css �|]}tdd��|�VqdS)cSr=r>r)r@r)r)r+rvrz2test_lambda_expression.<locals>.<genexpr>.<lambda>Nrrcr)r)r+reur�z)test_lambda_expression.<locals>.<genexpr>r�cS�g|]}|d�qS�r?r)rcr)r)r+r,xr~z*test_lambda_expression.<locals>.<listcomp>rh)r7r�r)r)r+�test_lambda_expressionps�rcs�d}d}d|�td||d�}|�fdd�t|�D��|jj|jjks&J�|jj|jjks0J�|�fdd�t|�D��|jj|jjksGJ�|jj|jjksQJ�d	S)320zCTest that a parallel backend correctly resets its batch statistics.r?i�g@r�)rxr`r7c3��|]321}ttj���VqdSr1r�rc��	task_timer)r+re�r�z6test_backend_batch_statistics_reset.<locals>.<genexpr>c3rr1r�rcrr)r+re�r�N)r!rir�Z_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r7r`Zn_inputsr�r)rr+�#test_backend_batch_statistics_reset{s��r!cCsdD]<}tt|d�j�t�usJ�t|dd�}t|j�tus J�t|dd�}t|j�tus/J�t|dd�}t|j�tus>J�qtdd	dd322�}t|j�tusOJ�|d	dd��tdd�}t|j�tusdJ�|jdkskJ�Wd�n1suwY|d	dd��tddd�}t|j�tus�J�|jdks�J�Wd�n1s�wY|d	dd��tdd
�}t|j�tus�J�|jdks�J�Wd�n1s�wY|d	dd��tddd�}t|j�tus�J�|jdks�J�Wd�dS1s�wYdS)N)rr?rrrs�threads�r`�prefer�	processes�	sharedmem�r`�requirer?r-)r`r7r$�r$rg�r(r)r[r!r�r3rrr`)rqr`r�r)r)r+�$test_backend_hinting_and_constraints�s<323��324�	"�r+r$)Nr%r"cCs�dd�td�D�}t||d�dd�td�D��}||ksJ�t||d��t�dd�td�D��}Wd�n1s;wY||ksFJ�dS)	NcSrrr)rcr)r)r+r,�r~z7test_backend_hinting_always_running.<locals>.<listcomp>r�r#csrar1rbrcr)r)r+re�rfz6test_backend_hinting_always_running.<locals>.<genexpr>)r$r`csrar1rbrcr)r)r+re�r�)rir!r&)r`r$Zexpected_resultsr�r)r)r+�#test_backend_hinting_always_running�s��r,cCszGdd�dt�}||���&tddd�}t|j�|usJ�tddd�}t|j�|us,J�Wd�n1s6wYGdd	�d	t�}||���Otddd�}t|j�|usXJ�|��\}}|d325ksdJ�|d326ksjJ�tdddd�}t|j�tuszJ�|��\}}d
}|��|ks�J�|d327ks�J�Wd�n1s�wYtt��t|�dd�Wd�dS1s�wYdS)Nc@�$eZdZdZdZdd�Zdd�ZdS)z[test_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackendTcSrCr1r)r�r)r)r+�apply_async�rJzgtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackend.apply_asynccS�|Sr1r)r]r)r)r+r$�rJzltest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomThreadingBackend.effective_n_jobsN�rLrMrNZsupports_sharedmemZuse_threadsr.r$r)r)r)r+�MyCustomThreadingBackend��328r1r?r%r#r&r'c@r-)z\test_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackendFcSrCr1r)r�r)r)r+r.�rJzhtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackend.apply_asynccSr/r1r)r]r)r)r+r$�rJzmtest_backend_hinting_and_constraints_with_custom_backends.<locals>.MyCustomProcessingBackend.effective_n_jobsNr0r)r)r)r+�MyCustomProcessingBackend�r2r3rUr�)r`r(rxz�Using ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.�r7r()	rr!r[r�Z329readouterrr�striprrR)Zcapsysrqr1r�r3�out�errr�r)r)r+�9test_backend_hinting_and_constraints_with_custom_backends�s2330�331��332"�r8cCstt��
tdd�Wd�n1swYtt��
tdd�Wd�n1s-wYtt��tddd�Wd�n1sGwYtdur�tt��tddd�Wd�n1sewYtt��td	dd�Wd�dS1s�wYdSdS)333N�invalidr)r*r%r&)r$r(r-r4r()rrRr!r
r)r)r)r+�,test_invalid_backend_hinting_and_constraintss"334�335�336�337�338"��r:cs�tdd��4}t|j�j|jjfg}�dkr|Wd�S|��fdd�td�D��}||dWd�S1s<wYdS)zCPerform nested parallel calls and introspect the backend on the wayr?rsrNc3s*�|]}tt�dd�di���VqdS)�limitrNr))r#�_recursive_backend_inforc�r�r;r)r+re's339��340�z*_recursive_backend_info.<locals>.<genexpr>r)r!r[r�rLr�ri)r;r�r�Z341this_levelr�r)r=r+r< s��342$�r<cCs^||dd��t�}Wd�n1swY|��d}|dfdddg}||ks-J�dS)Nr?rs�Backendr)rr)rr?)r<�title)rqr7Zbackend_types_and_levelsZtop_level_backend_typeZexpected_types_and_levelsr)r)r+�test_nested_parallelism_limit.s��r@cCst�dd�td�D��S)z6A horrible function that does recursive parallel callscsr}r1)r#�_recursive_parallelrcr)r)r+reAr~z&_recursive_parallel.<locals>.<genexpr>r?rh)Z
nesting_limitr)r)r+rA?srAc	Cs�||dd��tt��}t�Wd�n1swYWd�n1s(wY|j}|dkrFddlm}t||tf�rFt�	d�t|t343�sMJ�dS)Nr?rsr-r)�TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError)r�
BaseExceptionrAr�Z&joblib.externals.loky.process_executorrBr�rr��xfail�RecursionError)rqr7rW�excrBr)r)r+�test_thread_bomb_mitigationDs344���345rGcCs*i}dD]346}tj�|�||<q|td�fS)N)�OMP_NUM_THREADS�OPENBLAS_NUM_THREADS�MKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS�347ENABLE_IPCrz)rz�environr�r)Zenv_vars�varr)r)r+�_run_parallel_sum`s	rNzNeed OpenMP helper compiledcCs�td|d�dd�td�D��}tt�dd�}|D]-\}}||ks#J�|��D]\}}|�d�r9|t|�ks8J�q'|dks?J�|dksEJ�q'qdS)	Nr?r_csr}r1)r#rNr�r)r)r+rer��348349�z-test_parallel_thread_limit.<locals>.<genexpr>rZ_THREADSrK�1)r!rir\r"�items�endswithr�)r7r�Zexpected_num_threadsZworker_env_varsZomp_num_threadsr�r�r)r)r+�test_parallel_thread_limitos�350��rSzThis test requires daskcCs:ttdd��
|d�Wd�dS1swYdS)NzPlease install daskr�Zdask)rrRror)r)r+�)test_dask_backend_when_dask_not_installed�s351"�rTc	Cs�Gdd�dt�}d}||���6tjt|d��tdd�dd�td�D��Wd�n1s0wYWd�dSWd�dS1sHwYdS)	Nc@s&eZdZdd�Zddd�Zdd�ZdS)	z3test_zero_worker_backend.<locals>.ZeroWorkerBackendc_�dSrZr))rDr�r�r)r)r+r[�rJz=test_zero_worker_backend.<locals>.ZeroWorkerBackend.configureNcSr�)NzNo worker availabler)rD�func�callbackr)r)r+r.�r<z?test_zero_worker_backend.<locals>.ZeroWorkerBackend.apply_asynccSrUrZr)r]r)r)r+r$�rJzDtest_zero_worker_backend.<locals>.ZeroWorkerBackend.effective_n_jobsr1)rLrMrNr[r.r$r)r)r)r+�ZeroWorkerBackend�s352rXz&ZeroWorkerBackend has no active workerr�r?rscsrar1r�rcr)r)r+re�r�z+test_zero_worker_backend.<locals>.<genexpr>)rr�rr�r!ri)rqrXZexpected_msgr)r)r+�test_zero_worker_backend�s353��"�rYcs�dadd����dks
J�tdd��fdd�td�D��}t|�dhks&J�da��dks/J�tdd��fd	d�td�D��}t|�dhksHJ�dS)354Nzoriginal valuecSstSr1)�MY_GLOBAL_VARIABLEr)r)r)r+�
check_globals�sz@test_globals_update_at_each_parallel_call.<locals>.check_globalsr?rsc3��|]}t���VqdSr1rrc�r[r)r+re�rOz<test_globals_update_at_each_parallel_call.<locals>.<genexpr>z
changed valuec3r\r1rrcr]r)r+re�rO)rZr!rir�)Zworkers_global_variabler)r]r+�)test_globals_update_at_each_parallel_call�s��r^cCs4ddl}|j�dd�}|�||�t�d�}|��S)Nrrz�
threadpoolctl)�numpyr�r�r�r�ZimportorskipZthreadpool_info)rrEr_r)r)r+�_check_numpy_threadpool_limits�s355356racCs4|D]}|d|dkr|dSqtd�|���)N�filepath�num_threadsz,An unexpected module was loaded in child:357{})rRr�)�child_module�parent_info�
parent_moduler)r)r+�_parent_max_num_threads_for�s��rgcCs@|D]}|D]}t||�}t||�|h}|d|vsJ�qqdS)Nrc)rg�min)Zworkers_inforercZchild_threadpool_infordZparent_max_num_threadsr�r)r)r+�check_child_num_threads�s���ri)r?r�rwrrcCszt�}t|�dkrtjdd�td|d�dd�td�D��}t|�}|d	kr-|dd358}ntt�|d	�}t	|||�dS)Nr�&Need a version of numpy linked to BLASrSr-r�csr}r1�r#rarcr)r)r+re�rOz;test_threadpool_limitation_in_child_loky.<locals>.<genexpr>r?rrc�359rar�r�r�r!rir$r\r"ri)r`re�workers_threadpool_infos�expected_child_num_threadsr)r)r+�(test_threadpool_limitation_in_child_loky�s��ro�inner_max_num_threads)rr?r�NcCs�t�}t|�dkrtjdd�|d|d��t|d�dd�td	�D��}Wd�n1s.wYt|�}|d360krB|dd}n|durOtt�|d361�}n|}t	|||�dS)NrrjrSr-�rprscsr}r1rkrcr)r)r+rerOz>test_threadpool_limitation_in_child_context.<locals>.<genexpr>r?rrcrl)rqr`rprermrnr)r)r+�+test_threadpool_limitation_in_child_context�s"362363���rr�var_name)rIrJrHc	st|�dkrt�d�tdd���dd��tj���}z_dtj�<t|d���fd	d364�t	d�D��}|ddgks;J�|ddd
��t|d���fdd365�t	d�D��}Wd�n1s]wY|ddgksjJ�W|durutj�=dS|tj�<dS|dur�tj�=w|tj�<w)NrzSkip test when n_jobs == 1T�ZreusecSstj�|�Sr1)rzrLr�)rsr)r)r+�_get_env,r�z>test_threadpool_limitation_in_child_override.<locals>._get_env�4rsc3rr1rrc�rursr)r+re3r�z?test_threadpool_limitation_in_child_override.<locals>.<genexpr>r?r-rqc3rr1rrcrwr)r+re7rfrP)366r$r�r�r�shutdownrzrLr�r!ri)rqr`rsZoriginal_var_valuer�r)rwr+�,test_threadpool_limitation_in_child_overrides*	367368"369���ry)r?r�rrcCsJdd�}||�tdd�}td�D]}||�tdd�}||ks"J�qdS)Ncs,td��t|d��fdd�td�D��dS)Nr�rsc3r�r1)r#�sumrcr@r)r+reKr�zAtest_loky_reuse_workers.<locals>.parallel_call.<locals>.<genexpr>r�rsr)r@r+�
parallel_callIs$z.test_loky_reuse_workers.<locals>.parallel_callTrtr�)rri)r`r{Zfirst_executorr�r�r)r)r+�test_loky_reuse_workersCs370371�r|cCsd|t��<dS)N�initialized)rzr{)�statusr)r)r+�_set_initializedYr^rcCs�t��}|�|d�}|dvsJd|����|sdSd||<t��d}tdd�|��D��}t��|krR||krRt�d�tdd�|��D��}t��|krR||ks8t��|kr\td	��|S)372N)r}�startedz2worker should have been in initialized state, got r�rcS�g|]373\}}|dkr|�qS�r�r)�r*�pidr�r)r)r+r,i�z!_check_status.<locals>.<listcomp>r�cSr�r�r)r�r)r)r+r,lr�z-Waited more than 30s to start all the workers)rzr{r�rUr�rQrr)r~r`�wait_workersr��state�deadlineZ	n_startedr)r)r+�
_check_status]s"374�375�r�csft��}|���||�t�fd��t���fdd�td�D��Wd�dS1s,wYdS)N�r7r`ZinitializerZinitargsc3��|]376}tt����VqdSr1�r#r�rc�r`r~r)r+re�r�z+test_initializer_context.<locals>.<genexpr>rz)r
rB�dictrr!ri)r`r7rqrDr)r�r+�test_initializer_contextts� "�r�cs>t��}|���t|�t�fd���fdd�td�D��dS)Nr�c3r�r1r�rcr�r)r+re�r�z,test_initializer_parallel.<locals>.<genexpr>rz)r
rBr�r!rri)r`r7rDr)r�r+�test_initializer_parallel�s��r�csxd}t��}|���t�}t|�D]}td�t�fd���fdd�t��D��}|�t|��}qt|��ks:Jd��dS)Nrgr-r�c3�"�|]}tt���dd�VqdS�T)r�Nr�rcr�r)r+re��377��378�z*test_initializer_reused.<locals>.<genexpr>z=The workers should be reused when the initializer is the same)	r
rBr�r�rir!r�unionr��r`Z
n_repetitionsrDZpidsrdr�r)r�r+�test_initializer_reused�s&���	�r�cs|d}t��}t�}t|�D]"}|���td�t�fd���fdd�t��D��}|�t|��}q
t|�|�ks<Jd��dS)Nrgr-r�c3r�r�r�rcr�r)r+re�r�z.test_initializer_not_reused.<locals>.<genexpr>zFThe workers should not be reused when the initializer arguments change)	r
rBr�rir�r!rr�r�r�r)r�r+�test_initializer_not_reused�s&���	�r�)F)rr)rgr1)�rOr�rzr�r�r.rUr�rr�r�mathrr(r�picklerr�	tracebackr	r�r�r379rrZjoblib._multiprocessing_helpersr
Zjoblib.test.commonrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrr&r�posix�ImportErrorZ _openmp_test_helper.parallel_sumr�distributedZjoblib._parallel_backendsrrrrrZjoblib.parallelr r!r"r#r$r%r&r'r�ZRETURN_GENERATOR_BACKENDS�popr��keysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDS�hasattrr�r/r3r8r;rA�	ExceptionrBrTrWrYr]rkrmrp�markrvr�r�r�r�r�r�r�r�r�r�r�r�r�r�rrr
r�rrr%r(r@rErIrJrLrPrRrDrYrZr^r`rerfrgrirnrq�extendrirrrsrwryr|rZ_parallel_backendsr�r�r�r�r�r�r�r�r�r�r�r�r�r�ZSQUARE_LOCALZ
SQUARE_LAMBDAr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rrrrr	rrrrr!r+r,r8r:r<r@rAZno_coverrGrNrSrTrYr^rargrirorrryr|rr�r�r�r�r�r)r)r)r+�<module>s����,380381382�383*�384385386387388F����(389390
391	392		393394395 396 397398	399�400401 402 (�+403
�404
405406	407 408409�,"�.6$	
#
Aluode/PerceptionLabPortable · CoolFace