Aluode/PerceptionLabPortable
0
1"""2Test the hashing module.3"""4 5# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>6# Copyright (c) 2009 Gael Varoquaux7# License: BSD Style, 3 clauses.8 9import collections10import gc11import hashlib12import io13import itertools14import pickle15import random16import sys17import time18from concurrent.futures import ProcessPoolExecutor19from decimal import Decimal20 21from joblib.func_inspect import filter_args22from joblib.hashing import hash23from joblib.memory import Memory24from joblib.test.common import np, with_numpy25from joblib.testing import fixture, parametrize, raises, skipif26 27 28def unicode(s):29 return s30 31 32###############################################################################33# Helper functions for the tests34def time_func(func, *args):35 """Time function func on *args."""36 times = list()37 for _ in range(3):38 t1 = time.time()39 func(*args)40 times.append(time.time() - t1)41 return min(times)42 43 44def relative_time(func1, func2, *args):45 """Return the relative time between func1 and func2 applied on46 *args.47 """48 time_func1 = time_func(func1, *args)49 time_func2 = time_func(func2, *args)50 relative_diff = 0.5 * (abs(time_func1 - time_func2) / (time_func1 + time_func2))51 return relative_diff52 53 54class Klass(object):55 def f(self, x):56 return x57 58 59class KlassWithCachedMethod(object):60 def __init__(self, cachedir):61 mem = Memory(location=cachedir)62 self.f = mem.cache(self.f)63 64 def f(self, x):65 return x66 67 68###############################################################################69# Tests70 71input_list = [72 1,73 2,74 1.0,75 2.0,76 1 + 1j,77 2.0 + 1j,78 "a",79 "b",80 (1,),81 (82 1,83 1,84 ),85 [86 1,87 ],88 [89 1,90 1,91 ],92 {1: 1},93 {1: 2},94 {2: 1},95 None,96 gc.collect,97 [98 1,99 ].append,100 # Next 2 sets have unorderable elements in python 3.101 set(("a", 1)),102 set(("a", 1, ("a", 1))),103 # Next 2 dicts have unorderable type of keys in python 3.104 {"a": 1, 1: 2},105 {"a": 1, 1: 2, "d": {"a": 1}},106]107 108 109@parametrize("obj1", input_list)110@parametrize("obj2", input_list)111def test_trivial_hash(obj1, obj2):112 """Smoke test hash on various types."""113 # Check that 2 objects have the same hash only if they are the same.114 are_hashes_equal = hash(obj1) == hash(obj2)115 are_objs_identical = obj1 is obj2116 assert are_hashes_equal == are_objs_identical117 118 119def test_hash_methods():120 # Check that hashing instance methods works121 a = io.StringIO(unicode("a"))122 assert hash(a.flush) == hash(a.flush)123 a1 = collections.deque(range(10))124 a2 = collections.deque(range(9))125 assert hash(a1.extend) != hash(a2.extend)126 127 128@fixture(scope="function")129@with_numpy130def three_np_arrays():131 rnd = np.random.RandomState(0)132 arr1 = rnd.random_sample((10, 10))133 arr2 = arr1.copy()134 arr3 = arr2.copy()135 arr3[0] += 1136 return arr1, arr2, arr3137 138 139def test_hash_numpy_arrays(three_np_arrays):140 arr1, arr2, arr3 = three_np_arrays141 142 for obj1, obj2 in itertools.product(three_np_arrays, repeat=2):143 are_hashes_equal = hash(obj1) == hash(obj2)144 are_arrays_equal = np.all(obj1 == obj2)145 assert are_hashes_equal == are_arrays_equal146 147 assert hash(arr1) != hash(arr1.T)148 149 150def test_hash_numpy_dict_of_arrays(three_np_arrays):151 arr1, arr2, arr3 = three_np_arrays152 153 d1 = {1: arr1, 2: arr2}154 d2 = {1: arr2, 2: arr1}155 d3 = {1: arr2, 2: arr3}156 157 assert hash(d1) == hash(d2)158 assert hash(d1) != hash(d3)159 160 161@with_numpy162@parametrize("dtype", ["datetime64[s]", "timedelta64[D]"])163def test_numpy_datetime_array(dtype):164 # memoryview is not supported for some dtypes e.g. datetime64165 # see https://github.com/joblib/joblib/issues/188 for more details166 a_hash = hash(np.arange(10))167 array = np.arange(0, 10, dtype=dtype)168 assert hash(array) != a_hash169 170 171@with_numpy172def test_hash_numpy_noncontiguous():173 a = np.asarray(np.arange(6000).reshape((1000, 2, 3)), order="F")[:, :1, :]174 b = np.ascontiguousarray(a)175 assert hash(a) != hash(b)176 177 c = np.asfortranarray(a)178 assert hash(a) != hash(c)179 180 181@with_numpy182@parametrize("coerce_mmap", [True, False])183def test_hash_memmap(tmpdir, coerce_mmap):184 """Check that memmap and arrays hash identically if coerce_mmap is True."""185 filename = tmpdir.join("memmap_temp").strpath186 try:187 m = np.memmap(filename, shape=(10, 10), mode="w+")188 a = np.asarray(m)189 are_hashes_equal = hash(a, coerce_mmap=coerce_mmap) == hash(190 m, coerce_mmap=coerce_mmap191 )192 assert are_hashes_equal == coerce_mmap193 finally:194 if "m" in locals():195 del m196 # Force a garbage-collection cycle, to be certain that the197 # object is delete, and we don't run in a problem under198 # Windows with a file handle still open.199 gc.collect()200 201 202@with_numpy203@skipif(204 sys.platform == "win32",205 reason="This test is not stable under windows for some reason",206)207def test_hash_numpy_performance():208 """Check the performance of hashing numpy arrays:209 210 In [22]: a = np.random.random(1000000)211 212 In [23]: %timeit hashlib.md5(a).hexdigest()213 100 loops, best of 3: 20.7 ms per loop214 215 In [24]: %timeit hashlib.md5(pickle.dumps(a, protocol=2)).hexdigest()216 1 loops, best of 3: 73.1 ms per loop217 218 In [25]: %timeit hashlib.md5(cPickle.dumps(a, protocol=2)).hexdigest()219 10 loops, best of 3: 53.9 ms per loop220 221 In [26]: %timeit hash(a)222 100 loops, best of 3: 20.8 ms per loop223 """224 rnd = np.random.RandomState(0)225 a = rnd.random_sample(1000000)226 227 def md5_hash(x):228 return hashlib.md5(memoryview(x)).hexdigest()229 230 relative_diff = relative_time(md5_hash, hash, a)231 assert relative_diff < 0.3232 233 # Check that hashing an tuple of 3 arrays takes approximately234 # 3 times as much as hashing one array235 time_hashlib = 3 * time_func(md5_hash, a)236 time_hash = time_func(hash, (a, a, a))237 relative_diff = 0.5 * (abs(time_hash - time_hashlib) / (time_hash + time_hashlib))238 assert relative_diff < 0.3239 240 241def test_bound_methods_hash():242 """Make sure that calling the same method on two different instances243 of the same class does resolve to the same hashes.244 """245 a = Klass()246 b = Klass()247 assert hash(filter_args(a.f, [], (1,))) == hash(filter_args(b.f, [], (1,)))248 249 250def test_bound_cached_methods_hash(tmpdir):251 """Make sure that calling the same _cached_ method on two different252 instances of the same class does resolve to the same hashes.253 """254 a = KlassWithCachedMethod(tmpdir.strpath)255 b = KlassWithCachedMethod(tmpdir.strpath)256 assert hash(filter_args(a.f.func, [], (1,))) == hash(257 filter_args(b.f.func, [], (1,))258 )259 260 261@with_numpy262def test_hash_object_dtype():263 """Make sure that ndarrays with dtype `object' hash correctly."""264 265 a = np.array([np.arange(i) for i in range(6)], dtype=object)266 b = np.array([np.arange(i) for i in range(6)], dtype=object)267 268 assert hash(a) == hash(b)269 270 271@with_numpy272def test_numpy_scalar():273 # Numpy scalars are built from compiled functions, and lead to274 # strange pickling paths explored, that can give hash collisions275 a = np.float64(2.0)276 b = np.float64(3.0)277 assert hash(a) != hash(b)278 279 280def test_dict_hash(tmpdir):281 # Check that dictionaries hash consistently, even though the ordering282 # of the keys is not guaranteed283 k = KlassWithCachedMethod(tmpdir.strpath)284 285 d = {286 "#s12069__c_maps.nii.gz": [33],287 "#s12158__c_maps.nii.gz": [33],288 "#s12258__c_maps.nii.gz": [33],289 "#s12277__c_maps.nii.gz": [33],290 "#s12300__c_maps.nii.gz": [33],291 "#s12401__c_maps.nii.gz": [33],292 "#s12430__c_maps.nii.gz": [33],293 "#s13817__c_maps.nii.gz": [33],294 "#s13903__c_maps.nii.gz": [33],295 "#s13916__c_maps.nii.gz": [33],296 "#s13981__c_maps.nii.gz": [33],297 "#s13982__c_maps.nii.gz": [33],298 "#s13983__c_maps.nii.gz": [33],299 }300 301 a = k.f(d)302 b = k.f(a)303 304 assert hash(a) == hash(b)305 306 307def test_set_hash(tmpdir):308 # Check that sets hash consistently, even though their ordering309 # is not guaranteed310 k = KlassWithCachedMethod(tmpdir.strpath)311 312 s = set(313 [314 "#s12069__c_maps.nii.gz",315 "#s12158__c_maps.nii.gz",316 "#s12258__c_maps.nii.gz",317 "#s12277__c_maps.nii.gz",318 "#s12300__c_maps.nii.gz",319 "#s12401__c_maps.nii.gz",320 "#s12430__c_maps.nii.gz",321 "#s13817__c_maps.nii.gz",322 "#s13903__c_maps.nii.gz",323 "#s13916__c_maps.nii.gz",324 "#s13981__c_maps.nii.gz",325 "#s13982__c_maps.nii.gz",326 "#s13983__c_maps.nii.gz",327 ]328 )329 330 a = k.f(s)331 b = k.f(a)332 333 assert hash(a) == hash(b)334 335 336def test_set_decimal_hash():337 # Check that sets containing decimals hash consistently, even though338 # ordering is not guaranteed339 assert hash(set([Decimal(0), Decimal("NaN")])) == hash(340 set([Decimal("NaN"), Decimal(0)])341 )342 343 344def test_string():345 # Test that we obtain the same hash for object owning several strings,346 # whatever the past of these strings (which are immutable in Python)347 string = "foo"348 a = {string: "bar"}349 b = {string: "bar"}350 c = pickle.loads(pickle.dumps(b))351 assert hash([a, b]) == hash([a, c])352 353 354@with_numpy355def test_numpy_dtype_pickling():356 # numpy dtype hashing is tricky to get right: see #231, #239, #251 #1080,357 # #1082, and explanatory comments inside358 # ``joblib.hashing.NumpyHasher.save``.359 360 # In this test, we make sure that the pickling of numpy dtypes is robust to361 # object identity and object copy.362 363 dt1 = np.dtype("f4")364 dt2 = np.dtype("f4")365 366 # simple dtypes objects are interned367 assert dt1 is dt2368 assert hash(dt1) == hash(dt2)369 370 dt1_roundtripped = pickle.loads(pickle.dumps(dt1))371 assert dt1 is not dt1_roundtripped372 assert hash(dt1) == hash(dt1_roundtripped)373 374 assert hash([dt1, dt1]) == hash([dt1_roundtripped, dt1_roundtripped])375 assert hash([dt1, dt1]) == hash([dt1, dt1_roundtripped])376 377 complex_dt1 = np.dtype([("name", np.str_, 16), ("grades", np.float64, (2,))])378 complex_dt2 = np.dtype([("name", np.str_, 16), ("grades", np.float64, (2,))])379 380 # complex dtypes objects are not interned381 assert hash(complex_dt1) == hash(complex_dt2)382 383 complex_dt1_roundtripped = pickle.loads(pickle.dumps(complex_dt1))384 assert complex_dt1_roundtripped is not complex_dt1385 assert hash(complex_dt1) == hash(complex_dt1_roundtripped)386 387 assert hash([complex_dt1, complex_dt1]) == hash(388 [complex_dt1_roundtripped, complex_dt1_roundtripped]389 )390 assert hash([complex_dt1, complex_dt1]) == hash(391 [complex_dt1_roundtripped, complex_dt1]392 )393 394 395@parametrize(396 "to_hash,expected",397 [398 ("This is a string to hash", "71b3f47df22cb19431d85d92d0b230b2"),399 ("C'est l\xe9t\xe9", "2d8d189e9b2b0b2e384d93c868c0e576"),400 ((123456, 54321, -98765), "e205227dd82250871fa25aa0ec690aa3"),401 (402 [random.Random(42).random() for _ in range(5)],403 "a11ffad81f9682a7d901e6edc3d16c84",404 ),405 ({"abcde": 123, "sadfas": [-9999, 2, 3]}, "aeda150553d4bb5c69f0e69d51b0e2ef"),406 ],407)408def test_hashes_stay_the_same(to_hash, expected):409 # We want to make sure that hashes don't change with joblib410 # version. For end users, that would mean that they have to411 # regenerate their cache from scratch, which potentially means412 # lengthy recomputations.413 # Expected results have been generated with joblib 0.9.2414 assert hash(to_hash) == expected415 416 417@with_numpy418def test_hashes_are_different_between_c_and_fortran_contiguous_arrays():419 # We want to be sure that the c-contiguous and f-contiguous versions of the420 # same array produce 2 different hashes.421 rng = np.random.RandomState(0)422 arr_c = rng.random_sample((10, 10))423 arr_f = np.asfortranarray(arr_c)424 assert hash(arr_c) != hash(arr_f)425 426 427@with_numpy428def test_0d_array():429 hash(np.array(0))430 431 432@with_numpy433def test_0d_and_1d_array_hashing_is_different():434 assert hash(np.array(0)) != hash(np.array([0]))435 436 437@with_numpy438def test_hashes_stay_the_same_with_numpy_objects():439 # Note: joblib used to test numpy objects hashing by comparing the produced440 # hash of an object with some hard-coded target value to guarantee that441 # hashing remains the same across joblib versions. However, since numpy442 # 1.20 and joblib 1.0, joblib relies on potentially unstable implementation443 # details of numpy to hash np.dtype objects, which makes the stability of444 # hash values across different environments hard to guarantee and to test.445 # As a result, hashing stability across joblib versions becomes best-effort446 # only, and we only test the consistency within a single environment by447 # making sure:448 # - the hash of two copies of the same objects is the same449 # - hashing some object in two different python processes produces the same450 # value. This should be viewed as a proxy for testing hash consistency451 # through time between Python sessions (provided no change in the452 # environment was done between sessions).453 454 def create_objects_to_hash():455 rng = np.random.RandomState(42)456 # Being explicit about dtypes in order to avoid457 # architecture-related differences. Also using 'f4' rather than458 # 'f8' for float arrays because 'f8' arrays generated by459 # rng.random.randn don't seem to be bit-identical on 32bit and460 # 64bit machines.461 to_hash_list = [462 rng.randint(-1000, high=1000, size=50).astype("<i8"),463 tuple(rng.randn(3).astype("<f4") for _ in range(5)),464 [rng.randn(3).astype("<f4") for _ in range(5)],465 {466 -3333: rng.randn(3, 5).astype("<f4"),467 0: [468 rng.randint(10, size=20).astype("<i8"),469 rng.randn(10).astype("<f4"),470 ],471 },472 # Non regression cases for473 # https://github.com/joblib/joblib/issues/308474 np.arange(100, dtype="<i8").reshape((10, 10)),475 # Fortran contiguous array476 np.asfortranarray(np.arange(100, dtype="<i8").reshape((10, 10))),477 # Non contiguous array478 np.arange(100, dtype="<i8").reshape((10, 10))[:, :2],479 ]480 return to_hash_list481 482 # Create two lists containing copies of the same objects. joblib.hash483 # should return the same hash for to_hash_list_one[i] and484 # to_hash_list_two[i]485 to_hash_list_one = create_objects_to_hash()486 to_hash_list_two = create_objects_to_hash()487 488 e1 = ProcessPoolExecutor(max_workers=1)489 e2 = ProcessPoolExecutor(max_workers=1)490 491 try:492 for obj_1, obj_2 in zip(to_hash_list_one, to_hash_list_two):493 # testing consistency of hashes across python processes494 hash_1 = e1.submit(hash, obj_1).result()495 hash_2 = e2.submit(hash, obj_1).result()496 assert hash_1 == hash_2497 498 # testing consistency when hashing two copies of the same objects.499 hash_3 = e1.submit(hash, obj_2).result()500 assert hash_1 == hash_3501 502 finally:503 e1.shutdown()504 e2.shutdown()505 506 507def test_hashing_pickling_error():508 def non_picklable():509 return 42510 511 with raises(pickle.PicklingError) as excinfo:512 hash(non_picklable)513 excinfo.match("PicklingError while hashing")514 515 516def test_wrong_hash_name():517 msg = "Valid options for 'hash_name' are"518 with raises(ValueError, match=msg):519 data = {"foo": "bar"}520 hash(data, hash_name="invalid")521