Aluode/PerceptionLabPortable
0
1"""2This module contains function to analyse dynamic library3headers to extract system information4 5Currently only for MacOSX6 7Library file on macosx system starts with Mach-O or Fat field.8This can be distinguish by first 32 bites and it is called magic number.9Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means10reversed bytes order.11Both fields can occur in two types: 32 and 64 bytes.12 13FAT field inform that this library contains few version of library14(typically for different types version). It contains15information where Mach-O headers starts.16 17Each section started with Mach-O header contains one library18(So if file starts with this field it contains only one version).19 20After filed Mach-O there are section fields.21Each of them starts with two fields:22cmd - magic number for this command23cmdsize - total size occupied by this section information.24 25In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier)26and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting,27because them contains information about minimal system version.28 29Important remarks:30- For fat files this implementation looks for maximum number version.31 It not check if it is 32 or 64 and do not compare it with currently built package.32 So it is possible to false report higher version that needed.33- All structures signatures are taken form macosx header files.34- I think that binary format will be more stable than `otool` output.35 and if apple introduce some changes both implementation will need to be updated.36- The system compile will set the deployment target no lower than37 11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment38 target when the arm64 target is 11.0.39"""40 41from __future__ import annotations42 43import ctypes44import os45import sys46from io import BufferedIOBase47from typing import TYPE_CHECKING48 49if TYPE_CHECKING:50 from typing import Union51 52 StrPath = Union[str, os.PathLike[str]]53 54"""here the needed const and struct from mach-o header files"""55 56FAT_MAGIC = 0xCAFEBABE57FAT_CIGAM = 0xBEBAFECA58FAT_MAGIC_64 = 0xCAFEBABF59FAT_CIGAM_64 = 0xBFBAFECA60MH_MAGIC = 0xFEEDFACE61MH_CIGAM = 0xCEFAEDFE62MH_MAGIC_64 = 0xFEEDFACF63MH_CIGAM_64 = 0xCFFAEDFE64 65LC_VERSION_MIN_MACOSX = 0x2466LC_BUILD_VERSION = 0x3267 68CPU_TYPE_ARM64 = 0x0100000C69 70mach_header_fields = [71 ("magic", ctypes.c_uint32),72 ("cputype", ctypes.c_int),73 ("cpusubtype", ctypes.c_int),74 ("filetype", ctypes.c_uint32),75 ("ncmds", ctypes.c_uint32),76 ("sizeofcmds", ctypes.c_uint32),77 ("flags", ctypes.c_uint32),78]79"""80struct mach_header {81 uint32_t magic; /* mach magic number identifier */82 cpu_type_t cputype; /* cpu specifier */83 cpu_subtype_t cpusubtype; /* machine specifier */84 uint32_t filetype; /* type of file */85 uint32_t ncmds; /* number of load commands */86 uint32_t sizeofcmds; /* the size of all the load commands */87 uint32_t flags; /* flags */88};89typedef integer_t cpu_type_t;90typedef integer_t cpu_subtype_t;91"""92 93mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)]94"""95struct mach_header_64 {96 uint32_t magic; /* mach magic number identifier */97 cpu_type_t cputype; /* cpu specifier */98 cpu_subtype_t cpusubtype; /* machine specifier */99 uint32_t filetype; /* type of file */100 uint32_t ncmds; /* number of load commands */101 uint32_t sizeofcmds; /* the size of all the load commands */102 uint32_t flags; /* flags */103 uint32_t reserved; /* reserved */104};105"""106 107fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)]108"""109struct fat_header {110 uint32_t magic; /* FAT_MAGIC or FAT_MAGIC_64 */111 uint32_t nfat_arch; /* number of structs that follow */112};113"""114 115fat_arch_fields = [116 ("cputype", ctypes.c_int),117 ("cpusubtype", ctypes.c_int),118 ("offset", ctypes.c_uint32),119 ("size", ctypes.c_uint32),120 ("align", ctypes.c_uint32),121]122"""123struct fat_arch {124 cpu_type_t cputype; /* cpu specifier (int) */125 cpu_subtype_t cpusubtype; /* machine specifier (int) */126 uint32_t offset; /* file offset to this object file */127 uint32_t size; /* size of this object file */128 uint32_t align; /* alignment as a power of 2 */129};130"""131 132fat_arch_64_fields = [133 ("cputype", ctypes.c_int),134 ("cpusubtype", ctypes.c_int),135 ("offset", ctypes.c_uint64),136 ("size", ctypes.c_uint64),137 ("align", ctypes.c_uint32),138 ("reserved", ctypes.c_uint32),139]140"""141struct fat_arch_64 {142 cpu_type_t cputype; /* cpu specifier (int) */143 cpu_subtype_t cpusubtype; /* machine specifier (int) */144 uint64_t offset; /* file offset to this object file */145 uint64_t size; /* size of this object file */146 uint32_t align; /* alignment as a power of 2 */147 uint32_t reserved; /* reserved */148};149"""150 151segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)]152"""base for reading segment info"""153 154segment_command_fields = [155 ("cmd", ctypes.c_uint32),156 ("cmdsize", ctypes.c_uint32),157 ("segname", ctypes.c_char * 16),158 ("vmaddr", ctypes.c_uint32),159 ("vmsize", ctypes.c_uint32),160 ("fileoff", ctypes.c_uint32),161 ("filesize", ctypes.c_uint32),162 ("maxprot", ctypes.c_int),163 ("initprot", ctypes.c_int),164 ("nsects", ctypes.c_uint32),165 ("flags", ctypes.c_uint32),166]167"""168struct segment_command { /* for 32-bit architectures */169 uint32_t cmd; /* LC_SEGMENT */170 uint32_t cmdsize; /* includes sizeof section structs */171 char segname[16]; /* segment name */172 uint32_t vmaddr; /* memory address of this segment */173 uint32_t vmsize; /* memory size of this segment */174 uint32_t fileoff; /* file offset of this segment */175 uint32_t filesize; /* amount to map from the file */176 vm_prot_t maxprot; /* maximum VM protection */177 vm_prot_t initprot; /* initial VM protection */178 uint32_t nsects; /* number of sections in segment */179 uint32_t flags; /* flags */180};181typedef int vm_prot_t;182"""183 184segment_command_fields_64 = [185 ("cmd", ctypes.c_uint32),186 ("cmdsize", ctypes.c_uint32),187 ("segname", ctypes.c_char * 16),188 ("vmaddr", ctypes.c_uint64),189 ("vmsize", ctypes.c_uint64),190 ("fileoff", ctypes.c_uint64),191 ("filesize", ctypes.c_uint64),192 ("maxprot", ctypes.c_int),193 ("initprot", ctypes.c_int),194 ("nsects", ctypes.c_uint32),195 ("flags", ctypes.c_uint32),196]197"""198struct segment_command_64 { /* for 64-bit architectures */199 uint32_t cmd; /* LC_SEGMENT_64 */200 uint32_t cmdsize; /* includes sizeof section_64 structs */201 char segname[16]; /* segment name */202 uint64_t vmaddr; /* memory address of this segment */203 uint64_t vmsize; /* memory size of this segment */204 uint64_t fileoff; /* file offset of this segment */205 uint64_t filesize; /* amount to map from the file */206 vm_prot_t maxprot; /* maximum VM protection */207 vm_prot_t initprot; /* initial VM protection */208 uint32_t nsects; /* number of sections in segment */209 uint32_t flags; /* flags */210};211"""212 213version_min_command_fields = segment_base_fields + [214 ("version", ctypes.c_uint32),215 ("sdk", ctypes.c_uint32),216]217"""218struct version_min_command {219 uint32_t cmd; /* LC_VERSION_MIN_MACOSX or220 LC_VERSION_MIN_IPHONEOS or221 LC_VERSION_MIN_WATCHOS or222 LC_VERSION_MIN_TVOS */223 uint32_t cmdsize; /* sizeof(struct min_version_command) */224 uint32_t version; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */225 uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */226};227"""228 229build_version_command_fields = segment_base_fields + [230 ("platform", ctypes.c_uint32),231 ("minos", ctypes.c_uint32),232 ("sdk", ctypes.c_uint32),233 ("ntools", ctypes.c_uint32),234]235"""236struct build_version_command {237 uint32_t cmd; /* LC_BUILD_VERSION */238 uint32_t cmdsize; /* sizeof(struct build_version_command) plus */239 /* ntools * sizeof(struct build_tool_version) */240 uint32_t platform; /* platform */241 uint32_t minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */242 uint32_t sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */243 uint32_t ntools; /* number of tool entries following this */244};245"""246 247 248def swap32(x: int) -> int:249 return (250 ((x << 24) & 0xFF000000)251 | ((x << 8) & 0x00FF0000)252 | ((x >> 8) & 0x0000FF00)253 | ((x >> 24) & 0x000000FF)254 )255 256 257def get_base_class_and_magic_number(258 lib_file: BufferedIOBase,259 seek: int | None = None,260) -> tuple[type[ctypes.Structure], int]:261 if seek is None:262 seek = lib_file.tell()263 else:264 lib_file.seek(seek)265 magic_number = ctypes.c_uint32.from_buffer_copy(266 lib_file.read(ctypes.sizeof(ctypes.c_uint32))267 ).value268 269 # Handle wrong byte order270 if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]:271 if sys.byteorder == "little":272 BaseClass = ctypes.BigEndianStructure273 else:274 BaseClass = ctypes.LittleEndianStructure275 276 magic_number = swap32(magic_number)277 else:278 BaseClass = ctypes.Structure279 280 lib_file.seek(seek)281 return BaseClass, magic_number282 283 284def read_data(struct_class: type[ctypes.Structure], lib_file: BufferedIOBase):285 return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class)))286 287 288def extract_macosx_min_system_version(path_to_lib: str):289 with open(path_to_lib, "rb") as lib_file:290 BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0)291 if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]:292 return293 294 if magic_number in [FAT_MAGIC, FAT_CIGAM_64]:295 296 class FatHeader(BaseClass):297 _fields_ = fat_header_fields298 299 fat_header = read_data(FatHeader, lib_file)300 if magic_number == FAT_MAGIC:301 302 class FatArch(BaseClass):303 _fields_ = fat_arch_fields304 305 else:306 307 class FatArch(BaseClass):308 _fields_ = fat_arch_64_fields309 310 fat_arch_list = [311 read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch)312 ]313 314 versions_list: list[tuple[int, int, int]] = []315 for el in fat_arch_list:316 try:317 version = read_mach_header(lib_file, el.offset)318 if version is not None:319 if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1:320 # Xcode will not set the deployment target below 11.0.0321 # for the arm64 architecture. Ignore the arm64 deployment322 # in fat binaries when the target is 11.0.0, that way323 # the other architectures can select a lower deployment324 # target.325 # This is safe because there is no arm64 variant for326 # macOS 10.15 or earlier.327 if version == (11, 0, 0):328 continue329 versions_list.append(version)330 except ValueError:331 pass332 333 if len(versions_list) > 0:334 return max(versions_list)335 else:336 return None337 338 else:339 try:340 return read_mach_header(lib_file, 0)341 except ValueError:342 """when some error during read library files"""343 return None344 345 346def read_mach_header(347 lib_file: BufferedIOBase,348 seek: int | None = None,349) -> tuple[int, int, int] | None:350 """351 This function parses a Mach-O header and extracts352 information about the minimal macOS version.353 354 :param lib_file: reference to opened library file with pointer355 """356 base_class, magic_number = get_base_class_and_magic_number(lib_file, seek)357 arch = "32" if magic_number == MH_MAGIC else "64"358 359 class SegmentBase(base_class):360 _fields_ = segment_base_fields361 362 if arch == "32":363 364 class MachHeader(base_class):365 _fields_ = mach_header_fields366 367 else:368 369 class MachHeader(base_class):370 _fields_ = mach_header_fields_64371 372 mach_header = read_data(MachHeader, lib_file)373 for _i in range(mach_header.ncmds):374 pos = lib_file.tell()375 segment_base = read_data(SegmentBase, lib_file)376 lib_file.seek(pos)377 if segment_base.cmd == LC_VERSION_MIN_MACOSX:378 379 class VersionMinCommand(base_class):380 _fields_ = version_min_command_fields381 382 version_info = read_data(VersionMinCommand, lib_file)383 return parse_version(version_info.version)384 elif segment_base.cmd == LC_BUILD_VERSION:385 386 class VersionBuild(base_class):387 _fields_ = build_version_command_fields388 389 version_info = read_data(VersionBuild, lib_file)390 return parse_version(version_info.minos)391 else:392 lib_file.seek(pos + segment_base.cmdsize)393 continue394 395 396def parse_version(version: int) -> tuple[int, int, int]:397 x = (version & 0xFFFF0000) >> 16398 y = (version & 0x0000FF00) >> 8399 z = version & 0x000000FF400 return x, y, z401 402 403def calculate_macosx_platform_tag(archive_root: StrPath, platform_tag: str) -> str:404 """405 Calculate proper macosx platform tag basing on files which are included to wheel406 407 Example platform tag `macosx-10.14-x86_64`408 """409 prefix, base_version, suffix = platform_tag.split("-")410 base_version = tuple(int(x) for x in base_version.split("."))411 base_version = base_version[:2]412 if base_version[0] > 10:413 base_version = (base_version[0], 0)414 assert len(base_version) == 2415 if "MACOSX_DEPLOYMENT_TARGET" in os.environ:416 deploy_target = tuple(417 int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".")418 )419 deploy_target = deploy_target[:2]420 if deploy_target[0] > 10:421 deploy_target = (deploy_target[0], 0)422 if deploy_target < base_version:423 sys.stderr.write(424 "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than "425 "the version on which the Python interpreter was compiled ({}), and "426 "will be ignored.\n".format(427 ".".join(str(x) for x in deploy_target),428 ".".join(str(x) for x in base_version),429 )430 )431 else:432 base_version = deploy_target433 434 assert len(base_version) == 2435 start_version = base_version436 versions_dict: dict[str, tuple[int, int]] = {}437 for dirpath, _dirnames, filenames in os.walk(archive_root):438 for filename in filenames:439 if filename.endswith(".dylib") or filename.endswith(".so"):440 lib_path = os.path.join(dirpath, filename)441 min_ver = extract_macosx_min_system_version(lib_path)442 if min_ver is not None:443 min_ver = min_ver[0:2]444 if min_ver[0] > 10:445 min_ver = (min_ver[0], 0)446 versions_dict[lib_path] = min_ver447 448 if len(versions_dict) > 0:449 base_version = max(base_version, max(versions_dict.values()))450 451 # macosx platform tag do not support minor bugfix release452 fin_base_version = "_".join([str(x) for x in base_version])453 if start_version < base_version:454 problematic_files = [k for k, v in versions_dict.items() if v > start_version]455 problematic_files = "\n".join(problematic_files)456 if len(problematic_files) == 1:457 files_form = "this file"458 else:459 files_form = "these files"460 error_message = (461 "[WARNING] This wheel needs a higher macOS version than {} "462 "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least "463 + fin_base_version464 + " or recreate "465 + files_form466 + " with lower "467 "MACOSX_DEPLOYMENT_TARGET: \n" + problematic_files468 )469 470 if "MACOSX_DEPLOYMENT_TARGET" in os.environ:471 error_message = error_message.format(472 "is set in MACOSX_DEPLOYMENT_TARGET variable."473 )474 else:475 error_message = error_message.format(476 "the version your Python interpreter is compiled against."477 )478 479 sys.stderr.write(error_message)480 481 platform_tag = prefix + "_" + fin_base_version + "_" + suffix482 return platform_tag483