Aluode/PerceptionLabPortable
0
1"""setuptools.command.bdist_egg2 3Build .egg distributions"""4 5from __future__ import annotations6 7import marshal8import os9import re10import sys11import textwrap12from sysconfig import get_path, get_platform, get_python_version13from types import CodeType14from typing import TYPE_CHECKING, Literal15 16from setuptools import Command17from setuptools.extension import Library18 19from .._path import StrPathT, ensure_directory20 21from distutils import log22from distutils.dir_util import mkpath, remove_tree23 24if TYPE_CHECKING:25 from typing_extensions import TypeAlias26 27# Same as zipfile._ZipFileMode from typeshed28_ZipFileMode: TypeAlias = Literal["r", "w", "x", "a"]29 30 31def _get_purelib():32 return get_path("purelib")33 34 35def strip_module(filename):36 if '.' in filename:37 filename = os.path.splitext(filename)[0]38 if filename.endswith('module'):39 filename = filename[:-6]40 return filename41 42 43def sorted_walk(dir):44 """Do os.walk in a reproducible way,45 independent of indeterministic filesystem readdir order46 """47 for base, dirs, files in os.walk(dir):48 dirs.sort()49 files.sort()50 yield base, dirs, files51 52 53def write_stub(resource, pyfile) -> None:54 _stub_template = textwrap.dedent(55 """56 def __bootstrap__():57 global __bootstrap__, __loader__, __file__58 import sys, importlib.resources as irs, importlib.util59 with irs.as_file(irs.files(__name__).joinpath(%r)) as __file__:60 __loader__ = None; del __bootstrap__, __loader__61 spec = importlib.util.spec_from_file_location(__name__,__file__)62 mod = importlib.util.module_from_spec(spec)63 spec.loader.exec_module(mod)64 __bootstrap__()65 """66 ).lstrip()67 with open(pyfile, 'w', encoding="utf-8") as f:68 f.write(_stub_template % resource)69 70 71class bdist_egg(Command):72 description = 'create an "egg" distribution'73 74 user_options = [75 ('bdist-dir=', 'b', "temporary directory for creating the distribution"),76 (77 'plat-name=',78 'p',79 "platform name to embed in generated filenames "80 "(by default uses `sysconfig.get_platform()`)",81 ),82 ('exclude-source-files', None, "remove all .py files from the generated egg"),83 (84 'keep-temp',85 'k',86 "keep the pseudo-installation tree around after "87 "creating the distribution archive",88 ),89 ('dist-dir=', 'd', "directory to put final built distributions in"),90 ('skip-build', None, "skip rebuilding everything (for testing/debugging)"),91 ]92 93 boolean_options = ['keep-temp', 'skip-build', 'exclude-source-files']94 95 def initialize_options(self):96 self.bdist_dir = None97 self.plat_name = None98 self.keep_temp = False99 self.dist_dir = None100 self.skip_build = False101 self.egg_output = None102 self.exclude_source_files = None103 104 def finalize_options(self) -> None:105 ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info")106 self.egg_info = ei_cmd.egg_info107 108 if self.bdist_dir is None:109 bdist_base = self.get_finalized_command('bdist').bdist_base110 self.bdist_dir = os.path.join(bdist_base, 'egg')111 112 if self.plat_name is None:113 self.plat_name = get_platform()114 115 self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))116 117 if self.egg_output is None:118 # Compute filename of the output egg119 basename = ei_cmd._get_egg_basename(120 py_version=get_python_version(),121 platform=self.distribution.has_ext_modules() and self.plat_name,122 )123 124 self.egg_output = os.path.join(self.dist_dir, basename + '.egg')125 126 def do_install_data(self) -> None:127 # Hack for packages that install data to install's --install-lib128 self.get_finalized_command('install').install_lib = self.bdist_dir129 130 site_packages = os.path.normcase(os.path.realpath(_get_purelib()))131 old, self.distribution.data_files = self.distribution.data_files, []132 133 for item in old:134 if isinstance(item, tuple) and len(item) == 2:135 if os.path.isabs(item[0]):136 realpath = os.path.realpath(item[0])137 normalized = os.path.normcase(realpath)138 if normalized == site_packages or normalized.startswith(139 site_packages + os.sep140 ):141 item = realpath[len(site_packages) + 1 :], item[1]142 # XXX else: raise ???143 self.distribution.data_files.append(item)144 145 try:146 log.info("installing package data to %s", self.bdist_dir)147 self.call_command('install_data', force=False, root=None)148 finally:149 self.distribution.data_files = old150 151 def get_outputs(self):152 return [self.egg_output]153 154 def call_command(self, cmdname, **kw):155 """Invoke reinitialized command `cmdname` with keyword args"""156 for dirname in INSTALL_DIRECTORY_ATTRS:157 kw.setdefault(dirname, self.bdist_dir)158 kw.setdefault('skip_build', self.skip_build)159 kw.setdefault('dry_run', self.dry_run)160 cmd = self.reinitialize_command(cmdname, **kw)161 self.run_command(cmdname)162 return cmd163 164 def run(self): # noqa: C901 # is too complex (14) # FIXME165 # Generate metadata first166 self.run_command("egg_info")167 # We run install_lib before install_data, because some data hacks168 # pull their data path from the install_lib command.169 log.info("installing library code to %s", self.bdist_dir)170 instcmd = self.get_finalized_command('install')171 old_root = instcmd.root172 instcmd.root = None173 if self.distribution.has_c_libraries() and not self.skip_build:174 self.run_command('build_clib')175 cmd = self.call_command('install_lib', warn_dir=False)176 instcmd.root = old_root177 178 all_outputs, ext_outputs = self.get_ext_outputs()179 self.stubs = []180 to_compile = []181 for p, ext_name in enumerate(ext_outputs):182 filename, _ext = os.path.splitext(ext_name)183 pyfile = os.path.join(self.bdist_dir, strip_module(filename) + '.py')184 self.stubs.append(pyfile)185 log.info("creating stub loader for %s", ext_name)186 if not self.dry_run:187 write_stub(os.path.basename(ext_name), pyfile)188 to_compile.append(pyfile)189 ext_outputs[p] = ext_name.replace(os.sep, '/')190 191 if to_compile:192 cmd.byte_compile(to_compile)193 if self.distribution.data_files:194 self.do_install_data()195 196 # Make the EGG-INFO directory197 archive_root = self.bdist_dir198 egg_info = os.path.join(archive_root, 'EGG-INFO')199 self.mkpath(egg_info)200 if self.distribution.scripts:201 script_dir = os.path.join(egg_info, 'scripts')202 log.info("installing scripts to %s", script_dir)203 self.call_command('install_scripts', install_dir=script_dir, no_ep=True)204 205 self.copy_metadata_to(egg_info)206 native_libs = os.path.join(egg_info, "native_libs.txt")207 if all_outputs:208 log.info("writing %s", native_libs)209 if not self.dry_run:210 ensure_directory(native_libs)211 with open(native_libs, 'wt', encoding="utf-8") as libs_file:212 libs_file.write('\n'.join(all_outputs))213 libs_file.write('\n')214 elif os.path.isfile(native_libs):215 log.info("removing %s", native_libs)216 if not self.dry_run:217 os.unlink(native_libs)218 219 write_safety_flag(os.path.join(archive_root, 'EGG-INFO'), self.zip_safe())220 221 if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):222 log.warn(223 "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"224 "Use the install_requires/extras_require setup() args instead."225 )226 227 if self.exclude_source_files:228 self.zap_pyfiles()229 230 # Make the archive231 make_zipfile(232 self.egg_output,233 archive_root,234 verbose=self.verbose,235 dry_run=self.dry_run,236 mode=self.gen_header(),237 )238 if not self.keep_temp:239 remove_tree(self.bdist_dir, dry_run=self.dry_run)240 241 # Add to 'Distribution.dist_files' so that the "upload" command works242 getattr(self.distribution, 'dist_files', []).append((243 'bdist_egg',244 get_python_version(),245 self.egg_output,246 ))247 248 def zap_pyfiles(self):249 log.info("Removing .py files from temporary directory")250 for base, dirs, files in walk_egg(self.bdist_dir):251 for name in files:252 path = os.path.join(base, name)253 254 if name.endswith('.py'):255 log.debug("Deleting %s", path)256 os.unlink(path)257 258 if base.endswith('__pycache__'):259 path_old = path260 261 pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'262 m = re.match(pattern, name)263 path_new = os.path.join(base, os.pardir, m.group('name') + '.pyc')264 log.info(f"Renaming file from [{path_old}] to [{path_new}]")265 try:266 os.remove(path_new)267 except OSError:268 pass269 os.rename(path_old, path_new)270 271 def zip_safe(self):272 safe = getattr(self.distribution, 'zip_safe', None)273 if safe is not None:274 return safe275 log.warn("zip_safe flag not set; analyzing archive contents...")276 return analyze_egg(self.bdist_dir, self.stubs)277 278 def gen_header(self) -> Literal["w"]:279 return 'w'280 281 def copy_metadata_to(self, target_dir) -> None:282 "Copy metadata (egg info) to the target_dir"283 # normalize the path (so that a forward-slash in egg_info will284 # match using startswith below)285 norm_egg_info = os.path.normpath(self.egg_info)286 prefix = os.path.join(norm_egg_info, '')287 for path in self.ei_cmd.filelist.files:288 if path.startswith(prefix):289 target = os.path.join(target_dir, path[len(prefix) :])290 ensure_directory(target)291 self.copy_file(path, target)292 293 def get_ext_outputs(self):294 """Get a list of relative paths to C extensions in the output distro"""295 296 all_outputs = []297 ext_outputs = []298 299 paths = {self.bdist_dir: ''}300 for base, dirs, files in sorted_walk(self.bdist_dir):301 all_outputs.extend(302 paths[base] + filename303 for filename in files304 if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS305 )306 for filename in dirs:307 paths[os.path.join(base, filename)] = paths[base] + filename + '/'308 309 if self.distribution.has_ext_modules():310 build_cmd = self.get_finalized_command('build_ext')311 for ext in build_cmd.extensions:312 if isinstance(ext, Library):313 continue314 fullname = build_cmd.get_ext_fullname(ext.name)315 filename = build_cmd.get_ext_filename(fullname)316 if not os.path.basename(filename).startswith('dl-'):317 if os.path.exists(os.path.join(self.bdist_dir, filename)):318 ext_outputs.append(filename)319 320 return all_outputs, ext_outputs321 322 323NATIVE_EXTENSIONS: dict[str, None] = dict.fromkeys('.dll .so .dylib .pyd'.split())324 325 326def walk_egg(egg_dir):327 """Walk an unpacked egg's contents, skipping the metadata directory"""328 walker = sorted_walk(egg_dir)329 base, dirs, files = next(walker)330 if 'EGG-INFO' in dirs:331 dirs.remove('EGG-INFO')332 yield base, dirs, files333 yield from walker334 335 336def analyze_egg(egg_dir, stubs):337 # check for existing flag in EGG-INFO338 for flag, fn in safety_flags.items():339 if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)):340 return flag341 if not can_scan():342 return False343 safe = True344 for base, dirs, files in walk_egg(egg_dir):345 for name in files:346 if name.endswith('.py') or name.endswith('.pyw'):347 continue348 elif name.endswith('.pyc') or name.endswith('.pyo'):349 # always scan, even if we already know we're not safe350 safe = scan_module(egg_dir, base, name, stubs) and safe351 return safe352 353 354def write_safety_flag(egg_dir, safe) -> None:355 # Write or remove zip safety flag file(s)356 for flag, fn in safety_flags.items():357 fn = os.path.join(egg_dir, fn)358 if os.path.exists(fn):359 if safe is None or bool(safe) != flag:360 os.unlink(fn)361 elif safe is not None and bool(safe) == flag:362 with open(fn, 'wt', encoding="utf-8") as f:363 f.write('\n')364 365 366safety_flags = {367 True: 'zip-safe',368 False: 'not-zip-safe',369}370 371 372def scan_module(egg_dir, base, name, stubs):373 """Check whether module possibly uses unsafe-for-zipfile stuff"""374 375 filename = os.path.join(base, name)376 if filename[:-1] in stubs:377 return True # Extension module378 pkg = base[len(egg_dir) + 1 :].replace(os.sep, '.')379 module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0]380 skip = 16 # skip magic & reserved? & date & file size381 f = open(filename, 'rb')382 f.read(skip)383 code = marshal.load(f)384 f.close()385 safe = True386 symbols = dict.fromkeys(iter_symbols(code))387 for bad in ['__file__', '__path__']:388 if bad in symbols:389 log.warn("%s: module references %s", module, bad)390 safe = False391 if 'inspect' in symbols:392 for bad in [393 'getsource',394 'getabsfile',395 'getfile',396 'getsourcefile',397 'getsourcelines',398 'findsource',399 'getcomments',400 'getframeinfo',401 'getinnerframes',402 'getouterframes',403 'stack',404 'trace',405 ]:406 if bad in symbols:407 log.warn("%s: module MAY be using inspect.%s", module, bad)408 safe = False409 return safe410 411 412def iter_symbols(code):413 """Yield names and strings used by `code` and its nested code objects"""414 yield from code.co_names415 for const in code.co_consts:416 if isinstance(const, str):417 yield const418 elif isinstance(const, CodeType):419 yield from iter_symbols(const)420 421 422def can_scan() -> bool:423 if not sys.platform.startswith('java') and sys.platform != 'cli':424 # CPython, PyPy, etc.425 return True426 log.warn("Unable to analyze compiled code on this platform.")427 log.warn(428 "Please ask the author to include a 'zip_safe'"429 " setting (either True or False) in the package's setup.py"430 )431 return False432 433 434# Attribute names of options for commands that might need to be convinced to435# install to the egg build directory436 437INSTALL_DIRECTORY_ATTRS = ['install_lib', 'install_dir', 'install_data', 'install_base']438 439 440def make_zipfile(441 zip_filename: StrPathT,442 base_dir,443 verbose: bool = False,444 dry_run: bool = False,445 compress=True,446 mode: _ZipFileMode = 'w',447) -> StrPathT:448 """Create a zip file from all the files under 'base_dir'. The output449 zip file will be named 'base_dir' + ".zip". Uses either the "zipfile"450 Python module (if available) or the InfoZIP "zip" utility (if installed451 and found on the default search path). If neither tool is available,452 raises DistutilsExecError. Returns the name of the output zip file.453 """454 import zipfile455 456 mkpath(os.path.dirname(zip_filename), dry_run=dry_run) # type: ignore[arg-type] # python/mypy#18075457 log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)458 459 def visit(z, dirname, names):460 for name in names:461 path = os.path.normpath(os.path.join(dirname, name))462 if os.path.isfile(path):463 p = path[len(base_dir) + 1 :]464 if not dry_run:465 z.write(path, p)466 log.debug("adding '%s'", p)467 468 compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED469 if not dry_run:470 z = zipfile.ZipFile(zip_filename, mode, compression=compression)471 for dirname, dirs, files in sorted_walk(base_dir):472 visit(z, dirname, files)473 z.close()474 else:475 for dirname, dirs, files in sorted_walk(base_dir):476 visit(None, dirname, files)477 return zip_filename478