SciCodePile/SciCode-Domain-Code
DATA1: Domain-Specific Code Dataset Dataset Overview DATA1 is a large-scale domain-specific code dataset focusing on code samples from interdisciplinary fields such as biology, chemistry, materials science, and related areas. The dataset is collected and organized from GitHub repositories, covering 178 different domain topics with over 1.1 billion lines of code. Dataset Statistics Total Datasets: 178 CSV files Total Data Size: ~115 GB Total Lines… See the full description on the dataset page: https://huggingface.co/datasets/SciCodePile/SciCode-Domain-Code.
42.4k
1"keyword","repo_name","file_path","file_extension","file_size","line_count","content","language"
2"Glycomics","mobiusklein/glycresoft","setup.py",".py","14081","365","import sys3import traceback4import os5from setuptools import setup, Extension, find_packages6 7from distutils.command.build_ext import build_ext8from distutils.errors import (CCompilerError, DistutilsExecError,9 DistutilsPlatformError)10 11 12def has_option(name):13 try:14 sys.argv.remove('--%s' % name)15 return True16 except ValueError:17 pass18 # allow passing all cmd line options also as environment variables19 env_val = os.getenv(name.upper().replace('-', '_'), 'false').lower()20 if env_val == ""true"":21 return True22 return False23 24 25include_diagnostics = has_option(""include-diagnostics"")26force_cythonize = has_option(""force-cythonize"")27 28 29def make_extensions():30 is_ci = bool(os.getenv(""CI"", """"))31 try:32 import numpy33 except ImportError:34 print(""Installation requires `numpy`"")35 raise36 macros = []37 try:38 from Cython.Build import cythonize39 cython_directives = {40 'embedsignature': True,41 ""profile"": include_diagnostics42 }43 if include_diagnostics:44 macros.append((""CYTHON_TRACE_NOGIL"", ""1""))45 if is_ci and include_diagnostics:46 cython_directives['linetrace'] = True47 extensions = cythonize(48 [49 Extension(50 name=""glycresoft._c.structure.fragment_match_map"",51 sources=[""src/glycresoft/_c/structure/fragment_match_map.pyx""],52 include_dirs=[numpy.get_include()],53 ),54 Extension(55 name=""glycresoft._c.structure.intervals"",56 sources=[""src/glycresoft/_c/structure/intervals.pyx""],57 include_dirs=[numpy.get_include()],58 ),59 Extension(60 name=""glycresoft._c.scoring.shape_fitter"",61 sources=[""src/glycresoft/_c/scoring/shape_fitter.pyx""],62 include_dirs=[numpy.get_include()],63 ),64 Extension(65 name=""glycresoft._c.chromatogram_tree.mass_shift"",66 sources=[""src/glycresoft/_c/chromatogram_tree/mass_shift.pyx""],67 include_dirs=[numpy.get_include()],68 ),69 Extension(70 name=""glycresoft._c.chromatogram_tree.index"",71 sources=[""src/glycresoft/_c/chromatogram_tree/index.pyx""],72 include_dirs=[numpy.get_include()],73 ),74 Extension(75 name=""glycresoft._c.tandem.core_search"",76 sources=[""src/glycresoft/_c/tandem/core_search.pyx""],77 include_dirs=[numpy.get_include()],78 ),79 Extension(80 name=""glycresoft._c.database.mass_collection"",81 sources=[""src/glycresoft/_c/database/mass_collection.pyx""],82 include_dirs=[numpy.get_include()],83 ),84 Extension(85 name=""glycresoft._c.tandem.tandem_scoring_helpers"",86 libraries=[""npymath""],87 library_dirs=[os.path.join(os.path.dirname(numpy.get_include()), ""lib"")],88 sources=[""src/glycresoft/_c/tandem/tandem_scoring_helpers.pyx""],89 include_dirs=[numpy.get_include()],90 ),91 Extension(92 name=""glycresoft._c.tandem.spectrum_match"",93 sources=[""src/glycresoft/_c/tandem/spectrum_match.pyx""],94 include_dirs=[numpy.get_include()],95 ),96 Extension(97 name=""glycresoft._c.composition_network.graph"",98 sources=[""src/glycresoft/_c/composition_network/graph.pyx""],99 include_dirs=[numpy.get_include()],100 ),101 Extension(102 name=""glycresoft._c.composition_distribution_model.utils"",103 sources=[""src/glycresoft/_c/composition_distribution_model/utils.pyx""],104 include_dirs=[numpy.get_include()],105 ),106 Extension(107 name=""glycresoft._c.structure.lru"", sources=[""src/glycresoft/_c/structure/lru.pyx""]108 ),109 Extension(110 name=""glycresoft._c.tandem.target_decoy"",111 sources=[""src/glycresoft/_c/tandem/target_decoy.pyx""],112 include_dirs=[numpy.get_include()],113 ),114 Extension(115 name=""glycresoft._c.structure.structure_loader"",116 sources=[""src/glycresoft/_c/structure/structure_loader.pyx""],117 include_dirs=[numpy.get_include()],118 ),119 Extension(120 name=""glycresoft._c.tandem.oxonium_ions"",121 sources=[""src/glycresoft/_c/tandem/oxonium_ions.pyx""],122 include_dirs=[numpy.get_include()],123 ),124 Extension(125 name=""glycresoft._c.structure.probability"",126 sources=[""src/glycresoft/_c/structure/probability.pyx""],127 include_dirs=[numpy.get_include()],128 ),129 Extension(130 name=""glycresoft._c.tandem.peptide_graph"",131 sources=[""src/glycresoft/_c/tandem/peptide_graph.pyx""],132 include_dirs=[numpy.get_include()],133 ),134 Extension(135 name=""glycresoft._c.scoring.elution_time_grouping"",136 sources=[""src/glycresoft/_c/scoring/elution_time_grouping.pyx""],137 include_dirs=[numpy.get_include()],138 ),139 ],140 compiler_directives=cython_directives,141 force=force_cythonize,142 )143 except ImportError as err:144 print(err)145 extensions = [146 Extension(147 name=""glycresoft._c.structure.fragment_match_map"",148 sources=[""src/glycresoft/_c/structure/fragment_match_map.c""],149 include_dirs=[numpy.get_include()],150 ),151 Extension(152 name=""glycresoft._c.structure.intervals"",153 sources=[""src/glycresoft/_c/structure/intervals.c""],154 include_dirs=[numpy.get_include()],155 ),156 Extension(157 name=""glycresoft._c.scoring.shape_fitter"",158 sources=[""src/glycresoft/_c/scoring/shape_fitter.c""],159 include_dirs=[numpy.get_include()],160 ),161 Extension(162 name=""glycresoft._c.chromatogram_tree.mass_shift"",163 sources=[""src/glycresoft/_c/chromatogram_tree/mass_shift.c""],164 include_dirs=[numpy.get_include()],165 ),166 Extension(167 name=""glycresoft._c.chromatogram_tree.index"",168 sources=[""src/glycresoft/_c/chromatogram_tree/index.c""],169 include_dirs=[numpy.get_include()],170 ),171 Extension(172 name=""glycresoft._c.tandem.core_search"",173 sources=[""src/glycresoft/_c/tandem/core_search.c""],174 include_dirs=[numpy.get_include()],175 ),176 Extension(177 name=""glycresoft._c.database.mass_collection"",178 sources=[""src/glycresoft/_c/database/mass_collection.c""],179 include_dirs=[numpy.get_include()],180 ),181 Extension(182 name=""glycresoft._c.tandem.tandem_scoring_helpers"",183 libraries=[""npymath""],184 library_dirs=[os.path.join(os.path.dirname(numpy.get_include()), ""lib"")],185 sources=[""src/glycresoft/_c/tandem/tandem_scoring_helpers.c""],186 include_dirs=[numpy.get_include()],187 ),188 Extension(189 name=""glycresoft._c.tandem.spectrum_match"",190 sources=[""src/glycresoft/_c/tandem/spectrum_match.c""],191 include_dirs=[numpy.get_include()],192 ),193 Extension(194 name=""glycresoft._c.composition_network.graph"",195 sources=[""src/glycresoft/_c/composition_network/graph.c""],196 include_dirs=[numpy.get_include()],197 ),198 Extension(199 name=""glycresoft._c.composition_distribution_model.utils"",200 sources=[""src/glycresoft/_c/composition_distribution_model/utils.c""],201 include_dirs=[numpy.get_include()],202 ),203 Extension(name=""glycresoft._c.structure.lru"", sources=[""src/glycresoft/_c/structure/lru.c""]),204 Extension(205 name=""glycresoft._c.tandem.target_decoy"",206 sources=[""src/glycresoft/_c/tandem/target_decoy.c""],207 include_dirs=[numpy.get_include()],208 ),209 Extension(210 name=""glycresoft._c.structure.structure_loader"",211 sources=[""src/glycresoft/_c/structure/structure_loader.c""],212 include_dirs=[numpy.get_include()],213 ),214 Extension(215 name=""glycresoft._c.tandem.oxonium_ions"",216 sources=[""src/glycresoft/_c/tandem/oxonium_ions.c""],217 include_dirs=[numpy.get_include()],218 ),219 Extension(220 name=""glycresoft._c.structure.probability"",221 sources=[""src/glycresoft/_c/structure/probability.c""],222 include_dirs=[numpy.get_include()],223 ),224 Extension(225 name=""glycresoft._c.tandem.peptide_graph"",226 sources=[""src/glycresoft/_c/tandem/peptide_graph.c""],227 include_dirs=[numpy.get_include()],228 ),229 Extension(230 name=""glycresoft._c.scoring.elution_time_grouping"",231 sources=[""src/glycresoft/_c/scoring/elution_time_grouping.c""],232 include_dirs=[numpy.get_include()],233 ),234 ]235 return extensions236 237 238ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)239if sys.platform == 'win32':240 # 2.6's distutils.msvc9compiler can raise an IOError when failing to241 # find the compiler242 ext_errors += (IOError,)243 244 245class BuildFailed(Exception):246 247 def __init__(self):248 self.cause = sys.exc_info()[1] # work around py 2/3 different syntax249 250 def __str__(self):251 return str(self.cause)252 253 254class ve_build_ext(build_ext):255 # This class allows C extension building to fail.256 257 def run(self):258 try:259 build_ext.run(self)260 except DistutilsPlatformError:261 traceback.print_exc()262 raise BuildFailed()263 264 def build_extension(self, ext):265 try:266 build_ext.build_extension(self, ext)267 except ext_errors:268 traceback.print_exc()269 raise BuildFailed()270 except ValueError:271 # this can happen on Windows 64 bit, see Python issue 7511272 traceback.print_exc()273 if ""'path'"" in str(sys.exc_info()[1]): # works with both py 2/3274 raise BuildFailed()275 raise276 277 278cmdclass = {}279 280cmdclass['build_ext'] = ve_build_ext281 282 283def status_msgs(*msgs):284 print('*' * 75)285 for msg in msgs:286 print(msg)287 print('*' * 75)288 289 290with open(""src/glycresoft/version.py"") as version_file:291 version = None292 for line in version_file.readlines():293 if ""version = "" in line:294 version = line.split("" = "")[1].replace(""\"""", """").strip()295 print(""Version is: %r"" % (version,))296 break297 else:298 print(""Cannot determine version"")299 300 301requirements = []302with open(""requirements.txt"") as requirements_file:303 requirements.extend(requirements_file.readlines())304 305try:306 with open(""README.md"") as readme_file:307 long_description = readme_file.read()308except Exception as e:309 print(e)310 long_description = """"311 312 313def run_setup(include_cext=True):314 315 setup(316 name=""glycresoft"",317 version=version,318 packages=find_packages(where=""src""),319 package_dir={"""": ""src""},320 include_package_data=True,321 author="", "".join([""Joshua Klein""]),322 author_email=""jaklein@bu.edu"",323 description=""Glycan and Glycopeptide Mass Spectrometry Database Search Tool"",324 long_description=long_description,325 long_description_content_type=""text/markdown"",326 entry_points={327 ""console_scripts"": [""glycresoft = glycresoft.cli.__main__:main""],328 },329 extras_require={""fragmentation-modeling"": [""glycopeptide-feature-learning""]},330 package_data={331 ""glycresoft.models"": [""src/glycresoft/models/data/*""],332 ""glycresoft.database.prebuilt"": [""src/glycresoft/database/prebuilt/data/*""],333 ""glycresoft.output.report.glycan_lcms"": [""src/glycresoft/output/report/glycan_lcms/*""],334 ""glycresoft.output.report.glycopeptide_lcmsms"": [""src/glycresoft/output/report/glycopeptide_lcmsms/*""],335 },336 ext_modules=make_extensions() if include_cext else None,337 cmdclass=cmdclass,338 install_requires=requirements,339 classifiers=[340 ""Development Status :: 3 - Alpha"",341 ""Intended Audience :: Science/Research"",342 ""License :: OSI Approved :: Apache Software License"",343 ""Topic :: Scientific/Engineering :: Bio-Informatics"",344 ],345 zip_safe=False,346 python_requires="">3.8"",347 project_urls={348 ""Documentation"": ""https://mobiusklein.github.io/glycresoft"",349 ""Source Code"": ""https://github.com/mobiusklein/glycresoft"",350 ""Issue Tracker"": ""https://github.com/mobiusklein/glycresoft/issues"",351 },352 )353 354 355try:356 run_setup(True)357except Exception as exc:358 print(exc)359 run_setup(False)360 361 status_msgs(362 ""WARNING: The C extension could not be compiled, "" +363 ""speedups are not enabled."",364 ""Plain-Python build succeeded.""365 )366","Python"
367"Glycomics","mobiusklein/glycresoft","pyinstaller/make-pyinstaller.sh",".sh","576","15","#!/bin/bash368 369# NOTE: New versions of PyInstaller bundle more and more hooks, and throw370# errors if there are duplicate hooks. Mangle the names of hooks that have371# been superceded.372 373# NOTE: PyInstaller and newer versions of Anaconda-specific NumPy are incompatible.374# Even if you're using a conda environment, you must install NumPy with pip in order375# for the executable to work after deactivating the environment or moving it to another376# computer.377 378rm -rf ./build/ ./dist/379echo ""Beginning build""380python -m PyInstaller ./glycresoft-cli.spec --workpath build --distpath dist381","Shell"
382"Glycomics","mobiusklein/glycresoft","pyinstaller/install-from-git.py",".py","493","20","from os import path as ospath383import sys384import os385import shutil386 387repos = [388 ""https://github.com/mobiusklein/glypy"",389 ""https://github.com/mobiusklein/glycopeptidepy"",390 ""https://github.com/mobiusklein/ms_deisotope"",391]392 393clone_dir = ospath.join(ospath.dirname(__file__), ""gitsrc"")394 395origin_path = os.getcwd()396os.system(""rm -rf %s"" % clone_dir)397 398for repo in repos:399 repopath = ospath.join(clone_dir, ospath.splitext(ospath.basename(repo))[0])400 os.system(""pip install git+%s"" % repo)401","Python"
402"Glycomics","mobiusklein/glycresoft","pyinstaller/glycresoft-cli.py",".py","1522","55","import matplotlib403import os404import sys405import click406import platform407import multiprocessing408 409if platform.system().lower() != 'windows':410 os.environ[""NOWAL""] = ""1""411else:412 # while click.echo works when run under normal conditions,413 # it has started failing when packaged with PyInstaller. The414 # implementation of click's _winterm module seems to replicate415 # a lot of logic found in win_unicode_console.streams, but using416 # win_unicode_console seems to fix the problem, (found after tracing417 # why importing ipdb which imported IPython which called this fixed418 # the problem)419 import win_unicode_console420 # win_unicode_console.enable()421app_dir = click.get_app_dir(""glycresoft"")422_mpl_cache_dir = os.path.join(app_dir, 'mpl')423 424if not os.path.exists(_mpl_cache_dir):425 os.makedirs(_mpl_cache_dir)426 427os.environ[""MPLCONFIGDIR""] = _mpl_cache_dir428 429try:430 matplotlib.use(""agg"")431except Exception:432 pass433 434from rdflib.plugins import stores435from rdflib.plugins.stores import sparqlstore436 437from glycresoft.cli.__main__ import main438 439# import the fallback name440import glycan_profiling441 442try:443 from glycopeptide_feature_learning import (peak_relations, multinomial_regression, scoring)444 from glycopeptide_feature_learning._c import (amino_acid_classification, approximation, model_types, peak_relations)445except ImportError:446 pass447 448 449from glycresoft.cli.validators import strip_site_root450 451sys.excepthook = strip_site_root452 453if __name__ == '__main__':454 multiprocessing.freeze_support()455 main()456","Python"
457"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-glycopeptide_feature_learning.py",".py","278","7","from PyInstaller.utils.hooks import collect_submodules, collect_data_files458 459hiddenimports = collect_submodules(460 ""glycopeptide_feature_learning._c"") + collect_submodules(""glycopeptide_feature_learning.scoring._c"")461 462datas = collect_data_files('glycopeptide_feature_learning')463","Python"
464"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-psims.py",".py","92","4","from PyInstaller.utils.hooks import collect_data_files465 466datas = collect_data_files(""psims"")467","Python"
468"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-glycresoft.py",".py","314","8","469from PyInstaller.utils.hooks import collect_submodules, collect_data_files470 471hiddenimports = collect_submodules(""glycresoft._c"")472 473datas = list(filter(lambda x: ""test_data"" not in x[1] and not x[0].endswith(474 '.c') and not x[0].endswith('.html') and not x[0].endswith('.pyx'), collect_data_files(""glycresoft"")))475","Python"
476"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/_hook-rdflib.py",".py","110","5","477from PyInstaller.utils.hooks import collect_submodules478 479hiddenimports = collect_submodules(""rdflib.plugins"")480","Python"
481"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-sklearn.py",".py","108","4","from PyInstaller.utils.hooks import collect_submodules482 483hiddenimports = collect_submodules(""sklearn.utils"")484","Python"
485"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-ms_deisotope.py",".py","318","8","486from PyInstaller.utils.hooks import collect_submodules, collect_data_files487 488hiddenimports = collect_submodules(""ms_deisotope._c"")489 490datas = list(filter(lambda x: ""test_data"" not in x[1] and not x[0].endswith(491 '.c') and not x[0].endswith('.html') and not x[0].endswith('.pyx'), collect_data_files(""ms_deisotope"")))492","Python"
493"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-glypy.py",".py","305","9","494from PyInstaller.utils.hooks import collect_submodules, collect_data_files495 496hiddenimports = collect_submodules(""glypy._c"")497 498 499datas = list(filter(lambda x: ""test_data"" not in x[1] and not x[0].endswith(500 '.c') and not x[0].endswith('.html') and not x[0].endswith('.pyx'), collect_data_files(""glypy"")))501","Python"
502"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-brainpy.py",".py","104","3","from PyInstaller.utils.hooks import collect_submodules503hiddenimports = collect_submodules(""brainpy._c"")504","Python"
505"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-ms_peak_picker.py",".py","113","5","506from PyInstaller.utils.hooks import collect_submodules507 508hiddenimports = collect_submodules(""ms_peak_picker._c"")509","Python"
510"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-glycresoft_app.py",".py","171","5","511from PyInstaller.utils.hooks import collect_submodules, collect_data_files512 513datas = list(filter(lambda x: ""test_data"" not in x[1], collect_data_files(""glycresoft_app"")))514","Python"
515"Glycomics","mobiusklein/glycresoft","pyinstaller/hooks/hook-glycopeptidepy.py",".py","322","8","516from PyInstaller.utils.hooks import collect_submodules, collect_data_files517 518hiddenimports = collect_submodules(""glycopeptidepy._c"")519 520datas = list(filter(lambda x: ""test_data"" not in x[1] and not x[0].endswith(521 '.c') and not x[0].endswith('.html') and not x[0].endswith('.pyx'), collect_data_files(""glycopeptidepy"")))522","Python"
523"Glycomics","mobiusklein/glycresoft","src/glycresoft/task.py",".py","20110","694","from __future__ import print_function524import os525import logging526import pprint527import time528import traceback529import multiprocessing530import threading531 532from typing import Any, Generic, List, Optional, TypeVar, Union533from logging.handlers import QueueHandler, QueueListener534from multiprocessing.managers import SyncManager535from datetime import datetime536 537from queue import Empty538import warnings539 540import six541 542from glycresoft.version import version543 544 545 546logger = logging.getLogger(""glycresoft.task"")547 548T = TypeVar(""T"")549 550 551def display_version(print_fn):552 msg = ""glycresoft: version %s"" % version553 print_fn(msg)554 555 556def ensure_text(obj):557 if six.PY2:558 return six.text_type(obj)559 else:560 return str(obj)561 562 563def fmt_msg(*message):564 return u""%s %s"" % (ensure_text(datetime.now().isoformat(' ')), u', '.join(map(ensure_text, message)))565 566 567def printer(obj, *message, stacklevel=None):568 print(fmt_msg(*message))569 570 571def debug_printer(obj, *message, stacklevel=None):572 if obj.in_debug_mode():573 print(u""DEBUG:"" + fmt_msg(*message))574 575 576class CallInterval(object):577 """"""Call a function every `interval` seconds from578 a separate thread.579 580 Attributes581 ----------582 stopped: threading.Event583 A semaphore lock that controls when to run `call_target`584 call_target: callable585 The thing to call every `interval` seconds586 args: iterable587 Arguments for `call_target`588 interval: number589 Time between calls to `call_target`590 """"""591 592 def __init__(self, interval, call_target, *args):593 self.stopped = threading.Event()594 self.interval = interval595 self.call_target = call_target596 self.args = args597 self.thread = threading.Thread(target=self.mainloop)598 self.thread.daemon = True599 600 def mainloop(self):601 while not self.stopped.wait(self.interval):602 try:603 self.call_target(*self.args)604 except Exception as e:605 logger.exception(""An error occurred in %r"", self, exc_info=e)606 607 def start(self):608 self.thread.start()609 610 def stop(self):611 self.stopped.set()612 613 614class IPCLoggingManager:615 queue: multiprocessing.Queue616 listener: QueueListener617 618 def __init__(self, queue=None, *handlers):619 if queue is None:620 queue = multiprocessing.Queue()621 if not handlers:622 logger = logging.getLogger()623 handlers = logger.handlers624 625 self.queue = queue626 self.listener = QueueListener(627 queue, *handlers, respect_handler_level=True)628 self.listener.start()629 630 def sender(self, logger_name=""glycresoft""):631 return LoggingHandlerToken(self.queue, logger_name)632 633 def start(self):634 self.listener.start()635 636 def stop(self):637 try:638 self.listener.stop()639 except AttributeError:640 pass641 642 643class LoggingHandlerToken:644 queue: multiprocessing.Queue645 name: str646 configured: bool647 648 def __init__(self, queue: multiprocessing.Queue, name: str):649 self.queue = queue650 self.name = name651 self.configured = False652 653 def get_logger(self) -> logging.Logger:654 logger = logging.getLogger(self.name)655 return logger656 657 def clear_handlers(self, logger: logging.Logger):658 for handler in list(logger.handlers):659 logger.removeHandler(handler)660 if logger.parent is not None and logger.parent is not logger:661 self.clear_handlers(logger.parent)662 663 def log(self, *args, **kwargs):664 kwargs.setdefault('stacklevel', 2)665 self.get_logger().info(*args, **kwargs)666 667 def __call__(self, *args, **kwargs):668 kwargs.setdefault('stacklevel', 3)669 self.log(*args, **kwargs)670 671 def add_handler(self):672 if self.configured:673 return674 logger = self.get_logger()675 self.clear_handlers(logger)676 handler = QueueHandler(self.queue)677 handler.setLevel(logging.INFO)678 logger.addHandler(handler)679 logger.setLevel(logging.INFO)680 LoggingMixin.log_with_logger(logger)681 TaskBase.log_with_logger(logger)682 self.configured = True683 684 def __getstate__(self):685 return {686 ""queue"": self.queue,687 ""name"": self.name688 }689 690 def __setstate__(self, state):691 self.queue = state['queue']692 self.name = state['name']693 self.configured = False694 if multiprocessing.current_process().name == ""MainProcess"":695 return696 self.add_handler()697 698 699class MessageSpooler(object):700 """"""An IPC-based logging helper701 702 Attributes703 ----------704 halting : bool705 Whether the object is attempting to706 stop, so that the internal thread can707 tell when it should stop and tell other708 objects using it it is trying to stop709 handler : Callable710 A Callable object which can be used to do711 the actual logging712 message_queue : multiprocessing.Queue713 The Inter-Process Communication queue714 thread : threading.Thread715 The internal listener thread that will consume716 message_queue work items717 """"""718 def __init__(self, handler):719 self.handler = handler720 self.message_queue = multiprocessing.Queue()721 self.halting = False722 self.thread = threading.Thread(target=self.run)723 self.thread.start()724 725 def run(self):726 while not self.halting:727 try:728 message = self.message_queue.get(True, 2)729 self.handler(*message)730 except Exception:731 continue732 733 def stop(self):734 self.halting = True735 self.thread.join()736 737 def sender(self):738 return MessageSender(self.message_queue)739 740 741class MessageSender(object):742 """"""A simple callable for pushing objects into an IPC743 queue.744 745 Attributes746 ----------747 queue : multiprocessing.Queue748 The Inter-Process Communication queue749 """"""750 def __init__(self, queue):751 self.queue = queue752 753 def __call__(self, *message):754 self.send(*message)755 756 def send(self, *message):757 self.queue.put(message)758 759 760def humanize_class_name(name):761 parts = []762 i = 0763 last = 0764 while i < len(name):765 c = name[i]766 if c.isupper() and i != last:767 if i + 1 < len(name):768 if name[i + 1].islower():769 part = name[last:i]770 parts.append(part)771 last = i772 i += 1773 parts.append(name[last:i])774 return ' '.join(parts)775 776 777class LoggingMixin(object):778 logger_state = None779 print_fn = printer780 debug_print_fn = debug_printer781 error_print_fn = printer782 warn_print_fn = warnings.warn783 784 _debug_enabled = None785 786 @classmethod787 def log_with_logger(cls, logger):788 cls.logger_state = logger789 cls.print_fn = logger.info790 cls.debug_print_fn = logger.debug791 cls.error_print_fn = logger.error792 cls.warn_print_fn = logger.warning793 794 def instance_log_with_logger(self, logger):795 self.logger_state = logger796 self.print_fn = logger.info797 self.debug_print_fn = logger.debug798 self.error_print_fn = logger.error799 self.warn_print_fn = logger.warning800 801 @classmethod802 def log_to_stdout(cls):803 cls.logger_state = None804 cls.print_fn = printer805 cls.debug_print_fn = debug_printer806 cls.error_print_fn = printer807 cls.warn_print_fn = warnings.warn808 809 def log(self, *message):810 self.print_fn(u', '.join(map(ensure_text, message)), stacklevel=2)811 812 def debug(self, *message):813 self.debug_print_fn(u', '.join(map(ensure_text, message)), stacklevel=2)814 815 def error(self, *message, **kwargs):816 exception = kwargs.get(""exception"")817 self.error_print_fn(u', '.join(818 map(ensure_text, message)), stacklevel=2)819 if exception is not None:820 self.error_print_fn(traceback.format_exc())821 822 def warn(self, *message, **kwargs):823 self.warn_print_fn(u', '.join(map(ensure_text, message)), stacklevel=2)824 825 def ipc_logger(self, handler=None):826 return IPCLoggingManager()827 828 def in_debug_mode(self):829 if self._debug_enabled is None:830 logger_state = self.logger_state831 if logger_state is not None:832 self._debug_enabled = logger_state.isEnabledFor(""DEBUG"")833 return bool(self._debug_enabled)834 835 836class TaskBase(LoggingMixin):837 """"""A base class for a discrete, named step in a pipeline that838 executes in sequence.839 840 Attributes841 ----------842 debug_print_fn : Callable843 The function called to print debug messages844 display_fields : bool845 Whether to display fields at the start of execution846 end_time : datetime.datetime847 The time when the task ended848 error_print_fn : Callable849 The function called to print error messages850 logger_state : logging.Logger851 The Logger bound to this task852 print_fn : Callable853 The function called to print status messages854 start_time : datetime.datetime855 The time when the task began856 status : str857 The state of the executing task858 """"""859 860 status = ""new""861 862 display_fields = True863 864 _display_name = None865 866 @property867 def display_name(self):868 if self._display_name is None:869 return humanize_class_name(self.__class__.__name__)870 else:871 return self._display_name872 873 def in_debug_mode(self):874 if self._debug_enabled is None:875 logger_state = self.logger_state876 if logger_state is not None:877 self._debug_enabled = logger_state.isEnabledFor(logging.DEBUG)878 return bool(self._debug_enabled)879 880 def _format_fields(self):881 if self.display_fields:882 return '\n' + pprint.pformat(883 {k: v for k, v in self.__dict__.items()884 if not (k.startswith(""_"") or v is None)})885 else:886 return ''887 888 def display_header(self):889 display_version(self.log)890 891 def try_set_process_name(self, name=None):892 """"""893 This helper method may be used to try to change a process's name894 in order to make discriminating which role a particular process is895 fulfilling. This uses a third-party utility library that may not behave896 the same way on all platforms, and therefore this is done for convenience897 only.898 899 Parameters900 ----------901 name : str, optional902 A name to set. If not provided, will check the attribute ``process_name``903 for a non-null value, or else have no effect.904 """"""905 if name is None:906 name = getattr(self, 'process_name', None)907 if name is None:908 return909 _name_process(name)910 911 def _begin(self, verbose=True, *args, **kwargs):912 self.on_begin()913 self.start_time = datetime.now()914 self.status = ""started""915 if verbose:916 self.log(917 ""Begin %s%s"" % (918 self.display_name,919 self._format_fields()))920 921 def _end(self, verbose=True, *args, **kwargs):922 self.on_end()923 self.end_time = datetime.now()924 if verbose:925 self.log(""End %s"" % self.display_name)926 self.log(self.summarize())927 928 def on_begin(self):929 pass930 931 def on_end(self):932 pass933 934 def summarize(self):935 chunks = [936 ""Started at %s."" % self.start_time,937 ""Ended at %s."" % self.end_time,938 ""Total time elapsed: %s"" % (self.end_time - self.start_time),939 ""%s completed successfully."" % self.__class__.__name__ if self.status == 'completed' else940 ""%s failed with error message %r"" % (self.__class__.__name__, self.status),941 ''942 ]943 return '\n'.join(chunks)944 945 def start(self, *args, **kwargs):946 self._begin(*args, **kwargs)947 try:948 out = self.run()949 except (KeyboardInterrupt) as e:950 logger.exception(""An error occurred: %r"", e, exc_info=e)951 self.status = e952 out = e953 raise e954 else:955 self.status = 'completed'956 self._end(*args, **kwargs)957 return out958 959 def interact(self, **kwargs):960 from IPython.terminal.embed import InteractiveShellEmbed, load_default_config961 import sys962 config = kwargs.get('config')963 header = kwargs.pop('header', u'')964 compile_flags = kwargs.pop('compile_flags', None)965 if config is None:966 config = load_default_config()967 config.InteractiveShellEmbed = config.TerminalInteractiveShell968 kwargs['config'] = config969 frame = sys._getframe(1)970 shell = InteractiveShellEmbed.instance(971 _init_location_id='%s:%s' % (972 frame.f_code.co_filename, frame.f_lineno), **kwargs)973 shell(header=header, stack_depth=2, compile_flags=compile_flags,974 _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))975 InteractiveShellEmbed.clear_instance()976 977 978log_handle = TaskBase()979 980 981class TaskExecutionSequence(TaskBase, Generic[T]):982 """"""A task unit that executes in a separate thread or process.""""""983 984 def __call__(self) -> T:985 result = None986 try:987 if self._running_in_process:988 self.log(""%s running on PID %r"" % (self, multiprocessing.current_process().pid))989 if os.getenv(""GLYCRESOFTPROFILING""):990 import cProfile991 profiler = cProfile.Profile()992 result = profiler.runcall(self.run, standalone_mode=False)993 profiler.dump_stats('glycresoft_performance.profile')994 else:995 result = self.run()996 self.debug(""%r Done"" % self)997 except Exception as err:998 self.error(""An error occurred while executing %s"" %999 self, exception=err)1000 result = None1001 self.set_error_occurred()1002 try:1003 self.done_event.set()1004 except AttributeError:1005 pass1006 finally:1007 return result1008 1009 def run(self) -> T:1010 raise NotImplementedError()1011 1012 def _get_repr_details(self):1013 return ''1014 1015 _thread: Optional[Union[threading.Thread, multiprocessing.Process]] = None1016 _running_in_process: bool = False1017 _error_flag: Optional[threading.Event] = None1018 1019 def error_occurred(self) -> bool:1020 if self._error_flag is None:1021 return False1022 else:1023 return self._error_flag.is_set()1024 1025 def set_error_occurred(self):1026 if self._error_flag is None:1027 return False1028 else:1029 return self._error_flag.set()1030 1031 def __repr__(self):1032 template = ""{self.__class__.__name__}({details})""1033 return template.format(self=self, details=self._get_repr_details())1034 1035 def _make_event(self, provider=None) -> Union[threading.Event, multiprocessing.Event]:1036 if provider is None:1037 provider = threading1038 return provider.Event()1039 1040 def _name_for_execution_sequence(self):1041 return (""%s-%r"" % (self.__class__.__name__, id(self)))1042 1043 def start(self, process: bool=False, daemon: bool=False):1044 if self._thread is not None:1045 return self._thread1046 if process:1047 self._running_in_process = True1048 self._error_flag = self._make_event(multiprocessing)1049 t = multiprocessing.Process(1050 target=self, name=self._name_for_execution_sequence())1051 if daemon:1052 t.daemon = daemon1053 else:1054 self._error_flag = self._make_event(threading)1055 t = threading.Thread(1056 target=self, name=self._name_for_execution_sequence())1057 if daemon:1058 t.daemon = daemon1059 t.start()1060 self._thread = t1061 return t1062 1063 def join(self, timeout: Optional[float]=None) -> bool:1064 if self.error_occurred():1065 return True1066 try:1067 return self._thread.join(timeout)1068 except KeyboardInterrupt:1069 self.set_error_occurred()1070 return True1071 1072 def is_alive(self):1073 if self.error_occurred():1074 return False1075 return self._thread.is_alive()1076 1077 def stop(self):1078 if self.is_alive():1079 self.set_error_occurred()1080 1081 def kill_process(self):1082 if self._running_in_process:1083 if self.is_alive():1084 self._thread.terminate()1085 else:1086 self.log(""Cannot kill a process running in a thread"")1087 1088 1089class Pipeline(TaskExecutionSequence):1090 tasks: List[TaskExecutionSequence]1091 error_polling_rate: float1092 1093 def __init__(self, tasks, error_polling_rate=1.0):1094 self.tasks = tasks1095 self.error_polling_rate = error_polling_rate1096 1097 def start(self, *args, **kwargs):1098 for task in self:1099 task.start(*args, **kwargs)1100 1101 def join(self, timeout: Optional[float]=None):1102 if timeout is not None:1103 for task in self:1104 task.join(timeout)1105 else:1106 timeout = self.error_polling_rate1107 while True:1108 has_error = self.error_occurred()1109 if has_error:1110 self.log(""... Detected an error flag. Stopping!"")1111 self.stop()1112 break1113 alive = 01114 for task in self:1115 task.join(0.01)1116 is_alive = task.is_alive()1117 alive += is_alive1118 if alive == 0:1119 break1120 time.sleep(timeout)1121 1122 def is_alive(self):1123 alive = 01124 for task in self:1125 alive += task.is_alive()1126 return alive1127 1128 def error_occurred(self) -> int:1129 errors = 01130 for task in self.tasks:1131 errors += task.error_occurred()1132 return errors1133 1134 def stop(self):1135 for task in self.tasks:1136 task.stop()1137 1138 def __iter__(self):1139 return iter(self.tasks)1140 1141 def __len__(self):1142 return len(self.tasks)1143 1144 def __getitem__(self, i: Union[int, slice]):1145 return self.tasks[i]1146 1147 def add(self, task):1148 self.tasks.append(task)1149 return self1150 1151 1152class SinkTask(TaskExecutionSequence):1153 def __init__(self, in_queue, in_done_event):1154 self.in_queue = in_queue1155 self.in_done_event = in_done_event1156 self.done_event = self._make_event()1157 1158 def handle_item(self, task):1159 pass1160 1161 def process(self):1162 has_work = True1163 while has_work and not self.error_occurred():1164 try:1165 item = self.in_queue.get(True, 10)1166 self.handle_item(item)1167 except Empty:1168 if self.in_done_event.is_set():1169 has_work = False1170 break1171 self.done_event.set()1172 1173 1174def make_shared_memory_manager():1175 manager = SyncManager()1176 manager.start(_name_process, (""glycresoft-shm"", ))1177 return manager1178 1179 1180def _name_process(name):1181 try:1182 import setproctitle1183 setproctitle.setproctitle(name)1184 except (ImportError, AttributeError):1185 pass1186 1187 1188def elapsed(seconds):1189 '''Convert a second count into a human readable duration1190 1191 Parameters1192 ----------1193 seconds : :class:`int`1194 The number of seconds elapsed1195 1196 Returns1197 -------1198 :class:`str` :1199 A formatted, comma separated list of units of duration in days, hours, minutes, and seconds1200 '''