RASMUS/Finnish-ASR-Canary-v2

1# ! /usr/bin/python2# -*- coding: utf-8 -*-3 4# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.5#6# Licensed under the Apache License, Version 2.0 (the "License");7# you may not use this file except in compliance with the License.8# You may obtain a copy of the License at9#10#     http://www.apache.org/licenses/LICENSE-2.011#12# Unless required by applicable law or agreed to in writing, software13# distributed under the License is distributed on an "AS IS" BASIS,14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.15# See the License for the specific language governing permissions and16# limitations under the License.17 18"""Setup for pip package."""19 20import codecs21import importlib.util22import os23import subprocess24from distutils import cmd as distutils_cmd25from distutils import log as distutils_log26from itertools import chain27 28import setuptools29 30spec = importlib.util.spec_from_file_location('package_info', 'nemo/package_info.py')31package_info = importlib.util.module_from_spec(spec)32spec.loader.exec_module(package_info)33 34 35__contact_emails__ = package_info.__contact_emails__36__contact_names__ = package_info.__contact_names__37__description__ = package_info.__description__38__download_url__ = package_info.__download_url__39__homepage__ = package_info.__homepage__40__keywords__ = package_info.__keywords__41__license__ = package_info.__license__42__package_name__ = package_info.__package_name__43__repository_url__ = package_info.__repository_url__44__version__ = package_info.__version__45 46 47with open("README.md", "r", encoding='utf-8') as fh:48    long_description = fh.read()49    long_description_content_type = "text/markdown"50 51 52###############################################################################53#                             Dependency Loading                              #54# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #55 56 57def req_file(filename, folder="requirements"):58    files = [filename] if not isinstance(filename, list) else filename59    ans = []60    for file in files:61        with open(os.path.join(folder, file), encoding='utf-8') as f:62            ans.extend(list(map(str.strip, f.readlines())))63    return ans64 65 66install_requires = req_file("requirements.txt")67 68extras_require = {69    # User packages70    'test': req_file("requirements_test.txt"),71    'run': req_file("requirements_run.txt"),72    # Lightning Collections Packages73    'core': req_file(["requirements_lightning.txt"]),74    'lightning': req_file(["requirements_lightning.txt"]),75    'common-only': req_file('requirements_common.txt'),76    # domain packages77    'asr-only': req_file("requirements_asr.txt"),78    'nlp-only': req_file("requirements_nlp.txt"),79    'tts': req_file("requirements_tts.txt"),80    'slu': req_file("requirements_slu.txt"),81    'multimodal-only': req_file("requirements_multimodal.txt"),82    'audio': req_file("requirements_audio.txt"),83}84 85 86extras_require['all'] = list(chain(val for key, val in extras_require.items()))87 88# Add lightning requirements as needed89extras_require['common'] = extras_require['common-only']90 91extras_require['common'] = list(92    chain(93        extras_require['common'],94        extras_require['core'],95    )96)97extras_require['test'] = list(98    chain(99        extras_require['test'],100        extras_require['tts'],101        extras_require['common'],102    )103)104extras_require['asr'] = extras_require['asr-only']105extras_require['asr'] = list(106    chain(107        extras_require['asr'],108        extras_require['common'],109    )110)111extras_require['nlp'] = extras_require['nlp-only']112extras_require['nlp'] = list(113    chain(114        extras_require['nlp'],115        extras_require['common'],116    )117)118extras_require['llm'] = extras_require['nlp']119extras_require['tts'] = list(120    chain(121        extras_require['tts'],122        extras_require['asr'],123        extras_require['common'],124    )125)126extras_require['multimodal'] = extras_require['multimodal-only']127extras_require['multimodal'] = list(128    chain(129        extras_require['multimodal'],130        extras_require['nlp'],131        extras_require['common'],132    )133)134extras_require['audio'] = list(135    chain(136        extras_require['audio'],137        extras_require['common'],138    )139)140extras_require['slu'] = list(141    chain(142        extras_require['slu'],143        extras_require['asr'],144    )145)146 147 148###############################################################################149#                            Code style checkers                              #150# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #151 152 153class StyleCommand(distutils_cmd.Command):154    __ISORT_BASE = 'isort'155    __BLACK_BASE = 'black'156    description = 'Checks overall project code style.'157    user_options = [158        ('scope=', None, 'Folder of file to operate within.'),159        ('fix', None, 'True if tries to fix issues in-place.'),160    ]161 162    def __call_checker(self, base_command, scope, check):163        command = list(base_command)164 165        command.append(scope)166 167        if check:168            command.extend(['--check', '--diff'])169 170        self.announce(171            msg='Running command: %s' % str(' '.join(command)),172            level=distutils_log.INFO,173        )174 175        return_code = subprocess.call(command)176 177        return return_code178 179    def _isort(self, scope, check):180        return self.__call_checker(181            base_command=self.__ISORT_BASE.split(),182            scope=scope,183            check=check,184        )185 186    def _black(self, scope, check):187        return self.__call_checker(188            base_command=self.__BLACK_BASE.split(),189            scope=scope,190            check=check,191        )192 193    def _pass(self):194        self.announce(msg='\033[32mPASS\x1b[0m', level=distutils_log.INFO)195 196    def _fail(self):197        self.announce(msg='\033[31mFAIL\x1b[0m', level=distutils_log.INFO)198 199    # noinspection PyAttributeOutsideInit200    def initialize_options(self):201        self.scope = '.'202        self.fix = ''203 204    def run(self):205        scope, check = self.scope, not self.fix206        isort_return = self._isort(scope=scope, check=check)207        black_return = self._black(scope=scope, check=check)208 209        if isort_return == 0 and black_return == 0:210            self._pass()211        else:212            self._fail()213            exit(isort_return if isort_return != 0 else black_return)214 215    def finalize_options(self):216        pass217 218 219###############################################################################220 221setuptools.setup(222    name=__package_name__,223    # Versions should comply with PEP440.  For a discussion on single-sourcing224    # the version across setup.py and the project code, see225    # https://packaging.python.org/en/latest/single_source_version.html226    version=__version__,227    description=__description__,228    long_description=long_description,229    long_description_content_type=long_description_content_type,230    # The project's main homepage.231    url=__repository_url__,232    download_url=__download_url__,233    # Author details234    author=__contact_names__,235    author_email=__contact_emails__,236    # maintainer Details237    maintainer=__contact_names__,238    maintainer_email=__contact_emails__,239    # The licence under which the project is released240    license=__license__,241    classifiers=[242        # How mature is this project? Common values are243        #  1 - Planning244        #  2 - Pre-Alpha245        #  3 - Alpha246        #  4 - Beta247        #  5 - Production/Stable248        #  6 - Mature249        #  7 - Inactive250        'Development Status :: 5 - Production/Stable',251        # Indicate who your project is intended for252        'Intended Audience :: Developers',253        'Intended Audience :: Science/Research',254        'Intended Audience :: Information Technology',255        # Indicate what your project relates to256        'Topic :: Scientific/Engineering',257        'Topic :: Scientific/Engineering :: Mathematics',258        'Topic :: Scientific/Engineering :: Image Recognition',259        'Topic :: Scientific/Engineering :: Artificial Intelligence',260        'Topic :: Software Development :: Libraries',261        'Topic :: Software Development :: Libraries :: Python Modules',262        'Topic :: Utilities',263        # Pick your license as you wish (should match "license" above)264        'License :: OSI Approved :: Apache Software License',265        # Supported python versions266        'Programming Language :: Python :: 3',267        'Programming Language :: Python :: 3.10',268        # Additional Setting269        'Environment :: Console',270        'Natural Language :: English',271        'Operating System :: OS Independent',272    ],273    packages=setuptools.find_packages(),274    python_requires='>=3.10',275    install_requires=install_requires,276    # List additional groups of dependencies here (e.g. development277    # dependencies). You can install these using the following syntax,278    # $ pip install -e ".[all]"279    # $ pip install nemo_toolkit[all]280    extras_require=extras_require,281    # Add in any packaged data.282    include_package_data=True,283    exclude=['tools', 'tests'],284    package_data={'': ['*.tsv', '*.txt', '*.far', '*.fst', '*.cpp', 'Makefile']},285    zip_safe=False,286    # PyPI package information.287    keywords=__keywords__,288    # Custom commands.289    cmdclass={'style': StyleCommand},290    entry_points={291        "nemo_run.cli": [292            "llm = nemo.collections.llm",293        ],294    },295)296