Aluode/PerceptionLabPortable

1 2__all__ = [3    'BaseConstructor',4    'SafeConstructor',5    'FullConstructor',6    'UnsafeConstructor',7    'Constructor',8    'ConstructorError'9]10 11from .error import *12from .nodes import *13 14import collections.abc, datetime, base64, binascii, re, sys, types15 16class ConstructorError(MarkedYAMLError):17    pass18 19class BaseConstructor:20 21    yaml_constructors = {}22    yaml_multi_constructors = {}23 24    def __init__(self):25        self.constructed_objects = {}26        self.recursive_objects = {}27        self.state_generators = []28        self.deep_construct = False29 30    def check_data(self):31        # If there are more documents available?32        return self.check_node()33 34    def check_state_key(self, key):35        """Block special attributes/methods from being set in a newly created36        object, to prevent user-controlled methods from being called during37        deserialization"""38        if self.get_state_keys_blacklist_regexp().match(key):39            raise ConstructorError(None, None,40                "blacklisted key '%s' in instance state found" % (key,), None)41 42    def get_data(self):43        # Construct and return the next document.44        if self.check_node():45            return self.construct_document(self.get_node())46 47    def get_single_data(self):48        # Ensure that the stream contains a single document and construct it.49        node = self.get_single_node()50        if node is not None:51            return self.construct_document(node)52        return None53 54    def construct_document(self, node):55        data = self.construct_object(node)56        while self.state_generators:57            state_generators = self.state_generators58            self.state_generators = []59            for generator in state_generators:60                for dummy in generator:61                    pass62        self.constructed_objects = {}63        self.recursive_objects = {}64        self.deep_construct = False65        return data66 67    def construct_object(self, node, deep=False):68        if node in self.constructed_objects:69            return self.constructed_objects[node]70        if deep:71            old_deep = self.deep_construct72            self.deep_construct = True73        if node in self.recursive_objects:74            raise ConstructorError(None, None,75                    "found unconstructable recursive node", node.start_mark)76        self.recursive_objects[node] = None77        constructor = None78        tag_suffix = None79        if node.tag in self.yaml_constructors:80            constructor = self.yaml_constructors[node.tag]81        else:82            for tag_prefix in self.yaml_multi_constructors:83                if tag_prefix is not None and node.tag.startswith(tag_prefix):84                    tag_suffix = node.tag[len(tag_prefix):]85                    constructor = self.yaml_multi_constructors[tag_prefix]86                    break87            else:88                if None in self.yaml_multi_constructors:89                    tag_suffix = node.tag90                    constructor = self.yaml_multi_constructors[None]91                elif None in self.yaml_constructors:92                    constructor = self.yaml_constructors[None]93                elif isinstance(node, ScalarNode):94                    constructor = self.__class__.construct_scalar95                elif isinstance(node, SequenceNode):96                    constructor = self.__class__.construct_sequence97                elif isinstance(node, MappingNode):98                    constructor = self.__class__.construct_mapping99        if tag_suffix is None:100            data = constructor(self, node)101        else:102            data = constructor(self, tag_suffix, node)103        if isinstance(data, types.GeneratorType):104            generator = data105            data = next(generator)106            if self.deep_construct:107                for dummy in generator:108                    pass109            else:110                self.state_generators.append(generator)111        self.constructed_objects[node] = data112        del self.recursive_objects[node]113        if deep:114            self.deep_construct = old_deep115        return data116 117    def construct_scalar(self, node):118        if not isinstance(node, ScalarNode):119            raise ConstructorError(None, None,120                    "expected a scalar node, but found %s" % node.id,121                    node.start_mark)122        return node.value123 124    def construct_sequence(self, node, deep=False):125        if not isinstance(node, SequenceNode):126            raise ConstructorError(None, None,127                    "expected a sequence node, but found %s" % node.id,128                    node.start_mark)129        return [self.construct_object(child, deep=deep)130                for child in node.value]131 132    def construct_mapping(self, node, deep=False):133        if not isinstance(node, MappingNode):134            raise ConstructorError(None, None,135                    "expected a mapping node, but found %s" % node.id,136                    node.start_mark)137        mapping = {}138        for key_node, value_node in node.value:139            key = self.construct_object(key_node, deep=deep)140            if not isinstance(key, collections.abc.Hashable):141                raise ConstructorError("while constructing a mapping", node.start_mark,142                        "found unhashable key", key_node.start_mark)143            value = self.construct_object(value_node, deep=deep)144            mapping[key] = value145        return mapping146 147    def construct_pairs(self, node, deep=False):148        if not isinstance(node, MappingNode):149            raise ConstructorError(None, None,150                    "expected a mapping node, but found %s" % node.id,151                    node.start_mark)152        pairs = []153        for key_node, value_node in node.value:154            key = self.construct_object(key_node, deep=deep)155            value = self.construct_object(value_node, deep=deep)156            pairs.append((key, value))157        return pairs158 159    @classmethod160    def add_constructor(cls, tag, constructor):161        if not 'yaml_constructors' in cls.__dict__:162            cls.yaml_constructors = cls.yaml_constructors.copy()163        cls.yaml_constructors[tag] = constructor164 165    @classmethod166    def add_multi_constructor(cls, tag_prefix, multi_constructor):167        if not 'yaml_multi_constructors' in cls.__dict__:168            cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()169        cls.yaml_multi_constructors[tag_prefix] = multi_constructor170 171class SafeConstructor(BaseConstructor):172 173    def construct_scalar(self, node):174        if isinstance(node, MappingNode):175            for key_node, value_node in node.value:176                if key_node.tag == 'tag:yaml.org,2002:value':177                    return self.construct_scalar(value_node)178        return super().construct_scalar(node)179 180    def flatten_mapping(self, node):181        merge = []182        index = 0183        while index < len(node.value):184            key_node, value_node = node.value[index]185            if key_node.tag == 'tag:yaml.org,2002:merge':186                del node.value[index]187                if isinstance(value_node, MappingNode):188                    self.flatten_mapping(value_node)189                    merge.extend(value_node.value)190                elif isinstance(value_node, SequenceNode):191                    submerge = []192                    for subnode in value_node.value:193                        if not isinstance(subnode, MappingNode):194                            raise ConstructorError("while constructing a mapping",195                                    node.start_mark,196                                    "expected a mapping for merging, but found %s"197                                    % subnode.id, subnode.start_mark)198                        self.flatten_mapping(subnode)199                        submerge.append(subnode.value)200                    submerge.reverse()201                    for value in submerge:202                        merge.extend(value)203                else:204                    raise ConstructorError("while constructing a mapping", node.start_mark,205                            "expected a mapping or list of mappings for merging, but found %s"206                            % value_node.id, value_node.start_mark)207            elif key_node.tag == 'tag:yaml.org,2002:value':208                key_node.tag = 'tag:yaml.org,2002:str'209                index += 1210            else:211                index += 1212        if merge:213            node.value = merge + node.value214 215    def construct_mapping(self, node, deep=False):216        if isinstance(node, MappingNode):217            self.flatten_mapping(node)218        return super().construct_mapping(node, deep=deep)219 220    def construct_yaml_null(self, node):221        self.construct_scalar(node)222        return None223 224    bool_values = {225        'yes':      True,226        'no':       False,227        'true':     True,228        'false':    False,229        'on':       True,230        'off':      False,231    }232 233    def construct_yaml_bool(self, node):234        value = self.construct_scalar(node)235        return self.bool_values[value.lower()]236 237    def construct_yaml_int(self, node):238        value = self.construct_scalar(node)239        value = value.replace('_', '')240        sign = +1241        if value[0] == '-':242            sign = -1243        if value[0] in '+-':244            value = value[1:]245        if value == '0':246            return 0247        elif value.startswith('0b'):248            return sign*int(value[2:], 2)249        elif value.startswith('0x'):250            return sign*int(value[2:], 16)251        elif value[0] == '0':252            return sign*int(value, 8)253        elif ':' in value:254            digits = [int(part) for part in value.split(':')]255            digits.reverse()256            base = 1257            value = 0258            for digit in digits:259                value += digit*base260                base *= 60261            return sign*value262        else:263            return sign*int(value)264 265    inf_value = 1e300266    while inf_value != inf_value*inf_value:267        inf_value *= inf_value268    nan_value = -inf_value/inf_value   # Trying to make a quiet NaN (like C99).269 270    def construct_yaml_float(self, node):271        value = self.construct_scalar(node)272        value = value.replace('_', '').lower()273        sign = +1274        if value[0] == '-':275            sign = -1276        if value[0] in '+-':277            value = value[1:]278        if value == '.inf':279            return sign*self.inf_value280        elif value == '.nan':281            return self.nan_value282        elif ':' in value:283            digits = [float(part) for part in value.split(':')]284            digits.reverse()285            base = 1286            value = 0.0287            for digit in digits:288                value += digit*base289                base *= 60290            return sign*value291        else:292            return sign*float(value)293 294    def construct_yaml_binary(self, node):295        try:296            value = self.construct_scalar(node).encode('ascii')297        except UnicodeEncodeError as exc:298            raise ConstructorError(None, None,299                    "failed to convert base64 data into ascii: %s" % exc,300                    node.start_mark)301        try:302            if hasattr(base64, 'decodebytes'):303                return base64.decodebytes(value)304            else:305                return base64.decodestring(value)306        except binascii.Error as exc:307            raise ConstructorError(None, None,308                    "failed to decode base64 data: %s" % exc, node.start_mark)309 310    timestamp_regexp = re.compile(311            r'''^(?P<year>[0-9][0-9][0-9][0-9])312                -(?P<month>[0-9][0-9]?)313                -(?P<day>[0-9][0-9]?)314                (?:(?:[Tt]|[ \t]+)315                (?P<hour>[0-9][0-9]?)316                :(?P<minute>[0-9][0-9])317                :(?P<second>[0-9][0-9])318                (?:\.(?P<fraction>[0-9]*))?319                (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)320                (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)321 322    def construct_yaml_timestamp(self, node):323        value = self.construct_scalar(node)324        match = self.timestamp_regexp.match(node.value)325        values = match.groupdict()326        year = int(values['year'])327        month = int(values['month'])328        day = int(values['day'])329        if not values['hour']:330            return datetime.date(year, month, day)331        hour = int(values['hour'])332        minute = int(values['minute'])333        second = int(values['second'])334        fraction = 0335        tzinfo = None336        if values['fraction']:337            fraction = values['fraction'][:6]338            while len(fraction) < 6:339                fraction += '0'340            fraction = int(fraction)341        if values['tz_sign']:342            tz_hour = int(values['tz_hour'])343            tz_minute = int(values['tz_minute'] or 0)344            delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)345            if values['tz_sign'] == '-':346                delta = -delta347            tzinfo = datetime.timezone(delta)348        elif values['tz']:349            tzinfo = datetime.timezone.utc350        return datetime.datetime(year, month, day, hour, minute, second, fraction,351                                 tzinfo=tzinfo)352 353    def construct_yaml_omap(self, node):354        # Note: we do not check for duplicate keys, because it's too355        # CPU-expensive.356        omap = []357        yield omap358        if not isinstance(node, SequenceNode):359            raise ConstructorError("while constructing an ordered map", node.start_mark,360                    "expected a sequence, but found %s" % node.id, node.start_mark)361        for subnode in node.value:362            if not isinstance(subnode, MappingNode):363                raise ConstructorError("while constructing an ordered map", node.start_mark,364                        "expected a mapping of length 1, but found %s" % subnode.id,365                        subnode.start_mark)366            if len(subnode.value) != 1:367                raise ConstructorError("while constructing an ordered map", node.start_mark,368                        "expected a single mapping item, but found %d items" % len(subnode.value),369                        subnode.start_mark)370            key_node, value_node = subnode.value[0]371            key = self.construct_object(key_node)372            value = self.construct_object(value_node)373            omap.append((key, value))374 375    def construct_yaml_pairs(self, node):376        # Note: the same code as `construct_yaml_omap`.377        pairs = []378        yield pairs379        if not isinstance(node, SequenceNode):380            raise ConstructorError("while constructing pairs", node.start_mark,381                    "expected a sequence, but found %s" % node.id, node.start_mark)382        for subnode in node.value:383            if not isinstance(subnode, MappingNode):384                raise ConstructorError("while constructing pairs", node.start_mark,385                        "expected a mapping of length 1, but found %s" % subnode.id,386                        subnode.start_mark)387            if len(subnode.value) != 1:388                raise ConstructorError("while constructing pairs", node.start_mark,389                        "expected a single mapping item, but found %d items" % len(subnode.value),390                        subnode.start_mark)391            key_node, value_node = subnode.value[0]392            key = self.construct_object(key_node)393            value = self.construct_object(value_node)394            pairs.append((key, value))395 396    def construct_yaml_set(self, node):397        data = set()398        yield data399        value = self.construct_mapping(node)400        data.update(value)401 402    def construct_yaml_str(self, node):403        return self.construct_scalar(node)404 405    def construct_yaml_seq(self, node):406        data = []407        yield data408        data.extend(self.construct_sequence(node))409 410    def construct_yaml_map(self, node):411        data = {}412        yield data413        value = self.construct_mapping(node)414        data.update(value)415 416    def construct_yaml_object(self, node, cls):417        data = cls.__new__(cls)418        yield data419        if hasattr(data, '__setstate__'):420            state = self.construct_mapping(node, deep=True)421            data.__setstate__(state)422        else:423            state = self.construct_mapping(node)424            data.__dict__.update(state)425 426    def construct_undefined(self, node):427        raise ConstructorError(None, None,428                "could not determine a constructor for the tag %r" % node.tag,429                node.start_mark)430 431SafeConstructor.add_constructor(432        'tag:yaml.org,2002:null',433        SafeConstructor.construct_yaml_null)434 435SafeConstructor.add_constructor(436        'tag:yaml.org,2002:bool',437        SafeConstructor.construct_yaml_bool)438 439SafeConstructor.add_constructor(440        'tag:yaml.org,2002:int',441        SafeConstructor.construct_yaml_int)442 443SafeConstructor.add_constructor(444        'tag:yaml.org,2002:float',445        SafeConstructor.construct_yaml_float)446 447SafeConstructor.add_constructor(448        'tag:yaml.org,2002:binary',449        SafeConstructor.construct_yaml_binary)450 451SafeConstructor.add_constructor(452        'tag:yaml.org,2002:timestamp',453        SafeConstructor.construct_yaml_timestamp)454 455SafeConstructor.add_constructor(456        'tag:yaml.org,2002:omap',457        SafeConstructor.construct_yaml_omap)458 459SafeConstructor.add_constructor(460        'tag:yaml.org,2002:pairs',461        SafeConstructor.construct_yaml_pairs)462 463SafeConstructor.add_constructor(464        'tag:yaml.org,2002:set',465        SafeConstructor.construct_yaml_set)466 467SafeConstructor.add_constructor(468        'tag:yaml.org,2002:str',469        SafeConstructor.construct_yaml_str)470 471SafeConstructor.add_constructor(472        'tag:yaml.org,2002:seq',473        SafeConstructor.construct_yaml_seq)474 475SafeConstructor.add_constructor(476        'tag:yaml.org,2002:map',477        SafeConstructor.construct_yaml_map)478 479SafeConstructor.add_constructor(None,480        SafeConstructor.construct_undefined)481 482class FullConstructor(SafeConstructor):483    # 'extend' is blacklisted because it is used by484    # construct_python_object_apply to add `listitems` to a newly generate485    # python instance486    def get_state_keys_blacklist(self):487        return ['^extend$', '^__.*__$']488 489    def get_state_keys_blacklist_regexp(self):490        if not hasattr(self, 'state_keys_blacklist_regexp'):491            self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')')492        return self.state_keys_blacklist_regexp493 494    def construct_python_str(self, node):495        return self.construct_scalar(node)496 497    def construct_python_unicode(self, node):498        return self.construct_scalar(node)499 500    def construct_python_bytes(self, node):501        try:502            value = self.construct_scalar(node).encode('ascii')503        except UnicodeEncodeError as exc:504            raise ConstructorError(None, None,505                    "failed to convert base64 data into ascii: %s" % exc,506                    node.start_mark)507        try:508            if hasattr(base64, 'decodebytes'):509                return base64.decodebytes(value)510            else:511                return base64.decodestring(value)512        except binascii.Error as exc:513            raise ConstructorError(None, None,514                    "failed to decode base64 data: %s" % exc, node.start_mark)515 516    def construct_python_long(self, node):517        return self.construct_yaml_int(node)518 519    def construct_python_complex(self, node):520       return complex(self.construct_scalar(node))521 522    def construct_python_tuple(self, node):523        return tuple(self.construct_sequence(node))524 525    def find_python_module(self, name, mark, unsafe=False):526        if not name:527            raise ConstructorError("while constructing a Python module", mark,528                    "expected non-empty name appended to the tag", mark)529        if unsafe:530            try:531                __import__(name)532            except ImportError as exc:533                raise ConstructorError("while constructing a Python module", mark,534                        "cannot find module %r (%s)" % (name, exc), mark)535        if name not in sys.modules:536            raise ConstructorError("while constructing a Python module", mark,537                    "module %r is not imported" % name, mark)538        return sys.modules[name]539 540    def find_python_name(self, name, mark, unsafe=False):541        if not name:542            raise ConstructorError("while constructing a Python object", mark,543                    "expected non-empty name appended to the tag", mark)544        if '.' in name:545            module_name, object_name = name.rsplit('.', 1)546        else:547            module_name = 'builtins'548            object_name = name549        if unsafe:550            try:551                __import__(module_name)552            except ImportError as exc:553                raise ConstructorError("while constructing a Python object", mark,554                        "cannot find module %r (%s)" % (module_name, exc), mark)555        if module_name not in sys.modules:556            raise ConstructorError("while constructing a Python object", mark,557                    "module %r is not imported" % module_name, mark)558        module = sys.modules[module_name]559        if not hasattr(module, object_name):560            raise ConstructorError("while constructing a Python object", mark,561                    "cannot find %r in the module %r"562                    % (object_name, module.__name__), mark)563        return getattr(module, object_name)564 565    def construct_python_name(self, suffix, node):566        value = self.construct_scalar(node)567        if value:568            raise ConstructorError("while constructing a Python name", node.start_mark,569                    "expected the empty value, but found %r" % value, node.start_mark)570        return self.find_python_name(suffix, node.start_mark)571 572    def construct_python_module(self, suffix, node):573        value = self.construct_scalar(node)574        if value:575            raise ConstructorError("while constructing a Python module", node.start_mark,576                    "expected the empty value, but found %r" % value, node.start_mark)577        return self.find_python_module(suffix, node.start_mark)578 579    def make_python_instance(self, suffix, node,580            args=None, kwds=None, newobj=False, unsafe=False):581        if not args:582            args = []583        if not kwds:584            kwds = {}585        cls = self.find_python_name(suffix, node.start_mark)586        if not (unsafe or isinstance(cls, type)):587            raise ConstructorError("while constructing a Python instance", node.start_mark,588                    "expected a class, but found %r" % type(cls),589                    node.start_mark)590        if newobj and isinstance(cls, type):591            return cls.__new__(cls, *args, **kwds)592        else:593            return cls(*args, **kwds)594 595    def set_python_instance_state(self, instance, state, unsafe=False):596        if hasattr(instance, '__setstate__'):597            instance.__setstate__(state)598        else:599            slotstate = {}600            if isinstance(state, tuple) and len(state) == 2:601                state, slotstate = state602            if hasattr(instance, '__dict__'):603                if not unsafe and state:604                    for key in state.keys():605                        self.check_state_key(key)606                instance.__dict__.update(state)607            elif state:608                slotstate.update(state)609            for key, value in slotstate.items():610                if not unsafe:611                    self.check_state_key(key)612                setattr(instance, key, value)613 614    def construct_python_object(self, suffix, node):615        # Format:616        #   !!python/object:module.name { ... state ... }617        instance = self.make_python_instance(suffix, node, newobj=True)618        yield instance619        deep = hasattr(instance, '__setstate__')620        state = self.construct_mapping(node, deep=deep)621        self.set_python_instance_state(instance, state)622 623    def construct_python_object_apply(self, suffix, node, newobj=False):624        # Format:625        #   !!python/object/apply       # (or !!python/object/new)626        #   args: [ ... arguments ... ]627        #   kwds: { ... keywords ... }628        #   state: ... state ...629        #   listitems: [ ... listitems ... ]630        #   dictitems: { ... dictitems ... }631        # or short format:632        #   !!python/object/apply [ ... arguments ... ]633        # The difference between !!python/object/apply and !!python/object/new634        # is how an object is created, check make_python_instance for details.635        if isinstance(node, SequenceNode):636            args = self.construct_sequence(node, deep=True)637            kwds = {}638            state = {}639            listitems = []640            dictitems = {}641        else:642            value = self.construct_mapping(node, deep=True)643            args = value.get('args', [])644            kwds = value.get('kwds', {})645            state = value.get('state', {})646            listitems = value.get('listitems', [])647            dictitems = value.get('dictitems', {})648        instance = self.make_python_instance(suffix, node, args, kwds, newobj)649        if state:650            self.set_python_instance_state(instance, state)651        if listitems:652            instance.extend(listitems)653        if dictitems:654            for key in dictitems:655                instance[key] = dictitems[key]656        return instance657 658    def construct_python_object_new(self, suffix, node):659        return self.construct_python_object_apply(suffix, node, newobj=True)660 661FullConstructor.add_constructor(662    'tag:yaml.org,2002:python/none',663    FullConstructor.construct_yaml_null)664 665FullConstructor.add_constructor(666    'tag:yaml.org,2002:python/bool',667    FullConstructor.construct_yaml_bool)668 669FullConstructor.add_constructor(670    'tag:yaml.org,2002:python/str',671    FullConstructor.construct_python_str)672 673FullConstructor.add_constructor(674    'tag:yaml.org,2002:python/unicode',675    FullConstructor.construct_python_unicode)676 677FullConstructor.add_constructor(678    'tag:yaml.org,2002:python/bytes',679    FullConstructor.construct_python_bytes)680 681FullConstructor.add_constructor(682    'tag:yaml.org,2002:python/int',683    FullConstructor.construct_yaml_int)684 685FullConstructor.add_constructor(686    'tag:yaml.org,2002:python/long',687    FullConstructor.construct_python_long)688 689FullConstructor.add_constructor(690    'tag:yaml.org,2002:python/float',691    FullConstructor.construct_yaml_float)692 693FullConstructor.add_constructor(694    'tag:yaml.org,2002:python/complex',695    FullConstructor.construct_python_complex)696 697FullConstructor.add_constructor(698    'tag:yaml.org,2002:python/list',699    FullConstructor.construct_yaml_seq)700 701FullConstructor.add_constructor(702    'tag:yaml.org,2002:python/tuple',703    FullConstructor.construct_python_tuple)704 705FullConstructor.add_constructor(706    'tag:yaml.org,2002:python/dict',707    FullConstructor.construct_yaml_map)708 709FullConstructor.add_multi_constructor(710    'tag:yaml.org,2002:python/name:',711    FullConstructor.construct_python_name)712 713class UnsafeConstructor(FullConstructor):714 715    def find_python_module(self, name, mark):716        return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True)717 718    def find_python_name(self, name, mark):719        return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True)720 721    def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False):722        return super(UnsafeConstructor, self).make_python_instance(723            suffix, node, args, kwds, newobj, unsafe=True)724 725    def set_python_instance_state(self, instance, state):726        return super(UnsafeConstructor, self).set_python_instance_state(727            instance, state, unsafe=True)728 729UnsafeConstructor.add_multi_constructor(730    'tag:yaml.org,2002:python/module:',731    UnsafeConstructor.construct_python_module)732 733UnsafeConstructor.add_multi_constructor(734    'tag:yaml.org,2002:python/object:',735    UnsafeConstructor.construct_python_object)736 737UnsafeConstructor.add_multi_constructor(738    'tag:yaml.org,2002:python/object/new:',739    UnsafeConstructor.construct_python_object_new)740 741UnsafeConstructor.add_multi_constructor(742    'tag:yaml.org,2002:python/object/apply:',743    UnsafeConstructor.construct_python_object_apply)744 745# Constructor is same as UnsafeConstructor. Need to leave this in place in case746# people have extended it directly.747class Constructor(UnsafeConstructor):748    pass749