chendl/compositional_test

1# coding=utf-82# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.3# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.4#5# Licensed under the Apache License, Version 2.0 (the "License");6# you may not use this file except in compliance with the License.7# You may obtain a copy of the License at8#9#     http://www.apache.org/licenses/LICENSE-2.010#11# Unless required by applicable law or agreed to in writing, software12# distributed under the License is distributed on an "AS IS" BASIS,13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.14# See the License for the specific language governing permissions and15# limitations under the License.16""" Named entity recognition fine-tuning: utilities to work with CoNLL-2003 task. """17 18 19import logging20import os21from dataclasses import dataclass22from enum import Enum23from typing import List, Optional, Union24 25from filelock import FileLock26 27from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available28 29 30logger = logging.getLogger(__name__)31 32 33@dataclass34class InputExample:35    """36    A single training/test example for token classification.37 38    Args:39        guid: Unique id for the example.40        words: list. The words of the sequence.41        labels: (Optional) list. The labels for each word of the sequence. This should be42        specified for train and dev examples, but not for test examples.43    """44 45    guid: str46    words: List[str]47    labels: Optional[List[str]]48 49 50@dataclass51class InputFeatures:52    """53    A single set of features of data.54    Property names are the same names as the corresponding inputs to a model.55    """56 57    input_ids: List[int]58    attention_mask: List[int]59    token_type_ids: Optional[List[int]] = None60    label_ids: Optional[List[int]] = None61 62 63class Split(Enum):64    train = "train"65    dev = "dev"66    test = "test"67 68 69class TokenClassificationTask:70    @staticmethod71    def read_examples_from_file(data_dir, mode: Union[Split, str]) -> List[InputExample]:72        raise NotImplementedError73 74    @staticmethod75    def get_labels(path: str) -> List[str]:76        raise NotImplementedError77 78    @staticmethod79    def convert_examples_to_features(80        examples: List[InputExample],81        label_list: List[str],82        max_seq_length: int,83        tokenizer: PreTrainedTokenizer,84        cls_token_at_end=False,85        cls_token="[CLS]",86        cls_token_segment_id=1,87        sep_token="[SEP]",88        sep_token_extra=False,89        pad_on_left=False,90        pad_token=0,91        pad_token_segment_id=0,92        pad_token_label_id=-100,93        sequence_a_segment_id=0,94        mask_padding_with_zero=True,95    ) -> List[InputFeatures]:96        """Loads a data file into a list of `InputFeatures`97        `cls_token_at_end` define the location of the CLS token:98            - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]99            - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]100        `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)101        """102        # TODO clean up all this to leverage built-in features of tokenizers103 104        label_map = {label: i for i, label in enumerate(label_list)}105 106        features = []107        for ex_index, example in enumerate(examples):108            if ex_index % 10_000 == 0:109                logger.info("Writing example %d of %d", ex_index, len(examples))110 111            tokens = []112            label_ids = []113            for word, label in zip(example.words, example.labels):114                word_tokens = tokenizer.tokenize(word)115 116                # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.117                if len(word_tokens) > 0:118                    tokens.extend(word_tokens)119                    # Use the real label id for the first token of the word, and padding ids for the remaining tokens120                    label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1))121 122            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.123            special_tokens_count = tokenizer.num_special_tokens_to_add()124            if len(tokens) > max_seq_length - special_tokens_count:125                tokens = tokens[: (max_seq_length - special_tokens_count)]126                label_ids = label_ids[: (max_seq_length - special_tokens_count)]127 128            # The convention in BERT is:129            # (a) For sequence pairs:130            #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]131            #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1132            # (b) For single sequences:133            #  tokens:   [CLS] the dog is hairy . [SEP]134            #  type_ids:   0   0   0   0  0     0   0135            #136            # Where "type_ids" are used to indicate whether this is the first137            # sequence or the second sequence. The embedding vectors for `type=0` and138            # `type=1` were learned during pre-training and are added to the wordpiece139            # embedding vector (and position vector). This is not *strictly* necessary140            # since the [SEP] token unambiguously separates the sequences, but it makes141            # it easier for the model to learn the concept of sequences.142            #143            # For classification tasks, the first vector (corresponding to [CLS]) is144            # used as the "sentence vector". Note that this only makes sense because145            # the entire model is fine-tuned.146            tokens += [sep_token]147            label_ids += [pad_token_label_id]148            if sep_token_extra:149                # roberta uses an extra separator b/w pairs of sentences150                tokens += [sep_token]151                label_ids += [pad_token_label_id]152            segment_ids = [sequence_a_segment_id] * len(tokens)153 154            if cls_token_at_end:155                tokens += [cls_token]156                label_ids += [pad_token_label_id]157                segment_ids += [cls_token_segment_id]158            else:159                tokens = [cls_token] + tokens160                label_ids = [pad_token_label_id] + label_ids161                segment_ids = [cls_token_segment_id] + segment_ids162 163            input_ids = tokenizer.convert_tokens_to_ids(tokens)164 165            # The mask has 1 for real tokens and 0 for padding tokens. Only real166            # tokens are attended to.167            input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)168 169            # Zero-pad up to the sequence length.170            padding_length = max_seq_length - len(input_ids)171            if pad_on_left:172                input_ids = ([pad_token] * padding_length) + input_ids173                input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask174                segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids175                label_ids = ([pad_token_label_id] * padding_length) + label_ids176            else:177                input_ids += [pad_token] * padding_length178                input_mask += [0 if mask_padding_with_zero else 1] * padding_length179                segment_ids += [pad_token_segment_id] * padding_length180                label_ids += [pad_token_label_id] * padding_length181 182            assert len(input_ids) == max_seq_length183            assert len(input_mask) == max_seq_length184            assert len(segment_ids) == max_seq_length185            assert len(label_ids) == max_seq_length186 187            if ex_index < 5:188                logger.info("*** Example ***")189                logger.info("guid: %s", example.guid)190                logger.info("tokens: %s", " ".join([str(x) for x in tokens]))191                logger.info("input_ids: %s", " ".join([str(x) for x in input_ids]))192                logger.info("input_mask: %s", " ".join([str(x) for x in input_mask]))193                logger.info("segment_ids: %s", " ".join([str(x) for x in segment_ids]))194                logger.info("label_ids: %s", " ".join([str(x) for x in label_ids]))195 196            if "token_type_ids" not in tokenizer.model_input_names:197                segment_ids = None198 199            features.append(200                InputFeatures(201                    input_ids=input_ids, attention_mask=input_mask, token_type_ids=segment_ids, label_ids=label_ids202                )203            )204        return features205 206 207if is_torch_available():208    import torch209    from torch import nn210    from torch.utils.data import Dataset211 212    class TokenClassificationDataset(Dataset):213        """214        This will be superseded by a framework-agnostic approach215        soon.216        """217 218        features: List[InputFeatures]219        pad_token_label_id: int = nn.CrossEntropyLoss().ignore_index220        # Use cross entropy ignore_index as padding label id so that only221        # real label ids contribute to the loss later.222 223        def __init__(224            self,225            token_classification_task: TokenClassificationTask,226            data_dir: str,227            tokenizer: PreTrainedTokenizer,228            labels: List[str],229            model_type: str,230            max_seq_length: Optional[int] = None,231            overwrite_cache=False,232            mode: Split = Split.train,233        ):234            # Load data features from cache or dataset file235            cached_features_file = os.path.join(236                data_dir,237                "cached_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(max_seq_length)),238            )239 240            # Make sure only the first process in distributed training processes the dataset,241            # and the others will use the cache.242            lock_path = cached_features_file + ".lock"243            with FileLock(lock_path):244                if os.path.exists(cached_features_file) and not overwrite_cache:245                    logger.info(f"Loading features from cached file {cached_features_file}")246                    self.features = torch.load(cached_features_file)247                else:248                    logger.info(f"Creating features from dataset file at {data_dir}")249                    examples = token_classification_task.read_examples_from_file(data_dir, mode)250                    # TODO clean up all this to leverage built-in features of tokenizers251                    self.features = token_classification_task.convert_examples_to_features(252                        examples,253                        labels,254                        max_seq_length,255                        tokenizer,256                        cls_token_at_end=bool(model_type in ["xlnet"]),257                        # xlnet has a cls token at the end258                        cls_token=tokenizer.cls_token,259                        cls_token_segment_id=2 if model_type in ["xlnet"] else 0,260                        sep_token=tokenizer.sep_token,261                        sep_token_extra=False,262                        # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805263                        pad_on_left=bool(tokenizer.padding_side == "left"),264                        pad_token=tokenizer.pad_token_id,265                        pad_token_segment_id=tokenizer.pad_token_type_id,266                        pad_token_label_id=self.pad_token_label_id,267                    )268                    logger.info(f"Saving features into cached file {cached_features_file}")269                    torch.save(self.features, cached_features_file)270 271        def __len__(self):272            return len(self.features)273 274        def __getitem__(self, i) -> InputFeatures:275            return self.features[i]276 277 278if is_tf_available():279    import tensorflow as tf280 281    class TFTokenClassificationDataset:282        """283        This will be superseded by a framework-agnostic approach284        soon.285        """286 287        features: List[InputFeatures]288        pad_token_label_id: int = -100289        # Use cross entropy ignore_index as padding label id so that only290        # real label ids contribute to the loss later.291 292        def __init__(293            self,294            token_classification_task: TokenClassificationTask,295            data_dir: str,296            tokenizer: PreTrainedTokenizer,297            labels: List[str],298            model_type: str,299            max_seq_length: Optional[int] = None,300            overwrite_cache=False,301            mode: Split = Split.train,302        ):303            examples = token_classification_task.read_examples_from_file(data_dir, mode)304            # TODO clean up all this to leverage built-in features of tokenizers305            self.features = token_classification_task.convert_examples_to_features(306                examples,307                labels,308                max_seq_length,309                tokenizer,310                cls_token_at_end=bool(model_type in ["xlnet"]),311                # xlnet has a cls token at the end312                cls_token=tokenizer.cls_token,313                cls_token_segment_id=2 if model_type in ["xlnet"] else 0,314                sep_token=tokenizer.sep_token,315                sep_token_extra=False,316                # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805317                pad_on_left=bool(tokenizer.padding_side == "left"),318                pad_token=tokenizer.pad_token_id,319                pad_token_segment_id=tokenizer.pad_token_type_id,320                pad_token_label_id=self.pad_token_label_id,321            )322 323            def gen():324                for ex in self.features:325                    if ex.token_type_ids is None:326                        yield (327                            {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},328                            ex.label_ids,329                        )330                    else:331                        yield (332                            {333                                "input_ids": ex.input_ids,334                                "attention_mask": ex.attention_mask,335                                "token_type_ids": ex.token_type_ids,336                            },337                            ex.label_ids,338                        )339 340            if "token_type_ids" not in tokenizer.model_input_names:341                self.dataset = tf.data.Dataset.from_generator(342                    gen,343                    ({"input_ids": tf.int32, "attention_mask": tf.int32}, tf.int64),344                    (345                        {"input_ids": tf.TensorShape([None]), "attention_mask": tf.TensorShape([None])},346                        tf.TensorShape([None]),347                    ),348                )349            else:350                self.dataset = tf.data.Dataset.from_generator(351                    gen,352                    ({"input_ids": tf.int32, "attention_mask": tf.int32, "token_type_ids": tf.int32}, tf.int64),353                    (354                        {355                            "input_ids": tf.TensorShape([None]),356                            "attention_mask": tf.TensorShape([None]),357                            "token_type_ids": tf.TensorShape([None]),358                        },359                        tf.TensorShape([None]),360                    ),361                )362 363        def get_dataset(self):364            self.dataset = self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features)))365 366            return self.dataset367 368        def __len__(self):369            return len(self.features)370 371        def __getitem__(self, i) -> InputFeatures:372            return self.features[i]373