chendl/compositional_test

1import logging2from pathlib import Path3 4import numpy as np5import pytorch_lightning as pl6import torch7from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint8from pytorch_lightning.utilities import rank_zero_only9from utils_rag import save_json10 11 12def count_trainable_parameters(model):13    model_parameters = filter(lambda p: p.requires_grad, model.parameters())14    params = sum([np.prod(p.size()) for p in model_parameters])15    return params16 17 18logger = logging.getLogger(__name__)19 20 21def get_checkpoint_callback(output_dir, metric):22    """Saves the best model by validation EM score."""23    if metric == "rouge2":24        exp = "{val_avg_rouge2:.4f}-{step_count}"25    elif metric == "bleu":26        exp = "{val_avg_bleu:.4f}-{step_count}"27    elif metric == "em":28        exp = "{val_avg_em:.4f}-{step_count}"29    else:30        raise NotImplementedError(31            f"seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this"32            " function."33        )34 35    checkpoint_callback = ModelCheckpoint(36        dirpath=output_dir,37        filename=exp,38        monitor=f"val_{metric}",39        mode="max",40        save_top_k=3,41        every_n_epochs=1,  # maybe save a checkpoint every time val is run, not just end of epoch.42    )43    return checkpoint_callback44 45 46def get_early_stopping_callback(metric, patience):47    return EarlyStopping(48        monitor=f"val_{metric}",  # does this need avg?49        mode="min" if "loss" in metric else "max",50        patience=patience,51        verbose=True,52    )53 54 55class Seq2SeqLoggingCallback(pl.Callback):56    def on_batch_end(self, trainer, pl_module):57        lrs = {f"lr_group_{i}": param["lr"] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups)}58        pl_module.logger.log_metrics(lrs)59 60    @rank_zero_only61    def _write_logs(62        self, trainer: pl.Trainer, pl_module: pl.LightningModule, type_path: str, save_generations=True63    ) -> None:64        logger.info(f"***** {type_path} results at step {trainer.global_step:05d} *****")65        metrics = trainer.callback_metrics66        trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ["log", "progress_bar", "preds"]})67        # Log results68        od = Path(pl_module.hparams.output_dir)69        if type_path == "test":70            results_file = od / "test_results.txt"71            generations_file = od / "test_generations.txt"72        else:73            # this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json74            # If people want this it will be easy enough to add back.75            results_file = od / f"{type_path}_results/{trainer.global_step:05d}.txt"76            generations_file = od / f"{type_path}_generations/{trainer.global_step:05d}.txt"77            results_file.parent.mkdir(exist_ok=True)78            generations_file.parent.mkdir(exist_ok=True)79        with open(results_file, "a+") as writer:80            for key in sorted(metrics):81                if key in ["log", "progress_bar", "preds"]:82                    continue83                val = metrics[key]84                if isinstance(val, torch.Tensor):85                    val = val.item()86                msg = f"{key}: {val:.6f}\n"87                writer.write(msg)88 89        if not save_generations:90            return91 92        if "preds" in metrics:93            content = "\n".join(metrics["preds"])94            generations_file.open("w+").write(content)95 96    @rank_zero_only97    def on_train_start(self, trainer, pl_module):98        try:99            npars = pl_module.model.model.num_parameters()100        except AttributeError:101            npars = pl_module.model.num_parameters()102 103        n_trainable_pars = count_trainable_parameters(pl_module)104        # mp stands for million parameters105        trainer.logger.log_metrics({"n_params": npars, "mp": npars / 1e6, "grad_mp": n_trainable_pars / 1e6})106 107    @rank_zero_only108    def on_test_end(self, trainer: pl.Trainer, pl_module: pl.LightningModule):109        save_json(pl_module.metrics, pl_module.metrics_save_path)110        return self._write_logs(trainer, pl_module, "test")111 112    @rank_zero_only113    def on_validation_end(self, trainer: pl.Trainer, pl_module):114        save_json(pl_module.metrics, pl_module.metrics_save_path)115        # Uncommenting this will save val generations116        # return self._write_logs(trainer, pl_module, "valid")117