sidphbot/Researcher

1from src.Surveyor import Surveyor2 3import logging4logging.basicConfig()5logging.getLogger().setLevel(logging.ERROR)6 7 8if __name__ == '__main__':9    import argparse10 11    parser = argparse.ArgumentParser(description='Generate a survey just from a query !!')12    13    data = parser.add_mutually_exclusive_group(required=True)14    data.add_argument('--query', type=str, help='your research query/keywords')15    data.add_argument('--arxiv_ids', nargs='+', help='arxiv ids for your curated set of papers')16 17    parser.add_argument('--max_search', metavar='max_metadata_papers', type=int, default=None,18                        help='maximium number of papers to gaze at - defaults to 100')19    parser.add_argument('--num_papers', metavar='max_num_papers', type=int, default=None,20                        help='maximium number of papers to download and analyse - defaults to 25')21    parser.add_argument('--pdf_dir', metavar='pdf_dir', type=str, default=None,22                        help='pdf paper storage directory - defaults to arxiv_data/tarpdfs/')23    parser.add_argument('--txt_dir', metavar='txt_dir', type=str, default=None,24                        help='text-converted paper storage directory - defaults to arxiv_data/fulltext/')25    parser.add_argument('--img_dir', metavar='img_dir', type=str, default=None,26                        help='image storage directory - defaults to arxiv_data/images/')27    parser.add_argument('--tab_dir', metavar='tab_dir', type=str, default=None,28                        help='tables storage directory - defaults to arxiv_data/tables/')29    parser.add_argument('--dump_dir', metavar='dump_dir', type=str, default=None,30                        help='all_output_dir - defaults to arxiv_dumps/')31    parser.add_argument('--models_dir', metavar='save_models_dir', type=str, default=None,32                        help='directory to save models (> 5GB) - defaults to saved_models/')33    parser.add_argument('--title_model_name', metavar='title_model_name', type=str, default=None,34                        help='title model name/tag in hugging-face, defaults to \'Callidior/bert2bert-base-arxiv-titlegen\'')35    parser.add_argument('--ex_summ_model_name', metavar='extractive_summ_model_name', type=str, default=None,36                        help='extractive summary model name/tag in hugging-face, defaults to \'allenai/scibert_scivocab_uncased\'')37    parser.add_argument('--ledmodel_name', metavar='ledmodel_name', type=str, default=None,38                        help='led model(for abstractive summary) name/tag in hugging-face, defaults to \'allenai/led-large-16384-arxiv\'')39    parser.add_argument('--embedder_name', metavar='sentence_embedder_name', type=str, default=None,40                        help='sentence embedder name/tag in hugging-face, defaults to \'paraphrase-MiniLM-L6-v2\'')41    parser.add_argument('--nlp_name', metavar='spacy_model_name', type=str, default=None,42                        help='spacy model name/tag in hugging-face (if changed - needs to be spacy-installed prior), defaults to \'en_core_sci_scibert\'')43    parser.add_argument('--similarity_nlp_name', metavar='similarity_nlp_name', type=str, default=None,44                        help='spacy downstream model(for similarity) name/tag in hugging-face (if changed - needs to be spacy-installed prior), defaults to \'en_core_sci_lg\'')45    parser.add_argument('--kw_model_name', metavar='kw_model_name', type=str, default=None,46                        help='keyword extraction model name/tag in hugging-face, defaults to \'distilbert-base-nli-mean-tokens\'')47    parser.add_argument('--refresh_models', metavar='refresh_models', type=str, default=None,48                        help='Refresh model downloads with given names (needs atleast one model name param above), defaults to False')49    parser.add_argument('--high_gpu', metavar='high_gpu', type=str, default=None,50                        help='High GPU usage permitted, defaults to False')51 52    args = parser.parse_args()53 54    surveyor = Surveyor(55        pdf_dir=args.pdf_dir,56        txt_dir=args.txt_dir,57        img_dir=args.img_dir,58        tab_dir=args.tab_dir,59        dump_dir=args.dump_dir,60        models_dir=args.models_dir,61        title_model_name=args.title_model_name,62        ex_summ_model_name=args.ex_summ_model_name,63        ledmodel_name=args.ledmodel_name,64        embedder_name=args.embedder_name,65        nlp_name=args.nlp_name,66        similarity_nlp_name=args.similarity_nlp_name,67        kw_model_name=args.kw_model_name,68        refresh_models=args.refresh_models,69        high_gpu=args.high_gpu70 71    )72 73    surveyor.survey(query=args.query, id_list=args.arxiv_ids, max_search=args.max_search, num_papers=args.num_papers,74                                              debug=False, weigh_authors=False)75 76