193engineering/WebTableScraper

1import pandas as pd2import gradio as gr3import numpy as np4 5OriginalTable = None6 7#--------------------------------------------8def scrape(url, header, encode, index):9    global OriginalTable10 11    data = pd.read_html(url, header=header, encoding=encode)12    OriginalTable = data[index]13    OriginalTable.to_csv("temp.csv", index=False)14    row, col = OriginalTable.shape15 16    ret=[17        OriginalTable, 18        gr.File(label="Download csv", value="temp.csv", file_types=[".csv"], visible=True), 19        gr.Number(label="終了行", value=row-1, maximum=row-1), 20        gr.Number(label="終了列", value=col-1, maximum=col-1),21        gr.Dropdown(label="対象列", choices=list(OriginalTable.columns))22    ]23    return ret24 25#--------------------------------------------26def trim_table(start_row, end_row, start_column, end_column):27    global OriginalTable28 29    header = OriginalTable.columns[start_column:end_column+1]30    mat = OriginalTable.to_numpy()31    newMat = mat[start_row:end_row+1, start_column:end_column+1]32    ret = pd.DataFrame(newMat, index=None, columns=header)33    ret.to_csv("temp.csv", index=False)34 35    return ret, gr.File(label="Download csv", value="temp.csv", file_types=[".csv"], visible=True), gr.Dropdown(label="対象列", choices=list(header))36 37#--------------------------------------------38def trim_str(str, char, vector, offset):39    index = str.find(char) + offset40    if "前" == vector:41        return str[:index]42    else:43        return str[index:]44 45#--------------------------------------------46def trim_column(table, edit_column, trim_char, trim_index, trim_vector, trim_offset):47    if trim_index != 0:48        if "前" == trim_vector:49            table[edit_column] = table[edit_column].apply(lambda x: x[:trim_index+trim_offset])50        else:51            table[edit_column] = table[edit_column].apply(lambda x: x[trim_index+trim_offset:])52    else:53        table[edit_column] = table[edit_column].apply(lambda x: trim_str(x, trim_char, trim_vector, trim_offset))54 55    table.to_csv("temp.csv", index=False)56    return table, gr.File(label="Download csv", value="temp.csv", file_types=[".csv"], visible=True)57 58#--------------------------------------------59def replace_column(table, edit_column, replace_target, replace_new):60    table[edit_column] = table[edit_column].apply(lambda x: x.replace(replace_target, replace_new))61 62    table.to_csv("temp.csv", index=False)63    return table, gr.File(label="Download csv", value="temp.csv", file_types=[".csv"], visible=True)64 65 66with gr.Blocks(theme=gr.themes.Ocean()) as grApp:67    gr.Markdown("""68    <div style="text-align:center">69        <h1>Web Table Scraper</h1>70    </div>71    """)72 73    with gr.Row():74        with gr.Column(scale=1):75            url = gr.Textbox(label="URL")76            encode = gr.Textbox(label="文字コード", value="utf-8")77            header = gr.Number(label="Header", value=0, minimum=0)78            table_index = gr.Number(label="Table index", value=0, minimum=0)79            start_button = gr.Button(value="Read", variant='primary')80        with gr.Column(scale=2):81            table = gr.DataFrame(label="Table", )82            83            with gr.Column():84                with gr.Accordion(label="テーブル編集", open=False):85                    start_row = gr.Number(label="開始行", value=0, minimum=0)86                    end_row = gr.Number(label="終了行", value=1)87                    start_column = gr.Number(label="開始列", value=0, minimum=0)88                    end_column = gr.Number(label="終了列", value=1)89                    rebuild_table = gr.Button(value="再構成", variant='primary')90            91            with gr.Column():92                with gr.Accordion(label="列編集", open=False):93                    edit_column = gr.Dropdown(label="対象列", choices=[])94                    with gr.Accordion(label="トリミング", open=True):95                        trim_char = gr.Textbox(label="指定文字", value=None, max_length=1)96                        trim_index = gr.Number(label="指定インデックス", value=0, minimum=0)97                        trim_vector = gr.Radio(label="指定方向", choices=["前", "後"])98                        trim_offset = gr.Number(label="オフセット", value=0, minimum=0)99                        trim_button = gr.Button(value="トリミング", variant='primary')100                    with gr.Accordion(label="置き換え", open=True):101                        replace_target = gr.Textbox(label="指定文字列")102                        replace_new = gr.Textbox(label="新規文字列")103                        replace_button = gr.Button(value="置き換え", variant='primary')104            download = gr.File(visible=False)105 106    start_button.click(107        fn=scrape,108        inputs=[url, header, encode, table_index],109        outputs=[table, download, end_row, end_column, edit_column],110    )111 112    rebuild_table.click(113        fn=trim_table,114        inputs=[start_row, end_row, start_column, end_column],115        outputs=[table, download, edit_column],116    )117 118    trim_button.click(119        fn=trim_column,120        inputs=[table, edit_column, trim_char, trim_index, trim_vector, trim_offset],121        outputs=[table, download],122    )123 124    replace_button.click(125        fn=replace_column,126        inputs=[table, edit_column, replace_target, replace_new],127        outputs=[table, download],128    )129 130grApp.title = "Web Table Scraper"131 132if __name__ == '__main__':133    grApp.launch()