Hash101/Noncoding_variant_scoring
0
1import gradio as gr2import subprocess3import tempfile4import os5import shutil6import pandas as pd7from huggingface_hub import snapshot_download8 9# ============================================================10# Download data once11# ============================================================12DATA_DIR = "/app/data"13 14def download_data_if_needed():15 os.makedirs(DATA_DIR, exist_ok=True)16 existing = [f for f in os.listdir(DATA_DIR) if not f.startswith(".")]17 if not existing:18 print("Downloading data files from HF Dataset repo...", flush=True)19 snapshot_download(20 repo_id="Hash101/Noncod_dataset_v2",21 repo_type="dataset",22 local_dir=DATA_DIR23 )24 print("Data download complete.", flush=True)25 else:26 print(f"Data already present ({len(existing)} files), skipping download.", flush=True)27 28download_data_if_needed()29 30# ============================================================31# Validation32# ============================================================33def validate_variants(lines):34 errors = []35 for i, line in enumerate(lines, 1):36 parts = line.strip().split("\t")37 if len(parts) < 4:38 errors.append(f"Line {i}: expected 4 columns, got {len(parts)}")39 return errors40 41# ============================================================42# Find output43# ============================================================44def find_output_file(output_dir):45 all_files = []46 for root, dirs, files in os.walk(output_dir):47 for fname in files:48 if fname.endswith((".tsv", ".csv", ".txt")):49 all_files.append(os.path.join(root, fname))50 51 if not all_files:52 return None53 54 final_files = [f for f in all_files if "final" in os.path.basename(f).lower()]55 if final_files:56 return final_files[0]57 58 return max(all_files, key=os.path.getmtime)59 60# ============================================================61# PIPELINE (STREAMING VERSION)62# ============================================================63def run_pipeline(variant_text, tsv_file):64 65 yield "🚀 Starting pipeline...", None, None66 67 # -------- INPUT --------68 if tsv_file is not None:69 with open(tsv_file, "r") as f:70 raw_input = f.read()71 elif variant_text and variant_text.strip():72 raw_input = variant_text.strip()73 else:74 yield "❌ No input provided.", None, None75 return76 77 lines = [78 l for l in raw_input.strip().splitlines()79 if l.strip() and not l.upper().startswith("CHROM")80 ]81 82 if len(lines) == 0:83 yield "❌ No variants found.", None, None84 return85 86 if len(lines) > 10:87 yield f"❌ Max 10 variants allowed. Got {len(lines)}.", None, None88 return89 90 errors = validate_variants(lines)91 if errors:92 yield "❌ Format errors:\n" + "\n".join(errors), None, None93 return94 95 with tempfile.TemporaryDirectory() as tmpdir:96 97 input_path = os.path.join(tmpdir, "input.tsv")98 with open(input_path, "w") as f:99 f.write("\n".join(lines) + "\n")100 101 output_dir = os.path.join(tmpdir, "output")102 os.makedirs(output_dir)103 104 # -------- COMMAND --------105 cmd = [106 "nextflow", "run", "/app/main.nf",107 "--input_vcf", input_path,108 "--outdir", output_dir,109 "--output_dir", output_dir,110 111 "-params-file", "/app/params.yml",112 113 "-c", "/app/nextflow.config",114 "-work-dir", "/tmp/nextflow-work",115 "-ansi-log", "false",116 117 # 🔥 NEW (debug + tracking)118 "-with-trace",119 "-with-report",120 "-with-timeline",121 "-resume",122 ]123 124 yield "🚀 Running Nextflow...", None, None125 126 # -------- RUN PROCESS (STREAMING) --------127 process = subprocess.Popen(128 cmd,129 stdout=subprocess.PIPE,130 stderr=subprocess.STDOUT,131 text=True,132 cwd="/app"133 )134 135 logs = []136 current_step = "Initializing..."137 138 import select139 import time140 141 last_heartbeat = time.time() # ✅ ADD: track heartbeat time142 143 while True:144 reads, _, _ = select.select([process.stdout], [], [], 1)145 146 if reads:147 line = process.stdout.readline()148 if not line and process.poll() is not None: # ✅ only break when process truly done149 break150 151 print(line, flush=True)152 logs.append(line)153 154 if "process >" in line:155 try:156 step = line.split("process >")[1].strip()157 current_step = step158 last_heartbeat = time.time() # ✅ reset on real progress159 except:160 pass161 162 yield (163 f"🔄 Running: {current_step}\n\n"164 f"--- Recent logs ---\n"165 + "".join(logs[-15:]),166 None,167 None168 )169 else:170 # ✅ CHANGED: yield heartbeat every 20s to keep WebSocket alive171 elapsed = int(time.time() - last_heartbeat)172 yield (173 f"⏳ Still running: {current_step}... ({elapsed}s)\n\n"174 f"--- Recent logs ---\n"175 + "".join(logs[-15:]), # ✅ keep showing logs, not just spinner176 None,177 None178 )179 180 process.wait()181 # -------- FAILURE --------182 if process.returncode != 0:183 yield (184 "❌ Pipeline failed\n\n" + "".join(logs[-50:]),185 None,186 None187 )188 return189 190 # -------- FIND OUTPUT --------191 result_file = find_output_file(output_dir)192 193 if not result_file:194 yield "⚠️ No output file found.", None, None195 return196 197 # -------- READ OUTPUT --------198 try:199 df = pd.read_csv(result_file, sep="\t")200 except:201 df = None202 203 # 🔥 ALWAYS prepare debug zip204 debug_dir = os.path.join(output_dir, "full_workdir")205 debug_zip = os.path.join(output_dir, "debug_workdir.zip")206 207 if os.path.exists(debug_dir):208 shutil.make_archive(debug_zip.replace(".zip", ""), 'zip', debug_dir)209 download_file = debug_zip210 else:211 download_file = result_file212 213 # ✅ Final output214 if df is not None:215 yield (216 f"✅ Done! {len(df)} variants scored.",217 df,218 download_file219 )220 else:221 with open(result_file) as f:222 content = f.read()223 224 yield (225 f"✅ Done (raw output):\n\n{content[:2000]}",226 None,227 download_file228 )229 230# ============================================================231# UI232# ============================================================233with gr.Blocks(title="Noncoding Variant Scorer") as demo:234 235 gr.Markdown("""236# 🧬 Noncoding Variant Scorer237 238Max 10 variants.239 240Format:241chr1 925952 C T242""")243 244 with gr.Row():245 with gr.Column():246 variant_text = gr.Textbox(lines=10)247 tsv_upload = gr.File()248 submit_btn = gr.Button("🚀 Run")249 250 with gr.Column():251 status = gr.Textbox(lines=15)252 table = gr.Dataframe()253 file_out = gr.File()254 255 submit_btn.click(256 fn=run_pipeline,257 inputs=[variant_text, tsv_upload],258 outputs=[status, table, file_out]259 )260 261# 🔥 REQUIRED for streaming262demo.queue()263 264demo.launch(server_name="0.0.0.0", server_port=7860)