Nymbo/JSON-Crawl
0
1import gradio as gr2import requests3import bs44 5def sort_doc(in_list,steps_in=8,control=None):6 control_json={'control':'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ','char':'','leng':62}7 text=str(in_list)8 9######################################## 10 sen_list=in_list11###################################### 12 key_cnt=len(in_list)13 print(key_cnt)14 control_char=list(control_json['control'])15 char_len=len(control_char) 16 if not steps_in:17 n_cnt=018 nx=key_cnt19 while True:20 if nx >= 1:21 n_cnt+=122 nx = nx/char_len23 else:24 print("#######")25 print(n_cnt)26 print(nx)27 print("#######")28 steps=n_cnt29 break 30 if steps_in:31 steps=steps_in32 33 if control:34 control_len=control_json['leng']-steps35 control_char_val=list(control_json['control'][:control_len])36 control_val=list(control_json['control'][control_len:])37 val_len=len(control_val)38 39 json_out={}40 noun_list={}41 step_list=[]42 43 big_cnt=044 cnt=045 go=True46 47 48 step_cont_box=[]49 for ii in range(steps):50 print(ii)51 step_cont_box.append(0)52 #print (step_cont_box)53 mod=054 pos=len(step_cont_box)-155 56 57 58 if go:59 for i, ea in enumerate(in_list):60 if go:61 if cnt > char_len-1:62 #print(step_cont_box)63 go1=True64 for ii,ev in enumerate(step_cont_box):65 if go:66 if ev >= char_len-1:67 step_cont_box[ii]=068 if go1==True:69 step_cont_box[ii-1]=step_cont_box[ii-1]+170 go1=False71 cnt=172 else:73 step_cont_box[pos]=cnt74 cnt+=175 print(step_cont_box)76 out_js=""77 for iii,j in enumerate(step_cont_box):78 print(j)79 out_js = out_js+control_char[j]80 sen_obj=in_list[i] 81 #sen_obj=proc_sen(sen_list,i)82 83 #json_out[out_js]={'nouns':ea}84 json_out[out_js]=sen_obj85 print ("#################")86 print (out_js)87 print (sen_obj)88 print ("#################")89 90 big_cnt+=191 if big_cnt==key_cnt:92 print("DONE")93 go=False94 #noun_list=proc_nouns(json_out)95 return json_out96 97 98link_box = []99 100def link_find(url):101 out = []102 source = requests.get(url)103 if source.status_code ==200:104 print("YES")105 #soup = bs4.BeautifulSoup(source.content,'lxml')106 soup = bs4.BeautifulSoup(source.content,'html.parser')107 108 rawp=(f'RAW TEXT RETURNED: {soup.text}')109 cnt=0110 cnt+=len(rawp)111 rawt=soup.text112 #out.append(rawp)113 #out.append("HTML fragments: ")114 node1 = {"URL":url,"TITLE":soup.title,"STRING":soup.description,"TEXT":rawt,"LINKS":[],"TREE":[]}115 node2 = {"URL":url,"LINK_KEY":[],"LINKS":[],"TREE":[]}116 117 q=("a","p","span","content","article")118 for p in soup.find_all("a"):119 url0=p.get('href')120 if url0.startswith("//"):121 print(url0)122 uri1=url.split("//")[0]123 #uri2=url.split("//")[1]124 #uri3=uri2.split("/")[0]125 #uri=f'{uri1}//{uri3}'126 uri=f'{uri1}{url0}'127 print(uri)128 elif url0.startswith("/") and not url0.startswith("//"):129 uri1=url.split("//")[0]130 uri2=url.split("//")[1]131 uri3=uri2.split("/")[0]132 uri=f'{uri1}//{uri3}'133 uri=f'{uri}{url0}'134 print(uri) 135 else:136 uri=url0137 node1['LINKS'].append(uri) 138 node1['TREE'].append({"URL":uri,"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","LINKS":[],"TREE":[]}) 139 node2['TREE'].append({"URL":uri,"LINKS":[],"TREE":[]}) 140 node2['LINKS'].append(uri) 141 #node2['LINK_KEY'].append(uri_key) 142 link_box.append(uri)143 #out.append({"URL":p.get('href'),"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","TREE":[]})144 145 else:146 print("NO")147 pass148 149 return node1,node2150#https://huggingface.co/spaces/Omnibus/crawl151 152def sitemap(url,level):153 uri=""154 uri0=""155 if url != "" and url != None: 156 link1,link2=link_find(url)157 if level >=2:158 for i,ea in enumerate(link1['TREE']):159 print(ea)160 try:161 #if not ea['URL'].startswith("http"):162 # uri1=url.split("//")[0]163 # uri2=url.split("//")[1]164 # uri3=uri2.split("/")[0]165 # uri=f'{uri1}//{uri3}'166 # print(uri)167 out_list1,out_list2=link_find(f"{uri}{ea['URL']}")168 link1['TREE'][i]=out_list1169 link2['TREE'][i]=out_list2170 #link1['TREE'].append(out_list)171 172 if level>=3:173 for n,na in enumerate(link1['TREE'][i]['TREE']):174 print(na)175 try:176 #if not na['URL'].startswith("http"):177 # uri11=url.split("//")[0]178 # uri22=url.split("//")[1]179 # uri33=uri22.split("/")[0]180 # uri0=f'{uri11}//{uri33}'181 # print(uri0)182 out_list1,out_list2=link_find(f"{uri0}{na['URL']}")183 link1['TREE'][i]['TREE'][n]=out_list1184 link2['TREE'][i]['TREE'][n]=out_list2185 #link1['TREE'][i]['TREE'].append(out_list1)186 except Exception as e:187 print (e)188 except Exception as e:189 print (e)190 uri_key=sort_doc(link_box)191 192 return link1,link2,uri_key193 194 195 196def sitemap_OG(url,level):197 uri=""198 if url != "" and url != None: 199 link1=link_find(url)200 if level >=2:201 for i,ea in enumerate(link1):202 print(ea)203 try:204 if not ea['URL'].startswith("http"):205 uri1=url.split("//")[0]206 uri2=url.split("//")[1]207 uri3=uri2.split("/")[0]208 uri=f'{uri1}//{uri3}'209 print(uri)210 out_list=link_find(f"{uri}{ea['URL']}")211 link1[i]['TREE']=out_list212 if level>=3:213 for n,na in enumerate(link1[i]['TREE']):214 print(na)215 try:216 if not na['URL'].startswith("http"):217 uri11=url.split("//")[0]218 uri22=url.split("//")[1]219 uri33=uri22.split("/")[0]220 uri0=f'{uri11}//{uri33}'221 print(uri0)222 out_list1=link_find(f"{uri0}{na['URL']}")223 link1[i]['TREE'][n]['TREE']=out_list1224 except Exception as e:225 print (e)226 except Exception as e:227 print (e)228 return link1229with gr.Blocks(theme="Nymbo/Nymbo_Theme") as app:230 with gr.Row():231 with gr.Column(scale=3):232 with gr.Row():233 inp=gr.Textbox(label="URL")234 level=gr.Slider(minimum=1,maximum=2,step=1,value=1)235 btn=gr.Button()236 key_json=gr.JSON()237 outp=gr.JSON()238 with gr.Column(scale=1):239 outmap=gr.JSON()240 btn.click(sitemap,[inp,level],[outp,outmap,key_json])241app.launch()