d221/Linkedin_Content_Generator

1import json2from llm_helper import llm3from langchain_core.prompts import PromptTemplate4from langchain_core.output_parsers import JsonOutputParser5from langchain_core.exceptions import OutputParserException6 7 8def process_posts(raw_file_path, processed_file_path=None):9    with open(raw_file_path, encoding='utf-8') as file:10        posts = json.load(file)11        enriched_posts = []12        for post in posts:13            metadata = extract_metadata(post['text'])14            post_with_metadata = post | metadata15            enriched_posts.append(post_with_metadata)16 17    unified_tags = get_unified_tags(enriched_posts)18    for post in enriched_posts:19        current_tags = post['tags']20        new_tags = {unified_tags[tag] for tag in current_tags}21        post['tags'] = list(new_tags)22 23    with open(processed_file_path, encoding='utf-8', mode="w") as outfile:24        json.dump(enriched_posts, outfile, indent=4)25 26 27def extract_metadata(post):28    template = '''29    You are given a LinkedIn post. You need to extract number of lines, language of the post and tags.30    1. Return a valid JSON. No preamble. 31    2. JSON object should have exactly three keys: line_count, language and tags. 32    3. tags is an array of text tags. Extract maximum two tags.33    4. Language should be English or Hinglish (Hinglish means hindi + english)34    35    Here is the actual post on which you need to perform this task:  36    {post}37    '''38 39    pt = PromptTemplate.from_template(template)40    chain = pt | llm41    response = chain.invoke(input={"post": post})42 43    try:44        json_parser = JsonOutputParser()45        res = json_parser.parse(response.content)46    except OutputParserException:47        raise OutputParserException("Context too big. Unable to parse jobs.")48    return res49 50 51def get_unified_tags(posts_with_metadata):52    unique_tags = set()53    # Loop through each post and extract the tags54    for post in posts_with_metadata:55        unique_tags.update(post['tags'])  # Add the tags to the set56 57    unique_tags_list = ','.join(unique_tags)58 59    template = '''I will give you a list of tags. You need to unify tags with the following requirements,60    1. Tags are unified and merged to create a shorter list. 61       Example 1: "Jobseekers", "Job Hunting" can be all merged into a single tag "Job Search". 62       Example 2: "Motivation", "Inspiration", "Drive" can be mapped to "Motivation"63       Example 3: "Personal Growth", "Personal Development", "Self Improvement" can be mapped to "Self Improvement"64       Example 4: "Scam Alert", "Job Scam" etc. can be mapped to "Scams"65    2. Each tag should be follow title case convention. example: "Motivation", "Job Search"66    3. Output should be a JSON object, No preamble67    3. Output should have mapping of original tag and the unified tag. 68       For example: {{"Jobseekers": "Job Search",  "Job Hunting": "Job Search", "Motivation": "Motivation}}69    70    Here is the list of tags: 71    {tags}72    '''73    pt = PromptTemplate.from_template(template)74    chain = pt | llm75    response = chain.invoke(input={"tags": str(unique_tags_list)})76    try:77        json_parser = JsonOutputParser()78        res = json_parser.parse(response.content)79    except OutputParserException:80        raise OutputParserException("Context too big. Unable to parse jobs.")81    return res82 83 84if __name__ == "__main__":85    process_posts("raw_posts.json", "processed_posts.json")