CyberPeace-Institute/Cybersecurity-Knowledge-Graph-Extraction

1list_of_pos_tags = [2    "ADJ",3    "ADP",4    "ADV",5    "AUX",6    "CCONJ",7    "DET",8    "INTJ",9    "NOUN",10    "NUM",11    "PART",12    "PRON",13    "PROPN",14    "PUNCT",15    "SCONJ",16    "SYM",17    "VERB",18    "X"19]20 21realis_list = ["O",22    "Generic", 23    "Other", 24    "Actual"25]26 27 28event_args_list = ['O',29    'B-System',30    'I-System',31    'B-Organization',32    'B-Money',33    'I-Money',34    'B-Device',35    'B-Person',36    'I-Person',37    'B-Vulnerability',38    'I-Vulnerability',39    'B-Capabilities',40    'I-Capabilities',41    'I-Organization',42    'B-PaymentMethod',43    'I-PaymentMethod',44    'B-Data',45    'I-Data',46    'B-Number',47    'I-Number',48    'B-Malware',49    'I-Malware',50    'B-PII',51    'I-PII',52    'B-CVE',53    'I-CVE',54    'B-Purpose',55    'I-Purpose',56    'B-File',57    'I-File',58    'I-Device',59    'B-Time',60    'I-Time',61    'B-Software',62    'I-Software',63    'B-Patch',64    'I-Patch',65    'B-Version',66    'I-Version',67    'B-Website',68    'I-Website',69    'B-GPE',70    'I-GPE'71]72 73event_nugget_list = ['O',74 'B-Ransom',75 'I-Ransom',76 'B-DiscoverVulnerability',77 'I-DiscoverVulnerability',78 'B-PatchVulnerability',79 'I-PatchVulnerability',80 'B-Databreach',81 'I-Databreach',82 'B-Phishing',83 'I-Phishing'84]85 86arg_2_role = {87    "File" : ['Tool', 'Trusted-Entity'],88    "Person" : ['Victim', 'Attacker', 'Discoverer', 'Releaser', 'Trusted-Entity', 'Vulnerable_System_Owner'],89    "Capabilities" : ['Attack-Pattern', 'Capabilities', 'Issues-Addressed'],90    "Purpose" : ['Purpose'],91    "Time" : ['Time'],92    "PII" : ['Compromised-Data', 'Trusted-Entity'],93    "Data" : ['Compromised-Data', 'Trusted-Entity'],94    "Organization" : ['Victim', 'Releaser', 'Discoverer', 'Attacker', 'Vulnerable_System_Owner', 'Trusted-Entity'],95    "Patch" : ['Patch'],96    "Software" : ['Vulnerable_System', 'Victim', 'Trusted-Entity', 'Supported_Platform'],97    "Vulnerability" : ['Vulnerability'],98    "Version" : ['Patch-Number', 'Vulnerable_System_Version'],99    "Device" : ['Vulnerable_System', 'Victim', 'Supported_Platform'],100    "CVE" : ['CVE'],101    "Number" : ['Number-of-Data', 'Number-of-Victim'],102    "System" : ['Victim', 'Supported_Platform', 'Vulnerable_System', 'Trusted-Entity'],103    "Malware" : ['Tool'],104    "Money" : ['Price', 'Damage-Amount'],105    "PaymentMethod" : ['Payment-Method'],106    "GPE" : ['Place'],107    "Website" : ['Trusted-Entity', 'Tool', 'Vulnerable_System', 'Victim', 'Supported_Platform'],108}109 110def get_content(data):111    return data["content"]112 113def get_event_nugget(data):114    return [115        {"nugget" : event["nugget"], "type" : event["type"], "subtype" : event["subtype"], "realis" : event["realis"]}116        for hopper in data["cyberevent"]["hopper"] for event in hopper["events"]117    ]118def get_event_args(data):119    events = [event for hopper in data["cyberevent"]["hopper"] for event in hopper["events"]]120    args = []121    for event in events:122        if "argument" in event.keys():123            args.extend(event["argument"])124    return args125 126def get_idxs_from_text(text, text_tokenized):127    rest_text = text128    last_idx = 0129    result_dict = []130 131    for substring in text_tokenized:132        index = rest_text.find(substring)133        result_dict.append(134            {135                "word" : substring,136                "start_idx" : last_idx + index,137                "end_idx" : last_idx + index + len(substring)138            }139        )140        rest_text = rest_text[index + len(substring) : ]141        last_idx += index + len(substring)142    return result_dict143 144def get_entity_from_idx(start_idx, end_idx, event_nuggets):145    event_nuggets_idxs = [(nugget["nugget"]["startOffset"], nugget["nugget"]["endOffset"]) for nugget in event_nuggets]146    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):147        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):148            return "B-" + event_nuggets[idx]["subtype"]149        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):150            return "I-" + event_nuggets[idx]["subtype"]151    return "O"152 153def get_entity_and_realis_from_idx(start_idx, end_idx, event_nuggets):154    event_nuggets_idxs = [(nugget["nugget"]["startOffset"], nugget["nugget"]["endOffset"]) for nugget in event_nuggets]155    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):156        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):157            return "B-" + event_nuggets[idx]["subtype"], "B-" + event_nuggets[idx]["realis"]158        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):159            return "I-" + event_nuggets[idx]["subtype"], "I-" + event_nuggets[idx]["realis"]160    return "O", "O"161 162def get_args_entity_from_idx(start_idx, end_idx, event_args):163    event_nuggets_idxs = [(nugget["startOffset"], nugget["endOffset"]) for nugget in event_args]164    for idx, (nugget_start, nugget_end) in enumerate(event_nuggets_idxs):165        if (start_idx == nugget_start and end_idx == nugget_end) or (start_idx == nugget_start and end_idx <= nugget_end) or (start_idx == nugget_start and end_idx > nugget_end) or (end_idx == nugget_end and start_idx < nugget_start) or (start_idx <= nugget_start and end_idx <= nugget_end and end_idx > nugget_start):166            return "B-" + event_args[idx]["type"]167        elif (start_idx > nugget_start and end_idx <= nugget_end) or (start_idx > nugget_start and start_idx < nugget_end):168            return "I-" + event_args[idx]["type"]169    return "O"170 171def split_with_character(string, char):172    result = []173    start = 0174    for i, c in enumerate(string):175        if c == char:176            result.append(string[start:i])177            result.append(char)178            start = i + 1179    result.append(string[start:])180    return [x for x in result if x != '']181 182def extend_list_with_character(content_list, character):183    content_as_words = []184    for word in content_list:185        if character in word:186            split_list = split_with_character(word, character)187            content_as_words.extend(split_list)188        else:189            content_as_words.append(word)190    return content_as_words191 192def find_dict_by_overlap(list_of_dicts, key_value_pairs):193    for dictionary in list_of_dicts:194        if max(dictionary["start"], dictionary["end"]) >= min(key_value_pairs["start"], key_value_pairs["end"]) and max(key_value_pairs["start"], key_value_pairs["end"]) >= min(dictionary["start"], dictionary["end"]):195            return dictionary196    return None197