aravagarwal/CodeCloakPII

1from anoninterface import MAX_LENGTH, analyzer, anonymizer2from anoninterface.redaction import generate_constant_id, generate_hash_id3from anoninterface.treeutils import syntax_aware_replace_with_lambda, visualize_tree_sitter_matches, TREE_SITTER_QUERY_NAME_TO_QUERY4 5from presidio_anonymizer import OperatorConfig  6from copy import deepcopy7 8 9def anonymize_string(text_input, anon_fn = generate_constant_id):10    old_copy = deepcopy(text_input)11    redacted = ""12    for elem in range(0, len(text_input), MAX_LENGTH):13        chunks = text_input[elem:min(elem+MAX_LENGTH, len(text_input))]14        results = analyzer.analyze(chunks, "en", entities=["PERSON", "EMAIL_ADDRESS", "URL"], score_threshold=0.60)15        anonymized_text = anonymizer.anonymize(chunks, analyzer_results=results,  operators={"DEFAULT": OperatorConfig("custom", {"lambda" : anon_fn})})16        redacted = redacted + anonymized_text.text17    return redacted18 19 20def analyze_string(text_input):21    old_copy = deepcopy(text_input)22    results = []23    for elem in range(0, len(text_input), MAX_LENGTH):    24        chunks = text_input[elem:min(elem+MAX_LENGTH, len(text_input))]25        results.extend(analyzer.analyze(chunks, "en", entities=["PERSON", "EMAIL_ADDRESS", "URL"], score_threshold=0.60))26    read_loc = 027    labelled_text = []28    for capture in results:29        capture_start, capture_end = capture.start, capture.end30        if capture_start >= read_loc:31            labelled_text.append((text_input[read_loc:capture_start], None))32            labelled_text.append((text_input[capture_start:capture_end], capture.entity_type))33        read_loc = capture_end34    return labelled_text if len(labelled_text) != 0 else [(text_input,None)]35 36 37