RyanTietjen/Paper-Fragmentation

1"""
2Sep 2024
3Ryan Tietjen
4Contains helper functions to process user input for the demo
5"""
6import pandas as pd
7
8def split_abstract(abstract):
9    results = []
10
11    lines = abstract.split("\n")
12    for i, line in enumerate(lines):
13        entry = {
14                "target": 0,
15                "text": line.lower(),
16                "line_number": i + 1,
17                "total_lines": len(lines)
18                }
19        results.append(entry)
20    return results
21
22def split_abstract_original(abstract):
23    results = []
24
25    lines = abstract.split("\n")
26    for i, line in enumerate(lines):
27        entry = {
28                "target": 0,
29                "text": line,
30                "line_number": i + 1,
31                "total_lines": len(lines)
32                }
33        results.append(entry)
34    return results
35
36def split_sentences_by_characters(corpus):
37    return [" ".join(sentence) for sentence in corpus]
38
39def encode_labels(*datasets):
40    """
41    Encode labels for multiple datasets using a unified label mapping.
42    
43    Args:
44    *datasets: Arbitrary number of array-like structures containing labels.
45    
46    Returns:
47    tuple: Encoded labels as numpy arrays for each dataset.
48    """
49    # Collect all labels from all datasets into a single list
50    all_labels = pd.concat([pd.Series(data) for data in datasets])
51    
52    # Get unique labels and sort them to ensure consistency
53    unique_labels = pd.unique(all_labels)
54    unique_labels.sort()
55    
56    # Create mapping from labels to integers
57    label_to_index = {label: idx for idx, label in enumerate(unique_labels)}
58    
59    # Function to encode a single dataset
60    def encode_single_dataset(dataset, mapping):
61        return pd.Series(dataset).map(mapping).to_numpy()
62    
63    # Encode all datasets using the mapping
64    encoded_datasets = tuple(encode_single_dataset(dataset, label_to_index) for dataset in datasets)
65    
66    # Return only the encoded datasets
67    return encoded_datasets