CordwainerSmith/GolemGuard

{
  "id": "String",                    // Unique identifier
  "source_text": "String",          // Original text with PII
  "masked_text": "String",          // Text with PII entities masked
  "locale": "String",              // Always "IL"
  "language": "String",            // Always "he"
  "split": "String",              // "train" or "test"
  "privacy_mask": [               // List of PII entities with positions
    {
      "label": "String",         // Entity type
      "start": "Integer",       // Start position
      "end": "Integer",        // End position
      "value": "String",      // Original value
      "label_index": "Integer" // Index for multiple entities of same type
    }
  ],
  "span_labels": "List",        // Entity spans in [start, end, label] format
  "tokens": "List",            // Tokenized text
  "token_classes": "List",    // Token-level BIO tags
  "input_ids": "List",       // Model input token IDs
  "attention_mask": "List",  // Attention mask for padding
  "offset_mapping": "List", // Character offsets for tokens
  "template_type": "String" // Document type/template
}

{"id": "entry_000256", "source_text": "משתתף: דולב שם טוב\nתאריך לידה: 28.08.97\nכתובת: סחרוב דוד 158, קריית ים, 2676389\nאימייל: sagitbendor760@live.com\nטלפון: +972-58-2892208\n\nסיכום הפגישה: בפגישה זו נדונה חשיבות שיפור התקשורת הפנימית בין הצוותים. הוחלט לבצע סדנאות להכשרה נוספת בשבוע הבא.", "masked_text": "משתתף: [FIRST_NAME_1] [LAST_NAME_1]\nתאריך לידה: [DATE_1]\nכתובת: [STREET_1], [CITY_1], [POSTAL_CODE_1]\nאימייל: [EMAIL_1]\nטלפון: [PHONE_NUM_1]\n\nסיכום הפגישה: בפגישה זו נדונה חשיבות שיפור התקשורת הפנימית בין הצוותים. הוחלט לבצע סדנאות להכשרה נוספת בשבוע הבא.", "locale": "IL", "language": "he", "split": "train", "privacy_mask": [{"label": "FIRST_NAME", "start": 7, "end": 11, "value": "דולב", "label_index": 1}, {"label": "LAST_NAME", "start": 12, "end": 18, "value": "שם טוב", "label_index": 1}, {"label": "DATE", "start": 31, "end": 39, "value": "28.08.97", "label_index": 1}, {"label": "STREET", "start": 47, "end": 60, "value": "סחרוב דוד 158", "label_index": 1}, {"label": "CITY", "start": 62, "end": 70, "value": "קריית ים", "label_index": 1}, {"label": "POSTAL_CODE", "start": 72, "end": 79, "value": "2676389", "label_index": 1}, {"label": "EMAIL", "start": 88, "end": 111, "value": "sagitbendor760@live.com", "label_index": 1}, {"label": "PHONE_NUM", "start": 119, "end": 134, "value": "+972-58-2892208", "label_index": 1}], "span_labels": [[7, 11, "FIRST_NAME"], [12, 18, "LAST_NAME"], [31, 39, "DATE"], [47, 60, "STREET"], [62, 70, "CITY"], [72, 79, "POSTAL_CODE"], [88, 111, "EMAIL"], [119, 134, "PHONE_NUM"]], "tokens": ["<s>", "▁מ", "שתתף", ":", "▁דו", "לב", "▁שם", "▁טוב", "▁", "תאריך", "▁", "לידה", ":", "▁28.", "08.", "97", "▁כתובת", ":", "▁", "סחר", "וב", "▁דוד", "▁158", ",", "▁קרי", "ית", "▁", "ים", ",", "▁26", "76", "389", "▁אימייל", ":", "▁sa", "git", "ben", "dor", "760", "@", "live", ".", "com", "▁טלפון", ":", "▁+", "97", "2-", "58", "-28", "92", "208", "▁", "סיכום", "▁הפ", "גישה", ":", "▁בפ", "גישה", "▁זו", "▁", "נד", "ונה", "▁", "חשיבות", "▁", "שיפור", "▁התקשורת", "▁הפנימי", "ת", "▁בין", "▁הצוות", "ים", ".", "▁ה", "וח", "לט", "▁לבצע", "▁", "סדנאות", "▁ל", "הכשרה", "▁נוספת", "▁בשבוע", "▁הבא", ".", "</s>"], "token_classes": ["O", "O", "O", "O", "B-FIRST_NAME", "I-FIRST_NAME", "B-LAST_NAME", "I-LAST_NAME", "O", "O", "O", "O", "O", "B-DATE", "I-DATE", "I-DATE", "O", "O", "B-STREET", "I-STREET", "I-STREET", "I-STREET", "I-STREET", "O", "B-CITY", "I-CITY", "I-CITY", "I-CITY", "O", "B-POSTAL_CODE", "I-POSTAL_CODE", "I-POSTAL_CODE", "O", "O", "B-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "I-EMAIL", "O", "O", "B-PHONE_NUM", "I-PHONE_NUM", "I-PHONE_NUM", "I-PHONE_NUM", "I-PHONE_NUM", "I-PHONE_NUM", "I-PHONE_NUM", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"], "input_ids": [0, 874, 177834, 12, 18133, 22849, 16804, 19767, 6, 174081, 6, 119583, 12, 12511, 17331, 14773, 86185, 12, 6, 214797, 18340, 67512, 78373, 4, 46050, 2754, 6, 448, 4, 1381, 11835, 119861, 194921, 12, 57, 15769, 776, 1846, 110216, 981, 24056, 5, 277, 167153, 12, 997, 14773, 18504, 10057, 48590, 12231, 154782, 6, 186708, 18338, 58366, 12, 65412, 58366, 8248, 6, 16747, 15703, 6, 148055, 6, 154997, 185983, 214547, 609, 7501, 192819, 448, 5, 364, 15662, 21295, 107543, 6, 199930, 657, 226909, 122485, 132299, 54641, 5, 2], "attention_mask": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "offset_mapping": [[0, 0], [0, 1], [1, 5], [5, 6], [7, 9], [9, 11], [12, 14], [15, 18], [19, 20], [19, 24], [25, 26], [25, 29], [29, 30], [31, 34], [34, 37], [37, 39], [40, 45], [45, 46], [47, 48], [47, 50], [50, 52], [53, 56], [57, 60], [60, 61], [62, 65], [65, 67], [68, 69], [68, 70], [70, 71], [72, 74], [74, 76], [76, 79], [80, 86], [86, 87], [88, 90], [90, 93], [93, 96], [96, 99], [99, 102], [102, 103], [103, 107], [107, 108], [108, 111], [112, 117], [117, 118], [119, 120], [120, 122], [122, 124], [124, 126], [126, 129], [129, 131], [131, 134], [136, 137], [136, 141], [142, 144], [144, 148], [148, 149], [150, 152], [152, 156], [157, 159], [160, 161], [160, 162], [162, 165], [166, 167], [166, 172], [173, 174], [173, 178], [179, 186], [187, 193], [193, 194], [195, 198], [199, 204], [204, 206], [206, 207], [208, 209], [209, 211], [211, 213], [214, 218], [219, 220], [219, 225], [226, 227], [227, 232], [233, 238], [239, 244], [245, 248], [248, 249], [0, 0]], "template_type": "meeting_summary"}

from datasets import load_dataset

# Load dataset
dataset = load_dataset("GolemGuard")

# Example usage
sample = dataset['train'][0]
print(f"Original text: {sample['source_text']}")
print(f"Masked text: {sample['masked_text']}")

MIT License

Copyright (c) 2024 Liran Baba

Permission is hereby granted, free of charge, to any person obtaining a copy
of this dataset and associated documentation files (the "Dataset"), to deal
in the Dataset without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Dataset, and to permit persons to whom the Dataset is
furnished to do so, subject to the following conditions:

1. The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Dataset.

2. Any academic or professional work that uses this Dataset must include an 
appropriate citation as specified below.

THE DATASET IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE DATASET OR THE USE OR OTHER DEALINGS IN THE
DATASET.

GolemGuard Dataset by Liran Baba (https://huggingface.co/datasets/CordwainerSmith/GolemGuard)

Baba, L. (2024). GolemGuard: A Professional Hebrew PII Detection Dataset. 
Retrieved from https://huggingface.co/datasets/CordwainerSmith/GolemGuard

Related model: GolemPII-xlm-roberta-v1 (https://huggingface.co/CordwainerSmith/GolemPII-xlm-roberta-v1)

"""
This code uses the GolemGuard dataset by Liran Baba
(https://huggingface.co/datasets/CordwainerSmith/GolemGuard)
"""

from datasets import load_dataset
dataset = load_dataset("GolemGuard")

dataset_info:
  - name: GolemGuard
    author: Liran Baba
    url: https://huggingface.co/datasets/CordwainerSmith/GolemGuard
    year: 2024