thivy/norwegian-ner-combined

GPE_ORG → ORG  (Geo-political organizations)
GPE_LOC → LOC  (Geo-political locations)
PROD    → O    (Dropped: Products)
DRV     → O    (Dropped: Derived terms)
EVT     → O    (Dropped: Events)

Kept:
- PER (Person)
- ORG (Organization)
- LOC (Location)
- MISC (Miscellaneous)

0: O       (Outside any entity)
1: B-LOC   (Beginning of Location)
2: B-MISC  (Beginning of Miscellaneous)
3: B-ORG   (Beginning of Organization)
4: B-PER   (Beginning of Person)
5: I-LOC   (Inside Location)
6: I-MISC  (Inside Miscellaneous)
7: I-ORG   (Inside Organization)
8: I-PER   (Inside Person)

from datasets import load_dataset

# Load the dataset
dataset = load_dataset("thivy/norwegian-ner-combined")

# Access splits
train_data = dataset["train"]
val_data = dataset["validation"]
test_data = dataset["test"]

# Example
print(train_data[0])

{
    'id': 'train_0',
    'tokens': ['Oslo', 'er', 'hovedstaden', 'i', 'Norge', '.'],
    'ner_tags': [1, 0, 0, 0, 1, 0]  # B-LOC, O, O, O, B-LOC, O
}

from transformers import AutoTokenizer, AutoModelForTokenClassification, TrainingArguments, Trainer

# Load model and tokenizer
model_name = "NbAiLab/nb-bert-base"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForTokenClassification.from_pretrained(
    model_name, 
    num_labels=9  # 9 labels in IOB2 format
)

# Load dataset
dataset = load_dataset("thivy/norwegian-ner-combined")

# Tokenize and align labels
def tokenize_and_align_labels(examples):
    tokenized_inputs = tokenizer(
        examples["tokens"],
        truncation=True,
        is_split_into_words=True
    )
    
    labels = []
    for i, label in enumerate(examples["ner_tags"]):
        word_ids = tokenized_inputs.word_ids(batch_index=i)
        label_ids = []
        for word_idx in word_ids:
            if word_idx is None:
                label_ids.append(-100)
            else:
                label_ids.append(label[word_idx])
        labels.append(label_ids)
    
    tokenized_inputs["labels"] = labels
    return tokenized_inputs

tokenized_datasets = dataset.map(tokenize_and_align_labels, batched=True)

# Train
training_args = TrainingArguments(
    output_dir="./norwegian-ner-model",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    num_train_epochs=3,
    per_device_train_batch_size=8,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
)

trainer.train()

Min tokens: 1
Max tokens: 512
Mean tokens: 15.2
Median tokens: 12

@inproceedings{jorgensen-etal-2020-norne,
    title = "{N}or{NE}: Annotating Named Entities for {N}orwegian",
    author = "J{\o}rgensen, Fredrik  and
      Aasen, Tobias  and
      R{\o}nningstad, Anne-Katrine  and
      Indreb{\o}, Kristin Hagen  and
      Lyse, Gunn Inger",
    booktitle = "Proceedings of the 12th Language Resources and Evaluation Conference",
    month = may,
    year = "2020",
    address = "Marseille, France",
    publisher = "European Language Resources Association",
    pages = "4547--4556",
}

@inproceedings{pan-etal-2017-cross,
    title = "Cross-lingual Name Tagging and Linking for 282 Languages",
    author = "Pan, Xiaoman  and
      Zhang, Boliang  and
      May, Jonathan  and
      Nothman, Joel  and
      Knight, Kevin  and
      Ji, Heng",
    booktitle = "Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
    year = "2017",
}

@misc{kummervold2021operationalizing,
    title={Operationalizing a National Digital Library: The Case for a Norwegian Transformer Model},
    author={Per E Kummervold and Javier de la Rosa and Freddy Wetjen and Svein Arne Brygfjeld},
    year={2021},
    eprint={2104.09617},
    archivePrefix={arXiv},
    primaryClass={cs.CL}
}