latent-lab/got-activations-qwen2.5-0.5b

from lmprobe import pull_dataset, load_activation_dataset

# Option 1: Pull into local cache (enables probe training without re-extraction)
pull_dataset("latent-lab/got-activations-qwen2.5-0.5b")

# Option 2: Load tensors directly
tensors, info = load_activation_dataset("latent-lab/got-activations-qwen2.5-0.5b")
# tensors["hidden.layer_16"].shape => (N, hidden_dim)

import json
import pyarrow.parquet as pq
from safetensors import safe_open

# 1. Read the Parquet index
index = pq.read_table("index/train-00000-of-00001.parquet").to_pandas()
print(index.columns)  # text, label, num_tokens, shard_index, row_offset

# 2. Read tensor metadata
with open("lmprobe_info.json") as f:
    info = json.load(f)
print(list(info["tensors"].keys()))  # e.g. ["hidden_layers", "logits_topk"]

# 3. Load a shard — per-layer files: hidden_layer{L:03d}_shard{S:03d}.safetensors
with safe_open("tensors/hidden_layer000_shard000.safetensors", framework="pt") as f:
    print(f.keys())  # e.g. ["hidden.layer_0"]
    layer_0 = f.get_tensor("hidden.layer_0")

# 4. Map prompt index -> shard row
row = index.iloc[42]
tok_off, num_tok = row["token_offset"], row["num_tokens"]
# Slice full-sequence activations for this prompt
prompt_acts = layer_0[tok_off : tok_off + num_tok]  # (num_tokens, hidden_dim)

from datasets import load_dataset

# Shows prompt text + labels in Dataset Viewer
ds = load_dataset("latent-lab/got-activations-qwen2.5-0.5b")
print(ds["train"][0])  # {"text": "...", "label": ..., ...}