TheirStory/tokenizer

curl -X POST "https://theirstory-tokenizer.hf.space/tokenize" \
     -H "Content-Type: application/json" \
     -H "accept: application/json" \
     -d '{"text": "Hello world"}'

import requests

response = requests.post(
    "https://theirstory-tokenizer.hf.space/tokenize",
    json={"text": "Hello world"},
    headers={"accept": "application/json"}
)
tokens = response.json()
print(tokens)

const response = await fetch('https://theirstory-tokenizer.hf.space/tokenize', {
    method: 'POST',
    headers: {
        'Content-Type': 'application/json',
        'accept': 'application/json'
    },
    body: JSON.stringify({
        text: 'Hello world'
    })
});
const tokens = await response.json();
console.log(tokens);