1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

from tokenizers import Tokenizer

# (1) 选择一种分词模型，并创建实例
from tokenizers.models import BPE
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))

from tokenizers.trainers import BpeTrainer
trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])

# (2) 设置预分词器，这里按空白字符分割文本
from tokenizers.pre_tokenizers import Whitespace
tokenizer.pre_tokenizer = Whitespace()

# (3) 基于语料库，进行分词训练
copus_file = ["datasets/wikitext-103-raw-v1/wiki_train.csv"]
tokenizer.train(copus_file, trainer)

# (4) 保存并加载
tokenizer.save("datasets/wikitext-103-raw-v1/tokenizer-wiki.json")
tokenizer = Tokenizer.from_file("datasets/wikitext-103-raw-v1/tokenizer-wiki.json")
type(tokenizer)
# tokenizers.Tokenizer
# 与直接从预训练模型中提取的分词器类型不太一样
from transformers import AutoTokenizer
tokenizer2 = AutoTokenizer.from_pretrained("datasets/bert-base-uncased")
type(tokenizer2)
# transformers.models.bert.tokenization_bert_fast.BertTokenizerFast

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

## 单个词
print(tokenizer.token_to_id("[SEP]"))
# 2
print(tokenizer.id_to_token(1))
# [CLS]


## 一个文本句输入
output = tokenizer.encode("Hello, y'all! How are you 😁 ?")
print(output.tokens)
print(output.ids)
print(output.attention_mask)
# ['Hello', ',', 'y', "'", 'all', '!', 'How', 'are', 'you', '[UNK]', '?']
# [27195, 16, 93, 11, 5069, 5, 7929, 5084, 6191, 0, 35]
# [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]   

## 一个输入包含两个句子
output = tokenizer.encode("Hello, y'all!", "How are you 😁 ?")
print(output.tokens)
# ['Hello', ',', 'y', "'", 'all', '!', 'How', 'are', 'you', '[UNK]', '?']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

from tokenizers.processors import TemplateProcessing
tokenizer.post_processor = TemplateProcessing(
    # 单文本句处理
    single="[CLS] $A [SEP]",  
    # 一对文本句的处理
    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
    special_tokens=[
        ("[CLS]", tokenizer.token_to_id("[CLS]")),
        ("[SEP]", tokenizer.token_to_id("[SEP]")),
    ],
)

output = tokenizer.encode("Hello, y'all! How are you 😁 ?")

print(output.tokens)
# ['[CLS]', 'Hello', ',', 'y', "'", 'all', '!', 'How', 'are', 'you', '[UNK]', '?', '[SEP]']

print(output.type_ids)
# [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]

1
2
3
4
5
6
7

## 批处理
output = tokenizer.encode_batch(["Hello, y'all!", "How are you 😁 ?"])
# 返回list格式
output[0].tokens
# ['[CLS]', 'Hello', ',', 'y', "'", 'all', '!', '[SEP]']
output[1].tokens
# ['[CLS]', 'How', 'are', 'you', '[UNK]', '?', '[SEP]']

1
2
3
4
5
6
7

tokenizer.enable_padding(pad_id=3, pad_token="[PAD]")
output = tokenizer.encode_batch(["Hello, y'all!", "How are you 😁 ?"])

print(output[1].tokens)
# ['[CLS]', 'How', 'are', 'you', '[UNK]', '?', '[SEP]', '[PAD]']
print(output[1].attention_mask)
# [1, 1, 1, 1, 1, 1, 1, 0]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

from transformers import PreTrainedTokenizerFast

tokenizer3 = PreTrainedTokenizerFast(tokenizer_file="datasets/wikitext-103-raw-v1/tokenizer-wiki.json")
type(tokenizer3)
# transformers.tokenization_utils_fast.PreTrainedTokenizerFast

tokenizer3.convert_ids_to_tokens(3)

tokenizer3.cls_token = '[CLS]'
tokenizer3.pad_token = '[PAD]'
tokenizer3.sep_token = '[SEP]'
tokenizer3.sep_token_id

tokenizer3(
    ["Hello, y'all!", "How are you 😁 ?"],
    padding=True,
    truncation=True,
    max_length=10,
    return_tensors='pt'
)
# {'input_ids': tensor([[27195,    16,    93,    11,  5069,     5],
#         [ 7929,  5084,  6191,     0,    35,     3]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0],
#         [0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1],
#         [1, 1, 1, 1, 1, 0]])}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.trainers import WordLevelTrainer
from tokenizers.pre_tokenizers import Whitespace

# gene list
len(gene_names) #60694
gene_names[:5]
# ['bP-2171C21.3', 'bP-21264C1.2', 'ZZZ3', 'ZZEF1', 'ZYG11B']

# 初始化 WordLevel Tokenizer
tokenizer = Tokenizer(WordLevel())
# 使用空格预分词（其实应该用不到）
tokenizer.pre_tokenizer = Whitespace()

trainer = WordLevelTrainer(vocab_size=len(gene_names), special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
tokenizer.train_from_iterator(gene_names, trainer=trainer)

output = tokenizer.encode("BRCA1 TP53 EGFR")
print(output.tokens)
print(output.ids)
# ['BRCA1', 'TP53', 'EGFR']
# [17239, 46049, 5419]

output2 = tokenizer.encode(['APOE',"PTEN"], is_pretokenized=True)
print(output2.tokens)
print(output2.ids)
# ['APOE', 'PTEN']
# [16207, 37733]

out = tokenizer.encode_batch([["BRCA1 TP53 EGFR"],['APOE',"PTEN"]], is_pretokenized=True)