BERT를 이용한 빈 칸 채우기

!pip install transformers kobert-transformers Korpora

from Korpora import KoreanChatbotKorpus
data = KoreanChatbotKorpus()

from kobert_transformers import get_tokenizer

tokenizer = get_tokenizer()

from transformers import TFBertForMaskedLM

model = TFBertForMaskedLM.from_pretrained('monologg/kobert', from_pt=True)

from transformers import FillMaskPipeline

pipeline = FillMaskPipeline(model=model, tokenizer=tokenizer)

tokenizer.mask_token

pipeline('[MASK] 하루!')

from sklearn.model_selection import train_test_split

train_data, test_data = train_test_split(data.train, test_size=0.2, random_state=1234)

train_texts = [item.text for item in train_data]
test_texts = [item.text for item in test_data]

train_encodings = tokenizer(train_texts, padding=True, truncation=True)
test_encodings = tokenizer(test_texts, padding=True, truncation=True)

import random
import numpy as np

train_labels = []
for input_ids in train_encodings['input_ids']:
    n = input_ids.index(tokenizer.sep_token_id)
    i = random.randint(1, n-1)
    labels = np.ones_like(input_ids) * -100
    labels[i] = input_ids[i]
    input_ids[i] = tokenizer.mask_token_id
    train_labels.append(labels)

test_labels = []
for input_ids in test_encodings['input_ids']:
    n = input_ids.index(tokenizer.sep_token_id)
    i = random.randint(1, n-1)
    labels = np.ones_like(input_ids) * -100
    labels[i] = input_ids[i]
    input_ids[i] = tokenizer.mask_token_id
    test_labels.append(labels)

import tensorflow as tf

train_dataset = tf.data.Dataset.from_tensor_slices((
    dict(train_encodings),
    train_labels
))

test_dataset = tf.data.Dataset.from_tensor_slices((
    dict(test_encodings),
    test_labels
))

optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
model.compile(optimizer=optimizer, loss=model.compute_loss, metrics=['accuracy'])

model.evaluate(test_dataset.batch(32), batch_size=32)

model.fit(train_dataset.batch(32), epochs=1, batch_size=32)

pipeline('[MASK] 하루!')

model.evaluate(test_dataset.batch(32), batch_size=32)