트랜스포머 코드 예시

import torch
import torch.nn as nn

class SelfAttention(nn.Module):
    def __init__(self, emb_size, heads):
        super(SelfAttention, self).__init__()
        self.emb_size = emb_size
        self.heads = heads
        self.head_dim = emb_size // heads

        assert (
            self.head_dim * heads == emb_size
        ), "Embedding size needs to be divisible by heads"

        self.values = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.keys = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.queries = nn.Linear(self.head_dim, self.head_dim, bias=False)
        self.fc_out = nn.Linear(heads * self.head_dim, emb_size)

    def forward(self, values, keys, queries, mask):
        # Split the embedding into 'heads' pieces
        N = queries.shape[0]
        value_len, key_len, query_len = values.shape[1], keys.shape[1], queries.shape[1]

        # Split the embedding into self.heads pieces
        values = values.reshape(N, value_len, self.heads, self.head_dim)
        keys = keys.reshape(N, key_len, self.heads, self.head_dim)
        queries = queries.reshape(N, query_len, self.heads, self.head_dim)

        values = self.values(values)
        keys = self.keys(keys)
        queries = self.queries(queries)

        # Attention Mechanism
        energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
        if mask is not None:
            energy = energy.masked_fill(mask == 0, float("-1e20"))

        attention = torch.softmax(energy / (self.emb_size ** (1 / 2)), dim=3)

        out = torch.einsum("nhql,nlhd->nqhd", [attention, values]).reshape(
            N, query_len, self.heads * self.head_dim
        )

        out = self.fc_out(out)
        return out

import torch
import torch.nn.functional as F

class SelfAttention(torch.nn.Module):
    def __init__(self, embed_size):
        super(SelfAttention, self).__init__()
        self.embed_size = embed_size
        self.queries = torch.nn.Linear(embed_size, embed_size)
        self.keys = torch.nn.Linear(embed_size, embed_size)
        self.values = torch.nn.Linear(embed_size, embed_size)

    def forward(self, x):
        # x: (batch_size, seq_length, embed_size)
        Q = self.queries(x)
        K = self.keys(x)
        V = self.values(x)
        
        # 어텐션 스코어 계산
        attention_weights = torch.matmul(Q, K.permute(0, 2, 1)) / self.embed_size ** 0.5
        
        # 소프트맥스를 통한 스코어 정규화
        attention = F.softmax(attention_weights, dim=-1)
        
        # 밸류와 어텐션 가중치를 결합
        out = torch.matmul(attention, V)
        return out

import torch
import math

class PositionalEncoding(torch.nn.Module):
    def __init__(self, d_model, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.encoding = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len).unsqueeze(1).float()
        div_term = torch.exp(torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model))
        
        self.encoding[:, 0::2] = torch.sin(position * div_term)
        self.encoding[:, 1::2] = torch.cos(position * div_term)
        self.encoding = self.encoding.unsqueeze(0)

    def forward(self, x):
        return x + self.encoding[:, :x.size(1)].detach()

# 예시 사용:
d_model = 512  # 임베딩 벡터의 차원
max_len = 100  # 시퀀스의 최대 길이

pos_encoding = PositionalEncoding(d_model, max_len)
x = torch.zeros(1, max_len, d_model)
x = pos_encoding(x)

print(x)

from transformers import BertTokenizer, BertModel
import torch

# Tokenizer와 모델 초기화
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

# 입력 텍스트
input_text = "Here is some text to encode"
input_tokens = tokenizer(input_text, return_tensors='pt')

# BERT로 텍스트 인코딩
with torch.no_grad():
    outputs = model(**input_tokens)
    hidden_states = outputs.last_hidden_state

from transformers import GPT2Tokenizer, GPT2Model
import torch

# Tokenizer와 모델 초기화
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2Model.from_pretrained('gpt2')

# 입력 텍스트
input_text = "Here is some example text"
input_tokens = tokenizer(input_text, return_tensors='pt')

# GPT-2로 텍스트 인코딩
with torch.no_grad():
    outputs = model(**input_tokens)
    hidden_states = outputs.last_hidden_state