{"id":809,"date":"2021-01-23T23:03:16","date_gmt":"2021-01-23T14:03:16","guid":{"rendered":"http:\/\/cedartrees.co.kr\/?p=809"},"modified":"2021-04-03T19:07:18","modified_gmt":"2021-04-03T10:07:18","slug":"sequence2sequence-nlg","status":"publish","type":"post","link":"http:\/\/blog.cedartrees.co.kr\/index.php\/2021\/01\/23\/sequence2sequence-nlg\/","title":{"rendered":"Seq2Seq \ubb38\uc7a5\uc0dd\uc131"},"content":{"rendered":"\n

Sequence2Sequence \ubaa8\ub378\uc744 \ud65c\uc6a9\ud574\uc11c \ubb38\uc7a5\uc0dd\uc131\uc744 \uc218\ud589\ud558\ub294 \ud14c\uc2a4\ud2b8\ub97c \ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ud14c\uc2a4\ud2b8 \ud658\uacbd\uc740 Google Colab\uc758 GPU\ub97c \ud65c\uc6a9\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n

Google Drive\uc5d0 \uc5c5\ub85c\ub4dc\ub418\uc5b4 \uc788\ub294 text \ud30c\uc77c\uc744 \uc77d\uae30 \uc704\ud574\uc11c \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc784\ud3ec\ud2b8\ud569\ub2c8\ub2e4. \ud574\ub2f9 \ud30c\uc77c\uc744 \uc2e4\ud589\uc2dc\ud0a4\uba74 \uc544\ub798\uc640 \uac19\uc740 \uc774\ubbf8\uc9c0\uac00 \ud45c\uc2dc\ub429\ub2c8\ub2e4.<\/p>\n\n\n\n

\"\"<\/figure>\n\n\n\n

\ud574\ub2f9 \ub9c1\ud06c\ub97c \ud074\ub9ad\ud558\uace0 \ub4e4\uc5b4\uac00\uba74 \ucf54\ub4dc \uac12\uc774 \ub098\uc624\ub294\ub370 \ucf54\ub4dc\uac12\uc744 \ubcf5\uc0ac\ud574\uc11c \uc785\ub825\ud558\uba74 \uad6c\uae00 \ub4dc\ub77c\uc774\ube0c\uac00 \ub9c8\uc6b4\ud2b8 \ub418\uace0 \uad6c\uae00 \ub4dc\ub77c\uc774\ube0c\uc5d0 \uc800\uc7a5\ub41c \ud30c\uc77c\ub4e4\uc744 \uc0ac\uc6a9\ud560 \uc218 \uc788\uac8c\ub429\ub2c8\ub2e4. <\/p>\n\n\n\n

from google.colab import drive\ndrive.mount('\/content\/gdrive')<\/pre>\n\n\n\n
\uc815\uc0c1\uc801\uc73c\ub85c \ub9c8\uc6b4\ud2b8 \ub418\uba74 “Mounted at \/content\/gdrive”\uc640 \uac19\uc740 \ud14d\uc2a4\ud2b8\uac00 \ud45c\uc2dc\ub429\ub2c8\ub2e4.<\/p>\n\n\n\n
\ub9c8\uc6b4\ud2b8 \uc791\uc5c5\uc774 \ub05d\ub098\uba74 \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac \ub4e4\uc744 \uc784\ud3ec\ud2b8\ud569\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58(PyTorch)\ub97c \uc0ac\uc6a9\ud558\uae30 \ub54c\ubb38\uc5d0 \ud559\uc2b5\uc5d0 \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac \ub4e4\uc744 \uc784\ud3ec\ud2b8\ud558\uace0 \uae30\ud0c0 numpy, pandas\ub3c4 \ud568\uaed8 \uc784\ud3ec\ud2b8\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n
config \ud30c\uc77c\uc5d0\ub294 \ud559\uc2b5\uc5d0 \ud544\uc694\ud55c \uba87\uac00\uc9c0 \ud30c\ub77c\uba54\ud130\uac00 \uc815\uc758\ub418\uc5b4 \uc788\uc2b5\ub2c8\ub2e4. \ud559\uc2b5\uc774 \uc644\ub8cc\ub41c \ud6c4 \ubaa8\ub378\uc744 \uc800\uc7a5\ud558\uace0 \ub2e4\uc2dc \ubd88\ub7ec\uc62c \ub54c\uc5d0 config \ub370\uc774\ud130\uac00 \uc800\uc7a5\ub418\uc5b4 \uc788\uc73c\uba74 \ud559\uc2b5\ub41c \ubaa8\ub378\uc758 \uc815\ubcf4\ub97c \ud655\uc778\ud560 \uc218 \uc788\uc5b4 \ud3b8\ub9ac\ud569\ub2c8\ub2e4. <\/span><\/p>\n\n\n\n
import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nimport numpy as np\nimport pandas as pd\nimport os\nfrom argparse import Namespace\n\nfrom collections import Counter\n\nconfig = Namespace(\n    train_file='gdrive\/***\/book_of_genesis.txt', seq_size=7, batch_size=100...\n)<\/pre>\n\n\n\n
\uc774\uc81c \ud559\uc2b5\uc744 \uc704\ud55c \ud30c\uc77c\uc744 \uc77d\uc5b4\uc624\uaca0\uc2b5\ub2c8\ub2e4. \ud30c\uc77c\uc740 \uc131\uacbd “\ucc3d\uc138\uae30 1\uc7a5”\uc744 \ud559\uc2b5 \ub370\uc774\ud130\ub85c \ud65c\uc6a9\ud569\ub2c8\ub2e4. \ud14c\uc2a4\ud2b8 \ud30c\uc77c\uc740 \uc601\ubb38 \ubc84\uc804\uc744 \ud65c\uc6a9\ud569\ub2c8\ub2e4. \ud30c\uc77c\uc744 \uc77d\uc740 \ud6c4\uc5d0 \uacf5\ubc31\uc73c\ub85c \ubd84\ub9ac\ud574\uc11c \ubc30\uc5f4\uc5d0 \ub2f4\uc73c\uba74 \uc544\ub798\uc640 \uac19\uc740 \ud615\ud0dc\uc758 \uac12\uc744 \uac00\uc9c0\uac8c\ub429\ub2c8\ub2e4.<\/p>\n\n\n\n
with open(config.train_file, 'r', encoding='utf-8') as f:\n    text = f.read()\ntext = text.split()<\/pre>\n\n\n\n
['In', 'the', 'beginning,', 'God', 'created', 'the', 'heavens', 'and', 'the', 'earth.', 'The', 'earth', 'was', 'without', 'form', 'and', 'void,', 'and', 'darkness', 'was'...<\/code><\/pre>\n\n\n\n
\uc774\uc81c \ud559\uc2b5\uc744 \uc704\ud574 \uc911\ubcf5 \ub2e8\uc5b4\ub97c \uc81c\uac70\ud558\uace0 word2index, index2word \ud615\ud0dc\uc758 \ub370\uc774\ud130\uc14b\uc744 \uc0dd\uc131\ud569\ub2c8\ub2e4. \uc774\ub807\uac8c \ub9cc\ub4e4\uc5b4\uc9c4 \ub370\uc774\ud143\uc14b\uc744 \ud1b5\ud574\uc11c \uac01 \ubb38\uc7a5\uc744 \uc5b4\uc808 \ub2e8\uc704\ub85c \ubd84\ub9ac\ud558\uace0 \uac01 \ubc30\uc5f4\uc758 \uc778\ub371\uc2a4 \uac12\uc744 \ub9f5\ud551\ud574\uc11c \ubb38\uc7a5\uc744 \uc22b\uc790 \ud615\ud0dc\uc758 \uac12\uc744 \uac00\uc9c4 \ub370\uc774\ud130\ub85c \ubcc0\uacbd\ud574\uc90d\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc740 \uc790\uc5f0\uc5b4\ub97c \uc774\ud574\ud558\uc9c0 \ubabb\ud558\ub294 \ucef4\ud4e8\ud130\uac00 \uc5b4\ub5a0\ud55c \uc791\uc5c5\uc744 \uc218\ud589\ud560 \uc218 \uc788\ub3c4\ub85d \uc218\uce58 \ud615\ud0dc\uc758 \ub370\uc774\ud130\ub85c \ubcc0\uacbd\ud558\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n
word_counts = Counter(text)\nsorted_vocab = sorted(word_counts, key=word_counts.get, reverse=True)\nint_to_vocab = {k: w for k, w in enumerate(sorted_vocab)}\nvocab_to_int = {w: k for k, w in int_to_vocab.items()}\nn_vocab = len(int_to_vocab)\n\nprint('Vocabulary size', n_vocab)\n\nint_text = [vocab_to_int[w] for w in text] # \uc804\uccb4 \ud14d\uc2a4\ud2b8\ub97c index\ub85c \ubcc0\uacbd<\/pre>\n\n\n\n
\ub2e4\uc74c\uc740 \ud559\uc2b5\uc744 \uc704\ud55c \ub370\uc774\ud130\ub97c \ub9cc\ub4dc\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4. \uc774 \uacfc\uc815\uc774 \uc911\uc694\ud569\ub2c8\ub2e4. \ub370\uc774\ud130\ub294 source_word\uc640 target_word\ub85c \ubd84\ub9ac\ud569\ub2c8\ub2e4. source_word\ub294 [‘In’, ‘the’, ‘beginning,’, ‘God’, ‘created’, ‘the’, ‘heavens’], target_word\ub294 [ ‘the’, ‘beginning,’, ‘God’, ‘created’, ‘the’, ‘heavens’,’and’]\uc758 \ud615\ud0dc\uc785\ub2c8\ub2e4. 
\uc989, source_word \ubb38\uc7a5 \ubc30\uc5f4 \ub2e4\uc74c\uc5d0 target_word\uac00 \uc21c\uc11c\ub300\ub85c \ub4f1\uc7a5\ud55c\ub2e4\ub294 \uac83\uc744 \ubaa8\ub378\uc774 \ud559\uc2b5\ud558\ub3c4\ub85d \ud558\ub294 \uacfc\uc815\uc785\ub2c8\ub2e4. <\/p>\n\n\n\n
\uc5ec\uae30\uc11c \ubb38\uc7a5\uc758 \ud06c\uae30\ub294 7\ub85c \uc815\ud588\uc2b5\ub2c8\ub2e4. \ub354 \ud070 \uc0ac\uc774\uc988\ub85c \ud559\uc2b5\uc744 \uc9c4\ud589\ud558\uba74 \ubb38\uc7a5\uc744 \uc0dd\uc131\ud560 \ub54c \ub354 \uc88b\uc740 \uc608\uce21\uc744 \ud560 \uc218 \uc788\uaca0\uc73c\ub098 \uacc4\uc0b0\ub7c9\uc774 \ub9ce\uc544\uc838\uc11c \ud559\uc2b5 \uc2dc\uac04\uc774 \ub9ce\uc774 \ud544\uc694\ud569\ub2c8\ub2e4. \ud14c\uc2a4\ud2b8\ub97c \ud1b5\ud574\uc11c \uc801\uc815 \uc218\uc900\uc5d0\uc11c \uac12\uc744 \uc815\ud574\ubcf4\uc2dc\uae30 \ubc14\ub78d\ub2c8\ub2e4.<\/p>\n\n\n\n
source_words = []\ntarget_words = []\nfor i in range(len(int_text)):\n    ss_idx, se_idx, ts_idx, te_idx = i, (config.seq_size+i), i+1, (config.seq_size+i)+1\n    if len(int_text[ts_idx:te_idx]) >= config.seq_size:\n        source_words.append(int_text[ss_idx:se_idx])\n        target_words.append(int_text[ts_idx:te_idx])<\/pre>\n\n\n\n
\uc544\ub798\uc640 \uac19\uc774 \uc5b4\ub5bb\uac8c \uac12\uc774 \ub4e4\uc5b4\uac00 \uc788\ub294\uc9c0\ub97c \ud655\uc778\ud574\ubcf4\uae30 \uc704\ud574\uc11c \uac04\ub2e8\ud788 10\uac1c\uc758 \ub370\uc774\ud130\ub97c \ucd9c\ub825\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
for s,t in zip(source_words[0:10], target_words[0:10]):\n  print('source {} -> target {}'.format(s,t))<\/pre>\n\n\n\n
source [106, 0, 107, 3, 32, 0, 16] -> target [0, 107, 3, 32, 0, 16, 1]\nsource [0, 107, 3, 32, 0, 16, 1] -> target [107, 3, 32, 0, 16, 1, 0]\nsource [107, 3, 32, 0, 16, 1, 0] -> target [3, 32, 0, 16, 1, 0, 26]\nsource [3, 32, 0, 16, 1, 0, 26] -> target [32, 0, 16, 1, 0, 26, 62]\nsource [32, 0, 16, 1, 0, 26, 62] -> target [0, 16, 1, 0, 26, 62, 12]\nsource [0, 16, 1, 0, 26, 62, 12] -> target [16, 1, 0, 26, 62, 12, 4]\nsource [16, 1, 0, 26, 62, 12, 4] -> target [1, 0, 26, 62, 12, 4, 108]\nsource [1, 0, 26, 62, 12, 4, 108] -> target [0, 26, 62, 12, 4, 108, 109]\nsource [0, 26, 62, 12, 4, 108, 109] -> target [26, 62, 12, 4, 108, 109, 1]\nsource [26, 62, 12, 4, 108, 109, 1] -> target [62, 12, 4, 108, 109, 1, 110]<\/code><\/pre>\n\n\n\n
\uc774\uc81c \ud559\uc2b5\uc744 \uc704\ud574\uc11c \ubaa8\ub378\uc744 \uc0dd\uc131\ud569\ub2c8\ub2e4. \ubaa8\ub378\uc740 Encoder\uc640 Decoder\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4. \uc774 \ub450 \ubaa8\ub378\uc744 \uc0ac\uc6a9\ud558\ub294 \uac83\uc774 Sequence2Sequece\uc758 \uc804\ud615\uc801\uc778 \uad6c\uc870\uc785\ub2c8\ub2e4. \ud574\ub2f9 \ubaa8\ub378\uc5d0 \ub300\ud574\uc11c \uad81\uae08\ud558\uc2e0 \uc810\uc740 pytorch \uacf5\uc2dd \uc0ac\uc774\ud2b8\ub97c \ucc38\uc870\ud558\uc2dc\uae30 \ubc14\ub78d\ub2c8\ub2e4. \uc778\ucf54\ub354\uc640 \ub514\ucf54\ub354\uc5d0 \ub300\ud55c \uc790\uc138\ud55c \uc124\uba85\uc740 \uc544\ub798\uc758 \uadf8\ub9bc\uc73c\ub85c \ub300\uc2e0\ud558\uaca0\uc2b5\ub2c8\ub2e4. GRU \ub300\uc2e0\uc5d0 LSTM\uc744 \uc0ac\uc6a9\ud574\ub3c4 \ubb34\ubc29\ud569\ub2c8\ub2e4.<\/p>\n\n\n\n
\"\"\/
https:\/\/tutorials.pytorch.kr\/intermediate\/seq2seq_translation_tutorial.html<\/figcaption><\/figure><\/div>\n\n\n\n
\uc544\ub798\ub294 \uc778\ucf54\ub354\uc758 \uad6c\uc870\uc785\ub2c8\ub2e4. \uc704\uc758 \uadf8\ub9bc\uc5d0\uc11c\uc640 \uac19\uc774 \uc778\ucf54\ub354\ub294 \ub450\uac1c\uc758 \uac12\uc774 GRU \uc140(Cell)\ub85c \ub4e4\uc5b4\uac00\uac8c \ub429\ub2c8\ub2e4. \ud558\ub098\ub294 \uc785\ub825 \uac12\uc774 \uc784\ubca0\ub529 \ub808\uc774\uc5b4\ub97c \ud1b5\ud574\uc11c \ub098\uc624\ub294 \uac12\uacfc \ub610 \ud558\ub098\ub294 \uc774\uc804 \ub2e8\uacc4\uc758 hidden \uac12\uc785\ub2c8\ub2e4. \ucd5c\uc885 \ucd9c\ub825\uc740 \uc785\ub825\uc744 \ud1b5\ud574\uc11c \uc608\uce21\ub41c \uac12\uc778 output, \ub2e4\uc74c \ub2e8\uacc4\uc5d0 \uc785\ub825\uc73c\ub85c \ub4e4\uc5b4\uac00\ub294 hidden\uc774 \uadf8\uac83\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n
\uae30\ubcf8 \uad6c\uc870\uc758 seq2seq \ubaa8\ub378\uc5d0\uc11c\ub294 output \uac12\uc740 \uc0ac\uc6a9\ud558\uc9c0 \uc54a\uace0 \uc774\uc804 \ub2e8\uacc4\uc758 hidden \uac12\uc744 \uc0ac\uc6a9\ud569\ub2c8\ub2e4. \ucd5c\uc885 hidden \uac12\uc740 \uc785\ub825\ub41c \ubb38\uc7a5\uc758 \uc804\uccb4 \uc815\ubcf4\ub97c \uc5b4\ub5a4 \uace0\uc815\ub41c \ud06c\uae30\uc758 Context Vector\uc5d0 \ucd95\uc57d\ud558\uace0 \uc788\uae30 \ub54c\ubb38\uc5d0 \uc774 \uac12\uc744 Decoder\uc758 \uc785\ub825\uc73c\ub85c \uc0ac\uc6a9\ud569\ub2c8\ub2e4.<\/p>\n\n\n\n
\ucc38\uace0\ub85c \uc774\ud6c4\uc5d0 \ud14c\uc2a4\ud2b8\ud560 Attention \ubaa8\ub378\uc740 \uc774\ub7ec\ud55c \uad6c\uc870\uc640\ub294 \ub2ec\ub9ac encoder\uc758 \ucd9c\ub825 \uac12\uc744 \uc0ac\uc6a9\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4. \uc774 \uac12\uc744 \ud1b5\ud574\uc11c \uc5b4\ub514\uc5d0 \uc9d1\uc911\ud560\uc9c0\ub97c \uc815\ud558\uac8c \ub429\ub2c8\ub2e4.<\/p>\n\n\n\n
class Encoder(nn.Module):\n\n    def __init__(self, input_size, hidden_size):\n        super().__init__()\n        self.hidden_size = hidden_size\n        self.embedding = nn.Embedding(input_size, hidden_size) #199->10\n        self.gru = nn.GRU(hidden_size, hidden_size) #20-20\n\n    def forward(self, x, hidden):\n        x = self.embedding(x).view(1,1,-1)\n        #print('Encoder forward embedding size {}'.format(x.size()))\n        x, hidden = self.gru(x, hidden)\n        return x, hidden<\/pre>\n\n\n\n
\uc774\uc81c \uc544\ub798\uc758 \uadf8\ub9bc\uacfc \uac19\uc774 Decoder\ub97c \uc124\uacc4\ud569\ub2c8\ub2e4. Decoder \uc5ed\uc2dc GRU \uc140(Cell)\uc744 \uac00\uc9c0\uace0 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
\"\"\/
https:\/\/tutorials.pytorch.kr\/intermediate\/seq2seq_translation_tutorial.html<\/figcaption><\/figure><\/div>\n\n\n\n
class Decoder(nn.Module):\n    def __init__(self, hidden_size, output_size):\n        super().__init__()\n        self.hidden_size = hidden_size\n        self.embedding = nn.Embedding(output_size, hidden_size) #199->10\n        self.gru = nn.GRU(hidden_size, hidden_size) #10->10\n        self.out = nn.Linear(hidden_size, output_size) #10->199\n        self.softmax = nn.LogSoftmax(dim=1)\n        \n    def forward(self, x, hidden):\n        x = self.embedding(x).view(1,1,-1)\n        x, hidden = self.gru(x, hidden)\n        x = self.softmax(self.out(x[0]))\n        return x, hidden<\/pre>\n\n\n\n
\uc774\uc81c GPU\ub97c \uc0ac\uc6a9\ud558\uae30 \uc704\ud574\uc11c \uc124\uc815\uc744 \uc218\ud589\ud569\ub2c8\ub2e4. Google Colab\uc744 \ud65c\uc6a9\ud558\uc2dc\uba74 \ubcc4\ub3c4\uc758 \uc124\uc815\uc791\uc5c5 \uc5c6\uc774 GPU\ub97c \uc0ac\uc6a9\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)<\/pre>\n\n\n\n
\uc778\ucf54\ub354\uc640 \ub514\ucf54\ub354 \uc785\ucd9c\ub825 \uc815\ubcf4\ub97c \uc14b\ud305\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n
enc_hidden_size = 50\ndec_hidden_size = enc_hidden_size\nencoder = Encoder(n_vocab, enc_hidden_size).to(device) # source(199) -> embedding(10)\ndecoder = Decoder(dec_hidden_size, n_vocab).to(device) # embedding(199) -> target(199)\n\nencoder_optimizer = optim.SGD(encoder.parameters(), lr=0.01)\ndecoder_optimizer = optim.SGD(decoder.parameters(), lr=0.01)\n\ncriterion = nn.NLLLoss()<\/pre>\n\n\n\n
\ud574\ub2f9 \ubaa8\ub378\uc758 \uc774\ubbf8\uc9c0\ub97c \uc544\ub798\uc758 \uadf8\ub9bc\uacfc \uac19\uc774 \ub098\ud0c0\ub0bc \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
\"\"
\uadf8\ub9bc1 Sequence2Sequence Model<\/figcaption><\/figure><\/div>\n\n\n\n
Encoder(\n  (embedding): Embedding(199, 50)\n  (gru): GRU(50, 50)\n)\nDecoder(\n  (embedding): Embedding(199, 50)\n  (gru): GRU(50, 50)\n  (out): Linear(in_features=50, out_features=199, bias=True)\n  (softmax): LogSoftmax(dim=1)\n)<\/code><\/pre>\n\n\n\n
\ub370\uc774\ud130\ub97c 100\uac1c\uc529 \ub098\ub220\uc11c \ud6c8\ub828 \ud560 \uc218 \uc788\ub3c4\ub85d \ubc30\uce58 \ubaa8\ub378\uc744 \uc791\uc131\ud569\ub2c8\ub2e4.<\/p>\n\n\n\n
pairs = list(zip(source_words, target_words))\ndef get_batch(pairs, batch_size):\n  pairs_length = len(pairs)\n  for ndx in range(0, pairs_length, batch_size):\n    #print(ndx, min(ndx+batch_size, pairs_length))\n    yield pairs[ndx:min(ndx+batch_size, pairs_length)]<\/pre>\n\n\n\n
\ud574\ub2f9 \ubaa8\ub378\uc740 500\ubc88 \ud559\uc2b5\uc744 \uc218\ud589\ud569\ub2c8\ub2e4. \uac01 batch, epoch \ub9c8\ub2e4 loss \uc815\ubcf4\ub97c \ud45c\uc2dc\ud569\ub2c8\ub2e4. \ud45c1 \uc740 \ub9c8\uc9c0\ub9c9 \uc2a4\ud15d\uc758 loss\uc640 epoch \uc815\ubcf4\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n
number_of_epochs = 501\nfor epoch in range(number_of_epochs):\n    total_loss = 0\n    #for pair in get_batch(pairs, config.batch_size): # batch_size 100\n    for pair in get_batch(pairs, 100): # batch_size 100\n      batch_loss = 0\n       \n      for si, ti in pair:\n        x = torch.Tensor(np.array([si])).long().view(-1,1).to(device)\n        y = torch.Tensor(np.array([ti])).long().view(-1,1).to(device)\n        encoder_hidden = torch.zeros(1,1,enc_hidden_size).to(device)\n\n        for j in range(config.seq_size):\n            _, encoder_hidden = encoder(x[j], encoder_hidden)\n\n        decoder_hidden = encoder_hidden\n        decoder_input = torch.Tensor([[0]]).long().to(device)\n\n        loss = 0\n\n        for k in range(config.seq_size):\n            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n            decoder_input = y[k]\n            loss += criterion(decoder_output, y[k])\n\n        batch_loss += loss.item()\/config.seq_size\n        encoder_optimizer.zero_grad()\n        decoder_optimizer.zero_grad()\n\n        loss.backward()\n\n        encoder_optimizer.step()\n        decoder_optimizer.step()\n\n      total_loss += batch_loss\/config.batch_size\n      print('batch_loss {:.5f}'.format(batch_loss\/config.batch_size))\n    print('epoch {}, loss {:.10f}'.format(epoch, total_loss\/(len(pairs)\/\/config.batch_size)))<\/pre>\n\n\n\n
...\nbatch_loss 0.00523\nbatch_loss 0.00766\nbatch_loss 0.01120\nbatch_loss 0.00735\nbatch_loss 0.01218\nbatch_loss 0.00873\nbatch_loss 0.00352\nbatch_loss 0.00377\nepoch 500, loss 0.0085196330<\/code><\/pre>\n\n\n\n
\ud45c1. \ub9c8\uc9c0\ub9c9 batch, epoch \ud559\uc2b5 \uc815\ubcf4<\/p>\n\n\n\n
\ud559\uc2b5\uc774 \uc885\ub8cc\ub41c \ubaa8\ub378\uc744 \uc800\uc7a5\uc18c\uc5d0 \uc800\uc7a5\ud569\ub2c8\ub2e4. \uc800\uc7a5 \ud560 \ub54c\uc5d0 \ud559\uc2b5 \uc815\ubcf4\uac00 \uc800\uc7a5\ub418\uc5b4 \uc788\ub294 config \ub0b4\uc6a9\ub3c4 \ud3ec\ud568\ud558\ub294 \uac83\uc774 \uc88b\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
# Save best model weights.\ntorch.save({\n  'encoder': encoder.state_dict(), 'decoder':decoder.state_dict(),\n  'config': config,\n}, 'gdrive\/***\/model.genesis.210122')<\/pre>\n\n\n\n
\ud559\uc2b5\uc774 \uc644\ub8cc\ub41c \ud6c4\uc5d0 \ud574\ub2f9 \ubaa8\ub378\uc774 \uc798 \ud559\uc2b5\ub418\uc5c8\ub294\uc9c0 \ud655\uc778\ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ud559\uc2b5\uc740 “darkness was”\ub77c\ub294 \uba87\uac00\uc9c0 \ub2e8\uc5b4\ub97c \uc8fc\uace0 \ubaa8\ub378\uc774 \uc5b4\ub5a4 \ubb38\uc7a5\uc744 \uc0dd\uc131\ud558\ub294 \uc9c0\ub97c \uc54c\uc544 \ubcf4\ub294 \ubc29\uc2dd\uc73c\ub85c \uc218\ud589\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n
decoded_words = []\n\nwords = [vocab_to_int['darkness'], vocab_to_int['was']]\nx = torch.Tensor(words).long().view(-1,1).to(device)\n\nencoder_hidden = torch.zeros(1,1,enc_hidden_size).to(device)\n\nfor j in range(x.size(0)):\n    _, encoder_hidden = encoder(x[j], encoder_hidden)\n\ndecoder_hidden = encoder_hidden\ndecoder_input = torch.Tensor([[words[1]]]).long().to(device)  \n\nfor di in range(20):\n  decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n  _, top_index = decoder_output.data.topk(1)\n  decoded_words.append(int_to_vocab[top_index.item()])\n\n  decoder_input = top_index.squeeze().detach()\n\npredict_words = decoded_words    \npredict_sentence = ' '.join(predict_words)\nprint(predict_sentence)<\/pre>\n","protected":false},"excerpt":{"rendered":"
Sequence2Sequence \ubaa8\ub378\uc744 \ud65c\uc6a9\ud574\uc11c \ubb38\uc7a5\uc0dd\uc131\uc744 \uc218\ud589\ud558\ub294 \ud14c\uc2a4\ud2b8\ub97c \ud574\ubcf4\uaca0\uc2b5\ub2c8\ub2e4. \ud14c\uc2a4\ud2b8 \ud658\uacbd\uc740 Google Colab\uc758 GPU\ub97c \ud65c\uc6a9\ud569\ub2c8\ub2e4. Google Drive\uc5d0 \uc5c5\ub85c\ub4dc\ub418\uc5b4 \uc788\ub294 text \ud30c\uc77c\uc744 \uc77d\uae30 \uc704\ud574\uc11c \ud544\uc694\ud55c \ub77c\uc774\ube0c\ub7ec\ub9ac\ub97c \uc784\ud3ec\ud2b8\ud569\ub2c8\ub2e4. \ud574\ub2f9 \ud30c\uc77c\uc744 \uc2e4\ud589\uc2dc\ud0a4\uba74 \uc544\ub798\uc640 \uac19\uc740 \uc774\ubbf8\uc9c0\uac00 \ud45c\uc2dc\ub429\ub2c8\ub2e4. \ud574\ub2f9 \ub9c1\ud06c\ub97c \ud074\ub9ad\ud558\uace0 \ub4e4\uc5b4\uac00\uba74 \ucf54\ub4dc \uac12\uc774 \ub098\uc624\ub294\ub370 \ucf54\ub4dc\uac12\uc744 \ubcf5\uc0ac\ud574\uc11c \uc785\ub825\ud558\uba74 \uad6c\uae00 \ub4dc\ub77c\uc774\ube0c\uac00 \ub9c8\uc6b4\ud2b8 \ub418\uace0 \uad6c\uae00 \ub4dc\ub77c\uc774\ube0c\uc5d0 \uc800\uc7a5\ub41c \ud30c\uc77c\ub4e4\uc744 \uc0ac\uc6a9\ud560 \uc218 \uc788\uac8c\ub429\ub2c8\ub2e4. \uc815\uc0c1\uc801\uc73c\ub85c \ub9c8\uc6b4\ud2b8 \ub418\uba74 … <\/p>\n
\ub354 \ubcf4\uae30 “Seq2Seq \ubb38\uc7a5\uc0dd\uc131”<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[24,40,73,76,21,14],"tags":[97,96,83,86,61,74,72,55],"_links":{"self":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/809"}],"collection":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/comments?post=809"}],"version-history":[{"count":16,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/809\/revisions"}],"predecessor-version":[{"id":872,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/809\/revisions\/872"}],"wp:attachment":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/media?parent=809"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/categories?post=809"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/tags?post=809"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}