{"id":231,"date":"2020-08-01T14:51:33","date_gmt":"2020-08-01T05:51:33","guid":{"rendered":"http:\/\/cedartrees.co.kr\/?p=231"},"modified":"2021-04-03T19:19:32","modified_gmt":"2021-04-03T10:19:32","slug":"pytorch-lstm-example","status":"publish","type":"post","link":"http:\/\/blog.cedartrees.co.kr\/index.php\/2020\/08\/01\/pytorch-lstm-example\/","title":{"rendered":"PyTorch LSTM \uc608\uc81c"},"content":{"rendered":"\n

\uc774 \uc608\uc81c\ub294 \ud30c\uc774\ud1a0\uce58\ub97c \ud65c\uc6a9\ud574\uc11c LSTM\uc744 \uad6c\ud604\ud558\ub294 \uc608\uc81c\uc785\ub2c8\ub2e4.
LSTM\uc740 RNN(Recurrent Neural Network)\uc758 \ud558\ub098\ub85c \uc88b\uc740 \uc131\ub2a5\uc744 \ubc1c\ud718\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58\ub294 LSTM\ub97c \uc9c1\uad00\uc801\uc73c\ub85c \uad6c\ud604\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\ub294 \uc88b\uc740 \uc778\uacf5\uc9c0\ub2a5 \ud504\ub808\uc784\uc6cc\ud06c\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n

\ubcf8 \uc608\uc81c\ub294 \ud604\uc7ac \ubb38\uc7a5\uc744 \uc8fc\uace0 \ub2e4\uc74c \ubb38\uc7a5\uc744 \uc608\uce21\ud558\ub294 \uc54c\uace0\ub9ac\uc998\uc785\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uc608\uc81c\ub4e4\uc744 \ud65c\uc6a9\ud558\uba74 \ub2e4\uc591\ud55c \uc2dc\uacc4\uc5f4 \ub370\uc774\ud130\ub97c \ub2e4\ub8f0 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc2dc\uacc4\uc5f4 \ub370\uc774\ud130\ub77c \ud568\uc740 \ub370\uc774\ud130\uac00 \uc5b4\ub5a4 \uc2dc\uac04\uc758 \uc21c\uc11c\ub97c \uac00\uc9c4\ub2e4\ub294 \uac83\uc744 \uc758\ubbf8\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n

\uc608\ub97c \ub4e4\uc5b4 \ud55c \ubb38\uc7a5\uc758 \ub2e4\uc591\ud55c \ub2e8\uc5b4\ub4e4\uc740 \ube44\ub85d \uac19\uc740 \ub2e8\uc5b4\uc77c\uc9c0\ub77c\ub3c4 \uc55e\uc5d0 \uc624\ub290\ub0d0 \ub4a4\uc5d0 \uc624\ub290\ub0d0\uc5d0 \ub530\ub77c\uc11c \uadf8 \uc758\ubbf8\uac00 \ub2ec\ub77c\uc9c0\ub294 \uacbd\uc6b0\uac00 \uc788\uc2b5\ub2c8\ub2e4. \uadf8\ub807\uae30 \ub54c\ubb38\uc5d0 \ud604\uc7ac \ubb38\uc7a5\uc744 \uc720\ucd94\ud558\uae30 \uc704\ud574\uc11c\ub294 \uc55e\uc5d0 \uc5b4\ub5a4 \ub2e8\uc5b4\uac00 \uc788\ub294\uc9c0\ub97c \uc54c\uc544\ub0b4\ub294 \uac83\uc774 \uc911\uc694\ud569\ub2c8\ub2e4.
RNN\uc740 \uc774\ub7ec\ud55c \uc608\uce21\uc744 \uac00\ub2a5\ud558\uac8c \ud574\uc90d\ub2c8\ub2e4.<\/p>\n\n\n\n

RNN\uc740 \ub450\uac1c\uc758 Linear \ubaa8\ub378\uc774 \ud569\uccd0\uc9c4 \ud558\ub098\uc758 Activation Function\uc785\ub2c8\ub2e4. \uc544\ub798 \uadf8\ub9bc\uc774 \uc774\ub97c \uc798 \uc124\uba85\ud558\uace0 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n

\"\"\/
https:\/\/pytorch.org\/tutorials\/intermediate\/char_rnn_classification_tutorial.html<\/figcaption><\/figure>\n\n\n\n
import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport numpy as np<\/pre>\n\n\n\n
\uc785\ub825 \ubb38\uc7a5\uc740 “In the beginning God created the heavens and the earth”\ub77c\ub294 \ubb38\uc7a5\uc785\ub2c8\ub2e4.  x \ub370\uc774\ud130\ub294 \uc785\ub825 \ubb38\uc7a5\uc774\uace0 x \ub370\uc774\ud130\ub97c \ud1b5\ud574 \uc608\uce21\ud55c \uacb0\uacfc\ub97c \ube44\uad50\ud558\uae30 \uc704\ud574\uc11c \uc815\ub2f5 \ub370\uc774\ud130\uc14b y\ub97c \ub9cc\ub4ed\ub2c8\ub2e4. y \ub370\uc774\ud130\ub294 x \ub370\uc774\ud130\uc758 \uccab\ubc88\uc9f8 \uc785\ub825 ‘I’\uc758 \uacbd\uc6b0 ‘n’\ub97c \uc608\uce21\ud558\uace0 ‘n’\uac00 \uc785\ub825\ub41c \uacbd\uc6b0 ‘ ‘(\uacf5\ubc31) \ub97c \uc2dc\uc2a4\ud15c\uc774 \uc608\uce21\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\uae30 \uc704\ud568\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n
sentence = 'In the beginning God created the heavens and the earth'\nx = sentence[:-1]\ny = sentence[1:]\n\nchar_set = list(set(sentence))\ninput_size = len(char_set)\nhidden_size = len(char_set)\n\nindex2char = {i:c for i, c in enumerate(char_set)}\nchar2index = {c:i for i, c in enumerate(char_set)}<\/pre>\n\n\n\n
index2char\uacfc char2index\ub294 \uac01\uac01 \ubb38\uc790\ub97c \ubb38\uc790 \uc790\uccb4\ub85c \uc785\ub825\ud558\uc9c0 \uc54a\uace0 one-hot\uc758 \ud615\ud0dc\ub85c \uc785\ub825\ud558\uae30 \uc704\ud574\uc11c \ub9cc\ub4e4\uc5b4\uc900 python dict\uc785\ub2c8\ub2e4. 
char2index\ub97c \ucd9c\ub825\ud558\uba74 \uc544\ub798\uc640 \uac19\uc740 \ud615\ud0dc\uac00 \ub429\ub2c8\ub2e4.<\/p>\n\n\n\n
{‘s’: 0, ‘ ‘: 1, ‘t’: 2, ‘I’: 3, ‘o’: 4, ‘h’: 5, ‘e’: 6, ‘g’: 7, ‘d’: 8, ‘c’: 9, ‘b’: 10, ‘i’: 11, ‘n’: 12, ‘G’: 13, ‘v’: 14, ‘a’: 15, ‘r’: 16}<\/p>\n\n\n\n
one_hot = []\nfor i, tkn in enumerate(x):\n    one_hot.append(np.eye(len(char_set), dtype='int')[char2index[tkn]])\n\nx_train = torch.Tensor(one_hot)\nx_train = x_train.view(1,len(x),-1)<\/pre>\n\n\n\n
\uc785\ub825\ub41c sentence\ub294 \uadf8\ub300\ub85c \uc785\ub825\uac12\uc73c\ub85c \uc0ac\uc6a9\ud558\uc9c0 \uc54a\uace0 one-hot \ud615\ud0dc\ub85c \ubcc0\uacbd\ud574\uc11c \ucd5c\uc885 x_train \ud615\ud0dc\uc758 \ub370\uc774\ud130\ub97c \ub9cc\ub4ed\ub2c8\ub2e4. \ubb38\uc7a5\uc744 one-hot \ud615\ud0dc\ub85c \ub9cc\ub4e4\uae30 \uc704\ud574\uc11c numpy\uc758 eye\ud568\uc218\ub97c \uc0ac\uc6a9\ud569\ub2c8\ub2e4. <\/p>\n\n\n\n
print(x_train)\n\ntensor([[[0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n         [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n         [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n         [0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.] ...<\/pre>\n\n\n\n
x_train \ub370\uc774\ud130\ub97c \ucd9c\ub825\ud558\uba74 \uc704\uc640 \uac19\uc740 \ud615\ud0dc\uac00 \ub9cc\ub4e4\uc5b4\uc9d1\ub2c8\ub2e4.  \uc704\uc758 \ub370\uc774\ud130\uc14b\uc740 [1, 10, 8] \ud615\ud0dc\uc758 3\ucc28\uc6d0 \ub370\uc774\ud130\uc14b\uc785\ub2c8\ub2e4. NLP \ub370\uc774\ud130\ub97c 3\ucc28\uc6d0 \ud615\ud0dc\uc758 \uc785\ub825 \ub370\uc774\ud130\uc14b\uc744 \uac00\uc9d1\ub2c8\ub2e4. \uccab\ubc88\uc9f8 \ucc28\uc6d0\uc740 \ubb38\uc7a5\uc758 \uac2f\uc218, \ub450\ubc88\uc9f8\ub294 \ub2e8\uc5b4\uc758 \uac2f\uc218, \uc138\ubc88\uc9f8\ub294 \ub2e8\uc5b4\uc758 \uc785\ub825 \ucc28\uc6d0\uc785\ub2c8\ub2e4.<\/p>\n\n\n\n
\ucc38\uace0\ub85c CNN\uc758 \uacbd\uc6b0\uc5d0\ub294 4\ucc28\uc6d0 \ud615\ud0dc\uc758 \ub370\uc774\ud130\ub97c \uac00\uc9d1\ub2c8\ub2e4. 
\ub2e4\uc74c\uc73c\ub85c \uc544\ub798\uc640 \uac19\uc774 y_data\ub97c \ub9cc\ub4e4\uc5b4\uc90d\ub2c8\ub2e4.<\/p>\n\n\n\n
# y label\ny_data = [char2index[c] for c in y]\ny_data = torch.Tensor(y_data)<\/pre>\n\n\n\n
\uc774\uc81c \ubaa8\ub378\uc744 \ub9cc\ub4e4 \ucc28\ub840\uc785\ub2c8\ub2e4.
\ud30c\uc774\ud1a0\uce58\ub294 nn.Module\uc744 \uc0ac\uc6a9\ud574\uc11c Module\uc744 \ub9cc\ub4e4 \uc218 \uc788\uc2b5\ub2c8\ub2e4.<\/p>\n\n\n\n
class RNN(nn.Module):\n    \n    # (batch_size, n, ) torch already know, you don't need to let torch know\n    def __init__(self,input_size, hidden_size):\n        super().__init__()\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        \n        self.rnn = nn.LSTM(\n            input_size = input_size, \n            hidden_size = hidden_size, \n            num_layers = 4, \n            batch_first = True,\n            bidirectional = True\n        )\n        \n        self.layers = nn.Sequential(\n            nn.ReLU(),\n            nn.Linear(input_size*2, hidden_size),\n        )\n        \n    def forward(self, x):\n        y,_ = self.rnn(x)\n        y = self.layers(y)\n        return y\n    \nmodel = RNN(input_size, hidden_size)\nmodel<\/pre>\n\n\n\n
RNN \ud074\ub798\uc2a4\ub294 init, forward \ud568\uc218\ub85c \uad6c\uc131\ub429\ub2c8\ub2e4. init\ud568\uc218\ub294 LSTM \ubaa8\ub378\uc744 \uc120\uc5b8\ud558\ub294 \ubd80\ubd84\uacfc softamx \ud568\uc218\ub97c \uc120\uc5b8\ud558\ub294 \ub450\ubd80\ubd84\uc774 \uc788\uc2b5\ub2c8\ub2e4. LSTM \ud568\uc218\ub294 \ub450\uac1c\uc758 \uc778\uc790\uac12\uc744 \uae30\ubcf8\uc73c\ub85c \ubc1b\uc2b5\ub2c8\ub2e4. input_size, hidden_size\uc785\ub2c8\ub2e4. input_size\ub294 \uc785\ub825 \ubca1\ud130\uc758 \ud06c\uae30\uc774\uba70 hidden_size\ub294 \ucd9c\ub825 \ubca1\ud130\uc758 \ud06c\uae30\uc785\ub2c8\ub2e4. \ubcf8 \uc608\uc81c\uc5d0\uc11c\ub294 \uc785\ub825 \ubca1\ud130\uc640 \ucd9c\ub825 \ubca1\ud130\uac00 \ud06c\uae30\uac00 \uac19\uc2b5\ub2c8\ub2e4. \ubc30\uce58 \uc0ac\uc774\uc988\ub098 \uc2dc\ud000\uc2a4 \uc0ac\uc774\uc988\ub294 \ud30c\uc774\ud1a0\uce58\uc5d0\uc11c \uc790\ub3d9\uc73c\ub85c \uacc4\uc0b0\ud558\uae30 \ub54c\ubb38\uc5d0 \uc785\ub825\ud560 \ud544\uc694\uac00 \uc5c6\uc2b5\ub2c8\ub2e4. 
num_layers\ub294 RNN\uc758 \uce35\uc744 \uc758\ubbf8\ud569\ub2c8\ub2e4. \ubcf8 \uc608\uc81c\ub294 4\uac1c\uc758 \uce35\uc73c\ub85c \uad6c\uc131\ud588\uae30 \ub54c\ubb38\uc5d0 num_layers\ub97c 4\ub85c \uc124\uc815\ud588\uc2b5\ub2c8\ub2e4. \uadf8\ub9ac\uace0 bidirectional\uc744 True\ub85c \ud588\uae30 \ub54c\ubb38\uc5d0 \ub9c8\uc9c0\ub9c9 output\uc758 \ud615\ud0dc\ub294 input_size*2\uc758 \ud615\ud0dc\uac00 \ub429\ub2c8\ub2e4.
Linear \ub808\uc774\uc5b4\ub294 input_size\uc758 \ucc28\uc6d0\uc744 \uc904\uc774\uae30 \uc704\ud574\uc11c \uc120\uc5b8\ud569\ub2c8\ub2e4.<\/p>\n\n\n\n
\ub610 \ubaa8\ub378\uc744 \ub9cc\ub4e4\uba74\uc11c \uc911\uc694\ud55c \uac83\uc740 batch_first\ub97c True\ub85c \ud574\uc918\uc57c \ud55c\ub2e4\ub294 \uac83\uc785\ub2c8\ub2e4. \uadf8\ub807\uc9c0 \uc54a\uc73c\uba74 time-step(=sequence_length), batch_size, input_vector \uc758 \ud615\ud0dc\uac00 \ub429\ub2c8\ub2e4.<\/p>\n\n\n\n
\uc120\uc5b8\ud55c \ubaa8\ub378\uc758 \uc815\ubcf4\ub97c \ucd9c\ub825\ud574\ubcf4\uba74 \ub2e4\uc74c\uacfc \uac19\uc2b5\ub2c8\ub2e4.
RNN(
(rnn): LSTM(8, 8, num_layers=4, bidirectional=True)
(layers): Sequential(
(0): ReLU()
(1): Linear(in_features=16, out_features=8, bias=True)
)
)<\/p>\n\n\n\n
# loss & optimizer setting\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters())\n\n# start training\nfor i in range(5000):\n    model.train()\n    outputs = model(x_train)\n    loss = criterion(outputs.view(-1, input_size), y_data.view(-1).long())\n    \n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n\n    if i%500 == 0:\n        result = outputs.data.numpy().argmax(axis=2)\n        result_str = ''.join([char_set[c] for c in np.squeeze(result)])\n        print(i, \"loss: \", loss.item(), \"\\nprediction: \", result, \"\\ntrue Y: \", y_data, \"\\nprediction str: \", result_str,\"\\n\")\n<\/pre>\n\n\n\n
\uc704\uc758 \ucf54\ub4dc\uc640 \uac19\uc774 \ud559\uc2b5\uc744 \uc218\ud589\ud569\ub2c8\ub2e4.
x_train \ub370\uc774\ud130\ub97c \uc785\ub825 \ubc1b\uc544 \ub098\uc628 \uacb0\uacfc\ub97c y_data\uc640 \ube44\uad50\ud574\uc11c loss\ub97c \uacc4\uc0b0\ud558\uace0 \uc774 loss \uac12\uc744 Back-propagation\uc744 \uc218\ud589\ud558\uace0 Gradient\ub97c \ucd08\uae30\ud654\ud558\ub294 \uacfc\uc815\uc744 \ubc18\ubcf5\ud569\ub2c8\ub2e4.<\/p>\n\n\n\n
5000 \ud68c \ud559\uc2b5\ud560 \uacbd\uc6b0 \ub2e4\uc74c\uacfc \uac19\uc774 loss\uac00 \ub0b4\ub824\uac00\ub294 \uac83\uc744 \ud655\uc778 \ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4. <\/p>\n\n\n\n
\"\"<\/figure>\n\n\n\n
\uc2e4\uc81c\ub85c \ub370\uc774\ud130\ub97c \ub3cc\ub824\ubcf4\uba74 \uc57d 1500\ubc88 \uc815\ub3c4 \ud559\uc2b5\uc744 \uc644\ub8cc\ud558\uba74 \uc785\ub825 \ub2e8\uc5b4\ub97c \ud1b5\ud574\uc11c \uc815\ud655\ud788 \ub2e4\uc74c \ub2e8\uc5b4\ub97c \uc608\uce21\ud558\ub294 \uac83\uc744 \ud655\uc778\ud560 \uc218 \uc788\uc2b5\ub2c8\ub2e4.

1500 loss: 0.31987816095352173
prediction: [[12 1 2 5 6 1 10 6 7 11 12 12 11 12 7 1 13 4 8 1 9 16 6 15
2 6 8 1 2 5 6 1 5 6 15 14 6 12 0 1 15 12 8 1 2 5 6 1
6 15 16 2 5]]
true Y: tensor([12., 1., 2., 5., 6., 1., 10., 6., 7., 11., 12., 12., 11., 12.,
7., 1., 13., 4., 8., 1., 9., 16., 6., 15., 2., 6., 8., 1.,
2., 5., 6., 1., 5., 6., 15., 14., 6., 12., 0., 1., 15., 12.,
8., 1., 2., 5., 6., 1., 6., 15., 16., 2., 5.])
prediction str: n the beginning God created the heavens and the earth<\/p>\n","protected":false},"excerpt":{"rendered":"
\uc774 \uc608\uc81c\ub294 \ud30c\uc774\ud1a0\uce58\ub97c \ud65c\uc6a9\ud574\uc11c LSTM\uc744 \uad6c\ud604\ud558\ub294 \uc608\uc81c\uc785\ub2c8\ub2e4.LSTM\uc740 RNN(Recurrent Neural Network)\uc758 \ud558\ub098\ub85c \uc88b\uc740 \uc131\ub2a5\uc744 \ubc1c\ud718\ud558\ub294 \ubaa8\ub378\uc785\ub2c8\ub2e4. \ud30c\uc774\ud1a0\uce58\ub294 LSTM\ub97c \uc9c1\uad00\uc801\uc73c\ub85c \uad6c\ud604\ud560 \uc218 \uc788\ub3c4\ub85d \ud558\ub294 \uc88b\uc740 \uc778\uacf5\uc9c0\ub2a5 \ud504\ub808\uc784\uc6cc\ud06c\uc785\ub2c8\ub2e4. \ubcf8 \uc608\uc81c\ub294 \ud604\uc7ac \ubb38\uc7a5\uc744 \uc8fc\uace0 \ub2e4\uc74c \ubb38\uc7a5\uc744 \uc608\uce21\ud558\ub294 \uc54c\uace0\ub9ac\uc998\uc785\ub2c8\ub2e4. \uc774\ub7ec\ud55c \uc608\uc81c\ub4e4\uc744 \ud65c\uc6a9\ud558\uba74 \ub2e4\uc591\ud55c \uc2dc\uacc4\uc5f4 \ub370\uc774\ud130\ub97c \ub2e4\ub8f0 \uc218 \uc788\uc2b5\ub2c8\ub2e4. \uc2dc\uacc4\uc5f4 \ub370\uc774\ud130\ub77c \ud568\uc740 \ub370\uc774\ud130\uac00 \uc5b4\ub5a4 \uc2dc\uac04\uc758 \uc21c\uc11c\ub97c \uac00\uc9c4\ub2e4\ub294 \uac83\uc744 \uc758\ubbf8\ud569\ub2c8\ub2e4. \uc608\ub97c \ub4e4\uc5b4 … <\/p>\n
\ub354 \ubcf4\uae30 “PyTorch LSTM \uc608\uc81c”<\/span><\/a><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[28],"tags":[127,6,120,61,125,55],"_links":{"self":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/231"}],"collection":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/comments?post=231"}],"version-history":[{"count":7,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/231\/revisions"}],"predecessor-version":[{"id":306,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/posts\/231\/revisions\/306"}],"wp:attachment":[{"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/media?parent=231"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/categories?post=231"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/blog.cedartrees.co.kr\/index.php\/wp-json\/wp\/v2\/tags?post=231"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}