Character Recurrent Neural Network

• mimicing Shakespeare's writing style
• Naive RNN

!rm -r data
import os 

try:
  os.mkdir("./data")
except:
  pass

!wget https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt -P ./data

--2019-06-03 09:33:54--  https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1115394 (1.1M) [text/plain]
Saving to: ¡®./data/input.txt¡¯

input.txt           100%[===================>]   1.06M  --.-KB/s    in 0.08s   

2019-06-03 09:33:54 (13.9 MB/s) - ¡®./data/input.txt¡¯ saved [1115394/1115394]

1. Settings

1) Import required libraries

import torch
import torch.nn as nn

import unidecode
import string
import random
import re
import time, math

2) Hyperparameter

num_epochs = 2000
print_every = 100
plot_every = 10
chunk_len = 200
hidden_size = 100
batch_size = 1
num_layers = 1
embedding_size = 70
lr = 0.002

2. Data

1) Prepare characters

all_characters = string.printable
n_characters = len(all_characters)
print(all_characters)
print('num_chars = ', n_characters)

0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ 	

num_chars =  100

2) Get text data

file = unidecode.unidecode(open('./data/input.txt').read())
file_len = len(file)
print('file_len =', file_len)

file_len = 1115394

3. Functions for text processing

1) Random Chunk

def random_chunk():
    start_index = random.randint(0, file_len - chunk_len)
    end_index = start_index + chunk_len + 1
    return file[start_index:end_index]

print(random_chunk())

 speak.

QUEEN MARGARET:
Defy them then, or else hold close thy lips.

KING HENRY VI:
I prithee, give no limits to my tongue:
I am a king, and privileged to speak.

CLIFFORD:
My liege, the wound that b

2) Character to tensor

def char_tensor(string):
    tensor = torch.zeros(len(string)).long()
    for c in range(len(string)):
        tensor[c] = all_characters.index(string[c])
    return tensor

print(char_tensor('ABCdef'))

tensor([36, 37, 38, 13, 14, 15])

3) Chunk into input & label

def random_training_set():    
    chunk = random_chunk()
    inp = char_tensor(chunk[:-1])
    target = char_tensor(chunk[1:])
    return inp, target

class RNN(nn.Module):
    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers=1):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.embedding_size = embedding_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.num_layers = num_layers
        
        self.encoder = nn.Embedding(self.input_size, self.embedding_size)
        self.rnn = nn.GRU(self.embedding_size,self.hidden_size,self.num_layers)
        self.decoder = nn.Linear(self.hidden_size, self.output_size)
        
    
    def forward(self, input, hidden):
        out = self.encoder(input.view(1,-1))
        out,hidden = self.rnn(out,hidden)
        out = self.decoder(out.view(batch_size,-1))
        return out,hidden

    def init_hidden(self):
        hidden = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        return hidden
    
model = RNN(n_characters, embedding_size, hidden_size, n_characters, num_layers)

inp = char_tensor("A")
print(inp)
hidden = model.init_hidden()
print(hidden.size())

out,hidden = model(inp,hidden)
print(out.size())

tensor([36])
torch.Size([1, 1, 100])
torch.Size([1, 100])

2) Loss & Optimizer

optimizer = torch.optim.Adam(model.parameters(), lr=lr)
loss_func = nn.CrossEntropyLoss()

3) Test function

def test():
    start_str = "b"
    inp = char_tensor(start_str)
    hidden = model.init_hidden()
    x = inp

    print(start_str,end="")
    for i in range(200):
        output,hidden = model(x,hidden)

        output_dist = output.data.view(-1).div(0.8).exp()
        top_i = torch.multinomial(output_dist, 1)[0]
        predicted_char = all_characters[top_i]

        print(predicted_char,end="")

        x = char_tensor(predicted_char)

4. Train

for i in range(num_epochs):
    inp,label = random_training_set()
    hidden = model.init_hidden()

    loss = torch.tensor([0]).type(torch.FloatTensor)
    optimizer.zero_grad()
    for j in range(chunk_len-1):
        x  = inp[j]
        y_ = label[j].unsqueeze(0).type(torch.LongTensor)
        y,hidden = model(x,hidden)
        loss += loss_func(y,y_)

    loss.backward()
    optimizer.step()
    
    if i % 100 == 0:
        print("\n",loss/chunk_len,"\n")
        test()
        print("\n","="*100)

 tensor([4.6093], grad_fn=<DivBackward0>) 

b:u*_?F}*ME$+XB09|?c~\SRoM[CKyb{Q?Vf/TM2[SXW*[nmu
Z;MjWAnChQyoRZ:.|C
>>@fb&
68UqtmSY`5_XyB\bw
-
eS%k8D,CdVPMDC-!I@(?1t:I`
 ====================================================================================================

 tensor([2.6830], grad_fn=<DivBackward0>) 

bave fithnlim lh ald
 onend widisld venreord thi he,
Ofore an Ror thest mans sth hared this rhaOfee hean k lito bo theat asr shey fom aeed
L arEe fut anthe d ouls whou, wine nd thislle bo gon

This aio
 ====================================================================================================

 tensor([2.3287], grad_fn=<DivBackward0>) 

blly, me thouf ou the bell belopen meand fover haht wont, mus merw, lalt,
And a to tor sis of ar'd you kouk wast own my sarith bed is on wich the you, ands coam andy I the my this wedjs helt ad.

AQfov
 ====================================================================================================

 tensor([2.1773], grad_fn=<DivBackward0>) 

b@A
Thy syon theat ion
Thas tond fordive stos I in purover,
Horoud'st, I noleseor my my so and the me thou my wiles homy p toorvert,
I hing art in gomb
Tond fraing
Int om fithre yourn dopear, dandand a
 ====================================================================================================

 tensor([2.1543], grad_fn=<DivBackward0>) 

bo lothake of the to thou sith fape, the hoover,
I heat pelly hall chor gor manse, neoss and my kints and protime sollfore dacast sotere shath sberefe,
Thome
I not hat in hest chath you the, will
And t
 ====================================================================================================

 tensor([2.0411], grad_fn=<DivBackward0>) 

bucont lave my my losk, 'llose, wer me are what gushent you cowe,
As done s youl an! gish that be for theas will of thave Fake tarises!

Frow rashake for have ssere wint heis to us to that thit rage to
 ====================================================================================================

 tensor([2.1619], grad_fn=<DivBackward0>) 

bus agioke, my shative
Thty colt thing not noring cound in the the nor me that for hath, all with suke of so no some the didery.

BUMIO:
An fing advaing of come my ntier; that pord wenfiadswer then is 
 ====================================================================================================

 tensor([2.0667], grad_fn=<DivBackward0>) 

bat shate huth as bruy,
The fie.

MIINA:
De but by at noost, the rom a dood; of and faid a see, chamber ell sients well pood,
To for yue that I hill hely hance me he I hish bo.

CENTIO:
What is to his!
 ====================================================================================================

 tensor([1.8574], grad_fn=<DivBackward0>) 

b
As sider youth bugh be staild
Whee ar teath the sever?

CARDIN LIANGER:
Bround that ank seatread!

HANUS:
I't spird there cimment weres, and and dam what this the s to shall notious hast with eve wit
 ====================================================================================================

 tensor([1.7585], grad_fn=<DivBackward0>) 

bled
To hear a for hals do son, I gread wnere taran:
Thy my are Rus ening unis more uncion deat not no make grole,
But sell ridspreaw,
Thee frater the in of Mords,
And know is kut state hear stand on a
 ====================================================================================================

 tensor([1.9431], grad_fn=<DivBackward0>) 

ble beforisher thou hanmy bloding;
I plock meant, dries, mother, and agan and lesser wime the wooth frow saird'd cather
To crear stere to that him lath
red the my maine frow the auch for door let fore 
 ====================================================================================================

 tensor([1.8437], grad_fn=<DivBackward0>) 

beld is ever.

SOMEO:
The sloult that must heard; be ustar is more bought the sture now shour conghis come the reasemand.

Noth ill sentrous your wourd stentle, I mausper me have pleed me seink?

Sechi
 ====================================================================================================

 tensor([2.1247], grad_fn=<DivBackward0>) 

bito't not day madine,
And forgh mallow can And treath that.
To songent and sotinestingion with you hothed will, and be her
Now farces, her seable a,
Am to in noth the fants in desshus
And shall stand 
 ====================================================================================================

 tensor([1.6276], grad_fn=<DivBackward0>) 

bed would
so not I prance lige strand'd take is tran on
That not are parcess,
And not shaltion hat of them san:
How sloment lail peetter, misttaas so cains to maile.

SurcPright's then the quee!
Thee i
 ====================================================================================================

 tensor([1.7192], grad_fn=<DivBackward0>) 

bound come made for dester beace were own you and innow his some or will as arring's cold we for do him of he pessee of God consed dedon.

ONCHANT:
Shamfored lies be her we bere shall permborn?
she del
 ====================================================================================================

 tensor([1.7789], grad_fn=<DivBackward0>) 

by dish.

First I Rome, I fathat our your the thand
upon where and lay thel shall precome to not time on
Now me the suntes.

CAULAD RION, EDWARDI:
Mishin your eve his prook his driens, thee papsay.

MO
 ====================================================================================================

 tensor([2.2160], grad_fn=<DivBackward0>) 

breand,
The man is ast, say you are scour; a dead,
To the lare of prose the me courther he douncould,
What shave leave should is come theid lord.

BRINCEOT:
Ahas dowandber live a shour ancon, the coold
 ====================================================================================================

 tensor([1.6786], grad_fn=<DivBackward0>) 

bour with the wands trance-hose?

KING RICHIRI EDF II
LORKE:
You so reepidle? O dised is it stell feem:
The chillented in for a crimbery lust pries spire:
That the king beay, bloody.

SICHINICIO:
The d
 ====================================================================================================

 tensor([1.7443], grad_fn=<DivBackward0>) 

bed daughtents unponamess and Isaw hears
And be let I'll to it asse to thound to wrean:
Det your lassicest and a husblesford; and yent not.

LANTIO:
IZlant, I more daughted and the? To the rantly but h
 ====================================================================================================

 tensor([1.8941], grad_fn=<DivBackward0>) 

bly was it my pother.

KING RICHARD I II:
I seet me:
The man he his be king shall to must them canders ances,
Good way, I hath rair, had man: and lant not soold genteds
!m, you shall been send I man on
 ====================================================================================================