Character Recurrent Neural Network

• mimicing Shakespeare's writing style
• Naive RNN

!rm -r data
import os 

try:
  os.mkdir("./data")
except:
  pass

!wget https://raw.githubusercontent.com/GunhoChoi/PyTorch-FastCampus/master/05_RNN/2_Char_RNN/data/linux.txt -P ./data

--2019-06-03 09:56:14--  https://raw.githubusercontent.com/GunhoChoi/PyTorch-FastCampus/master/05_RNN/2_Char_RNN/data/linux.txt
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 33756 (33K) [text/plain]
Saving to: ¡®./data/linux.txt¡¯

linux.txt           100%[===================>]  32.96K  --.-KB/s    in 0.01s   

2019-06-03 09:56:15 (2.62 MB/s) - ¡®./data/linux.txt¡¯ saved [33756/33756]

1. Settings

1) Import required libraries

import torch
import torch.nn as nn

import unidecode
import string
import random
import re
import time, math

2) Hyperparameter

num_epochs = 2000
print_every = 100
plot_every = 10
chunk_len = 200
hidden_size = 100
batch_size = 1
num_layers = 1
embedding_size = 70
lr = 0.002

2. Data

1) Prepare characters

all_characters = string.printable
n_characters = len(all_characters)
print(all_characters)
print('num_chars = ', n_characters)

0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ 	

num_chars =  100

2) Get text data

file = unidecode.unidecode(open('./data/linux.txt').read())
file_len = len(file)
print('file_len =', file_len)

file_len = 33756

3. Functions for text processing

1) Random Chunk

def random_chunk():
    start_index = random.randint(0, file_len - chunk_len)
    end_index = start_index + chunk_len + 1
    return file[start_index:end_index]

print(random_chunk())

e",
		.private = offsetof(struct bfq_group, stats.time),
		.seq_show = bfqg_print_stat_recursive,
	},
	{
		.name = "bfq.sectors_recursive",
		.seq_show = bfqg_print_stat_sectors_recursive,
	},
	{
		.na

2) Character to tensor

def char_tensor(string):
    tensor = torch.zeros(len(string)).long()
    for c in range(len(string)):
        tensor[c] = all_characters.index(string[c])
    return tensor

print(char_tensor('ABCdef'))

tensor([36, 37, 38, 13, 14, 15])

3) Chunk into input & label

def random_training_set():    
    chunk = random_chunk()
    inp = char_tensor(chunk[:-1])
    target = char_tensor(chunk[1:])
    return inp, target

3. Model & Optimizer

1) Model

class RNN(nn.Module):
    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers=1):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.embedding_size = embedding_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.num_layers = num_layers
        
        self.encoder = nn.Embedding(self.input_size, self.embedding_size)
        self.rnn = nn.GRU(self.embedding_size,self.hidden_size,self.num_layers)
        self.decoder = nn.Linear(self.hidden_size, self.output_size)
        
    
    def forward(self, input, hidden):
        out = self.encoder(input.view(1,-1))
        out,hidden = self.rnn(out,hidden)
        out = self.decoder(out.view(batch_size,-1))
        return out,hidden

    def init_hidden(self):
        hidden = torch.zeros(self.num_layers, batch_size, self.hidden_size)
        return hidden
    
model = RNN(n_characters, embedding_size, hidden_size, n_characters, num_layers)

inp = char_tensor("A")
print(inp)
hidden = model.init_hidden()
print(hidden.size())

out,hidden = model(inp,hidden)
print(out.size())

tensor([36])
torch.Size([1, 1, 100])
torch.Size([1, 100])

2) Loss & Optimizer

optimizer = torch.optim.Adam(model.parameters(), lr=lr)
loss_func = nn.CrossEntropyLoss()

3) Test function

def test():
    start_str = "b"
    inp = char_tensor(start_str)
    hidden = model.init_hidden()
    x = inp

    print(start_str,end="")
    for i in range(200):
        output,hidden = model(x,hidden)

        output_dist = output.data.view(-1).div(0.8).exp()
        top_i = torch.multinomial(output_dist, 1)[0]
        predicted_char = all_characters[top_i]

        print(predicted_char,end="")

        x = char_tensor(predicted_char)

4. Train

for i in range(num_epochs):
    inp,label = random_training_set()
    hidden = model.init_hidden()

    loss = torch.tensor([0]).type(torch.FloatTensor)
    optimizer.zero_grad()
    for j in range(chunk_len-1):
        x  = inp[j]
        y_ = label[j].unsqueeze(0).type(torch.LongTensor)
        y,hidden = model(x,hidden)
        loss += loss_func(y,y_)

    loss.backward()
    optimizer.step()
    
    if i % 100 == 0:
        print("\n",loss/chunk_len,"\n")
        test()
        print("\n","="*100)

 tensor([4.5848], grad_fn=<DivBackward0>) 

b,KQfAgcw1'8;(\`8!Z@q/;OZk9pNMYrAu
0)\X+m0dI36m`L
n|Fq2tBRB:2B@<w )06,zy38-wQ(\8hc`NCr\73Min@=\im-rRSB^lRb#U Ev@tKEzhz$P1$e,g^M)8JVaFxp
 ====================================================================================================

 tensor([2.4355], grad_fn=<DivBackward0>) 

blkg &of (s->sstigibfqqg_rf->ogrond bfqg>aue & ing ohe (ned thtnty *a bfqg ne tablg.  lee bfq_stue *tee ist a			 * fh o#no et, int bfqqgeuitap dntity)
{
										} bfq_eimeapityonth  s to tbfqgrethtit
 ====================================================================================================

 tensor([2.1628], grad_fn=<DivBackward0>) 

blkcg_prhe me_tat_rew, blkq_s(&nd(bfqg)

	 * stats->et;
	 * se tatred_restturuit);
	bfqg) {s->princs_wBentite))lime se (stat old *bfqg_ind T= falitice *bfqq_uproup we of prine_bfqg, bfqg, !entity){
		y
 ====================================================================================================

 tensor([1.8952], grad_fn=<DivBackward0>) 

bfqq = bfqg->encss_stat_clolkg(start_qup *bfqq_group =/
								 / struct bfq_group(struct clonc_bfq_group *blkg_statsic_tatic bfq_group *bfq_group, Thed assty_entity = blkg_daet_mpreat_stat_tornw, the
 ====================================================================================================

 tensor([1.9491], grad_fn=<DivBackward0>) 

bfqg_stats->stats);
			bfqg_data *sed = of bfq_group *bfqq_data *bfqg = &bfqg to meqg = bfq_group_sturt = bfqg->stat_prio_stats_cllackig, gesursecurss, foro_dalg_data *s->blkg->sched_data_structs(stats
 ====================================================================================================

 tensor([0.8923], grad_fn=<DivBackward0>) 

bfqg) stats it maup %hinct becy
 * lock cherium the ofprio_restats_pore morent ile = 0;
}

statsel csive stats ber in bfqg_stats anclurent blkg_size = bfqg_stats_upe.w(struct bfqg_to_bfqg_printht io Ma
 ====================================================================================================

 tensor([1.5306], grad_fn=<DivBackward0>) 

bfq.
					      se con it bfq_io_size_show, parent(bfqg->torive_trivate_entity_entity_entity = bfq_amyne_entity;
	entity(stats);
	entity->entity);
	bfq_prime);
};
}

void bfqg, stats->group_bfqq_blkg_b
 ====================================================================================================

 tensor([1.6940], grad_fn=<DivBackward0>) 

bjeq_seight {,
		6f (f)
{
	bfqg(bfqd_uporoment_aux([pwat = blkg_print_stat_init(sf,
		.namp = bfqq_io_show = uelack, bfqq) {
},
		private = bfq_group *bfqg, fistroot stats);

	return 0;

	if loke_afqd(
 ====================================================================================================

 tensor([1.1762], grad_fn=<DivBackward0>) 

bfqg);
	return hee *sh->blkg_queue_size(struct bfq_group *bfqg)
{
	blkg *bfqg);
	blkg_print_stat_recursive_stats_canclude *s bfq.io_print == bfqg_print_luctive,
		.pof(struct bfqg_stats_cgroup *blkg_st
 ====================================================================================================

 tensor([1.7709], grad_fn=<DivBackward0>) 

bfqg);
}

void bfqg->ret(&stats->stat_add(&bfqg, bfqd->root_group(walloc_NFLAG_FISn_bfqg_to_bfqg(bfqg);
}
{
	return bfqg_prigrt_bfqd, pration _fio_cpd(velic _bfqq_group *bfqd, blkg_to_bfqq(entity->nhee
 ====================================================================================================

 tensor([0.9089], grad_fn=<DivBackward0>) 

blkg_prfill_stoichis of the sius, &bfqg->avg_to_bfqg_move,
		.name = bfqg_prentint_blkg_print_writime);
	blkg_rwstat_recursive,
	{
		.prequesentern for no flags
		 * blkg_resaved,
	.private, statio to 
 ====================================================================================================

 tensor([1.6995], grad_fn=<DivBackward0>) 

blkg_rwstat_group;
}

void bfq_group *bfqd->root_group *bfqg *pd->lock_log_group *bfqq = bfqg_stats_stats_update_idle_time)))
			blkcg_policy_bfq.io_cud_group *bfqg = offer u64 bfqq_root_print_aux(&sta
 ====================================================================================================

 tensor([1.3504], grad_fn=<DivBackward0>) 

blkg_group_liack_weight = blkg_prof, to *bfqg_print_stat_recursive",
		.private = = "bfqng);

	entity;
	entity->sched_to_bfq_end_exit(&stats->idle_time),
		.seq_shoid_to_bfqg_print Tolis = bfq_entity(e
 ====================================================================================================

 tensor([1.0711], grad_fn=<DivBackward0>) 

blkg);
	return;
	struct bfq_queue *bfqd->incloct_stat_reset_dellack entity *entity to move to blkg = bfqg_print_rwstat_service blkg_rwstat_to_blkg(pd),
		.
							     * The u64 bfq_group, stats->eset(
 ====================================================================================================

 tensor([1.2710], grad_fn=<DivBackward0>) 

bfqg->sched_data = &bfqd->in_seq_show = bfqg_print_rwstat_init(&stats->folloci_entity);
	blkg_stat_active_recursive,
		.namp = pof,
		.
	 * blkg_rwstat_logs (is = "bfq.io_blkgs(sf, gfp);
}

static void
 ====================================================================================================

 tensor([0.4265], grad_fn=<DivBackward0>) 

blkcg_policy_bfq, NULL;

	return NULL);
	blkg;
}

struct blkg_rwstat_init(&stats->data *bfqg;
}

void bfq_end_wr_arent,
	{
		.prime(bfqg);
		rFu_stats->arequeue);
	blkg_leach_change_pd_set_ireed */
	bl
 ====================================================================================================

 tensor([0.9161], grad_fn=<DivBackward0>) 

bytes);
	}
	struct bfq_group *bfqd->idle_time(struct bfq_group *bfqg = bfq_print_rwstat_repare_time);
		blkg_rwstat_add(&bfqg->stats);

	root group *blkcg = bfqg(blkg);
											\

static void bfq_gr
 ====================================================================================================

 tensor([0.5478], grad_fn=<DivBackward0>) 

blkcg_policy_datat = "bfq.enit(&stats->merged_weight = offsetof(struct bfq_cilall_stat,
}

/**
 * bfq_io_set_time),
		.seq_show = bfq_end_weight "bfq_no_blkcg_print_rwstat_add_aux(&to->get(bfqd->entity
 ====================================================================================================

 tensor([0.4530], grad_fn=<DivBackward0>) 

blkg_to_bfqg(pd);
}

void bfqg_stats_update_group_data *bfqg, unsigned int op)
{
	struct bfq_data *bfqg, unsigned int op)
{
	blkg_rwstat_init(&stats->blkg_get of the
	 * pinre cosetofely with new_weelo
 ====================================================================================================

 tensor([1.0063], grad_fn=<DivBackward0>) 

blkcg)
		bfq_entity;
	entity->new_weight",
							  ctats.
entive alserencgs
					 * stats ->mertorecter of the the parent,
						     blkg_rwstat_exit(&stats->wait_time(struct bfq_group. */
#inc_ct(sf
 ====================================================================================================