Karpathy Video: makemore 1
Karpathy video - simple neural net bigram, neural-net model to predict names.
1
16
References
Notes
What is MakeMore? It makes more of what you give it. We're building character level language model - predict next character given a sequence of characters. Bigram language model - working with two characters at a time. With a bigram language model, you look at only the previous character to predict the next one.
words = open("names.txt",'r').read().splitlines()
words[0:10]
out[2]
len(words)
out[3]
min(len(w) for w in words)
out[4]
max(len(w) for w in words)
out[5]
b = {} # Counting frequencies of bigrams
for w in words:
# Special start character + list characters + special end character
chs =['<S>'] + list(w) + ["<E>"]
for ch1, ch2 in zip(chs,chs[1:]):
bigram = (ch1, ch2)
b[bigram] = b.get(bigram,0) + 1
out[6]
sorted(b.items(),key=lambda kv: kv[1],reverse=True)
out[7]
import torch
N = torch.zeros((27,27),dtype=torch.int32)
out[8]
# Character Lookup Array
chars = sorted(list(set(''.join(words))))
stoi = {s:i+1 for i, s in enumerate(chars)} # Mapping of characters to integers
stoi['.'] = 0
itos = {i:s for s,i in stoi.items()}
b = {} # Counting frequencies of bigrams
for w in words:
# Special start character + list characters + special end character
chs =['.'] + list(w) + ["."]
for ch1, ch2 in zip(chs,chs[1:]):
ix1 = stoi[ch1]
ix2 = stoi[ch2]
N[ix1,ix2] += 1
out[9]
import matplotlib.pyplot as plt
%matplotlib inline
plt.figure(figsize=(16,16))
plt.imshow(N,cmap='Blues')
for i in range(27):
for j in range(27):
chstr = itos[i] + itos[j]
plt.text(j,i,chstr,ha="center",va="bottom",color="gray")
plt.text(j,i,N[i,j].item(),ha="center",va="top",color="gray")
plt.axis("off")
out[10]
N[0,:]
p = N[0].float()
p = p / p.sum()
p
out[11]
g = torch.Generator().manual_seed(214783647)
ix = torch.multinomial(p,num_samples=1,replacement=True,generator=g).item()
itos[ix]
out[12]
g = torch.Generator().manual_seed(214783647)
p = torch.randn(3,generator=g)
p = p /p.sum()
p
out[13]
torch.multinomial(p,num_samples=100,replacement=True,generator=g)
out[14]
p.shape
out[15]
P = N.float()
P = P / P.sum(1,keepdim=True)
out[16]
P[0].sum()
out[17]
g = torch.Generator().manual_seed(214783647)
ix = 0
for i in range(20):
out = []
while True:
p = N[ix].float()
p = p / p.sum()
ix = torch.multinomial(p,num_samples=1,replacement=True,generator=g).item()
out.append(itos[ix])
if ix==0:
break
print(''.join(out))
out[18]
# Goal": maximize likelihood of the data wrt to model parameters (statistical modeling); equivakent to maximizing the log lokelihood (because log is monotonic); equivalent to minimizing the negative log likelihood; eqivalent to minimizing the negative log likelihood
# log(a*b*c) = log(a) + log(b) + log(c)
log_likelihood = 0.0
for w in words:
chs = ['.'] + list(w) + ['.']
for ch1, ch2 in zip(chs, chs[1:]):
ix1 = stoi[ch1]
ix2 = stoi[ch2]
prob = P[ix1, ix2]
logprop = torch.log(prob)
log_likelihood += logprop
n+=1
# print(f'{ch1}{ch2}: {prob :.4f} {logprop :.4f}')
print(f"{log_likelihood=}")
nll = -log_likelihood
print(f"{nll=}")
print(f"{nll/n}")
out[19]
xs, ys = [], []
for w in words[:1]:
chs = ['.'] + list(w) + ['.']
for ch1, ch2 in zip(chs, chs[1:]):
ix1 = stoi[ch1]
ix2 = stoi[ch2]
print(ch1,ch2)
xs.append(ix1)
ys.append(ix2)
xs = torch.tensor(xs)
ys = torch.tensor(ys)
out[20]
xs
out[21]
ys
out[22]
import torch.nn.functional as F
xenc = F.one_hot(xs,num_classes=27).float()
xenc
out[23]
xenc.shape
out[24]
plt.imshow(xenc)
out[25]
W = torch.randn((27,27))
xenc @ W
out[26]
out[27]
Font Weight
Font Family
Undo Last Change
Redo Last Change
Clear Content
Insert Line Break
Indent Content
Outdent Content
Left Align
Center Align
Right Align
Justify Align
Bold Text
Italicize Text
Strikethrough Text
Underline Text
Superscript Text
Subscript Text
Insert Header
Change Background Color
Change Text Color
Highlight Text
Insert Saved Version of Article
Insert Image
Insert Audio
Insert Video
Upload GIF
Embed News or Youtube Video
Insert a Quote
Insert Lists
Insert
<aside> Element
Insert
<details> Element
Insert Code Markup
Insert Math Markup
Insert Table
Insert Chart
Insert Horizontal Rule
Insert Paragraph at End of Editor
View Result at Different Screen Sizes
Edit the Style of the Block Elements in Selection
Insert Link
Code Text
Clear Text Formatting
Speech To Text
Download Editor State
Upload Editor State
Insert Keyboard Command
Insert Abbreviation
Insert Inline Quote
Insert Section Heading
Insert Custom HTML
Insert Columns Layout
Copy Editor Contents
Add a Comment
Annotate Article
Comments
You have to be logged in to add a comment
User Comments
There are currently no comments for this article.