Karpathy_ZtH_Learning/micrograd_from_scratch.ipynb

In [90]:

import math
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

In [91]:

def f(x):
    return 3*x**2 - 4*x + 5

In [92]:

f(3.0)

Out [92]:

20.0

In [93]:

xs = np.arange(-5, 5, 0.25)
ys = f(xs)
ys
plt.plot(xs,ys)

Out [93]:

[<matplotlib.lines.Line2D at 0x7a6b3b8029d0>]

In [94]:

h = 0.000001
x = 2/3
(f(x + h) - f(x)) / h

Out [94]:

2.999378523327323e-06

In [95]:

a = 2.0
b = -3.0
c = 10.0
d = a*b + c
print(d)

4.0

In [96]:

h = 0.0001

a = 2.0
b = -3.0
c = 10.0

d1 = a*b + c
c += h
d2 = a*b + c

print('d1', d1)
print('d2', d2)
print('slope', (d2 - d1)/h )

d1 4.0
d2 4.0001
slope 0.9999999999976694

In [97]:

class Value:
    def __init__(self, data, _children=(), _op='', label=''):
        self.data = data
        self.grad = 0.0
        self._backward = lambda: None
        self._prev = set(_children)
        self._op = _op
        self.label = label
    def __repr__(self):
        return f"Value(data={self.data}, label={self.label})"
    
    def __add__(self, other):
        out = Value(self.data + other.data, (self, other), '+')

        def _backward():
            self.grad = 1.0 * out.grad
            other.grad = 1.0 * out.grad
        out._backward = _backward
        
        return out
    
    def __mul__(self, other):
        out = Value(self.data * other.data, (self, other), '*')

        def _backward():
            self.grad = other.data * out.grad
            other.grad = self.data * out.grad
        out._backward = _backward
        
        return out
    
    def tanh(self):
        x = self.data
        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)
        out = Value(t,(self, ), 'tanh')

        def _backward():
            self.grad = (1 - t**2) * out.grad
        out._backward = _backward

        return out
    def backward(self):


        topo = []
        visited = set()
        def build_topo(v):
            if v not in visited:
                visited.add(v)
                for child in v._prev:
                    build_topo(child)
                topo.append(v)
        build_topo(self)
        self.grad = 1.0
        for node in reversed(topo):
            node._backward()
    


a = Value(2.0, label='a')
b = Value(-3.0, label='b')
c = Value(10.0, label='c')
e = a*b; e.label = 'e'
d = e + c; d.label = 'd'
f = Value(-2, label='f')
L = d * f; L.label='L'
L

Out [97]:

Value(data=-8.0, label=L)

In [ ]:

In [98]:

d._prev

Out [98]:

{Value(data=-6.0, label=e), Value(data=10.0, label=c)}

In [99]:

d._op

Out [99]:

'+'

In [100]:

from graphviz import Digraph

def trace(root):
    nodes, edges, = set(), set()
    def build(v):
        if v not in nodes:
            nodes.add(v)
            for child in v._prev:
                edges.add((child, v))
                build(child)
    build(root)
    return nodes, edges

def draw_dot(root):
    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'})

    nodes, edges = trace(root)
    for n in nodes:
        uid = str(id(n))
        dot.node(name = uid, label = "{ %s | data %.4f | grad %.4f }" % (n.label , n.data, n.grad), shape = 'record')
        if n._op:
            dot.node(name = uid + n._op, label = n._op)
            dot.edge(uid + n._op, uid)

    for n1, n2 in edges:
        dot.edge(str(id(n1)), str(id(n2)) + n2._op)

    return dot

In [101]:

draw_dot(L)

Out [101]:

In [102]:

L.grad = 1
d.grad = -2
f.grad = 4
c.grad = -2
e.grad = -2
a.grad = -2 * -3
b.grad = -2 * 2

In [103]:

plt.plot(np.arange(-5,5,0.2), np.tanh(np.arange(-5,5,0.2))); plt.grid()

In [104]:

x1 = Value(2.0, label='x1')
x2 = Value(0.0, label='x2')

w1 = Value(-3.0, label='w1')
w2 = Value(1.0, label='w2')

b = Value(6.8813735870195432, label='b')

x1w1 = x1*w1; x1w1.label = 'x1*w1'
x2w2 = x2*w2; x2w2.label = 'x2*w2'

x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'

n = x1w1x2w2 + b; n.label = 'n'
o = n.tanh(); o.label = 'o'

In [105]:

'''
o.grad = 1.0
o._backward()
n._backward()
b._backward()
x1w1x2w2._backward()
x1w1._backward()
x2w2._backward()
'''

Out [105]:

'\no.grad = 1.0\no._backward()\nn._backward()\nb._backward()\nx1w1x2w2._backward()\nx1w1._backward()\nx2w2._backward()\n'

In [ ]:

In [108]:

o.backward()

In [109]:

draw_dot(o)

Out [109]:

In [107]:

o.grad = 1.0
# 1 - o.data**2
n.grad = 0.5
x1w1x2w2.grad = 0.5
b.grad = 0.5
x1w1.grad = 0.5
x2w2.grad = 0.5
x2.grad = w2.data * x2w2.grad
w2.grad = x2.data * x2w2.grad
x1.grad = w1.data * x1w1.grad
w1.grad = x1.data * x1w1.grad