Karpathy_ZtH_Learning/micrograd_from_scratch.ipynb

In [97]:

import math
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

In [98]:

def f(x):
    return 3*x**2 - 4*x + 5

In [99]:

f(3.0)

Out [99]:

20.0

In [100]:

xs = np.arange(-5, 5, 0.25)
ys = f(xs)
ys
plt.plot(xs,ys)

Out [100]:

[<matplotlib.lines.Line2D at 0x72b9b1b9c4d0>]

In [101]:

h = 0.000001
x = 2/3
(f(x + h) - f(x)) / h

Out [101]:

2.999378523327323e-06

In [102]:

a = 2.0
b = -3.0
c = 10.0
d = a*b + c
print(d)

4.0

In [103]:

h = 0.0001

a = 2.0
b = -3.0
c = 10.0

d1 = a*b + c
c += h
d2 = a*b + c

print('d1', d1)
print('d2', d2)
print('slope', (d2 - d1)/h )

d1 4.0
d2 4.0001
slope 0.9999999999976694

In [104]:

class Value:
    def __init__(self, data, _children=(), _op='', label=''):
        self.data = data
        self.grad = 0.0
        self._prev = set(_children)
        self._op = _op
        self.label = label
    def __repr__(self):
        return f"Value(data={self.data}, label={self.label})"
    
    def __add__(self, other):
        out = Value(self.data + other.data, (self, other), '+')
        return out
    
    def __mul__(self, other):
        out = Value(self.data * other.data, (self, other), '*')
        return out
    
    def tanh(self):
        x = self.data
        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)
        out = Value(t,(self, ), 'tanh')
        return out
    
a = Value(2.0, label='a')
b = Value(-3.0, label='b')
c = Value(10.0, label='c')
e = a*b; e.label = 'e'
d = e + c; d.label = 'd'
f = Value(-2, label='f')
L = d * f; L.label='L'
L

Out [104]:

Value(data=-8.0, label=L)

In [105]:

d._prev

Out [105]:

{Value(data=-6.0, label=e), Value(data=10.0, label=c)}

In [106]:

d._op

Out [106]:

'+'

In [107]:

from graphviz import Digraph

def trace(root):
    nodes, edges, = set(), set()
    def build(v):
        if v not in nodes:
            nodes.add(v)
            for child in v._prev:
                edges.add((child, v))
                build(child)
    build(root)
    return nodes, edges

def draw_dot(root):
    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'})

    nodes, edges = trace(root)
    for n in nodes:
        uid = str(id(n))
        dot.node(name = uid, label = "{ %s | data %.4f | grad %.4f }" % (n.label , n.data, n.grad), shape = 'record')
        if n._op:
            dot.node(name = uid + n._op, label = n._op)
            dot.edge(uid + n._op, uid)

    for n1, n2 in edges:
        dot.edge(str(id(n1)), str(id(n2)) + n2._op)

    return dot

In [108]:

draw_dot(L)

Out [108]:

In [109]:

L.grad = 1
d.grad = -2
f.grad = 4
c.grad = -2
e.grad = -2
a.grad = -2 * -3
b.grad = -2 * 2

In [110]:

plt.plot(np.arange(-5,5,0.2), np.tanh(np.arange(-5,5,0.2))); plt.grid()

In [111]:

x1 = Value(2.0, label='x1')
x2 = Value(0.0, label='x2')

w1 = Value(-3.0, label='w1')
w2 = Value(1.0, label='w2')

b = Value(6.8813735870195432, label='b')

x1w1 = x1*w1; x1w1.label = 'x1*w1'
x2w2 = x2*w2; x2w2.label = 'x2*w2'

x1w1x2w2 = x1w1 + x2w2; x1w1x2w2.label = 'x1*w1 + x2*w2'

n = x1w1x2w2 + b; n.label = 'n'
o = n.tanh(); o.label = 'o'
draw_dot(o)

Out [111]:

In [ ]: