model.py

import numpy as np
from utils import ReLU, softmax, onehotencoder, ReLUderivative


def init():
  w1 = np.random.rand(128, 784)-0.5
  b1 = np.random.rand(128, 1)-0.5
  w2 = np.random.rand(10, 128)-0.5
  b2 = np.random.rand(10, 1)-0.5
  vw1 = np.zeros_like(w1)
  vb1 = np.zeros_like(b1)
  vw2 = np.zeros_like(w2)
  vb2 = np.zeros_like(b2)
  return w1, w2, b1, b2, vw1, vw2, vb1, vb2
  
def forwardprop(w1, b1, w2, b2, features):
  z1 = w1.dot(features)+b1
  a1 = ReLU(z1)
  z2 = w2.dot(a1)+b2
  a2 = softmax(z2)
  return z1, z2, a1, a2

def backprop(w1, w2, z1, a1, a2, features, labels, lambda_reg):
  encodedlabels = onehotencoder(labels)
  count = labels.size
  dz2 = a2-encodedlabels
  dw2 = (dz2.dot(a1.T))/count
  db2 = (np.sum(dz2, 1, keepdims=True))/count
  dz1 = w2.T.dot(dz2)*ReLUderivative(z1)
  dw1 = (dz1.dot(features.T))/count
  db1 = (np.sum(dz1, 1, keepdims=True))/count
  dw1 += lambda_reg*w1
  dw2 += lambda_reg*w2
  return dw1, dw2, db1, db2

def update_parameters(w1, w2, b1, b2, dw1, dw2, db1, db2, vw1, vw2, vb1, vb2, learningrate, beta_mom):
  vw1 = beta_mom*vw1 - learningrate*dw1
  vw2 = beta_mom*vw2 - learningrate*dw2
  vb1 = beta_mom*vb1 - learningrate*db1
  vb2 = beta_mom*vb2 - learningrate*db2
  
  w1 += vw1
  w2 += vw2
  b1 += vb1
  b2 += vb2
  return w1, w2, b1, b2, vw1, vw2, vb1, vb2