separate cost computation from backprop

This commit is contained in:
dadel 2018-01-23 22:48:52 +01:00
parent 79c2cbe69b
commit 4bc26caef6
1 changed files with 34 additions and 28 deletions

62
mlp.py
View File

@ -342,18 +342,36 @@ class MultiLayerPerceptron(object):
self.propagate() self.propagate()
return self._A[self._L] return self._A[self._L]
def back_propagation(self, get_cost_function=False): def compute_cost_function(self):
"""Back propagation """Compute cost function:
J = -1/m((Y(A.T)) + (1-Y)(A.T))
if self._regularization is True will add:
J += lamda/(2*m)*Wnorm
:param get_cost_function: if True the cost function J :return: the cost function result
will be computed and returned.
J = -1/m((Y(A.T)) + (1-Y)(A.T))
if self._regularization will add:
J += lamda/(2*m)*Wnorm
:return: the cost function if get_cost_function==True else None
""" """
L = self._L
m = self._m
J = None J = None
# Compute cost function
if self._softmax:
# case of softmax multi-class
loss = -np.sum(self._Y * np.log(self._A[L]), axis=0)
J = 1/m * np.sum(loss)
else:
J = -1/m * np.sum(( np.dot(self._Y, np.log(self._A[L]).T) + \
np.dot((1 - self._Y), np.log(1-self._A[L]).T) ), axis=1)
# add regularization
if self._regularization:
wnorms = 0
for w in self._W[1:]:
wnorms += np.linalg.norm(w)
J += self._lambda_regul/(2*m) * wnorms
return J
def back_propagation(self, get_cost_function=False):
"""Back propagation"""
L = self._L L = self._L
m = self._m m = self._m
dW = [None] + [None] * self._L dW = [None] + [None] * self._L
@ -361,22 +379,6 @@ class MultiLayerPerceptron(object):
dA = [None] + [None] * self._L dA = [None] + [None] * self._L
dA[L] = -self._Y/self._A[L] + ((1-self._Y)/(1-self._A[L])) dA[L] = -self._Y/self._A[L] + ((1-self._Y)/(1-self._A[L]))
# Compute cost function
if get_cost_function:
if self._softmax:
# case of softmax multi-class
loss = -np.sum(self._Y * np.log(self._A[L]), axis=0)
J = 1/m * np.sum(loss)
else:
J = -1/m * np.sum(( np.dot(self._Y, np.log(self._A[L]).T) + \
np.dot((1 - self._Y), np.log(1-self._A[L]).T) ), axis=1)
# add regularization
if self._regularization:
wnorms = 0
for w in self._W[1:]:
wnorms += np.linalg.norm(w)
J += self._lambda_regul/(2*m) * wnorms
# Compute weights derivatives # Compute weights derivatives
for l in range(L, 0, -1): for l in range(L, 0, -1):
if self._softmax and l == L: if self._softmax and l == L:
@ -417,30 +419,34 @@ class MultiLayerPerceptron(object):
self._W[l] = self._W[l] - self._alpha * w_factor self._W[l] = self._W[l] - self._alpha * w_factor
self._b[l] = self._b[l] - self._alpha * b_factor self._b[l] = self._b[l] - self._alpha * b_factor
return J
def minimize_cost(self, min_cost, max_iter=100000, alpha=None, plot=False): def minimize_cost(self, min_cost, max_iter=100000, alpha=None, plot=False):
"""Propagate forward then backward in loop while minimizing the cost function. """Propagate forward then backward in loop while minimizing the cost function.
:param min_cost: cost function value to reach in order to stop algo. :param min_cost: cost function value to reach in order to stop algo.
:param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000). :param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000).
:param alpha: learning rate, if None use the instance alpha value. Default None. :param alpha: learning rate, if None use the instance alpha value. Default None.
:param plot: if True will plot the graph cost function depending on iteration
""" """
nb_iter = 0 nb_iter = 0
if alpha is None: if alpha is None:
alpha = self._alpha alpha = self._alpha
# forward propagation
self.propagate() self.propagate()
if plot: if plot:
y=[] y=[]
x=[] x=[]
for i in range(max_iter): for i in range(max_iter):
J = self.back_propagation(True) nb_iter = i + 1
# compute cost function
J = self.compute_cost_function()
if plot: if plot:
y.append(J) y.append(J)
x.append(nb_iter) x.append(nb_iter)
# back propagation
self.back_propagation()
# forward propagation
self.propagate() self.propagate()
nb_iter = i + 1
if J <= min_cost: if J <= min_cost:
break break
if mp and plot: if mp and plot: