You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

699 lines
24 KiB

#!/usr/bin/env python3
__author__ = "Adel Daouzli"
__licence__ = "GPLv3"
import struct
import numpy as np
try:
import matplotlib.pyplot as mp
except:
mp = None
def sigmoid(x):
return 1/(1+np.exp(-x))
def deriv_sigmoid(x):
a = sigmoid(x)
return a * (1 - a)
def tanh(x):
ep = np.exp(x)
en = np.exp(-x)
#print("ep:{}\nen:{}\n".format(ep,en))
return (ep - en)/(ep + en)
def deriv_tanh(x):
a = tanh(x)
return 1 - (a * a)
def relu(x):
return np.maximum(x, np.zeros(x.shape))
def deriv_relu(x):
ret = x
ret[ret > 0] = 1
ret[ret < 0] = 0
return ret
def leaky_relu(x):
ret = 0.01 * x
#fixme should map to compare
if x > 0:
ret = x
elif type(x) is np.ndarray:
ret = np.ones(x.shape)*0.01
return ret
def softmax(x):
t = np.exp(x)
sum_t = np.sum(t, axis=0)
return t / sum_t
def w_rand_tanh(n, l, xavier_init=True):
""" Initialize weights of a layer for tanH activation function
:param n: vector of number of units per layer
:param l: current layer
:param xavier_init: if True will use Xavier initialization
"""
if xavier_init:
print("tanh factor={}".format(np.sqrt(1/n[l-1])))
ret = np.random.randn(n[l], n[l-1]) * np.sqrt(1/n[l-1])
else:
print("tanh factor={}".format(np.sqrt(2/(n[l-1]+n[l]))))
ret = np.random.randn(n[l], n[l-1]) * np.sqrt(2/(n[l-1]+n[l]))
return ret
def w_rand_relu(n, l):
""" Initialize weights of a layer for tanH activation function
:param n: vector of number of units per layer
:param l: current layer
"""
print("relu factor={}".format(np.sqrt(2/n[l-1])))
return np.random.randn(n[l], n[l-1]) * np.sqrt(2/n[l-1])
def w_rand_sigmoid(n, l):
print("sigmoid factor={}".format(1/(n[l-1]*n[l])))
return np.random.randn(n[l], n[l-1]) * (1/(n[l-1]*n[l]))
def w_rand_softmax(n, l, factor=0.01):
print("softmax factor={}".format(factor))
return np.random.randn(n[l], n[l-1]) * factor
class MultiLayerPerceptron(object):
param_type = {
"uint32": {"fmt": 'I', "size": 4},
"float64": {"fmt": 'd', "size": 8},
}
functions = {
"sigmoid": {"function": sigmoid, "derivative": deriv_sigmoid, "w_rand": w_rand_sigmoid, "name": "sigmoid"},
"tanh": {"function": tanh, "derivative": deriv_tanh, "w_rand": w_rand_tanh, "name": "tanh"},
"relu": {"function": relu, "derivative": deriv_relu, "w_rand": w_rand_relu, "name": "relu"},
"softmax": {"function": softmax, "derivative": None, "w_rand": w_rand_softmax, "name": "softmax"},
}
def __init__(self, L=1, n=None, g=None, alpha=0.01, set_random_w=True, use_formula_w=False, w_rand_factor=1):
"""Initializes network geometry and parameters
:param L: number of layers including output and excluding input. Defaut 1.
:type L: int
:param n: list of number of units per layer including input. Default [2, 1].
:type n: list
:param g: list of activation functions name per layer excluding input.
Possible names are: "sigmoid", "tanh". Default ["sigmoid"].
:type g: list
:param alpha: learning rate. Default 0.01.
:param set_random_w: if True will initialize randomly weights using either w_rand_factor
or a formula depending on activation functions if use_formula_w is True
"""
#w_rand_factor = 1
self._prepared = False
self._L = L
if n is None:
n = [2, 1]
self._n = n
if g is None:
g = [MultiLayerPerceptron.functions["sigmoid"]]
else:
g = [MultiLayerPerceptron.functions[fct] for fct in g]
self._g = [None] + g
# check if softmax multi-class classification
self._softmax = False
if g[-1]["name"] == "softmax":
self._softmax = True
self._A = None
self._X = None
self._Y = None
self._Z = None
self._m = 0
self._alpha = alpha
# optimization
self._lambda = 0
self._regularization = False
self._momentum = False
self._rmsprop = False
self._adam = False
# initialise weights
self._b = [None] + [np.zeros((n[l+1], 1)) for l in range(L)]
self._W = [None] + [np.zeros((n[l+1], n[l])) for l in range(L)]
if set_random_w:
self.init_random_weights(use_formula_w, w_rand_factor)
assert(len(self._g) == len(self._W))
assert(len(self._g) == len(self._b))
assert(len(self._g) == len(self._n))
self._bacthes_x = []
self._bacthes_y = []
self._current_batch = 0
self._nb_batches = 0
def init_random_weights(self, use_formula=False, w_rand_factor=1):
"""Initialize randomly weights using a factor or using some formula
:param w_rand_factor: factorize random weights with this (default 1)
:param use_formula: if True will use formules corresponding to the activation functions
"""
if use_formula:
for l0 in range(self._L):
l = l0 + 1
self._W[l] = self._g[l]["w_rand"](self._n, l)
else:
if type(w_rand_factor) is list:
self._W = [None] + [np.random.randn(self._n[l+1], self._n[l])*w_rand_factor[l] for l in range(self._L)]
else:
self._W = [None] + [np.random.randn(self._n[l+1], self._n[l])*w_rand_factor for l in range(self._L)]
def use_regularization(self, lambd):
"""Activates regularization for backpropagation
:param lambd: the lambda parameter value for regularization
"""
self._regularization = True
self._lambda_regul = lambd
def use_momentum(self, beta=0.9, v_dw=0., v_db=0.):
"""Activates momentum optimization for backpropagation
:param beta: the beta parameter value for momentum (default 0.9)
:param v_dw: v_dw initial value for momentum (default 0.0)
:param v_db: v_db initial value for momentum (default 0.0)
"""
self._momentum = True
self._beta_momentum = beta
n = self._n
self._v_dw_momentum = [None] + [v_dw * np.ones((n[l+1], n[l])) for l in range(self._L)]
self._v_db_momentum = [None] + [v_db * np.ones((n[l+1], 1)) for l in range(self._L)]
def use_rmsprop(self, beta=0.999, s_dw=0., s_db=0., epsilon=1.0e-8):
"""Activates RMSProp optimization for backpropagation
:param beta: the beta parameter value for RMSProp (default 0.999)
:param s_dw: s_dw initial value for RMSProp (default 0.0)
:param s_db: s_db initial value for RMSProp (default 0.0)
:param epsilon: epsilon value for RMSProp (default 1.0e-8)
"""
self._rmsprop = True
self._beta_rmsprop = beta
n = self._n
self._s_dw_rmsprop = [None] + [s_dw * np.ones((n[l+1], n[l])) for l in range(self._L)]
self._s_db_rmsprop = [None] + [s_db * np.ones((n[l+1], 1)) for l in range(self._L)]
self._epsilon_rmsprop = epsilon
def use_adam(self, beta_m=0.9, v_dw=0., v_db=0., beta_r=0.999, s_dw=0., s_db=0., epsilon=1.0e-8):
"""Activates Adam optimization for backpropagation
:param beta_m: the beta parameter value for momentum (default 0.9)
:param v_dw: v_dw initial value for momentum (default 0.0)
:param v_db: v_db initial value for momentum (default 0.0)
:param beta_r: the beta parameter value for RMSProp (default 0.999)
:param s_dw: s_dw initial value for RMSProp (default 0.0)
:param s_db: s_db initial value for RMSProp (default 0.0)
:param epsilon: epsilon value for RMSProp (default 1.0e-8)
"""
self._adam = True
self.use_momentum(beta_m, v_dw, v_db)
self.use_rmsprop(beta_r, s_dw, s_db, epsilon)
def set_all_input_examples(self, X, m=None):
"""Set the input examples.
:param X: matrix of dimensions (n[0], m). Accepts also a list (len m) of lists (len n[0])
:param m: number of training examples.
:type m: int
"""
if m is None:
m = self._m
if type(X) is list:
assert(len(X) == m)
self._X = np.matrix(X).T
else:
#print(X.shape, self._n[0], m)
assert(X.shape == (self._n[0], m))
self._X = X
self._m = m
assert((self._m == m) or (self._m == 0))
self._m = m
self._prepared = False
def set_all_expected_output_examples(self, Y, m=None):
"""Set the output examples
:param Y: matrix of dimensions (n[L], m). Accepts also a list (len m) of lists (len n[L])
:param m: number of training examples.
:type m: int
"""
if m is None:
m = self._m
if type(Y) is list:
assert(len(Y) == m)
self._Y = np.matrix(Y).T
else:
#print(Y.shape, self._n[self._L], m)
assert(Y.shape == (self._n[self._L], m))
self._Y = Y
assert((self._m == m) or (self._m == 0))
self._m = m
def set_all_training_examples(self, X, Y, m=1):
"""Set all training examples
:param X: matrix of dimensions (n[0], m). Accepts also a list (len m) of lists (len n[0])
:param Y: matrix of dimensions (n[L], m). Accepts also a list (len m) of lists (len n[L])
:param m: number of training examples.
:type m: int
"""
self._m = m
self.set_all_input_examples(X, m)
self.set_all_expected_output_examples(Y, m)
def set_batches(self, batches):
"""Set the batch for training. dimensions are nb_minibatches, exemples (in/out->2), minibatch lengths
"""
self._batch_x = []
self._batch_y = []
self._batch_m = []
for b in batches:
self._batch_x.append(np.array(b[0]).T)
self._batch_y.append(np.array(b[1]).T)
self._batch_m.append(len(b[0]))
self._current_batch = 0
self._nb_batches = len(batches)
def get_next_batch(self):
"""Get the current mini-bacth and update to next the current. If reaches the end then go back to first.
"""
X = self._batch_x[self._current_batch]
Y = self._batch_y[self._current_batch]
self._current_batch += 1
if self._current_batch >= self._nb_batches:
self._current_batch = 0
return X, Y
def prepare(self, X=None, m=None, force=False):
"""Prepare network for propagation"""
if X is not None:
force = True
#print("sforce,prep =", force, self._prepared)
if force or self._prepared == False:
if not force:
self._prepared = True
if X is None:
assert(self._X is not None)
X = self._X
if m is None:
m = self._m
if m == 0:
if type(X) is list:
m = 1
else:
m = X.shape[1]
else:
if m is None:
if type(X) is list:
m = 1
else:
m = X.shape[1]
if type(X) is list:
X = np.array(X).reshape(len(X), 1)
if m is None:
assert(self._m > 0)
m = self._m
#print("m prep =", m)
self._A = [X]
self._A += [np.empty((self._n[l+1], m)) for l in range(self._L)]
self._Z = [None] + [np.empty((self._n[l+1], m)) for l in range(self._L)]
def propagate(self):
"""Forward propagation
:return: matrix of computed outputs (n[L], m)
"""
for l0 in range(self._L):
l = l0 + 1
self._Z[l] = np.dot(self._W[l], self._A[l-1]) + self._b[l]
self._A[l] = self._g[l]["function"](self._Z[l])
return self._A[self._L]
def compute_outputs(self, X=None, m=None):
"""Compute outputs with forward propagation.
Note: if no input provided, then the input should have been set using
either `set_all_input_examples()` or `set_all_training_examples()`.
:param X: if None will use self._X
:return: the computed output
"""
if X is not None:
if m is None:
if type(X) is list:
m = 1
else:
m = X.shape[1]
#print("lenX,m",len(X),m)
self.prepare(X, m)
else:
self.prepare()
self.propagate()
return self._A[self._L]
def compute_cost_function(self):
"""Compute cost function:
J = -1/m((Y(A.T)) + (1-Y)(A.T))
if self._regularization is True will add:
J += lamda/(2*m)*Wnorm
:return: the cost function result
"""
L = self._L
m = self._m
J = None
# Compute cost function
if self._softmax:
# case of softmax multi-class
loss = -np.sum(self._Y * np.log(self._A[L]), axis=0)
J = 1/m * np.sum(loss)
else:
J = -1/m * np.sum(( np.dot(self._Y, np.log(self._A[L]).T) + \
np.dot((1 - self._Y), np.log(1-self._A[L]).T) ), axis=1)
# add regularization
if self._regularization:
wnorms = 0
for w in self._W[1:]:
wnorms += np.linalg.norm(w)
J += self._lambda_regul/(2*m) * wnorms
return J
def back_propagation(self):
"""Back propagation"""
L = self._L
m = self._m
dW = [None] + [None] * self._L
db = [None] + [None] * self._L
dA = [None] + [None] * self._L
dA[L] = -self._Y/self._A[L] + ((1-self._Y)/(1-self._A[L]))
# Compute weights derivatives
for l in range(L, 0, -1):
if self._softmax and l == L:
# output layer for softmax multi-class
dZ = self._A[L] - self._Y
else:
dZ = dA[l] * self._g[l]["derivative"](self._Z[l])
dW[l] = 1/self._m * np.dot(dZ, self._A[l-1].T)
db[l] = 1/m * np.sum(dZ, axis=1, keepdims=True)
dA[l-1] = np.dot(self._W[l].T, dZ)
# Update weights
for l in range(L, 0, -1):
w_factor = dW[l]
b_factor = db[l]
# add momentum
if self._momentum:
self._v_dw_momentum[l] = self._beta_momentum * self._v_dw_momentum[l] + \
(1 - self._beta_momentum) * dW[l]
self._v_db_momentum[l] = self._beta_momentum * self._v_db_momentum[l] + \
(1 - self._beta_momentum) * db[l]
w_factor = self._v_dw_momentum[l]
b_factor = self._v_db_momentum[l]
# add RMSProp
if self._rmsprop:
self._s_dw_rmsprop[l] = self._beta_rmsprop * self._s_dw_rmsprop[l] + \
(1 - self._beta_rmsprop) * (dW[l]**2)
self._s_db_rmsprop[l] = self._beta_rmsprop * self._s_db_rmsprop[l] + \
(1 - self._beta_rmsprop) * (db[l]**2)
# if adam optimization is use the formula will work as w/b_factor are set in momentum
w_factor = w_factor / (np.sqrt(self._s_dw_rmsprop[l]) + self._epsilon_rmsprop)
b_factor = b_factor / (np.sqrt(self._s_db_rmsprop[l]) + self._epsilon_rmsprop)
# add regularization
if self._regularization:
self._W[l] = self._W[l] - self._alpha * w_factor - \
(self._alpha*self._lambda_regul/m) * self._W[l]
else:
self._W[l] = self._W[l] - self._alpha * w_factor
self._b[l] = self._b[l] - self._alpha * b_factor
def minimize_cost(self, min_cost, max_iter=100000, alpha=None, plot=False):
"""Propagate forward then backward in loop while minimizing the cost function.
:param min_cost: cost function value to reach in order to stop algo.
:param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000).
:param alpha: learning rate, if None use the instance alpha value. Default None.
:param plot: if True will plot the graph cost function depending on iteration
"""
nb_iter = 0
if alpha is None:
alpha = self._alpha
# forward propagation
self.propagate()
if plot:
y=[]
x=[]
for i in range(max_iter):
nb_iter = i + 1
# compute cost function
J = self.compute_cost_function()
if plot:
y.append(J)
x.append(nb_iter)
# back propagation
self.back_propagation()
# forward propagation
self.propagate()
if J <= min_cost:
break
if mp and plot:
mp.plot(x,y)
mp.show()
return {"iterations": nb_iter, "cost_function": J}
def minimize_cost_batches(self, min_cost, max_iter=100000, alpha=None, plot=False):
"""Propagate forward then backward in loop while minimizing the cost function.
:param min_cost: cost function value to reach in order to stop algo.
:param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000).
:param alpha: learning rate, if None use the instance alpha value. Default None.
:param plot: if True will plot the graph cost function depending on iteration
"""
nb_iter = 0
if alpha is None:
alpha = self._alpha
if plot:
y=[]
x=[]
for i in range(max_iter):
nb_iter = i + 1
tot_j = 0
for t in range(self._nb_batches):
X, Y = self.get_next_batch()
self.set_all_training_examples(X, Y, self._batch_m[t])
self.prepare()
# forward propagation
self.propagate()
# compute cost function
J = self.compute_cost_function()
# back propagation
self.back_propagation()
tot_j += J / self._nb_batches
if plot:
y.append(tot_j)
x.append(nb_iter)
#if tot_j <= min_cost:
# break
if mp and plot:
mp.plot(x,y)
mp.show()
return {"iterations": nb_iter, "cost_function": tot_j}
def learning(self, X, Y, m, min_cost=0.05, max_iter=100000, alpha=None, plot=False):
"""Tune parameters in order to learn examples by propagate and backpropagate.
:param X: the inputs training examples
:param Y: the expected outputs training examples
:param m: the number of examples
:param min_cost: cost function value to reach in order to stop algo. Default 0.0.5
:param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000).
:param alpha: learning rate, if None use the instance alpha value. Default None.
"""
self.set_all_training_examples(X, Y, m)
self.prepare()
res = self.minimize_cost(min_cost, max_iter, alpha, plot)
return res
def learning_batches(self, min_cost=0.05, max_iter=100000, alpha=None, plot=False):
"""Tune parameters in order to learn examples by propagate and backpropagate.
:param X: the inputs training examples
:param Y: the expected outputs training examples
:param m: the number of examples
:param min_cost: cost function value to reach in order to stop algo. Default 0.0.5
:param max_iter: maximum number of iterations to reach min cost befor stoping algo. (Default 100000).
:param alpha: learning rate, if None use the instance alpha value. Default None.
"""
res = self.minimize_cost_batches(min_cost, max_iter, alpha, plot)
return res
def get_output(self):
return self._A[self._L]
def get_expected_output(self):
return self._Y
def get_input(self):
return self._X
def get_weights(self):
return self._W[1:]
def get_bias(self):
return self._b[1:]
def set_flatten_weights(self, W):
"""Set weights from a flatten list"""
shapes = [w.shape for w in self._W[1:]]
sizes = [w.size for w in self._W[1:]]
flat_size = sum(sizes)
assert(len(W) == flat_size)
ini = 0
for i, (size, shape) in enumerate(zip(sizes, shapes)):
self._W[1+i] = np.reshape(W[ini:ini+size], shape)
ini += size
def set_flatten_bias(self, B):
"""Set bias from a flatten list"""
shapes = [b.shape for b in self._b[1:]]
sizes = [b.size for b in self._b[1:]]
flat_size = sum(sizes)
assert(len(B) == flat_size)
ini = 0
for i, (size, shape) in enumerate(zip(sizes, shapes)):
self._b[1+i] = np.reshape(B[ini:ini+size], shape)
ini += size
def load_weights(self, filename):
""""Load weights from file
:param filename: the file name
:return: True if success else False
"""
ret = False
res = MultiLayerPerceptron.load_params(filename)
if res["result"]:
ret = True
self.set_flatten_weights(res["params"])
return ret
def save_weights(self, filename):
"""Save weights to file
:param filename: the file name
:return: True if success else False
"""
return MultiLayerPerceptron.save_params(self._W[1:], filename)
def load_bias(self, filename):
""""Load bias from file
:param filename: the file name
:return: True if success else False
"""
ret = False
res = MultiLayerPerceptron.load_params(filename)
if res["result"]:
ret = True
self.set_flatten_bias(res["params"])
return ret
def save_bias(self, filename):
"""Save bias to file
:param filename: the file name
:return: True if success else False
"""
return MultiLayerPerceptron.save_params(self._b[1:], filename)
@staticmethod
def save_params(params, name):
"""Save features to file
:param params: the features to save
:param name: the file name
:return: True if success else False
"""
ret = False
param_type = MultiLayerPerceptron.param_type
# nb layers
nb = len(params)
# nbs features per layer
sizes = [p.size for p in params]
# packing format string
fmt = "<{fnb}{fsizes}{fparams}".format(
fnb=param_type["uint32"]["fmt"],
fsizes=param_type["uint32"]["fmt"]*nb,
fparams="".join([param_type["float64"]["fmt"]*n for n in sizes])
)
# build the list of sizes and all parameters
data = [nb] + sizes
for p in params:
data += p.flatten().tolist()
print(sizes)
# packing raw data
raw = struct.pack(fmt, *data)
print(len(raw))
# save
with open(name, "wb") as f:
f.write(raw)
ret = True
return ret
@staticmethod
def load_params(name):
"""Load features from file.
A main list contains lists of all parameters for each layer
:return: dict with results with key: 'result' (True or False), 'params' (if success), 'sizes' (if success)
"""
ret = {"result": False}
param_type = MultiLayerPerceptron.param_type
with open(name, "rb") as f:
# get nb layers
raw = f.read(param_type["uint32"]["size"])
fmt = "<{fnb}".format(fnb=param_type["uint32"]["fmt"])
nb = struct.unpack(fmt, raw)[0]
# get nbs features per layer
raw = f.read(param_type["uint32"]["size"]*nb)
fmt = "<{fsizes}".format(fsizes=param_type["uint32"]["fmt"]*nb)
sizes = struct.unpack(fmt, raw)
# get params
total = sum(sizes)
raw = f.read(param_type["float64"]["size"]*total)
fmt = "<{fparams}".format(fparams=param_type["float64"]["fmt"]*total)
params = struct.unpack(fmt, raw)
# result
ret["params"] = params
ret["sizes"] = sizes
ret["result"] = True
return ret