this article in the previous article: the construction of the depth of the neural network python (DNN) on the basis of , adding the following contents:


2) regularization) loss function v intermediate output

3) builds cross loss function

4) will be trained the network is saved, and call to test the new data


 data preprocessing #! /usr/bin/env Python coding: UTF-8 # -*- -*- # @Time: 2017-03-12 @Author: CC @File # 15:11 #: from numpy import numpy as NP import * import cPickle def (load_data): "load the decompressed data. And read" with open ('data/mnist_pkl/mnist.pkl','rb') as f: try: train_data, validation_data, test_data = cPickle.load (f) print the fil E open sucessfully print train_data[0].shape # # (50000784) # print train_data[1].shape # (50000, return) (train_data, validation_data, test_data) except EOFError: print'the file open error'return None def (data_transform): "the data into the calculation format" "t_d, va_d, te_d = load_data (print) # t_d[0].shape # (50000784) # print te_d[0].shape # # # va_d[0].shape (10000784) print (10000784) # N1 = [np.reshape (x, 784,1) for X in t_d[0]] # 50 thousand data respectively into one by one out (784,1), n (x = [np.reshape arranged one by one, (784, 1)) for X in t_d[0]] # the 50 thousand data were removed one by one into (784,1), print'n1' n1[0].shape # arranged one by one, # print'n', n[0].shape M = [vectors (y) for y in t_d[1 "# 50 thousand label (50000,1) to train_data (1050000) = zip (n, m) # package and label data into yuan Group (x, [np.reshape = n (784, 1)) for X in va_d[0]] # 50 thousand data respectively into one by one out (784,1), with validation_data = zip (n, va_d[1]) # not label data vector (x, n = [np.reshape (784, 1)) for X in te_d[0]] # 50 thousand data respectively into one by one out (784,1), test_data = zip (n, order te_d[1]) will not # tag data vector # print train_data [0][0].shape # (784 #), print len (train_data[0]), len (train_data[0]) #2 # print" len (train_data[100]), len (train_data[100]) #2 # print "len (train_data[0][0]), len (train_data[0][0]) #784 # print" train_data [0][0].shape ", train_data[0][0].shape # (784,1) # print "len (train_data), len (train_data) #50000 print train_data[0][1].shape (10,1) # # # # print test_data[0][1] 7 return (train_data, validation_data, test_data) def vectors (y):" given the label "label = np.zeros ((10,1)) label[y] = 1 return label 

# floating-point calculation the definition of

2 network and train the

 /usr/bin/env Python #! # -*- coding: UTF-8 -*- # @Time: 2017-03-28 @Author: CC @File # 10:18 #: from numpy import numpy as NP import * import operator import JSON import sys class QuadraticCost (#):" the definition of the two cost function class method "and" @staticmethod "def FN (a, y): cost = 0.5*np.linalg.norm (a-y) **2 return cost @staticmethod def Delta (Z, a, y): delta = (a-y) *sig_derivate (z) return delta class (CrossEntroyCost): "the definition of the cross entropy function method" and "@staticmethod def FN" (a, y): cost = np.sum (np.nan_to_num (-y*np.log (a) - (1-y) *np.log (1-A))) not a # number---0, inf---larger number return cost @staticmethod def Delta (Z, a, y): delta = (a - y) return delta class Network (object): "the definition of the network structure and the method of" def __init__ "(self, sizes, cost): self.num_layer = len (sizes) self.sizes = sizes self.cost =" self.cost.__name__ print cost # "self.cost.__name__ CrossEntropyCost, # self.default_weight_initializer (DEF) default_weight_initializer (self):" initialization "self.bias = [np.random.rand (x, 1) for X in self.sizes[1:]] self.weigh T = [np.random.randn (y, x) /float (np.sqrt (x)) for (x, y) in zip (self.sizes[: -1], self.sizes[1:])] def large_weight_initializer (self): "another weight initialization" self.bias = [np.random.rand (x, 1) for X in (self.sizes[1:]] self.weight = [np.random.randn y, for X, y x) in zip (self.sizes[: -1], self.sizes[1:])] def forward (self, a): "forward the network" for W, B in zip (self.weight, self.bias): a=sigmoid ( (W, a) +b return a def SGD (self). Train_data, min_batch_size, epochs, ETA, test_data=False, lambd = 0, monitor_train_cost = False, monitor_train_accuracy = False, monitor_test_cost=False, monitor_test_accuracy=False): "Set the train_data shuffle, 1); 2) loop the epoches, 3) s Et the min_batches, and rule of update "if test_data: n_test=len (test_data) n = len (train_data) for I in xrange (epochs): random.shuffle (train_data) min_batches = [train_data[k:k+min_batch_size] for K in xrange (0, N, min_batch_size)] for min_batch in min_batches: # extraction time of a batch of samples (self.update_minbatch_parameter min_batch, ETA, lambd, n) train_cost = if monitor_train_cost: cost1 [] = self.total_cost (train_data, lambd, cont=False) train_cost.append (cost1) print Epoche {0}, train_cost: {1}.Format (I, cost1) if monitor_train_accuracy: accuracy = self.accuracy (train_data, cont=True) train_cost.append (accuracy) print Epoche {0}/{1}, train_accuracy: {2}.Format (I, epoch S, accuracy) test_cost if monitor_test_cost: cost1 = [] = self.total_cost (test_data, lambd) test_cost.append (cost1) print Epoche {0}, test_cost: {1}.Format (I, cost1) test_accuracy = if monitor_test_accuracy: accuracy [] = self.accuracy (test_data) test_cost.append (accuracy) print "epoche:{0}/{1}, test_accuracy:{2}".Format (I, epochs, accuracy) (filename= net_save) # save network parameters of def total_cost (self, train_data, lambd, cont, =True): cost1 = 0 for X, y in train_data: a self.forward (x) if = cont: y = vectors (y) # will test the sample label for matrix cost1 =.Fn (a (self.cost) y), /len (train_data) cost1 = lambd/len (train_data) *np.sum (np.linalg.norm (weight) **2 for we Ight in self.weight #) plus cost1 def accuracy return weight (self, train_data, cont=False): if cont: output1 = [(np.argmax (self.forward (x)), np.argmax (y)) for (x, y) in train_data] else: output1 (np.argmax = [(self.forward (x)), y for) (x, y) in train_data] return sum (int (out1 = = y) for (out1, y) in output1) def update_minbatch_parameter (self, min_batch, ETA, lambd, n): "determine the weight and 1) bias 2) calculate the the delta 3) update the data" and "able_b" = [np.zeros (b.shape) for B in self.bias] able_w=[np.zeros (w.shape) for W in self.weight] for X, y in min_batch: # only take a sample? Deltab, DeltaW = self.backprop (x, y) able_b =[a_b+dab for a_b, dab in zip (able_b, deltab)] # actually DW, DB batch accumulate, finally take the average small batches of able_w = [a_w + Daw for a_w, Daw in zip (able_w, DeltaW)] - eta (self.weight = [weight * DW) / len (min_batch) - eta* (lambd*weight) /n for weight, DW in zip (self.weight, able_w)] # increase the regularization term: eta*lambda/m *weight self.bias = [bias - ETA * dB / len (min_batch) for bias, DB in zip (self.bias, able_b)] def BackProp (self, x, y): "1 clacu the forward) value calcu the (2) delta:delta = y-f (z)); Deltak = delta*w (k) *fz (k-1) 3) clacu the Delta in 'every layer: deltab=delta deltaw=delta*fz (k-1);" deltab = [np.zeros (b.shape) for B in self.bias] DeltaW = [np. Zeros (w.shape) for W in self.weight] ZS activate = x activates = [] = [x] for W, B in zip (self.weight, self.bias): Z (W, activate) +b zs.append (z) activate = sigmoid (z) activates.append (activate) # BackProp delta = (zs[-1], activates[-1], y) method # call different cost function gradient deltab[-1] deltaw[-1] = (delta = Delta, activates[-2].transpose) (for) I in xrange (2, self.num_layer): z = zs[-i] delta = (self.weight[-i+1].transpose), (delta) * sig_derivate (z) deltab[-i] = Delta deltaw[-i] = (Delta, activates[-i-1].transpose) (return) (deltab, DeltaW, DEF) save (self, filename): "the network trained by JSON (java script object notation) Saving objects into strings, for the production of the deployment of encoder=json.dumps (data) Python primitive types (no array type) to control conversion of type JSON Python JSON dict object list/tuple: arrary int/long/float number (.Tolist) array into a list of > > > a = np.array ([[1, 2], [3, 4]]) > > > list (a) [array ([1, 2]), array ([3, 4]) > > > a.tolist ([[1), 2], [3, 4]] "data = {" sizes ": self.sizes" weight "[weight.tolist (for): weight in self.weight]." bias ": ([bias.tolist) for (bias in self.bias])," cost ": str (self.cost.__name__) #} save the trained network weights, bias, cross entropy. F = open (filename, w) json.dump (data, f) f.close (DEF) load_net (filename): "by data=json.load (json.dumps (data) decoder = json.load) decoding, encoding and decoding (encoder) after the key does not order the original data according to the arrangement, but each key corresponding the value will not change into the trained network to test the" F = open (filename, R) data = json.load (f) f.close (print) # "data[cost]" (sys.modules[__name__], getattr, data["cost"]) # won the __main__.CrossEntropyCost property of # print "data[cost]" data["cost" data["sizes"]], net = Network ("data[sizes", cost=data["cost"]) # network initialization of net.weight = [np.array (W) for W in data["weight" "# with trained weights, and list---> array net.bias = [np.array (b) for B in data[B" "IAS" return net def sig_derivate (z): "derivate sigmoid" return sigmoid (z) * (1-sigmoid (z)) def sigmoid (x): sigm=1.0/ (1.0+exp (-x)) return sIgM def vectors (y): "given the label" label (= np.zeros (10,1) label[y]) = 1 return label 

3 # floating-point calculation) network test

 /usr/bin/env Python #! # -*- coding: UTF-8 -*- # @Time: 2017-03-12 @Author: CC @File # 15:24 #: import net_load_data net_load_data.load_data (train_data) #, validation_data, test_data = net_load_data.data_transform (import) net_network2 as net cost = net.QuadraticCost cost net.CrossEntroyCost = lambd = 0 = net.Network ([784,50,1 Net1

This concludes the body part