吴恩达作业6：梯度检验

梯度检验的目的就是看反向传播过程中的导数有没有较大的误差，首先看J=theta*x的梯度检验：代码如下

import numpy as np
"""
J=x*theta的前向传播
"""
def forward_propagation(x,theta):J=x*thetareturn J
"""
J=x*theta的后向传播
"""
def back_propagation(x,theta):d_theta=xreturn d_theta
#简单的梯度检验 利用数值估计的梯度和真实的梯度二范数之差除以二范数值之和就是误差
def gradient_check(x,theta,epsilon):thetaplus=theta+epsilonthetaminus = theta -epsilonJ_plus=forward_propagation(x, thetaplus)J_minus = forward_propagation(x, thetaminus)grad_approx=(J_plus-J_minus)/(2*epsilon)grad=back_propagation(x,theta)numerator=np.linalg.norm(grad-grad_approx)  ####np.linalg.norm 二范数denominator=np.linalg.norm(grad_approx)+np.linalg.norm(grad)difference=numerator/denominatorif difference<1e-7:print('the gradient is ok')else:print('the gradient is wrong')return difference
if __name__ == '__main__':x, theta = 2, 4difference = gradient_check(x, theta, epsilon=1e-7)print('difference is {}'.format(difference))

打印结果：

接下来用三层神经网络演示梯度：

首先看产生数据集和权值的函数放在testCases.py里

def gradient_check_n_test_case(): np.random.seed(1)x = np.random.randn(4,3)y = np.array([1, 1, 0])W1 = np.random.randn(5,4) b1 = np.random.randn(5,1) W2 = np.random.randn(3,5) b2 = np.random.randn(3,1) W3 = np.random.randn(1,3) b3 = np.random.randn(1,1) parameters = {"W1": W1,"b1": b1,"W2": W2,"b2": b2,"W3": W3,"b3": b3}return x, y, parameters

激活函数，字典变成一列向量的函数，向量变成字典的函数，放在gc_utils.py

代码如下：

import numpy as npdef sigmoid(x):"""Compute the sigmoid of xArguments:x -- A scalar or numpy array of any size.Return:s -- sigmoid(x)"""s = 1/(1+np.exp(-x))return sdef relu(x):"""Compute the relu of xArguments:x -- A scalar or numpy array of any size.Return:s -- relu(x)"""s = np.maximum(0,x)return s
#变成一列向量
def dictionary_to_vector(parameters):"""Roll all our parameters dictionary into a single vector satisfying our specific required shape."""keys = []count = 0for key in ["W1", "b1", "W2", "b2", "W3", "b3"]:# flatten parameternew_vector = np.reshape(parameters[key], (-1,1))keys = keys + [key]*new_vector.shape[0]if count == 0:theta = new_vectorelse:         #####np.concatenate 拼接theta = np.concatenate((theta, new_vector), axis=0)count = count + 1return theta, keysdef vector_to_dictionary(theta):"""Unroll all our parameters dictionary from a single vector satisfying our specific required shape."""parameters = {}parameters["W1"] = theta[:20].reshape((5,4))parameters["b1"] = theta[20:25].reshape((5,1))parameters["W2"] = theta[25:40].reshape((3,5))parameters["b2"] = theta[40:43].reshape((3,1))parameters["W3"] = theta[43:46].reshape((1,3))parameters["b3"] = theta[46:47].reshape((1,1))return parameters
#变成一列向量
def gradients_to_vector(gradients):"""Roll all our gradients dictionary into a single vector satisfying our specific required shape."""count = 0for key in ["dW1", "db1", "dW2", "db2", "dW3", "db3"]:# flatten parameternew_vector = np.reshape(gradients[key], (-1,1))if count == 0:theta = new_vectorelse:theta = np.concatenate((theta, new_vector), axis=0)count = count + 1return theta

最终代码：

import numpy as np
import testCases
import gc_utils
"""
J=x*theta的前向传播
"""
def forward_propagation(x,theta):J=x*thetareturn J
"""
三层神经网络的前向传播
"""
def forward_propagation_n(X,Y,parameters):m=X.shape[1]W1 = parameters['W1']b1 = parameters['b1']W2 = parameters['W2']b2 = parameters['b2']W3 = parameters['W3']b3 = parameters['b3']###forward processZ1=np.dot(W1,X)+b1A1=gc_utils.relu(Z1)Z2=np.dot(W2,A1)+b2A2 = gc_utils.relu(Z2)Z3 = np.dot(W3, A2) + b3A3=gc_utils.sigmoid(Z3)##compute costlogprobs = np.multiply(-np.log(A3), Y) + np.multiply(-np.log(1 - A3), 1 - Y)cost = 1. / m * np.nansum(logprobs)cache=(Z1,A1,W1,b1,Z2,A2,W2,b2,Z3,A3,W3,b3)return cost,cache
"""
J=x*theta的后向传播
"""
def back_propagation(x,theta):d_theta=xreturn d_theta
"""
三层神经网络的后向传播
"""
def back_propagation_n(X,Y,cache):(Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)=cachem=X.shape[1]dZ3 = (A3 - Y)dW3 = 1. / m * np.dot(dZ3, A2.T)db3 = 1. / m * np.sum(dZ3, axis=1, keepdims=True)dA2 = np.dot(W3.T, dZ3)dZ2 = np.multiply(dA2, np.int64(A2 > 0))dW2 = 1. / m * np.dot(dZ2, A1.T)db2 = 1. / m * np.sum(dZ2, axis=1, keepdims=True)dA1 = np.dot(W2.T, dZ2)dZ1 = np.multiply(dA1, np.int64(A1 > 0))dW1 = 1. / m * np.dot(dZ1, X.T)db1 = 1. / m * np.sum(dZ1, axis=1, keepdims=True)gradients = {'dZ3': dZ3, 'dW3': dW3, 'db3': db3, 'dA2': dA2, 'dZ2': dZ2,'dW2': dW2, 'db2': db2, 'dA1': dA1, 'dZ1': dZ1, 'dW1': dW1, 'db1': db1}return gradients
"""
三层神经网络的梯度检验
"""
def gradient_check_n(parameters,gradients,X,Y,epsilon):parameters_values,_=gc_utils.dictionary_to_vector(parameters)grad=gc_utils.gradients_to_vector(gradients)num_parameters=parameters_values.shape[0]J_plus=np.zeros((num_parameters,1))J_minus = np.zeros((num_parameters, 1))grad_approx=np.zeros((num_parameters, 1))for i in range(num_parameters):theta_plus = np.copy(parameters_values)###之所以放在里面是因为对每一个权重和偏置梯度检验theta_plus[i][0]=parameters_values[i][0]+epsilonJ_plus[i],_=forward_propagation_n(X,Y,gc_utils.vector_to_dictionary(theta_plus))theta_minus = np.copy(parameters_values)theta_minus[i][0] = parameters_values[i][0] - epsilonJ_minus[i], _ = forward_propagation_n(X, Y, gc_utils.vector_to_dictionary(theta_minus))grad_approx[i]=(J_plus[i]-J_minus[i])/(2*epsilon)numerator = np.linalg.norm(grad - grad_approx)  ####np.linalg.norm 二范数denominator = np.linalg.norm(grad_approx) + np.linalg.norm(grad)difference = numerator / denominatorif difference<1e-7:print('diference ={} there is no mistake in the back_propagation'.format(difference))else:print('diference = {} there is a mistake in the back_propagation'.format(difference))return difference
#简单的梯度检验 利用数值估计的梯度和真实的梯度二范数之差除以二范数值之和就是误差
def gradient_check(x,theta,epsilon):thetaplus=theta+epsilonthetaminus = theta -epsilonJ_plus=forward_propagation(x, thetaplus)J_minus = forward_propagation(x, thetaminus)grad_approx=(J_plus-J_minus)/(2*epsilon)grad=back_propagation(x,theta)numerator=np.linalg.norm(grad-grad_approx)  ####np.linalg.norm 二范数denominator=np.linalg.norm(grad_approx)+np.linalg.norm(grad)difference=numerator/denominatorif difference<1e-7:print('the gradient is ok')else:print('the gradient is wrong')return difference
def test():
###########test forward_propagation# x, theta = 2, 4# J = forward_propagation(x, theta)# print('J={}'.format(J))
####################
###########test back_propagation# x, theta = 2, 4# d_theta=back_propagation(x, theta)# print('d_theta={}'.format(d_theta))
####################test gradient_check# x, theta = 2, 4# difference=gradient_check(x, theta, epsilon=1e-7)# print('difference is {}'.format(difference))
###################
####################test gradient_check_nX, Y, parameters=testCases.gradient_check_n_test_case()cost, cache=forward_propagation_n(X,Y,parameters)gradients=back_propagation_n(X,Y,cache)difference=gradient_check_n(parameters, gradients, X, Y, epsilon=1e-6)
############################
if __name__=='__main__':test()

结果如下：