这次实践的前半部分是，用水库水位的变化，来预测大坝的出水量。
给数据集拟合一条直线，可能得到一个逻辑回归拟合，但它并不能很好地拟合数据，这是高偏差（high bias）的情况，也称为“欠拟合”（underfitting）
相反，如果我们拟合一个非常复杂的分类器，比如深度神经网络或含有隐藏单元的神经网络，可能非常适用于这个数据，但是这看起来也不是一种很好的拟合方式分类器----方差较高（high variance）
数据过拟合（over fitting）

• 高偏差和高方差是两种不同的情况，通常会用训练验证集来诊断算法是否存在偏差或方差问题。
• 在机器学习初级阶段，会有很多关于偏差方差的讨论，能尝试的方法很多。在当前深度学习和大数据时代，只需持续训练更大的网络，就能不影响方差减少偏差；准备更多的数据，就能不影响偏差减少方差。

Regularized Linear Regression

Visualizing the dataset

data = loadmat('ex5data1.mat')
# Training set
X, y = data['X'], data['y']
# Cross validation set
Xval, yval = data['Xval'], data['yval']
# Test set
Xtest, ytest = data['Xtest'], data['ytest']X = np.insert(X, 0, 1, axis=1)
Xval = np.insert(Xval, 0, 1, axis=1)
Xtest = np.insert(Xtest, 0, 1, axis=1)def plot_data():plt.figure(figsize=(6, 4))plt.scatter(X[:, 1:], y, c='r', marker='x')plt.xlabel('change in water level(x)')plt.ylabel('Water flowing out of the dam (y)')plt.grid(True)plot_data()
plt.show()

Regularized linear regression cost function

def regularized_cost(theta, X, y, l):cost = ((X.dot(theta) - y.flatten()) **2).sum()/(2*len(X))regularized_theta = l * (theta[1:].dot(theta[1:]))/(2 * len(X))return cost + regularized_thetatheta = np.ones(X.shape[1])
print(regularized_cost(theta, X, y, 1))

Regularized linear regression gradient

def regularized_gradient(theta, X, y, l):grad = (X.dot(theta) - y.flatten()).dot(X)regularized_theta = l * thetareturn (grad + regularized_theta) / len(X)print(regularized_gradient(theta, X, y, 1))

def train_linear_regularized(X, y, l):theta = np.zeros(X.shape[1])res = opt.minimize(fun=regularized_cost,x0=theta,args=(X, y, l),method='TNC',jac=regularized_gradient)return res.x

Fitting linear regression

拟合线性回归，画出拟合线

final_theta = train_linear_regularized(X, y, 0)
plot_data()
plt.plot(X[:, 1], X.dot(final_theta))
plt.show()

Bias-variance

Learning curves

画出学习曲线

def plot_learning_curve(X, y, Xval, yval, l):training_cost, cross_cost = [], []for i in range(1, len(X)):res = train_linear_regularized(X[:i], y[:i], l)training_cost_item = regularized_cost(res, X[:i], y[:i], 0)cross_cost_item = regularized_cost(res, Xval, yval, 0)training_cost.append(training_cost_item)cross_cost.append(cross_cost_item)plt.figure(figsize=(6, 4))plt.plot([i for i in range(1, len(X))], training_cost, label='training cost')plt.plot([i for i in range(1, len(X))], cross_cost, label='cross cost')plt.legend()plt.xlabel('Number of training examples')plt.ylabel('Error')plt.title('Learning curve for linear regression')plt.grid(True)plot_learning_curve(X, y, Xval, yval, 0)
plt.show()

Polynomial regression

Learning Polynomial Regression

使用多项式回归，规定假设函数如下：

def genPolyFeatures(X, power):Xpoly = X.copy()for i in range(2, power + 1):Xpoly = np.insert(Xpoly, Xpoly.shape[1], np.power(Xpoly[:,1], i), axis=1)return Xpoly#获取训练集的均值和误差
def get_means_std(X):means = np.mean(X, axis=0)stds = np.std(X, axis=0, ddof=1) # ddof=1  样本标准差return means, stds# 标准化
def featureNormalize(myX, means, stds):X_norm = myX.copy()X_norm[:,1:] = X_norm[:,1:] - means[1:]X_norm[:,1:] = X_norm[:,1:] / stds[1:]return X_normpower = 6  # 扩展到x的6次方train_means, train_stds = get_means_std(genPolyFeatures(X,power))
X_norm = featureNormalize(genPolyFeatures(X,power), train_means, train_stds)
Xval_norm = featureNormalize(genPolyFeatures(Xval,power), train_means, train_stds)
Xtest_norm = featureNormalize(genPolyFeatures(Xtest,power), train_means, train_stds)def plot_fit(means, stds, l):"""拟合曲线"""theta = train_linear_regularized(X_norm, y, l)x = np.linspace(-75, 55, 50)xmat = x.reshape(-1, 1)xmat = np.insert(xmat, 0, 1, axis=1)Xmat = genPolyFeatures(xmat, power)Xmat_norm = featureNormalize(Xmat, means, stds)plot_data()plt.plot(x, Xmat_norm @ theta, 'b--')plot_fit(train_means, train_stds, 0)
plot_learning_curve(X_norm, y, Xval_norm, yval, 0)

Adjusting the regularization parameter

plot_fit(train_means, train_stds, 1)
plot_learning_curve(X_norm, y, Xval_norm, yval, 1)
plt.show()

Selecting λ using a cross validation set

尝试用不同的lambda调试，进行交叉验证

lambdas = [0., 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1., 3., 10.]
errors_train, errors_val = [], []
for l in lambdas:theta = train_linear_regularized(X_norm, y, l)errors_train.append(regularized_cost(theta, X_norm, y, 0))errors_val.append(regularized_cost(theta, Xval_norm, yval, 0))plt.figure(figsize=(8, 5))
plt.plot(lambdas, errors_train, label='Train')
plt.plot(lambdas, errors_val, label='Cross Validation')
plt.legend()
plt.xlabel('lambda')
plt.ylabel('Error')
plt.grid(True)