参考链接：Click Here

显著性检验（Significance Test）主要分为两个类别：

• Statistical Significance Test (统计显著性检验)

  计量方式：p-value < 0.05

  目的：检验原始分布与目标分布之间是否具有显著差异性

• Practical Significance Test (现实显著性检验)

  计量方式：effect size（cohen's d）（统计效应）

  目的：检验原始分布与目标分布之间的差异性有多大“

NLPStatTest: A Toolkit for Comparing NLP System Performance”中提出在NLP领域除了Statistical Significance，做Practical Significance也是有必要的

2.2.3 Effect Size Estimation

In most experimental NLP papers employing significance testing, the p-value is the only quantity reported. However, the p-value is often misused and misinterpreted. For instance, statistical significance is easily conflated with practical significance; as a result, NLP researchers often run significance tests to show that the performances of two NLP systems are different (i.e., statistical significance), without measuring the degree or the importance of such a difference (i.e., practical significance).

使用说明：

Statistical Significance Test (统计显著性检验)：

python Statistical_significance.py file1 file2 0.05

import sys
import numpy as np
from scipy import stats### Normality Check
# H0: data is normally distributed
def normality_check(data_A, data_B, name, alpha):if(name=="Shapiro-Wilk"):# Shapiro-Wilk: Perform the Shapiro-Wilk test for normality.shapiro_results = stats.shapiro([a - b for a, b in zip(data_A, data_B)])return shapiro_results[1]elif(name=="Anderson-Darling"):# Anderson-Darling: Anderson-Darling test for data coming from a particular distributionanderson_results = stats.anderson([a - b for a, b in zip(data_A, data_B)], 'norm')sig_level = 2if(float(alpha) <= 0.01):sig_level = 4elif(float(alpha)>0.01 and float(alpha)<=0.025):sig_level = 3elif(float(alpha)>0.025 and float(alpha)<=0.05):sig_level = 2elif(float(alpha)>0.05 and float(alpha)<=0.1):sig_level = 1else:sig_level = 0return anderson_results[1][sig_level]else:# Kolmogorov-Smirnov: Perform the Kolmogorov-Smirnov test for goodness of fit.ks_results = stats.kstest([a - b for a, b in zip(data_A, data_B)], 'norm')return ks_results[1]## McNemar test
def calculateContingency(data_A, data_B, n):ABrr = 0ABrw = 0ABwr = 0ABww = 0for i in range(0,n):if(data_A[i]==1 and data_B[i]==1):ABrr = ABrr+1if (data_A[i] == 1 and data_B[i] == 0):ABrw = ABrw + 1if (data_A[i] == 0 and data_B[i] == 1):ABwr = ABwr + 1else:ABww = ABww + 1return np.array([[ABrr, ABrw], [ABwr, ABww]])def mcNemar(table):statistic = float(np.abs(table[0][1]-table[1][0]))**2/(table[1][0]+table[0][1])pval = 1-stats.chi2.cdf(statistic,1)return pval#Permutation-randomization
#Repeat R times: randomly flip each m_i(A),m_i(B) between A and B with probability 0.5, calculate delta(A,B).
# let r be the number of times that delta(A,B)<orig_delta(A,B)
# significance level: (r+1)/(R+1)
# Assume that larger value (metric) is better
def rand_permutation(data_A, data_B, n, R):delta_orig = float(sum([ x - y for x, y in zip(data_A, data_B)]))/nr = 0for x in range(0, R):temp_A = data_Atemp_B = data_Bsamples = [np.random.randint(1, 3) for i in xrange(n)] #which samples to swap without repetitionsswap_ind = [i for i, val in enumerate(samples) if val == 1]for ind in swap_ind:temp_B[ind], temp_A[ind] = temp_A[ind], temp_B[ind]delta = float(sum([ x - y for x, y in zip(temp_A, temp_B)]))/nif(delta<=delta_orig):r = r+1pval = float(r+1.0)/(R+1.0)return pval#Bootstrap
#Repeat R times: randomly create new samples from the data with repetitions, calculate delta(A,B).
# let r be the number of times that delta(A,B)<2*orig_delta(A,B). significance level: r/R
# This implementation follows the description in Berg-Kirkpatrick et al. (2012),
# "An Empirical Investigation of Statistical Significance in NLP".
def Bootstrap(data_A, data_B, n, R):delta_orig = float(sum([x - y for x, y in zip(data_A, data_B)])) / nr = 0for x in range(0, R):temp_A = []temp_B = []samples = np.random.randint(0,n,n) #which samples to add to the subsample with repetitionsfor samp in samples:temp_A.append(data_A[samp])temp_B.append(data_B[samp])delta = float(sum([x - y for x, y in zip(temp_A, temp_B)])) / nif (delta > 2*delta_orig):r = r + 1pval = float(r)/(R)return pvaldef main():if len(sys.argv) < 3:print("You did not give enough arguments\n ")sys.exit(1)filename_A = sys.argv[1]filename_B = sys.argv[2]alpha = sys.argv[3]with open(filename_A) as f:data_A = f.read().splitlines()with open(filename_B) as f:data_B = f.read().splitlines()data_A = list(map(float,data_A))data_B = list(map(float,data_B))print("\nPossible statistical tests: Shapiro-Wilk, Anderson-Darling, Kolmogorov-Smirnov, t-test, Wilcoxon, McNemar, Permutation, Bootstrap")name = input("\nEnter name of statistical test: ")### Normality Checkif(name=="Shapiro-Wilk" or name=="Anderson-Darling" or name=="Kolmogorov-Smirnov"):output = normality_check(data_A, data_B, name, alpha)if(float(output)>float(alpha)):answer = input("\nThe normal test is significant, would you like to perform a t-test for checking significance of difference between results? (Y\\N) ")if(answer=='Y'):# two sided t-testt_results = stats.ttest_rel(data_A, data_B)# correct for one sided testpval = t_results[1]/2if(float(pval)<=float(alpha)):print("\nTest result is significant with p-value: {}".format(pval))returnelse:print("\nTest result is not significant with p-value: {}".format(pval))returnelse:answer2 = input("\nWould you like to perform a different test (permutation or bootstrap)? If so enter name of test, otherwise type 'N' ")if(answer2=='N'):print("\nbye-bye")returnelse:name = answer2else:answer = input("\nThe normal test is not significant, would you like to perform a non-parametric test for checking significance of difference between results? (Y\\N) ")if (answer == 'Y'):answer2 = input("\nWhich test (Permutation or Bootstrap)? ")name = answer2else:print("\nbye-bye")return### Statistical tests# Paired Student's t-test: Calculate the T-test on TWO RELATED samples of scores, a and b. for one sided test we multiply p-value by halfif(name=="t-test"):t_results = stats.ttest_rel(data_A, data_B)# correct for one sided testpval = float(t_results[1]) / 2if (float(pval) <= float(alpha)):print("\nTest result is significant with p-value: {}".format(pval))returnelse:print("\nTest result is not significant with p-value: {}".format(pval))return# Wilcoxon: Calculate the Wilcoxon signed-rank test.if(name=="Wilcoxon"):wilcoxon_results = stats.wilcoxon(data_A, data_B)if (float(wilcoxon_results[1]) <= float(alpha)):print("\nTest result is significant with p-value: {}".format(wilcoxon_results[1]))returnelse:print("\nTest result is not significant with p-value: {}".format(wilcoxon_results[1]))returnif(name=="McNemar"):print("\nThis test requires the results to be binary : A[1, 0, 0, 1, ...], B[1, 0, 1, 1, ...] for success or failure on the i-th example.")f_obs = calculateContingency(data_A, data_B, len(data_A))mcnemar_results = mcNemar(f_obs)if (float(mcnemar_results) <= float(alpha)):print("\nTest result is significant with p-value: {}".format(mcnemar_results))returnelse:print("\nTest result is not significant with p-value: {}".format(mcnemar_results))returnif(name=="Permutation"):R = max(10000, int(len(data_A) * (1 / float(alpha))))pval = rand_permutation(data_A, data_B, len(data_A), R)if (float(pval) <= float(alpha)):print("\nTest result is significant with p-value: {}".format(pval))returnelse:print("\nTest result is not significant with p-value: {}".format(pval))returnif(name=="Bootstrap"):R = max(10000, int(len(data_A) * (1 / float(alpha))))pval = Bootstrap(data_A, data_B, len(data_A), R)if (float(pval) <= float(alpha)):print("\nTest result is significant with p-value: {}".format(pval))returnelse:print("\nTest result is not significant with p-value: {}".format(pval))returnelse:print("\nInvalid name of statistical test")sys.exit(1)if __name__ == "__main__":main()

Practical Significance Test (现实显著性检验):

python Practical_significance.py file1 file2

import sys
import numpy as np
from numpy import mean, std, sqrtdef read_data_from_file(file_name):with open(file_name, 'r', encoding='utf-8') as reader:data_file = []try:lines = reader.readlines()data_file = [float(line.strip()) for line in lines]except:print('Data format error, please check')if len(data_file) == 0:print('Empty file, exit')sys.exit(0)return data_filedef two_side_data_reader(file1_name, file2_name):data_file1 = read_data_from_file(file1_name)data_file2 = read_data_from_file(file2_name)return data_file1, data_file2def cal_cohen_d(data1, data2):def cohen_d(x, y):return (mean(x) - mean(y)) / sqrt((std(x) ** 2 + std(y) ** 2) / 2.0)mean1 = np.mean(data1)mean2 = np.mean(data2)# print(type(mean1))std1 = np.std(data1)std2 = np.std(data2)cohen = cohen_d(data1, data2)print('Data1 [mean:%.4f, std:%.4f]' % (mean1, std1))print('Data2 [mean:%.4f, std:%.4f]' % (mean2, std2))print("cohen's d value = %.4f" % (cohen))return cohenif __name__ == '__main__':file_1 = sys.argv[1]file_2 = sys.argv[2]data1, data2 = two_side_data_reader(file_1, file_2)res = cal_cohen_d(data1, data2)