采用Nagao算法统计各个子字符串的频次，然后基于这些频次统计每个字符串的词频、左右邻个数、左右熵、交互信息(内部凝聚度)。
名词解释：
Nagao算法：一种快速的统计文本里所有子字符串频次的算法。详细算法可见http://www.doc88.com/p-664123446503.html
词频：该字符串在文档中出现的次数。出现次数越多越重要。
左右邻个数：文档中该字符串的左边和右边出现的不同的字的个数。左右邻越多，说明字符串成词概率越高。
左右熵：文档中该字符串的左边和右边出现的不同的字的数量分布的熵。类似上面的指标，有一定区别。
交互信息：每次将某字符串分成两部分，左半部分字符串和右半部分字符串，计算其同时出现的概率除于其各自独立出现的概率，最后取所有的划分里面概率最小值。这个值越大，说明字符串内部凝聚度越高，越可能成词。
算法具体流程：
1. 将输入文件逐行读入，按照非汉字([^\u4E00-\u9FA5]+)以及停词“的很了么呢是嘛个都也比还这于不与才上用就好在和对挺去后没说”，
分成一个个字符串，代码如下：
String[] phrases = line.split("[^\u4E00-\u9FA5]+|["+stopwords+"]");
停用词可以修改。
2. 获取所有切分后的字符串的左子串和右子串，分别加入左、右PTable
3. 对PTable排序，并计算LTable。LTable记录的是，排序后的PTable中，下一个子串同上一个子串具有相同字符的数量
4. 遍历PTable和LTable，即可得到所有子字符串的词频、左右邻
5. 根据所有子字符串的词频、左右邻结果，输出字符串的词频、左右邻个数、左右熵、交互信息
1. NagaoAlgorithm.java
package com.algo.word;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class NagaoAlgorithm {
private int N;
private List leftPTable;
private int[] leftLTable;
private List rightPTable;
private int[] rightLTable;
private double wordNumber;
private Map wordTFNeighbor;
private final static String stopwords = "的很了么呢是嘛个都也比还这于不与才上用就好在和对挺去后没说";
private NagaoAlgorithm(){
//default N = 5
N = 5;
leftPTable = new ArrayList();
rightPTable = new ArrayList();
wordTFNeighbor = new HashMap();
}
//reverse phrase
private String reverse(String phrase) {
StringBuilder reversePhrase = new StringBuilder();
for (int i = phrase.length() - 1; i >= 0; i--)
reversePhrase.append(phrase.charAt(i));
return reversePhrase.toString();
}
//co-prefix length of s1 and s2
private int coPrefixLength(String s1, String s2){
int coPrefixLength = 0;
for(int i = 0; i < Math.min(s1.length(), s2.length()); i++){
if(s1.charAt(i) == s2.charAt(i)) coPrefixLength++;
else break;
}
return coPrefixLength;
}
//add substring of line to pTable
private void addToPTable(String line){
//split line according to consecutive none Chinese character
String[] phrases = line.split("[^\u4E00-\u9FA5]+|["+stopwords+"]");
for(String phrase : phrases){
for(int i = 0; i < phrase.length(); i++)
rightPTable.add(phrase.substring(i));
String reversePhrase = reverse(phrase);
for(int i = 0; i < reversePhrase.length(); i++)
leftPTable.add(reversePhrase.substring(i));
wordNumber += phrase.length();
}
}
//count lTable
private void countLTable(){
Collections.sort(rightPTable);
rightLTable = new int[rightPTable.size()];
for(int i = 1; i < rightPTable.size(); i++)
rightLTable[i] = coPrefixLength(rightPTable.get(i-1), rightPTable.get(i));
Collections.sort(leftPTable);
leftLTable = new int[leftPTable.size()];
for(int i = 1; i < leftPTable.size(); i++)
leftLTable[i] = coPrefixLength(leftPTable.get(i-1), leftPTable.get(i));
System.out.println("Info: [Nagao Algorithm Step 2]: having sorted PTable and counted left and right LTable");
}
//according to pTable and lTable, count statistical result: TF, neighbor distribution
private void countTFNeighbor(){
//get TF and right neighbor
for(int pIndex = 0; pIndex < rightPTable.size(); pIndex++){
String phrase = rightPTable.get(pIndex);
for(int length = 1 + rightLTable[pIndex]; length <= N && length <= phrase.length(); length++){
String word = phrase.substring(0, length);
TFNeighbor tfNeighbor = new TFNeighbor();
tfNeighbor.incrementTF();
if(phrase.length() > length)
tfNeighbor.addToRightNeighbor(phrase.charAt(length));
for(int lIndex = pIndex+1; lIndex < rightLTable.length; lIndex++){
if(rightLTable[lIndex] >= length){
tfNeighbor.incrementTF();
String coPhrase = rightPTable.get(lIndex);
if(coPhrase.length() > length)
tfNeighbor.addToRightNeighbor(coPhrase.charAt(length));
}
else break;
}
wordTFNeighbor.put(word, tfNeighbor);
}
}
//get left neighbor
for(int pIndex = 0; pIndex < leftPTable.size(); pIndex++){
String phrase = leftPTable.get(pIndex);
for(int length = 1 + leftLTable[pIndex]; length <= N && length <= phrase.length(); length++){
String word = reverse(phrase.substring(0, length));
TFNeighbor tfNeighbor = wordTFNeighbor.get(word);
if(phrase.length() > length)
tfNeighbor.addToLeftNeighbor(phrase.charAt(length));
for(int lIndex = pIndex + 1; lIndex < leftLTable.length; lIndex++){
if(leftLTable[lIndex] >= length){
String coPhrase = leftPTable.get(lIndex);
if(coPhrase.length() > length)
tfNeighbor.addToLeftNeighbor(coPhrase.charAt(length));
}
else break;
}
}
}
System.out.println("Info: [Nagao Algorithm Step 3]: having counted TF and Neighbor");
}
//according to wordTFNeighbor, count MI of word
private double countMI(String word){
if(word.length() <= 1) return 0;
double coProbability = wordTFNeighbor.get(word).getTF()/wordNumber;
List mi = new ArrayList(word.length());
for(int pos = 1; pos < word.length(); pos++){
String leftPart = word.substring(0, pos);
String rightPart = word.substring(pos);
double leftProbability = wordTFNeighbor.get(leftPart).getTF()/wordNumber;
double rightProbability = wordTFNeighbor.get(rightPart).getTF()/wordNumber;
mi.add(coProbability/(leftProbability*rightProbability));
}
return Collections.min(mi);
}
//save TF, (left and right) neighbor number, neighbor entropy, mutual information
private void saveTFNeighborInfoMI(String out, String stopList, String[] threshold){
try {
//read stop words file
Set stopWords = new HashSet();
BufferedReader br = new BufferedReader(new FileReader(stopList));
String line;
while((line = br.readLine()) != null){
if(line.length() > 1)
stopWords.add(line);
}
br.close();
//output words TF, neighbor info, MI
BufferedWriter bw = new BufferedWriter(new FileWriter(out));
for(Map.Entry entry : wordTFNeighbor.entrySet()){
if( entry.getKey().length() <= 1 || stopWords.contains(entry.getKey()) ) continue;
TFNeighbor tfNeighbor = entry.getValue();
int tf, leftNeighborNumber, rightNeighborNumber;
double mi;
tf = tfNeighbor.getTF();
leftNeighborNumber = tfNeighbor.getLeftNeighborNumber();
rightNeighborNumber = tfNeighbor.getRightNeighborNumber();
mi = countMI(entry.getKey());
if(tf > Integer.parseInt(threshold[0]) && leftNeighborNumber > Integer.parseInt(threshold[1]) &&
rightNeighborNumber > Integer.parseInt(threshold[2]) && mi > Integer.parseInt(threshold[3]) ){
StringBuilder sb = new StringBuilder();
sb.append(entry.getKey());
sb.append(",").append(tf);
sb.append(",").append(leftNeighborNumber);
sb.append(",").append(rightNeighborNumber);
sb.append(",").append(tfNeighbor.getLeftNeighborEntropy());
sb.append(",").append(tfNeighbor.getRightNeighborEntropy());
sb.append(",").append(mi).append("\n");
bw.write(sb.toString());
}
}
bw.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
System.out.println("Info: [Nagao Algorithm Step 4]: having saved to file");
}
//apply nagao algorithm to input file
public static void applyNagao(String[] inputs, String out, String stopList){
NagaoAlgorithm nagao = new NagaoAlgorithm();
//step 1: add phrases to PTable
String line;
for(String in : inputs){
try {
BufferedReader br = new BufferedReader(new FileReader(in));
while((line = br.readLine()) != null){
nagao.addToPTable(line);
}
br.close();
} catch (IOException e) {
throw new RuntimeException();
}
}
System.out.println("Info: [Nagao Algorithm Step 1]: having added all left and right substrings to PTable");
//step 2: sort PTable and count LTable
nagao.countLTable();
//step3: count TF and Neighbor
nagao.countTFNeighbor();
//step4: save TF NeighborInfo and MI
nagao.saveTFNeighborInfoMI(out, stopList, "20,3,3,5".split(","));
}
public static void applyNagao(String[] inputs, String out, String stopList, int n, String filter){
NagaoAlgorithm nagao = new NagaoAlgorithm();
nagao.setN(n);
String[] threshold = filter.split(",");
if(threshold.length != 4){
System.out.println("ERROR: filter must have 4 numbers, seperated with ',' ");
return;
}
//step 1: add phrases to PTable
String line;
for(String in : inputs){
try {
BufferedReader br = new BufferedReader(new FileReader(in));
while((line = br.readLine()) != null){
nagao.addToPTable(line);
}
br.close();
} catch (IOException e) {
throw new RuntimeException();
}
}
System.out.println("Info: [Nagao Algorithm Step 1]: having added all left and right substrings to PTable");
//step 2: sort PTable and count LTable
nagao.countLTable();
//step3: count TF and Neighbor
nagao.countTFNeighbor();
//step4: save TF NeighborInfo and MI
nagao.saveTFNeighborInfoMI(out, stopList, threshold);
}
private void setN(int n){
N = n;
}
public static void main(String[] args) {
String[] ins = {"E://test//ganfen.txt"};
applyNagao(ins, "E://test//out.txt", "E://test//stoplist.txt");
}
}
2. TFNeighbor.java
package com.algo.word;
import java.util.HashMap;
import java.util.Map;
public class TFNeighbor {
private int tf;
private Map leftNeighbor;
private Map rightNeighbor;
TFNeighbor(){
leftNeighbor = new HashMap();
rightNeighbor = new HashMap();
}
//add word to leftNeighbor
public void addToLeftNeighbor(char word){
//leftNeighbor.put(word, 1 + leftNeighbor.getOrDefault(word, 0));
Integer number = leftNeighbor.get(word);
leftNeighbor.put(word, number == null? 1: 1+number);
}
//add word to rightNeighbor
public void addToRightNeighbor(char word){
//rightNeighbor.put(word, 1 + rightNeighbor.getOrDefault(word, 0));
Integer number = rightNeighbor.get(word);
rightNeighbor.put(word, number == null? 1: 1+number);
}
//increment tf
public void incrementTF(){
tf++;
}
public int getLeftNeighborNumber(){
return leftNeighbor.size();
}
public int getRightNeighborNumber(){
return rightNeighbor.size();
}
public double getLeftNeighborEntropy(){
double entropy = 0;
int sum = 0;
for(int number : leftNeighbor.values()){
entropy += number*Math.log(number);
sum += number;
}
if(sum == 0) return 0;
return Math.log(sum) - entropy/sum;
}
public double getRightNeighborEntropy(){
double entropy = 0;
int sum = 0;
for(int number : rightNeighbor.values()){
entropy += number*Math.log(number);
sum += number;
}
if(sum == 0) return 0;
return Math.log(sum) - entropy/sum;
}
public int getTF(){
return tf;
}
}
3. Main.java
package com.algo.word;
public class Main {
public static void main(String[] args) {
//if 3 arguments, first argument is input files splitting with ','
//second argument is output file
//output 7 columns split with ',' , like below:
//word, term frequency, left neighbor number, right neighbor number, left neighbor entropy, right neighbor entropy, mutual information
//third argument is stop words list
if(args.length == 3)
NagaoAlgorithm.applyNagao(args[0].split(","), args[1], args[2]);
//if 4 arguments, forth argument is the NGram parameter N
//5th argument is threshold of output words, default is "20,3,3,5"
//output TF > 20 && (left | right) neighbor number > 3 && MI > 5
else if(args.length == 5)
NagaoAlgorithm.applyNagao(args[0].split(","), args[1], args[2], Integer.parseInt(args[3]), args[4]);
}
}
以上所述就是本文的全部内容了，希望大家能够喜欢。
