搜狗语料库数据整编

#!/bin/bash
if [ -z "$1" ] ; then
  echo "请指定输入目录"
  exit 1
elif [ ! -d $1 ] ; then
  echo "[$1]不是一个合法的输入路径"
  exit 1
fi
if [ -z "$2" ] ; then
  echo "请指定输出目录"
  exit 1
elif [ ! -d $2 ] ; then
  echo "[$2]不是一个合法的输出目录"
  exit 1
fi

echo "处理开始"

for filename in `ls $1`
  do
    if [ ! -f "$1/$filename" ] ; then
       echo "[$filename]不是一个合法的文件!将略过"
       continue
    fi
    echo $filename
    iconv -f gbk -t utf-8 -c "$1/$filename"| awk '{if(NR%6==1||NR%6==5){ORS=""}else{ORS=NR%6?"|":"\n"};gsub("</?doc>|</?url>|</?docno>|</?contenttitle>|</?content>","",$0);print}' >> "$2/$filename"
  done

echo "处理结束"