java 中文文本分词

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/XnCSD/article/details/81516032

java 中文文本分词

本文使用 classifier4J 以及 IKAnalyzer2012_u6 实现中文分词。可以增加自定义词库，词库保存为 “exdict.dic” 文件，一个词一行。

// MyTokenizer.java 文件

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import net.sf.classifier4J.ITokenizer;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.wltea.analyzer.cfg.Configuration;
import org.wltea.analyzer.cfg.DefaultConfig;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.lucene.IKTokenizer;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

/**
 * 中文分词器类
 * 
 * @author CSD
 *
 */
@SuppressWarnings("deprecation")
public class MyTokenizer implements ITokenizer {

    private static final Logger logger = LogManager.getLogger(MyTokenizer.class);

    private List<String> list;
    private String[] strArray;
    private static Collection<String> exwordc = new ArrayList<>();
    private static String exdict = "exdict.dic";

    // 加载新增词库
    static {

        try {
            File file = new File(exdict);
            FileInputStream fin = new FileInputStream(file);
            BufferedReader reader = new BufferedReader(new InputStreamReader(fin));
            String line = "";
            while ((line = reader.readLine()) != null) {
                exwordc.add(line.trim());
            }
            reader.close();
            logger.info("加载词典::" + exdict);
            // 增加词库
            Configuration cfg = DefaultConfig.getInstance();
            Dictionary dict = Dictionary.initial(cfg);
            dict.addWords(exwordc);
        } catch (IOException e) {
            logger.error(e + "------------------加载词典出错，请确认词典文件！------------------");
        }
    }

    /**
     * 分词，返回分词数组
     * 
     * @param input
     *            文本字符串
     * @return String[]
     */
    public String[] tokenize(String input) {
        list = new ArrayList<String>();

        IKTokenizer tokenizer = new IKTokenizer(new StringReader(input), true);
        try {
            while (tokenizer.incrementToken()) {
                TermAttribute termAtt = (TermAttribute) tokenizer.getAttribute(TermAttribute.class);
                String str = termAtt.term();
                list.add(str);
            }
        } catch (IOException e) {
            logger.error(e + "------------------分词出错------------------");
        }
        strArray = new String[list.size()];
        for (int i = 0; i < list.size(); i++) {
            strArray[i] = (String) list.get(i);
        }

        return strArray;
    }

}

// Segmentation.java 文件
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

import net.sf.classifier4J.ITokenizer;

/**
 * 中文语料分词
 * 
 * @author CSD
 *
 */
public class Segmentation {

    private static final Logger logger = LogManager.getLogger(Segmentation.class);

    public static void main(String[] args) throws IOException {

        String path = "1.txt";
        File file = new File(path);
        FileInputStream fin = new FileInputStream(file);
        String input = getString(fin);

        logger.info("开始分词::" + path);
        ITokenizer tokenizer = new MyTokenizer();
        String[] words = tokenizer.tokenize(input);
        for (String word : words) {
            System.out.println(word);
        }

    }

    /**
     * 从 inputStream 读取文本并转为一个字符串。
     * 
     * @param is
     *            inputStream 输入流
     * @return String 文本字符串
     * @throws IOException
     */
    public static String getString(InputStream is) throws IOException {

        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
        String line = "";
        StringBuffer stringBuffer = new StringBuffer();
        while ((line = reader.readLine()) != null) {
            stringBuffer.append(line);
            stringBuffer.append(" ");
        }

        reader.close();

        return stringBuffer.toString().trim();
    }
}

程序需依赖 IKAnalyzer2012_u6.jar 以及添加 pom.xml 文件

<!-- https://mvnrepository.com/artifact/classifier4j/classifier4j -->
        <dependency>
            <groupId>classifier4j</groupId>
            <artifactId>classifier4j</artifactId>
            <version>0.6</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.apache.lucene/lucene-analyzers -->
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-analyzers</artifactId>
            <version>3.2.0</version>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.5</version>
        </dependency>