Lucene实现分词

版权声明：Arno https://blog.csdn.net/yu342107056/article/details/87896310

1、我使用的jar为IKAnalyzer2012_u6.jar 请自行百度下载，并引用到项目中

2、pom.xml 添加引用

<dependency> <!-- 查询相关jar包 -->
	<groupId>org.apache.lucene</groupId>
	<artifactId>lucene-queryparser</artifactId>
	<version>6.0.0</version>
	</dependency>
	<dependency> <!-- lucene自带只能中文分词器jar包 -->
		<groupId>org.apache.lucene</groupId>
		<artifactId>lucene-analyzers-smartcn</artifactId>
		<version>6.0.0</version>
	</dependency>
	<dependency> <!-- 测试用到的lucene工具包 -->
		<groupId>org.apache.lucene</groupId>
		<artifactId>lucene-analyzers-common</artifactId>
		<version>6.0.0</version>
	</dependency>
	<dependency> <!-- 测试用到的lucene核心包 -->
		<groupId>org.apache.lucene</groupId>
		<artifactId>lucene-core</artifactId>
		<version>6.0.0</version>
	</dependency>

3、创建IKAnalyzer6x.java 和IKTokenizer6x.java

IKAnalyzer6x.java

package com.arno.ik;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;

public class IKAnalyzer6x extends Analyzer{
	private boolean useSmart;
	public boolean useSmart(){
		return useSmart;
	}
	public void setUseSmart(boolean useSmart){
		this.useSmart=useSmart;
	}
	public IKAnalyzer6x(){
		this(false);//IK分词器lucene analyzer接口实现类，默认细粒度切分算法
	}
	//重写最新版本createComponents；重载analyzer接口，构造分词组件
	@Override
	protected TokenStreamComponents createComponents(String filedName) {
		Tokenizer _IKTokenizer=new IKTokenizer6x(this.useSmart);
		return new TokenStreamComponents(_IKTokenizer);
	}
	public IKAnalyzer6x(boolean useSmart){
		super();
		this.useSmart=useSmart;
	}
	
}

IKTokenizer6x.java

package com.arno.ik;

import java.io.IOException;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

public class IKTokenizer6x extends Tokenizer{
	//ik分词器实现
	private IKSegmenter _IKImplement;
	//词元文本属性
	private final CharTermAttribute termAtt;
	//词元位移属性
	private final OffsetAttribute offsetAtt;
	//词元分类属性
	private final TypeAttribute typeAtt;
	//记录最后一个词元的结束位置
	private int endPosition;
	//构造函数，实现最新的Tokenizer
	public IKTokenizer6x(boolean useSmart){
		super();
		offsetAtt=addAttribute(OffsetAttribute.class);
		termAtt=addAttribute(CharTermAttribute.class);
		typeAtt=addAttribute(TypeAttribute.class);
		_IKImplement=new IKSegmenter(input, useSmart);
	}

	@Override
	public final boolean incrementToken() throws IOException {
		//清除所有的词元属性
		clearAttributes();
		Lexeme nextLexeme=_IKImplement.next();
		if(nextLexeme!=null){
			//将lexeme转成attributes
			termAtt.append(nextLexeme.getLexemeText());
			termAtt.setLength(nextLexeme.getLength());
			offsetAtt.setOffset(nextLexeme.getBeginPosition(), 
					nextLexeme.getEndPosition());
			//记录分词的最后位置
			endPosition=nextLexeme.getEndPosition();
			typeAtt.setType(nextLexeme.getLexemeText());
			return true;//告知还有下个词元
		}
		return false;//告知词元输出完毕
	}
	
	@Override
	public void reset() throws IOException {
		super.reset();
		_IKImplement.reset(input);
	}
	
	@Override
	public final void end(){
		int finalOffset = correctOffset(this.endPosition);
		offsetAtt.setOffset(finalOffset, finalOffset);
	}

}

创建好后。如果报错，请查看4个依赖是否已经加载完成和下载的IKAnalyzer2012_u6.jar是否引入

4、创建5个不同的分词器进行分词测试

package com.arno.test;

import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import com.arno.ik.IKAnalyzer6x;

/*
 * 通过不同分词器analyzer实现对同一个
 * 字符串的分词计算查看结果
 */
public class AnalyzerTest {
	
	//编写静态方法,实现打印分词词项的逻辑
	public static void printResult(Analyzer analyzer,String str) throws Exception{
		//将str转化成字符串输入流
		StringReader reader=new  StringReader(str);
		//anlyzer就是lucene的分词器接口,从analyzer中获取分词的流
		//TokenStream
		TokenStream tokenStream = analyzer.
				tokenStream("test", reader);
		tokenStream.reset();
		//打印词项,拿到字符属性对象
		CharTermAttribute attr=
				tokenStream.getAttribute(CharTermAttribute.class);
		//挨个打印词项
		while(tokenStream.incrementToken()){
			System.out.println(attr.toString());
		}
	}
	
	public static void main(String[] args) throws Exception{
		//准备lucene提供的各种分词器对象
		Analyzer a1=new SmartChineseAnalyzer();
		Analyzer a2=new WhitespaceAnalyzer();
		Analyzer a3=new SimpleAnalyzer();
		Analyzer a4=new StandardAnalyzer();
		Analyzer a5=new IKAnalyzer6x();
		String msg="大部分年轻人,现在都喜欢在,业余时间玩王者荣耀";
		
		System.out.println("**********智能中文分词器**********");
		printResult(a1, msg);
		System.out.println("**********IK分词器**********");
		printResult(a5, msg);
		System.out.println("**********空格分词器**********");
		printResult(a2, msg);
		System.out.println("**********简单分词器**********");
		printResult(a3, msg);
		System.out.println("**********标准分词器**********");
		printResult(a4, msg);
		
	}
}