准备工作:
- 最新NLPIR分词系统下载包http://ictclas.nlpir.org/downloads
- 授权文件更新:https://github.com/NLPIR-team/NLPIR/tree/431b6351f30ed7d606ac50fde1f4456c596998df/License或者https://github.com/NLPIR-team/NLPIR/tree/master/License
在eclipse中导入中科院分词系统下载包中sample->JnaTest_NLPIR项目,优化NlpirTest.java文件代码。
其中,官方原始demo代码需要更改的地方:
CLibrary Instance = (CLibrary) Native.loadLibrary(
"D:\\tools\\NLPIR\\lib\\win64\\NLPIR", CLibrary.class);
中路径改为自己操作系统的.dll和.lib的上一级目录,并加上NLPIR,不要后缀名
public static void main(String[] args) throws Exception {
String argu = "D:\\tools\\NLPIR";
路径改为Data文件夹的上一级目录
☆注:遇到初始化问题,如果是文件过期问题,在上面链接中下载对应的最新授权文件,并替换掉项目里Data文件下对应的授权文件;如果是配置问题,可能是导入的项目路径中有中文。
本代码可实现功能:
- 普通批量分词
- 添加用户词典txt后批量
- 批量提取关键字
package code;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import com.sun.jna.Library;
import com.sun.jna.Native;
public class Nlpir {
// 定义接口CLibrary,继承自com.sun.jna.Library
public interface CLibrary extends Library {
// 定义并初始化接口的静态变量
CLibrary Instance = (CLibrary) Native.loadLibrary(
"D:\\tools\\NLPIR\\lib\\win64\\NLPIR", CLibrary.class);
public int NLPIR_Init(String sDataPath, int encoding,
String sLicenceCode);
public String NLPIR_ParagraphProcess(String sSrc, int bPOSTagged); //对字符串内容进行分词
public String NLPIR_GetKeyWords(String sLine, int nMaxKeyLimit,
boolean bWeightOut); //从字符串中提取关键词
public String NLPIR_GetFileKeyWords(String sLine, int nMaxKeyLimit,
boolean bWeightOut); //从txt文件中提取关键词,本代码未用到
public int NLPIR_AddUserWord(String sWord);
public String NLPIR_GetLastErrorMsg();
public void NLPIR_Exit();
}
public static String transString(String aidString, String ori_encoding,
String new_encoding) {
try {
return new String(aidString.getBytes(ori_encoding), new_encoding);
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return null;
}
public static void main(String[] args) throws Exception {
String argu = "D:\\tools\\NLPIR"; //Data的上一级目录
String system_charset = "UTF-8";
int charset_type = 1;
int init_flag = CLibrary.Instance.NLPIR_Init(argu, charset_type, "0");
String nativeBytes = null;
if (0 == init_flag) {
nativeBytes = CLibrary.Instance.NLPIR_GetLastErrorMsg();
System.err.println("初始化失败!fail reason is "+nativeBytes);
return;
}
/*添加用户词典功能*/
String userFilePath = "D:\\tools\\userdic.txt";
File userFile = new File(userFilePath);
InputStreamReader readerUser = new InputStreamReader(new FileInputStream(userFile), "gbk");
BufferedReader brUser = new BufferedReader(readerUser);
String lineUser="";
List<String> list = new ArrayList<String>();
while((lineUser = brUser.readLine()) != null){
String[] strings = lineUser.split("\n");
for(int i = 0; i < strings.length; i++){
list.add(strings[i]);
}
}
brUser.close();
/*实现批量读取多个要分词的TXT文件*/
String sInput = "";
String path = "D:\\tools\\source";
String outName = "";
File file = new File(path);
File[] files = file.listFiles();
String out = "";
Arrays.sort(files);
for(int i = 0; i < files.length; i++){
if (files[i].isFile()){
out = "";
outName = files[i].getName();
InputStreamReader reader = new InputStreamReader(new FileInputStream(files[i]), "gbk");
BufferedReader br = new BufferedReader(reader);
String line="";
sInput = "";
line = br.readLine();
while (line != null){
sInput += line;
line = br.readLine();
}
br.close();
try {
nativeBytes = CLibrary.Instance.NLPIR_ParagraphProcess(sInput, 1);
out += "未添加用户词典的分词结果是: "; //对应TXT的内容
out += nativeBytes;
out += "\r\n";
for(int k = 0; k < list.size(); k++){
CLibrary.Instance.NLPIR_AddUserWord(list.get(k)); //用户词典txt中的词用空格隔开
}
/*屏蔽这三行OR上段未添加用户词典的分词结果,使result结果中只出现一种结果,否则两者都出现在TXT中
out += "增加用户词典后分词结果是: "; //对应TXT的内容
out += nativeBytes;
out += "\r\n";
*/
int nCountKey = 0;
String nativeByte = CLibrary.Instance.NLPIR_GetKeyWords(sInput, 10,false);
File fp1 = new File("D:\\tools\\keyword\\" + outName);
PrintWriter pfp1 = new PrintWriter(fp1);
pfp1.write("关键词为:" + nativeByte); //对应TXT内容
pfp1.close();
//分词结果(添加和未添加用户词典的分词结果)
File fp = new File("D:\\tools\\result\\" + outName);
PrintWriter pfp = new PrintWriter(fp);
pfp.write(out);
pfp.close();
} catch (Exception ex) {
// TODO Auto-generated catch block
ex.printStackTrace();
}
}
}
CLibrary.Instance.NLPIR_Exit();
}
}
还有需要完善的地方,希望多多交流!!!