Lucene笔记37-Lucene如何通过NRTManager和SearchManager实现近实时搜索

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_36059561/article/details/84075518

一、思路分析

如何实现近实时搜索呢？每次更新完索引都commit？那恐怕太浪费资源了，当数据量非常庞大的时候，几乎不可能。这里有两种方案。

1. 使用SearchManager来管理IndexSearcher对象，当发现索引更新之后，searchManager会调用maybeReopen()方法，来获取最新的IndexSearcher对象，索引search()就是最新的结果了。
2. 使用NRTManager和SearcherManager，NRTManager会定时执行maybeReopen()方法来保持最新的索引。它有一个NRTManagerReopenThread类，这个类是一个线程类，用来跟踪索引内存的变化。

二、使用SearcherManager

package com.wsy;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Executors;

public class IndexUtilPlus {
    private String[] ids = {"1", "2", "3", "4", "5", "6"};
    private String[] emails = {"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"};
    private String[] contents = {"content 1", "content 2", "content 3", "content 4", "content 5", "content 6"};
    private int[] attachFiles = {1, 2, 3, 4, 5, 6};
    private String[] names = {"qianyi", "zhaoer", "zhangsan", "lisi", "wangwu", "liuliu"};
    private Directory directory = null;
    private Map<String, Float> scores = new HashMap<>();
    private Date[] dates = null;
    private SearcherManager searcherManager;
    private IndexWriter indexWriter;

    public IndexUtilPlus() {
        try {
            InitDate();
            directory = FSDirectory.open(new File("E:\\Lucene\\IndexLibrary"));
            indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
            searcherManager = new SearcherManager(directory, new SearcherWarmer() {
                // 索引更新的时候，需要重新获取IndexSearcher，就会调用这个方法
                // 执行maybeReopen()方法的时候就会执行warm()方法，在warm()方法里可以对资源进行控制
                @Override
                public void warm(IndexSearcher indexSearcher) throws IOException {
                    System.out.println("index has changed");
                }
            }, Executors.newCachedThreadPool());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void InitDate() {
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
        dates = new Date[ids.length];
        for (int i = 0; i < ids.length; i++) {
            try {
                dates[i] = simpleDateFormat.parse("2018-10-" + i + 1);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
    }

    public void index() {
        IndexWriter indexWriter = null;
        try {
            indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
            indexWriter.deleteAll();
            Document document = null;
            for (int i = 0; i < ids.length; i++) {
                document = new Document();
                document.add(new Field("id", ids[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                document.add(new Field("email", emails[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
                document.add(new Field("content", contents[i], Field.Store.NO, Field.Index.ANALYZED));
                document.add(new Field("name", names[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                String emailType = emails[i].substring(emails[i].lastIndexOf("@") + 1);
                if (scores.containsKey(emailType)) {
                    document.setBoost(scores.get(emailType));
                } else {
                    document.setBoost(0.1f);
                }
                indexWriter.addDocument(document);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (indexWriter != null) {
                try {
                    indexWriter.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    public void search() {
        // 使用searcherManager来管理indexSearcher，使用acquire()来获取IndexSearcher对象
        IndexSearcher indexSearcher = searcherManager.acquire();
        try {
            // 当searcherManager检测到硬盘上的索引发生变化的时候
            // 通过执行maybeReopen()方法来重新加载IndexSearcher
            // 执行maybeReopen()方法的时候，会调用SearcherWarmer中的warm()方法
            searcherManager.maybeReopen();
            // 搜索content域包含“content”的
            TermQuery termQuery = new TermQuery(new Term("content", "content"));
            TopDocs topDocs = indexSearcher.search(termQuery, 10);
            for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                Document document = indexSearcher.doc(scoreDoc.doc);
                System.out.println(scoreDoc.doc + " " + document.get("name"));
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                // 使用release()来释放IndexSearcher对象
                searcherManager.release(indexSearcher);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}

三、使用SearchManager和NRTManager

package com.wsy;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

public class IndexUtilPlus {
    private String[] ids = {"1", "2", "3", "4", "5", "6"};
    private String[] emails = {"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"};
    private String[] contents = {"content 1", "content 2", "content 3", "content 4", "content 5", "content 6"};
    private int[] attachFiles = {1, 2, 3, 4, 5, 6};
    private String[] names = {"qianyi", "zhaoer", "zhangsan", "lisi", "wangwu", "liuliu"};
    private Directory directory = null;
    private Map<String, Float> scores = new HashMap<>();
    private Date[] dates = null;
    private SearcherManager searcherManager;
    private NRTManager nrtManager;
    private IndexWriter indexWriter;

    public IndexUtilPlus() {
        try {
            InitDate();
            directory = FSDirectory.open(new File("E:\\Lucene\\IndexLibrary"));
            indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
            nrtManager = new NRTManager(indexWriter, new SearcherWarmer() {
                @Override
                public void warm(IndexSearcher indexSearcher) throws IOException {
                    System.out.println("index has changed");
                }
            });
            NRTManagerReopenThread nrtManagerReopenThread = new NRTManagerReopenThread(nrtManager, 5.0, 0.025);
            nrtManagerReopenThread.setDaemon(true);
            nrtManagerReopenThread.setName("NRTManager Reopen Thread");
            nrtManagerReopenThread.start();
            // true表示允许所有更新
            searcherManager = nrtManager.getSearcherManager(true);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void InitDate() {
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
        dates = new Date[ids.length];
        for (int i = 0; i < ids.length; i++) {
            try {
                dates[i] = simpleDateFormat.parse("2018-10-" + i + 1);
            } catch (ParseException e) {
                e.printStackTrace();
            }
        }
    }

    public void index() {
        IndexWriter indexWriter = null;
        try {
            indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
            indexWriter.deleteAll();
            Document document = null;
            for (int i = 0; i < ids.length; i++) {
                document = new Document();
                document.add(new Field("id", ids[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                document.add(new Field("email", emails[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
                document.add(new Field("content", contents[i], Field.Store.NO, Field.Index.ANALYZED));
                document.add(new Field("name", names[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
                String emailType = emails[i].substring(emails[i].lastIndexOf("@") + 1);
                if (scores.containsKey(emailType)) {
                    document.setBoost(scores.get(emailType));
                } else {
                    document.setBoost(0.1f);
                }
                indexWriter.addDocument(document);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (indexWriter != null) {
                try {
                    indexWriter.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    // 使用NRTManager来删除索引
    public void deletePlus() {
        try {
            nrtManager.deleteDocuments(new Term("id","1"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    // 使用NRTManager来更新索引
    public void updatePlus(){
        Document document = new Document();
        document.add(new Field("id", "11", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
        document.add(new Field("email", emails[0], Field.Store.YES, Field.Index.NOT_ANALYZED));
        document.add(new Field("content", contents[0], Field.Store.NO, Field.Index.ANALYZED));
        document.add(new Field("name", names[0], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
        try {
            nrtManager.updateDocument(new Term("id", "1"), document);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void search() {
        // 使用searcherManager来管理indexSearcher，使用acquire()来获取IndexSearcher对象
        IndexSearcher indexSearcher = searcherManager.acquire();
        try {
            // 当searcherManager检测到硬盘上的索引发生变化的时候
            // 通过执行maybeReopen()方法来重新加载IndexSearcher
            // 执行maybeReopen()方法的时候，会调用SearcherWarmer中的warm()方法
            searcherManager.maybeReopen();
            // 搜索content域包含“content”的
            TermQuery termQuery = new TermQuery(new Term("content", "content"));
            TopDocs topDocs = indexSearcher.search(termQuery, 10);
            for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
                Document document = indexSearcher.doc(scoreDoc.doc);
                System.out.println(scoreDoc.doc + " " + document.get("name"));
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                // 使用release()来释放IndexSearcher对象
                searcherManager.release(indexSearcher);
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}