需求
在检索系统中，遇到了分组统计（Grouping/GroupBy）的需求，比如将搜索结果按照栏目分类，统计每个栏目下各有多少条结果。以前的做法很愚蠢，先发起一次search统计出有多少组，然后在每个组里发起一次search；这样在有N组的情况下一共执行了N+1此搜索，效率低下。
改进
最近发现Lucene提供了分组的功能，是通过Collector实现的，最多可以在2次search的时候得出结果，如果内存够用，CachingCollector还可以节约一次查询。
两次检索
第一次
第一次的目的是收集符合条件的组，创建一个FirstPassGroupingCollector送入search接口即可。在此处使用CachingCollector对其cache的话，可以节省一次查询：
- TermFirstPassGroupingCollector c1 = new TermFirstPassGroupingCollector("catalog", groupSort, topNGroups);
- boolean cacheScores = true;
- double maxCacheRAMMB = 16.0;
- CachingCollector cachedCollector = CachingCollector.create(c1, cacheScores, maxCacheRAMMB);
- searcher.search(query, cachedCollector);
第二次
第二次的目的是收集每个组里面符合条件的文档，此时利用第一次的分组结果创建TermSecondPassGroupingCollector，并执行/replay搜索。
完整实例
-
package com.hankcs; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.*; import org.apache.lucene.search.grouping.GroupDocs; import org.apache.lucene.search.grouping.SearchGroup; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.search.grouping.term.TermAllGroupsCollector; import org.apache.lucene.search.grouping.term.TermFirstPassGroupingCollector; import org.apache.lucene.search.grouping.term.TermSecondPassGroupingCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import java.util.Collection; /** * 演示faceting * * @author hankcs */ public class FacetingDemo { public static void main(String[] args) throws Exception { // Lucene Document的主要域名 String mainFieldName = "text"; // Lucene版本 Version ver = Version.LUCENE_48; // 实例化Analyzer分词器 Analyzer analyzer = new StandardAnalyzer(ver); Directory directory; IndexWriter writer; IndexReader reader; IndexSearcher searcher; //索引过程********************************** //建立内存索引对象 directory = new RAMDirectory(); //配置IndexWriterConfig IndexWriterConfig iwConfig = new IndexWriterConfig(ver, analyzer); iwConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); writer = new IndexWriter(directory, iwConfig); for (int i = 0; i < 100; ++i) { Document doc = new Document(); doc.add(new TextField(mainFieldName, "Banana is sweet " + i, Field.Store.YES)); doc.add(new TextField("catalog", "fruit", Field.Store.YES)); writer.addDocument(doc); } for (int i = 0; i < 50; ++i) { Document doc = new Document(); doc.add(new TextField(mainFieldName, "Juice is sweet " + i, Field.Store.YES)); doc.add(new TextField("catalog", "drink", Field.Store.YES)); writer.addDocument(doc); } for (int i = 0; i < 25; ++i) { Document doc = new Document(); doc.add(new TextField(mainFieldName, "Hankcs is here " + i, Field.Store.YES)); doc.add(new TextField("catalog", "person", Field.Store.YES)); writer.addDocument(doc); } writer.close(); //搜索过程********************************** //实例化搜索器 reader = DirectoryReader.open(directory); searcher = new IndexSearcher(reader); String keyword = "sweet"; //使用QueryParser查询分析器构造Query对象 QueryParser qp = new QueryParser(ver, mainFieldName, analyzer); Query query = qp.parse(keyword); System.out.println("Query = " + query); //搜索相似度最高的5条记录并且分组 int topNGroups = 10; // 每页需要多少个组 int groupOffset = 0; // 起始的组 boolean fillFields = true; Sort docSort = Sort.RELEVANCE; // groupSort用于对组进行排序，docSort用于对组内记录进行排序，多数情况下两者是相同的，但也可不同 Sort groupSort = docSort; int docOffset = 0; // 用于组内分页，起始的记录 int docsPerGroup = 2;// 每组返回多少条结果 boolean requiredTotalGroupCount = true; // 是否需要计算总的组的数量 // 如果需要对Lucene的score进行修正，则需要重载TermFirstPassGroupingCollector TermFirstPassGroupingCollector c1 = new TermFirstPassGroupingCollector("catalog", groupSort, topNGroups); boolean cacheScores = true; double maxCacheRAMMB = 16.0; CachingCollector cachedCollector = CachingCollector.create(c1, cacheScores, maxCacheRAMMB); searcher.search(query, cachedCollector); Collection<SearchGroup<BytesRef>> topGroups = c1.getTopGroups(groupOffset, fillFields); if (topGroups == null) { // No groups matched return; } Collector secondPassCollector = null; boolean getScores = true; boolean getMaxScores = true; // 如果需要对Lucene的score进行修正，则需要重载TermSecondPassGroupingCollector TermSecondPassGroupingCollector c2 = new TermSecondPassGroupingCollector("catalog", topGroups, groupSort, docSort, docsPerGroup, getScores, getMaxScores, fillFields); // 是否需要计算一共有多少个分类，这一步是可选的 TermAllGroupsCollector allGroupsCollector = null; if (requiredTotalGroupCount) { allGroupsCollector = new TermAllGroupsCollector("catalog"); secondPassCollector = MultiCollector.wrap(c2, allGroupsCollector); } else { secondPassCollector = c2; } if (cachedCollector.isCached()) { // 被缓存的话，就用缓存 cachedCollector.replay(secondPassCollector); } else { // 超出缓存大小，重新执行一次查询 searcher.search(query, secondPassCollector); } int totalGroupCount = -1; // 所有组的数量 int totalHitCount = -1; // 所有满足条件的记录数 int totalGroupedHitCount = -1; // 所有组内的满足条件的记录数(通常该值与totalHitCount是一致的) if (requiredTotalGroupCount) { totalGroupCount = allGroupsCollector.getGroupCount(); } System.out.println("一共匹配到多少个分类: " + totalGroupCount); TopGroups<BytesRef> groupsResult = c2.getTopGroups(docOffset); totalHitCount = groupsResult.totalHitCount; totalGroupedHitCount = groupsResult.totalGroupedHitCount; System.out.println("groupsResult.totalHitCount:" + totalHitCount); System.out.println("groupsResult.totalGroupedHitCount:" + totalGroupedHitCount); int groupIdx = 0; // 迭代组 for (GroupDocs<BytesRef> groupDocs : groupsResult.groups) { groupIdx++; System.out.println("group[" + groupIdx + "]:" + groupDocs.groupValue); // 组的标识 System.out.println("group[" + groupIdx + "]:" + groupDocs.totalHits); // 组内的记录数 int docIdx = 0; // 迭代组内的记录 for (ScoreDoc scoreDoc : groupDocs.scoreDocs) { docIdx++; System.out.println("group[" + groupIdx + "][" + docIdx + "]:" + scoreDoc.doc + "/" + scoreDoc.score); Document doc = searcher.doc(scoreDoc.doc); System.out.println("group[" + groupIdx + "][" + docIdx + "]:" + doc); } } } }
输出
- Query = text:sweet
- 一共匹配到多少个分类: 2
- groupsResult.totalHitCount:150
- groupsResult.totalGroupedHitCount:150
- group[1]:[66 72 75 69 74]
- group[1]:100
- group[1][1]:0/0.573753
- group[1][1]:Document<stored,indexed,tokenized<text:Banana is sweet 0> stored,indexed,tokenized<catalog:fruit>>
- group[1][2]:1/0.573753
- group[1][2]:Document<stored,indexed,tokenized<text:Banana is sweet 1> stored,indexed,tokenized<catalog:fruit>>
- group[2]:[64 72 69 6e 6b]
- group[2]:50
- group[2][1]:100/0.573753
- group[2][1]:Document<stored,indexed,tokenized<text:Juice is sweet 0> stored,indexed,tokenized<catalog:drink>>
- group[2][2]:101/0.573753
- group[2][2]:Document<stored,indexed,tokenized<text:Juice is sweet 1> stored,indexed,tokenized<catalog:drink>>
Reference
