Lucene---全文检索(处理一对多去重问题 )
在处理如＂问答＂功能时，以答进行搜索，这时假就会出现去重问题－－－＞http://www.iteye.com/problems/56869
?
?
解决方案：
１，写个线程管理器，用来存储当前查出的重复数据
?
/* * CopyRright (c) www.fdauto.com */package com.fdauto.bws.business.module.lucene.duplicate;import java.util.ArrayList;import java.util.List;/** * *用于管理在一个线程内共用一个Map * * @author 09817(wu_quanyin) * @date 2011-01-06 上午17:02:44 * @version 1.0 */public class DuplicateManager {private static ThreadLocal<List<DuplicateModel>> threadLocalList = new ThreadLocal<List<DuplicateModel>>() {// 该函数说明，在当前线程内，不管调用多次，只实例化一次protected synchronized List<DuplicateModel> initialValue() {return new ArrayList<DuplicateModel>();}};public static List<DuplicateModel> getCurrentThreadList() {return threadLocalList.get();}/**清空当前线程中设置进的list*/public static void removeCurrentThreadListValue(){threadLocalList.remove();}public static void main(String[] args) throws Exception {Thread thread1=new Thread(){public void run(){DuplicateManager.getCurrentThreadList().add(new DuplicateModel());System.out.println(DuplicateManager.getCurrentThreadList().size());DuplicateManager.removeCurrentThreadListValue();DuplicateManager.removeCurrentThreadListValue();System.out.println("当前线程的值被删除了．"+DuplicateManager.getCurrentThreadList().size());}};Thread thread2=new Thread(){public void run(){System.out.println(DuplicateManager.getCurrentThreadList().size());}};thread1.start();Thread.sleep(1000L);thread2.start();System.out.println("-------------------------------");Runnable runnable1=new Runnable(){public void run(){DuplicateManager.getCurrentThreadList().add(new DuplicateModel());System.out.println(DuplicateManager.getCurrentThreadList().size());}};Thread thread3=new Thread(runnable1);Thread thread4=new Thread(runnable1);thread3.start();Thread.sleep(1000L);thread4.start();}}二， 设置一个model存储相同值的doc，并设置开关，只出现一次
?
/* * CopyRright (c) www.fdauto.com */package com.fdauto.bws.business.module.lucene.duplicate;import java.util.ArrayList;import java.util.List;/** * *用于存放一组具有相同值的字段 * * @author 09817(wu_quanyin) * @date 2011-01-06 上午17:02:44 * @version 1.0 */public class DuplicateModel {private int mainDoc;//用于进行判断该重复值是否比较过，如果比较过则不在比较private boolean isChecked=false;private List<Integer> assistantDocs=new ArrayList<Integer>();public DuplicateModel(){}public DuplicateModel(int mainDoc){this.mainDoc=mainDoc;}public int getMainDoc() {return mainDoc;}public void setMainDoc(int mainDoc) {this.mainDoc = mainDoc;}public boolean isChecked() {return isChecked;}public void setChecked(boolean isChecked) {this.isChecked = isChecked;}public List<Integer> getAssistantDocs() {return assistantDocs;}public void setAssistantDocs(List<Integer> assistantDocs) {this.assistantDocs = assistantDocs;}}?三，扩展duplicatefilter类
?
?
/* * CopyRright (c) www.fdauto.com */package com.fdauto.bws.business.module.lucene.duplicate;import java.io.IOException;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Term;import org.apache.lucene.index.TermDocs;import org.apache.lucene.index.TermEnum;import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.Filter;import org.apache.lucene.util.OpenBitSet;/** * 在对重复的项进行以集合的形式区分存放在list中待在DuplicateQuery进行比较 * * @author wu_quanyin(09817) * @version 1.0 * @date 2011-01-06 上午10:54:01 */public class DuplicateExtendFilter extends Filter {private static final long serialVersionUID = 1601875802819099276L;String fieldName;public DuplicateExtendFilter(String fieldName) {this.fieldName = fieldName;}@Overridepublic DocIdSet getDocIdSet(IndexReader reader) throws IOException {return correctBits(reader);}private OpenBitSet correctBits(IndexReader reader) throws IOException {OpenBitSet bits = new OpenBitSet(reader.maxDoc()); // assume all are// INvalidTerm startTerm = new Term(fieldName);TermEnum te = reader.terms(startTerm);if (te != null) {Term currTerm = te.term();while ((currTerm != null)&& (currTerm.field() == startTerm.field())) // term// fieldnames// are interned{int firstDoc = -1;// set non duplicatesTermDocs td = reader.termDocs(currTerm);// 第一个全部添加if (td.next()) {firstDoc = td.doc();bits.set(firstDoc);}// --------------当有重复键时，实行对具有重复的进行保留比较DuplicateModel duplicateModel = new DuplicateModel();boolean isDuplicate = false;while (td.next()) {isDuplicate = true;duplicateModel.getAssistantDocs().add(td.doc());}if (isDuplicate) {duplicateModel.setMainDoc(firstDoc);duplicateModel.getAssistantDocs().add(firstDoc);DuplicateManager.getCurrentThreadList().add(duplicateModel);} else {duplicateModel = null;}// -----------------------if (!te.next()) {break;}currTerm = te.term();}}return bits;}public String getFieldName() {return fieldName;}public void setFieldName(String fieldName) {this.fieldName = fieldName;}@Overridepublic boolean equals(Object obj) {if (this == obj)return true;if ((obj == null) || (obj.getClass() != this.getClass()))return false;DuplicateExtendFilter other = (DuplicateExtendFilter) obj;return (fieldName == other.fieldName || (fieldName != null && fieldName.equals(other.fieldName)));}@Overridepublic int hashCode() {int hash = 217;hash = 31 * hash + fieldName.hashCode();return hash;}}??
四，在query中，根据自己的逻辑进行判断，扩展filterquery类
?
/* * CopyRright (c) www.fdauto.com */package com.fdauto.bws.business.module.lucene.duplicate;import java.io.IOException;import java.util.Iterator;import java.util.Set;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.Term;import org.apache.lucene.search.DocIdSet;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.Explanation;import org.apache.lucene.search.Filter;import org.apache.lucene.search.FilteredQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.Scorer;import org.apache.lucene.search.Searcher;import org.apache.lucene.search.Similarity;import org.apache.lucene.search.Weight;import org.apache.lucene.util.ToStringUtils;/** * 与DuplicateExtendFilter配合使用， * 对有重复的一个个比较过去．． * * @author 09817(wu_quanyin) * @date 2010-12-25 上午09:02:44 * @version 1.0 */public class DuplicateQuery extends Query {/** * */private static final long serialVersionUID = 4610299680666587077L;Query query;Filter filter;/** * Constructs a new query which applies a filter to the results of the * original query. Filter.getDocIdSet() will be called every time this query * is used in a search. * * @param query * Query to be filtered, cannot be <code>null</code>. * @param filter * Filter to apply to query results, cannot be <code>null</code>. */public DuplicateQuery(Query query, Filter filter) {this.query = query;this.filter = filter;}/** * Returns a Weight that applies the filter to the enclosed query's Weight. * This is accomplished by overriding the Scorer returned by the Weight. */@Overridepublic Weight createWeight(final Searcher searcher) throws IOException {final Weight weight = query.createWeight(searcher);final Similarity similarity = query.getSimilarity(searcher);return new Weight() {private static final long serialVersionUID = 3001781092877864947L;private float value;// pass these methods through to enclosed query's weight@Overridepublic float getValue() {return value;}@Overridepublic float sumOfSquaredWeights() throws IOException {return weight.sumOfSquaredWeights() * getBoost() * getBoost();}@Overridepublic void normalize(float v) {weight.normalize(v);value = weight.getValue() * getBoost();}@Overridepublic Explanation explain(IndexReader ir, int i)throws IOException {Explanation inner = weight.explain(ir, i);if (getBoost() != 1) {Explanation preBoost = inner;inner = new Explanation(inner.getValue() * getBoost(),"product of:");inner.addDetail(new Explanation(getBoost(), "boost"));inner.addDetail(preBoost);}Filter f = DuplicateQuery.this.filter;DocIdSet docIdSet = f.getDocIdSet(ir);DocIdSetIterator docIdSetIterator = docIdSet == null ? DocIdSet.EMPTY_DOCIDSET.iterator(): docIdSet.iterator();if (docIdSetIterator == null) {docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator();}if (docIdSetIterator.advance(i) == i) {return inner;} else {Explanation result = new Explanation(0.0f,"failure to match filter: " + f.toString());result.addDetail(inner);return result;}}// return this query@Overridepublic Query getQuery() {return DuplicateQuery.this;}// return a filtering scorer@Overridepublic Scorer scorer(IndexReader indexReader,boolean scoreDocsInOrder, boolean topScorer)throws IOException {final Scorer scorer = weight.scorer(indexReader, true, false);if (scorer == null) {return null;}DocIdSet docIdSet = filter.getDocIdSet(indexReader);if (docIdSet == null) {return null;}final DocIdSetIterator docIdSetIterator = docIdSet.iterator();if (docIdSetIterator == null) {return null;}return new Scorer(similarity) {private int doc = -1;private int advanceToCommon(int scorerDoc, int disiDoc)throws IOException {while (scorerDoc != disiDoc) {if (scorerDoc < disiDoc) {scorerDoc = scorer.advance(disiDoc);} else {disiDoc = docIdSetIterator.advance(scorerDoc);}}return scorerDoc;}@Overridepublic int nextDoc() throws IOException {int scorerDoc = -1;//对每一个重复的进行比较while ((scorerDoc = scorer.nextDoc()) != NO_MORE_DOCS) {boolean ignoreDocment=false;for (Iterator<DuplicateModel> duplicateIterator = DuplicateManager.getCurrentThreadList().iterator(); duplicateIterator.hasNext();) {DuplicateModel duplicateModel = duplicateIterator.next();if(duplicateModel.getAssistantDocs().contains(scorerDoc)){if(!duplicateModel.isChecked()){duplicateModel.setChecked(true);return scorerDoc;}else{//如果已经被检查过的忽略ignoreDocment=true;break;}}}// 如果该doc已经加入过，则忽略if (ignoreDocment) {continue;}return scorerDoc;}return NO_MORE_DOCS;}@Overridepublic int docID() {return doc;}@Overridepublic int advance(int target) throws IOException {int disiDoc, scorerDoc;return doc = (disiDoc = docIdSetIterator.advance(target)) != NO_MORE_DOCS&& (scorerDoc = scorer.advance(disiDoc)) != NO_MORE_DOCS&& advanceToCommon(scorerDoc, disiDoc) != NO_MORE_DOCS ? scorer.docID(): NO_MORE_DOCS;}@Overridepublic float score() throws IOException {return getBoost() * scorer.score();}};}};}/** Rewrites the wrapped query. */@Overridepublic Query rewrite(IndexReader reader) throws IOException {Query rewritten = query.rewrite(reader);if (rewritten != query) {DuplicateQuery clone = (DuplicateQuery) this.clone();clone.query = rewritten;return clone;} else {return this;}}public Query getQuery() {return query;}public Filter getFilter() {return filter;}// inherit javadoc@Overridepublic void extractTerms(Set<Term> terms) {getQuery().extractTerms(terms);}/** Prints a user-readable version of this query. */@Overridepublic String toString(String s) {StringBuilder buffer = new StringBuilder();buffer.append("filtered(");buffer.append(query.toString(s));buffer.append(")->");buffer.append(filter);buffer.append(ToStringUtils.boost(getBoost()));return buffer.toString();}/** Returns true iff <code>o</code> is equal to this. */@Overridepublic boolean equals(Object o) {if (o instanceof FilteredQuery) {DuplicateQuery fq = (DuplicateQuery) o;return (query.equals(fq.query) && filter.equals(fq.filter) && getBoost() == fq.getBoost());}return false;}/** Returns a hash code value for this object. */@Overridepublic int hashCode() {return query.hashCode() ^ filter.hashCode()+ Float.floatToRawIntBits(getBoost());}}使用：
DuplicateExtendFilter filter = new DuplicateExtendFilter(uniqueField);duplicateQuery = new DuplicateQuery(query, filter);1 楼 confident_f 2011-10-11 通过你写的这篇文章，可以看出你对lucene的一套机制肯定做了深入的研究，关于lucene的去重问题，我想请教你一个问题，它去重的具体过程是怎么样的？