lucene 3.5学习笔记
最近一段时间在研究lucene的使用，可以说lucene的功能确实很强大，我只是略沾皮毛，下面是我学习lucene的过程。
开发环境：eclipse3.7；
lucene版本：3.5
功能需求是：对50万的简历数据可以进行多关键字的查询，查询响应时间控制在3S中以内。
需要说明的是，我的简历数据是放在数据库表里面的，数据库对于like%的查询效率太低。所以这里我就想到了用lucene的全文检索来实现。
简历表：tb_resume(resumeId,.....);对应实体类：ResumeModel
CREATE TABLE `tb_resume` (
`resumeId` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(200) DEFAULT NULL,
`sex` varchar(11) DEFAULT NULL,
`degree` varchar(200) DEFAULT NULL,
`jingyan` varchar(200) DEFAULT NULL,
`birthday` datetime DEFAULT NULL,
`nowPlace` varchar(200) DEFAULT NULL,
`birthPlace` varchar(200) DEFAULT NULL,
`email` varchar(200) DEFAULT NULL,
`tel` varchar(200) DEFAULT NULL,
`workExp` longtext,
`educationExp` longtext,
`projectExp` longtext,
`selfEvaluation` longtext,
`state` int(11) DEFAULT NULL,
`qz` int(11) DEFAULT '1',
PRIMARY KEY (`resumeId`)
) ENGINE=InnoDB AUTO_INCREMENT=500058 DEFAULT CHARSET=utf8;
第一步，创建索引
由于数据来源是在数据库，所以需要先取出数据库里面的数据。
然后对这些数据创建索引：
import java.io.IOException;import java.util.Date;import java.util.List;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.store.FSDirectory;import org.apache.lucene.util.Version;import org.apache.lucene.index.IndexWriterConfig.OpenMode; import scott.resume.model.ResumeModel;import util.Constant;/** * 创建索引类 * @author Administrator * */public class CreateIndex {//private static final File INDEX_DIR = new File("E:\\新的设计目录\\txlpf\\lucene");// private static final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_35); /** * 将简历信息添加到索引库中 * @param list * @throws IOException */ public static void index(List<ResumeModel> list) throws IOException { long start1 = new Date().getTime(); IndexWriter writer = openIndexWriter(); try { for (ResumeModel rm : list) { Document document = builderDocument(rm); writer.addDocument(document); } } finally { writer.close(); } long end1 = new Date().getTime(); System.out.println("创建索引花费时间：" + (double) (end1 - start1) / 1000 + "秒"); } private static IndexWriter openIndexWriter() throws IOException { IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_35, Constant.analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);//增量型索引 ，这个地方是关于你创建索引是增量的还是始终创建的。一般来说，选择create_or_append比较好，无论你是否已经创建了索引，都满足 return new IndexWriter(FSDirectory.open(Constant.INDEX_DIR), iwc); } /** * 创建索引 * @param obj 要创建索引的对象 * @return */public static Document builderDocument(ResumeModel rm) { Document document = new Document(); String state = null; if(rm.getState()==0){ state = "未录用"; } else{ state = "已录用"; } Field id = new Field("id", String.valueOf(rm.getResumeId()), Field.Store.YES, Field.Index.ANALYZED); Field content = new Field("content", rm.getDegree()+rm.getJingyan()+rm.getWorkExp()+rm.getEducationExp()+rm.getProjectExp()+rm.getSelfEvaluation()+state, Field.Store.YES, Field.Index.ANALYZED); document.add(id); document.add(content); return document; } }import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.PhraseQuery;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;/** * 查询类 * @author Administrator * */public class QueryUtil {/** * 功能：按词条搜索 － TermQuery * @param field field的名称 * @param searchKey 查询的词 * @return */public static Query searchByTeam(String field,String searchKey){System.err.println("TermQuery test demo------");Term term = new Term(field, searchKey);Query query= new TermQuery(term);System.err.println("查询条件:" + query);System.err.println("查询语义:查询在AlarmType中出现""+searchKey+""这个词的Document.");return query;}/** * 功能：查询俩个子查询的交集 * @param qy1 * @param qy2 * @return */public static Query booleanAndSearch(Query qy1,Query qy2){BooleanQuery query = new BooleanQuery();//新建一个布尔查询query.add(qy1, BooleanClause.Occur.MUST);query.add(qy2, BooleanClause.Occur.MUST);return query;}/** * 功能：查询俩个子查询的交集 * @param fieldOne 第一个子查询field * @param searchKeyOne 第一个子查询关键词 * @param fieldTwo 第二个子查询的field * @param searchKeyTwo 第二个子查询的关键词 * @return */public static Query searchByBooleanAnd(String fieldOne ,String searchKeyOne,String fieldTwo,String searchKeyTwo){BooleanQuery query = new BooleanQuery();//新建一个布尔查询Query qy1 = QueryUtil.searchByTeam(fieldOne, searchKeyOne);Query qy2 = QueryUtil.searchByTeam(fieldTwo, searchKeyTwo);query.add(qy1, BooleanClause.Occur.MUST);query.add(qy2, BooleanClause.Occur.MUST);System.err.println("查询条件:" + query);System.err.println("查询语义:查询在"+fieldOne+"中包含""+searchKeyOne+""这个词但是不能包含查询在"+fieldTwo+"中包含""+searchKeyTwo+""的Document.");return query;}/** * 功能：查询俩个子查询的并集 * @param fieldOne 第一个子查询field * @param searchKeyOne 第一个子查询关键词 * @param fieldTwo 第二个子查询的field * @param searchKeyTwo 第二个子查询的关键词 * @return */public static Query searchByBooleanOr(String fieldOne ,String searchKeyOne,String fieldTwo,String searchKeyTwo){BooleanQuery query = new BooleanQuery();//新建一个布尔查询Query qy1 = QueryUtil.searchByTeam(fieldOne, searchKeyOne);Query qy2 = QueryUtil.searchByTeam(fieldTwo, searchKeyTwo);query.add(qy1, BooleanClause.Occur.SHOULD);query.add(qy2, BooleanClause.Occur.SHOULD);System.err.println("查询条件:" + query);System.err.println("查询语义:包含查询在"+fieldOne+"中包含""+searchKeyOne+""这个词而且包含查询在"+fieldTwo+"中包含""+searchKeyTwo+""的Document.");return query;}/** * 功能：查询俩个子查询的 差集,第一个子查询的结果不包含第二个子查询 * @param fieldOne 第一个子查询field * @param searchKeyOne 第一个子查询关键词 * @param fieldTwo 第二个子查询的field * @param searchKeyTwo 第二个子查询的关键词 * @return */public static Query searchByBooleanNot(String fieldOne ,String searchKeyOne,String fieldTwo,String searchKeyTwo){BooleanQuery query = new BooleanQuery();//新建一个布尔查询Query qy1 = QueryUtil.searchByTeam(fieldOne, searchKeyOne);Query qy2 = QueryUtil.searchByTeam(fieldTwo, searchKeyTwo);query.add(qy1, BooleanClause.Occur.MUST);query.add(qy2, BooleanClause.Occur.MUST_NOT);System.err.println("查询条件:" + query);System.err.println("查询语义:查询在"+fieldOne+"中包含""+searchKeyOne+""这个词但是不包含查询在"+fieldTwo+"中包含""+searchKeyTwo+""的Document.");return query;}/** * 功能：词组查询也称短语查询 * @param fields 查询的field * @param keywords 关键词数组 * @return */public static Query searchByManyKeywords(String fields,String[] keywords){PhraseQuery query = new PhraseQuery();String str = null;for(int i=0;i<keywords.length;i++){query.add(new Term(fields, keywords[i]));str = str +keywords[i];}query.setSlop(5);System.err.println("查询条件:" + query);System.err.println("查询语义:查询"+fields+"中含有:"+str+"的数据，这几个词至少移动五步才能构成词组，返回符合条件的Document.");return query;}}/** * 根据输入的关键字返回关键词字符串数组 * @param key * @return */public static String[] getKeywords(String key){String[] resultKeys = null;key = key.replace(" ", " ");//第一步，将所有的全角空格换个半角空格key = key.replace("，", ",");//第二部，将所有的全角逗号，换成半角逗号key = key.replaceAll(",", " ");//将所有的半角逗号换成空格key = key.replaceAll( "(\\s+) ", " "); //将多个空格换成一个空格key = key.trim(); //去除俩端的空格resultKeys = key.split("[ ,]");//以空格分隔字符串return resultKeys;}/** * 功能：获得多个关键词的查询的query * @param field 要查询的字段 * @param keys 查询的关键词数组 * @return * @throws CorruptIndexException * @throws ParseException * @throws IOException */public static Query searchByManyKey(String field,String keys[]) throws CorruptIndexException, ParseException, IOException{Query qy1 = null;Query qy2 = null;Query temp = null;for(int i=0;i<keys.length-1;i++){qy1 = QueryForLucene.buildQuery(field, keys[i]);qy2 = QueryForLucene.buildQuery(field, keys[i+1]);temp = qy1;temp = QueryUtil.booleanAndSearch(temp, qy2);System.out.println(temp+"\n__________________");}return temp;}/** * 功能：多关键词查询，返回查询结果集 * @param field 查询的field * @param key 查询的关键词数组 * @return * @throws CorruptIndexException * @throws ParseException * @throws IOException */public static ScoreDoc[] getSearchResultAnd(String field,String key[]) throws CorruptIndexException, ParseException, IOException{IndexReader reader = IndexReader.open(FSDirectory.open(Constant.INDEX_DIR)); IndexSearcher searcher = new IndexSearcher(reader); try { Query query = searchByManyKey(field, key); TopDocs topDocs = searcher.search(query, 510000); int total = topDocs.totalHits; System.out.println("在"+field+"中搜索："+query.toString()+"total=" + total); ScoreDoc[] scoreDocs = topDocs.scoreDocs; return scoreDocs; } finally { searcher.close(); } }import org.apache.lucene.store.Directory;import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.Term;import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; import scott.resume.model.ResumeModel;import util.Constant;/** * 更新索引 * @author Administrator * */public class UpdateIndex {/** * 功能：删除原先的索引之后，新增新的索引 * @param rm */public static void updateAndNewIndex(ResumeModel rm){try{if(rm.getState()==1){//已经存在了简历deleteIndex("id",String.valueOf(rm.getResumeId()));}Document doc = CreateIndex.builderDocument(rm);Directory dir = FSDirectory.open(Constant.INDEX_DIR); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_35,Constant.analyzer).setOpenMode(OpenMode.APPEND); IndexWriter writer = new IndexWriter(dir, config); writer.addDocument(doc); writer.close();}catch(Exception e){e.printStackTrace();}}/** * 删除索引，这里主要是对resumeId进行查询，然后删除这个结果 * @param field * @param keyword * @throws Exception */public static void deleteIndex(String field , String keyword) throws Exception{ long startTime = System.currentTimeMillis(); //首先，我们需要先将相应的document删除 Directory dir = FSDirectory.open(Constant.INDEX_DIR); IndexReader reader = IndexReader.open(dir,false); Term term = new Term(field,keyword); reader.deleteDocuments(term); reader.close(); long endTime = System.currentTimeMillis(); System.out.println("total time: " + (endTime - startTime) + " ms"); } }