二十七、Luence在项目中运用
public class LuceneContext {
??? private static LuceneContext instance;
??? private static final String INDEX_PATH = "d:/lucene/test";
??? private static IndexWriter writer;
??? private static Analyzer analyzer;
??? private static Version version;
??? private static NRTManager nrtMgr;
??? private static SearcherManager mgr;
??? private static Directory directory;
??? private LuceneContext(){}
???
??? public static LuceneContext getInstance() {
??? ??? if(instance==null){
??? ??? ??? System.out.println("init");
??? ??? ??? init();
??? ??? ??? instance = new LuceneContext();
??? ??? }
??? ??? return instance;
??? }
???
??? private static void init() {
??? ??? try {
??? ??? ??? directory = FSDirectory.open(new File(INDEX_PATH));
??? ??? ??? version = Version.LUCENE_35;
??? ??? ??? String dicUrl = LuceneContext.class.getClassLoader().getResource("data").getPath();
??? ??? ??? analyzer = new MMSegAnalyzer(dicUrl);
??? ??? ??? writer = new IndexWriter(directory,new IndexWriterConfig(version,analyzer));
??? ??? ??? nrtMgr = new NRTManager(writer, new SearcherWarmer() {
??? ??? ??? ??? @Override
??? ??? ??? ??? public void warm(IndexSearcher arg0) throws IOException {
??? ??? ??? ??? ??? System.out.println("reopen index");
??? ??? ??? ??? }
??? ??? ??? });
??? ??? ??? mgr = nrtMgr.getSearcherManager(true);
??? ??? ???
??? ??? ??? NRTManagerReopenThread reopenThread = new NRTManagerReopenThread(nrtMgr, 5.0,0.025);
??? ??? ??? reopenThread.setName("NRTManager reopen thread");
??? ??? ??? reopenThread.setDaemon(true);
??? ??? ??? reopenThread.start();
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
???
??? public IndexSearcher getSearcher() {
??? ??? return mgr.acquire();
??? }
???
??? public void releaseSearcher(IndexSearcher searcher) {
??? ??? try {
??? ??? ??? mgr.release(searcher);
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
???
??? public void commitIndex() {
??? ??? try {
??? ??? ??? writer.commit();
??? ??? ??? writer.forceMerge(3);
??? ??? } catch (CorruptIndexException e) {
??? ??? ??? e.printStackTrace();
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
???
??? public NRTManager getNRTManager() {
??? ??? return nrtMgr;
??? }
???
??? public Version getVersion() {
??? ??? return version;
??? }
???
??? public Analyzer getAnalyzer() {
??? ??? return analyzer;
??? }
}
public class IndexField {
??? private String id;
??? private String title;
??? private List<String> content;
??? private List<String> atths;
??? private int parentId;
??? private int objId;
??? private Date createDate;
??? private String type;
public class Index {
???
??? private int msgId;
??? private String title;
??? private String summary;
??? private Date createDate;
@Entity
@Table(name="temp_index")
public class TempIndex {
???
??? private int id;
??? private int objId;
??? private String type;//Message,Attachment
??? private int operator;
@Service("indexService")
public class IndexService implements IIndexService {
??? private Document field2Doc(IndexField field) {
??? ??? Document doc = new Document();
??? ??? doc.add(new Field("id",field.getId(),Field.Store.YES,Field.Index.NOT_ANALYZED_NO_NORMS));
??? ??? doc.add(new Field("title",field.getTitle(),Field.Store.YES,Field.Index.ANALYZED));
??? ??? for(String content:field.getContent()) {
??? ??? ??? doc.add(new Field("content",content,Field.Store.NO,Field.Index.ANALYZED));
??? ??? }
??? ??? if(field.getAtths()!=null) {
??? ??? ??? for(String att:field.getAtths()) {
??? ??? ??? ??? doc.add(new Field("atts",att,Field.Store.YES,Field.Index.NO));
??? ??? ??? }
??? ??? }
??? ??? doc.add(new NumericField("objId",Field.Store.YES,true).setIntValue(field.getObjId()));
??? ??? doc.add(new NumericField("createDate",Field.Store.YES,true).setLongValue(field.getCreateDate().getTime()));
??? ??? return doc;
??? }
??? @Override
??? public void addIndex(IndexField fields,boolean inDatabase) {
??? ??? try {
??? ??? ??? if(inDatabase) {
??? ??? ??? ??? TempIndex ti = new TempIndex();
??? ??? ??? ??? ti.setAdd();
??? ??? ??? ??? ti.setObjId(fields.getObjId());
??? ??? ??? ??? ti.setType(fields.getType());
??? ??? ??? ??? tempIndexDao.add(ti);
??? ??? ??? }
??? ??? ??? NRTManager nrtMgr = LuceneContext.getInstance().getNRTManager();
??? ??? ??? Document doc = field2Doc(fields);
??? ??? ??? nrtMgr.addDocument(doc);
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
??? @Override
??? public void deleteIndex(String id,String type) {
??? ??? try {
??? ??? ??? TempIndex ti = new TempIndex();
??? ??? ??? ti.setDelete();
??? ??? ??? ///xx_xx
??? ??? ??? ti.setId(Integer.parseInt(id));
??? ??? ??? ti.setType(type);
??? ??? ??? tempIndexDao.add(ti);
??? ??? ??? LuceneContext.getInstance().getNRTManager().deleteDocuments(new Term("id",id));
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
??? @Override
??? public void updateIndex(IndexField fields) {
??? ??? try {
??? ??? ??? TempIndex ti = new TempIndex();
??? ??? ??? ti.setDelete();
??? ??? ??? ///xx_xx
??? ??? ??? ti.setId(fields.getObjId());
??? ??? ??? ti.setType(fields.getType());
??? ??? ??? tempIndexDao.add(ti);
??? ??? ??? NRTManager nrtMgr = LuceneContext.getInstance().getNRTManager();
??? ??? ??? Document doc = field2Doc(fields);
??? ??? ??? nrtMgr.updateDocument(new Term("id",fields.getId()), doc);
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
??? private String highligher(String text,Query query,String field) {
??? ??? try {
??? ??? ??? QueryScorer scorer = new QueryScorer(query);
??? ??? ??? Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
??? ??? ??? Formatter formatter = new SimpleHTMLFormatter("<span class='lighter'>","</span>");
??? ??? ??? Highlighter lighter = new Highlighter(formatter,scorer);
??? ??? ??? lighter.setTextFragmenter(fragmenter);
??? ??? ??? String ht = lighter.getBestFragment(LuceneContext.getInstance().getAnalyzer(),
??? ??? ??? ??? ??? field,text);
??? ??? ??? if(ht==null) {
??? ??? ??? ??? if(text.length()>=200) {
??? ??? ??? ??? ??? text = text.substring(0, 200);
??? ??? ??? ??? ??? text=text+"....";
??? ??? ??? ??? }
??? ??? ??? ??? return text;
??? ??? ??? }
??? ??? ??? else return ht.trim();
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? } catch (InvalidTokenOffsetsException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? ??? return text;
??? }
??? private ScoreDoc getLastDoc(int pageOffset,IndexSearcher searcher,Query query) {
??? ??? if(pageOffset<=0) return null;
??? ??? try {
??? ??? ??? TopDocs tds = searcher.search(query,pageOffset-1);
??? ??? ??? return tds.scoreDocs[pageOffset-1];
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? ??? return null;
??? }
??? @Override
??? public PageObject<Index> findByIndex(String condition) {
??? ??? if(condition==null) condition = "";
??? ??? IndexSearcher searcher = LuceneContext.getInstance().getSearcher();
??? ??? PageObject<Index> pages = new PageObject<Index>();
??? ??? List<Index> datas = new ArrayList<Index>();
??? ??? try {
??? ??? ??? int pageSize = SystemContext.getPageSize();
??? ??? ??? int pageOffset = SystemContext.getPageOffset();
??? ??? ??? MultiFieldQueryParser parser = new MultiFieldQueryParser(LuceneContext.getInstance().getVersion(),
??? ??? ??? ??? ??? ??? new String[]{"title","content"}, LuceneContext.getInstance().getAnalyzer());
??? ??? ??? Query query = parser.parse(condition);
??? ??? ??? TopDocs tds = searcher.searchAfter(getLastDoc(pageOffset,searcher,query),
??? ??? ??? ??? ??? ??? ? query, pageSize);
??? ??? ??? int totalRecord = tds.totalHits;
??? ??? ??? List<Integer> msgs = new ArrayList<Integer>();
??? ??? ??? for(ScoreDoc sd:tds.scoreDocs) {
??? ??? ??? ??? Document doc = searcher.doc(sd.doc);
??? ??? ??? ??? Index index = new Index();
??? ??? ??? ??? index.setCreateDate(new Date(Long.parseLong(doc.get("createDate"))));
??? ??? ??? ??? String title = doc.get("title");
??? ??? ??? ??? index.setTitle(highligher(title,query,"title"));
??? ??? ??? ??? String[] ans = doc.getValues("atts");
??? ??? ??? ??? StringBuffer content = new StringBuffer();
??? ??? ??? ??? if(ans!=null) {
??? ??? ??? ??? ??? for(String fn:ans) {
??? ??? ??? ??? ??? ??? content.append(IndexUtil.file2String(fn));
??? ??? ??? ??? ??? }
??? ??? ??? ??? }
??? ??? ??? ??? index.setSummary(content.toString());
??? ??? ??? ??? int msgId = Integer.parseInt(doc.get("id"));
??? ??? ??? ??? index.setMsgId(msgId);
??? ??? ??? ??? msgs.add(msgId);
??? ??? ??? ??? datas.add(index);
??? ??? ??? }
??? ??? ??? Map<Integer,String> contents = messageDao.listMessageContent(msgs);
??? ??? ??? for(int i=0;i<datas.size();i++) {
??? ??? ??? ??? Index index = datas.get(i);
??? ??? ??? ??? String content = contents.get(index.getMsgId());
??? ??? ??? ??? content = content+index.getSummary();
??? ??? ??? ??? datas.get(i).setSummary(highligher(content,query,"content"));
??? ??? ??? }
??? ??? ???
??? ??? ??? pages.setDatas(datas);
??? ??? ??? pages.setOffset(pageOffset);
??? ??? ??? pages.setPageSize(pageSize);
??? ??? ??? pages.setTotalRecord(totalRecord);
??? ??? } catch (ParseException e) {
??? ??? ??? e.printStackTrace();
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? } catch (TikaException e) {
??? ??? ??? e.printStackTrace();
??? ??? } finally {
??? ??? ??? LuceneContext.getInstance().releaseSearcher(searcher);
??? ??? }
??? ??? return pages;
??? }
??? private void indexMessage(Message msg) {
??? ??? List<String> contents = new ArrayList<String>();
??? ??? List<String> ans = new ArrayList<String>();
??? ??? contents.add(msg.getContent());
??? ??? List<Attachment> atts = attachmentDao.listByMessage(msg.getId());
??? ??? for(Attachment att:atts) {
??? ??? ??? IndexUtil.attach2Index(contents, ans,att);
??? ??? }
??? ??? IndexField field = IndexUtil.msg2IndexField(msg);
??? ??? field.setContent(contents);
??? ??? field.setAtths(ans);
??? ??? addIndex(field,false);
??? }
??? private void indexMessages(List<Message> messages) {
??? ??? for(Message msg:messages) {
??? ??? ??? indexMessage(msg);
??? ??? }
??? }
??? @Override
??? public void updateReconstructorIndex() {
??? ??? /**
??? ??? ?* 将数据库中的所有对象取出，创建相应的IndexField完成索引的重构
??? ??? ?*/
??? ??? try {
??? ??? ??? LuceneContext.getInstance().getNRTManager().deleteAll();
??? ??? ??? List<Message> messages = messageDao.list("from Message");
??? ??? ??? indexMessages(messages);
??? ??? ??? LuceneContext.getInstance().commitIndex();
??? ??? ??? tempIndexDao.delAll();
??? ??? } catch (IOException e) {
??? ??? ??? e.printStackTrace();
??? ??? }
??? }
??? @Override
??? public void updateSetIndex() {
??? ??? List<TempIndex> tis = tempIndexDao.list("from TempIndex");
??? ??? for(TempIndex ti:tis) {
??? ??? ??? if(ti.getType().equals(IndexUtil.MSG_TYPE)) {
??? ??? ??? ??? Message msg = messageDao.load(ti.getObjId());
??? ??? ??? ??? indexMessage(msg);
??? ??? ??? }
??? ??? }
??? ??? LuceneContext.getInstance().commitIndex();
??? ??? tempIndexDao.delAll();
??? }
??? @Override
??? public void updateCommitIndex() {
??? ??? tempIndexDao.delAll();
??? ??? LuceneContext.getInstance().commitIndex();
??? }
}
public class CleanListener implements ServletContextListener {
??? private Timer indexTimer;
??? private WebApplicationContext wac = null;
??? private class IndexCommit extends TimerTask {
??? ??? @Override
??? ??? public void run() {
??? ??? ??? SystemContext.setRealPath(realPath);
??? ??? ??? System.out.println("索引进行了提交"+new Date());
??? ??? ??? IIndexService indexService = (IIndexService)wac.getBean("indexService");
??? ??? ??? indexService.updateCommitIndex();
??? ??? }
??? }
??? @Override
??? public void contextInitialized(ServletContextEvent sce) {
??? ??? //可以获取spring中BeanFactory,这个BeanFactory是在系统启动的时候就完成存储了
??? ??? wac = WebApplicationContextUtils.getWebApplicationContext(sce.getServletContext());
??? ??? System.out.println("------------------清理的启动程序已经开启（已经获取了"+wac+"）---------------------");
??? ??? realPath = sce.getServletContext().getRealPath("");
??? ??? timer = new Timer();
??? ??? timer.scheduleAtFixedRate(new ClearDataTask(),50000, 300000);
??? ??? indexTimer = new Timer();
??? ??? indexTimer.scheduleAtFixedRate(new IndexCommit(), 600000, 600000);
??? }
