softwarerec/src/main/java/com/ow2/rec/lucene/LuceneIndex.java

//Copyright (c) 2016 Yarong Zeng, NUDT.
package com.ow2.rec.lucene;

import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;

import javax.annotation.Resource;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.dao.UserTagDao;
import com.ow2.rec.model.Project;
import com.ow2.rec.util.Normalizer;

@Component("luceneindex")
public class LuceneIndex {
	
	private Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
	public static final String PRJ_INDEX_PATH = "prjLuceneIndex";
	public static final String TEST_INDEX = "testIndex";
	public static String tagFieldName = "userTags";
	public static String userIdFieldName = "userId";
	public static String prjIdFieldName = "prjId";
	public static String prjNameFieldName = "prjName";
	public static String prjDescFieldName = "prjDesc";
	private int step = 1000;
	
	@Resource
	private UserTagDao userTagDao;
	@Resource
	private ProjectDao projectDao;
	
	public void run() {
		logger.info("start creating index!!!");
		long start = System.currentTimeMillis();
		Analyzer ikanalyzer = new IKAnalyzer(true);//智能分词
		try {
			createPrjIndex(ikanalyzer);
		} catch (IOException e) {
			logger.error("IOException: " + e);
		}
		long end = System.currentTimeMillis();
		logger.info("createIndex time: " + (end - start) / 1000 + "s");
	}
    //创建索引文件
	public IndexWriter createIndexWriter(String indexPath, Analyzer analyzer)
			throws IOException {
		Directory dire = FSDirectory.open(Paths.get(indexPath));
		IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
		iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
		IndexWriter iw = new IndexWriter(dire, iwc);
		return iw;
	}
	

	public void createPrjIndex(Analyzer analyzer)
			throws IOException {
		int startId = 0;
		int endId = startId + step;//step=100000;
		IndexWriter writer = createIndexWriter(PRJ_INDEX_PATH, analyzer);
		int maxPrjId = projectDao.getNewLast();
		while (startId < maxPrjId) {
			try {
				List<Project> projects = projectDao.getBatchPrjs(startId, endId);
				for(Project project : projects) {
					if (project.getName() == null) {
						continue;
					}
					Document doc = new Document();
					doc.add(new StringField(prjIdFieldName, String.valueOf(project.getId()), Store.YES));
					//对Title和desc设定权重
					String  prjName = project.getName().replaceAll("[-]", " ");
					Field prjNameField = new TextField(prjNameFieldName, prjName, Store.YES);
					prjNameField.setBoost(2.0f);
					doc.add(prjNameField);
					
					String prjDescString = project.getDescription();
					if (prjDescString == null) {
						prjDescString = "";
					}

					Field prjDescField = new TextField(prjDescFieldName,prjDescString,Store.YES);
					prjDescField.setBoost(0.8f);
					doc.add(prjDescField);
					writer.addDocument(doc);
				}
				logger.info("lucene indexed project: "+ startId + "-->"+ endId);
			} catch (IOException e) {
				logger.error("createPrjIndex IOException: " + e);
			}
			//(startId,endId]
			if ((maxPrjId - endId) <= step) {
				startId = endId;
				endId = maxPrjId;
			}
			else {
				startId = endId;
				endId += step;
			}
		}
		writer.commit();
		writer.close();
	}

	
	public static void main(String []args) throws IOException, ParseException {
		Directory directory = FSDirectory.open(Paths.get(TEST_INDEX));
		IndexWriterConfig iwc = new IndexWriterConfig(new IKAnalyzer(false));
		iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
		IndexWriter iw = new IndexWriter(directory, iwc);
		//第一个doc 
		Document doc = new Document();
		doc.add(new StringField("prjId", String.valueOf(11), Store.YES));
		Field prjNameField = new TextField("name", "baidu books", Store.NO);
		prjNameField.setBoost(60.0f);
		doc.add(prjNameField);
		iw.updateDocument(new Term("prjId",String.valueOf(11)),doc);
		//第二个doc 
		Document doc1 = new Document();
		doc1.add(new StringField("prjId", String.valueOf(22), Store.YES));
		Field prjNameField1 = new TextField("name", "google books", Store.YES);
		//prjNameField1.setBoost(0.5f);
		doc1.add(prjNameField1);
		List<String> synonymsList = Normalizer.tagsSegmentation("<Android SDK>,<Android UI>");
		for(String synonym : synonymsList){
			TextField textField = new TextField("prjsynonyms", synonym, Store.YES);
			textField.setBoost(2f);
			doc1.add(textField);
		}
		iw.updateDocument(new Term("prjId",String.valueOf(22)), doc1);
		iw.commit();
		iw.close();
		
		QueryParser parser = new QueryParser("prjsynonyms", new IKAnalyzer());
		IndexReader ir = DirectoryReader.open(directory);
		IndexSearcher is = new IndexSearcher(ir);
		parser.setDefaultOperator(QueryParser.Operator.AND);
		Query query = parser.parse("android UI");
		TopDocs tds = is.search(query, 10);
		ScoreDoc[] sds = tds.scoreDocs;
		for(ScoreDoc sd : sds){
			Document d = is.doc(sd.doc);
			String[] aString = d.getValues("prjsynonyms");
			Explanation explanation = is.explain(query, sd.doc);
			System.out.println(explanation.toString());
			System.out.println(d.getField("prjId"));
			System.out.println(d.getField("name"));
			System.out.println(sd.score);
		}
	}

}