first version

This commit is contained in:
nigel 2016-09-02 09:32:44 +08:00
commit 72c5414b4f
45 changed files with 286729 additions and 0 deletions

31
.classpath Executable file
View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" path="src/main/assembly"/>
<classpathentry including="**/*.java" kind="src" path="src/main/resources"/>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
<accessrules>
<accessrule kind="accessible" pattern="javax/annotation/**"/>
</accessrules>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>

6
.gitignore vendored Executable file
View File

@ -0,0 +1,6 @@
/target/
#.project
#/.settings
/log/*
/prjLuceneIndex/
/testIndex/

23
.project Executable file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>rec</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>

View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
encoding/<project>=UTF-8

View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.7

View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

BIN
lib/IKAnalyzer-5.0.jar Executable file

Binary file not shown.

176
pom.xml Executable file
View File

@ -0,0 +1,176 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ow2</groupId>
<artifactId>rec</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>rec</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<build>
<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<archive>
<manifest>
<mainClass>com.alan.myfunction.ProcedureTag</mainClass>
</manifest>
</archive>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.7</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-smartcn</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.30</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<version>5.3.0</version>
</dependency>
<dependency>
<groupId>c3p0</groupId>
<artifactId>c3p0</artifactId>
<version>0.9.1.2</version>
</dependency>
<dependency>
<groupId>org.wltea.analyzer</groupId>
<artifactId>IKAnalyzer</artifactId>
<version>5.0</version>
<scope>system</scope>
<systemPath>${project.basedir}/lib/IKAnalyzer-5.0.jar</systemPath>
</dependency>
<dependency>
<groupId>commons-dbcp</groupId>
<artifactId>commons-dbcp</artifactId>
<version>1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjrt</artifactId>
<version>1.8.3</version>
</dependency>
<dependency>
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
<version>1.8.3</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>3.1.2.RELEASE</version>
</dependency>
</dependencies>
</project>

25
src/main/assembly/assembly.xml Executable file
View File

@ -0,0 +1,25 @@
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<!-- TODO: a jarjar format would be better -->
<id>jar-with-dependencies-without-resources</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<dependencySets>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<unpack>false</unpack>
<scope>runtime</scope>
</dependencySet>
<dependencySet>
<outputDirectory>/</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<unpack>false</unpack>
<scope>system</scope>
</dependencySet>
</dependencySets>
</assembly>

View File

@ -0,0 +1,13 @@
package com.ow2.rec;
/**
* Hello world!
*
*/
public class App
{
public static void main( String[] args )
{
System.out.println( "Hello World!" );
}
}

View File

@ -0,0 +1,19 @@
package com.ow2.rec.dao;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import com.ow2.rec.model.MatchItem;
public interface MatchResultDao {
//写入匹配结果
@Insert("insert into sta_git_matches (userId,prjId,match_score,match_time) values (#{item.userId},#{item.projectId},#{item.matchScore},now())")
public void insertMatchResult( @Param("item") MatchItem item);
@Select("SELECT prjId FROM usertagtoprj where userId=#{uid};")
public List<Integer> GetMatchPrjList(@Param("uid") int uid);
}

View File

@ -0,0 +1,34 @@
package com.ow2.rec.dao;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import com.ow2.rec.model.Project;
public interface ProjectDao {
@Select("select id,name,description from projects where id>#{startId} AND id<=#{endId} AND forked_from is null and deleted = 0")
public List<Project> getBatchPrjs(@Param("startId") int startId,
@Param("endId") int endId);
//项目原创没有fork且没有删除的
@Select("select MAX(id) from projects where forked_from is null and deleted = 0")
public Integer getNewLast();
@Select("select id,name,description from projects where id = #{prjId}")
public Project getPrjById(@Param("prjId") int prjId);
@Select("select count(user_id) from watchers where repo_id=#{prjId}")
public Integer getWatchers(@Param("prjId") int prjId);
}

View File

@ -0,0 +1,27 @@
package com.ow2.rec.dao;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import com.ow2.rec.model.Tag;
public interface TagDao {
@Select("select Count from Tags where TagName = #{tagname}")
public Integer getTagCounts(@Param("tagname")String tagname);
@Select("select sum(Count) from Tags")
public Integer getTagAllCounts();
//某用户的帖子总数
@Select("select count(distinct PostId) from ${tablename} where UserId =#{uid}")
public Integer getTagTf2(@Param("tablename")String tablename,@Param("uid")int uid);
//tag在某用户的所有帖子中的累积出现次数
@Select("select count(*) from ${tablename} where UserId =#{uid} and Tag=#{tagname}")
public Integer getTagTf1(@Param("tablename")String tablename,@Param("uid")int uid,@Param("tagname")String tagname);
}

View File

@ -0,0 +1,18 @@
package com.ow2.rec.dao;
import java.util.List;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import com.ow2.rec.model.UserTag;
public interface UserTagDao {
@Select("select UserId,AllTags from user_tags where Id>#{startId} AND Id<=#{endId}")
public List<UserTag> getUserTag(@Param("startId") int startId, @Param("endId") int endId);
@Select("select max(Id) from user_tags")
public Integer getMaxId();
}

View File

@ -0,0 +1,26 @@
package com.ow2.rec.datasource;
import org.springframework.stereotype.Component;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
@Aspect
@Component
public class DataSourceInterceptor {
//切点表达式ret-type-pattern name-pattern(param_pattern) 返回类型方法名称参数三部分来匹配
// * 表示任意返回类型任意方法名任意一个参数类型 .. 连续两个点表示0个或多个包路径还有0个或多个参数
@Pointcut("execution(public * com.ow2.rec.dao.ProjectDao.*(..))")
public void dataSourceGit(){};
@Before("dataSourceGit()")
public void before(JoinPoint jp) {
DataSourceTypeManager.set(DataSources.stackoverflow);
}
@After("dataSourceGit()")
public void after(JoinPoint jp) {
DataSourceTypeManager.set(DataSources.ossean_production);
}
}

View File

@ -0,0 +1,22 @@
package com.ow2.rec.datasource;
public class DataSourceTypeManager {
private static final ThreadLocal<DataSources> dataSourceTypes = new ThreadLocal<DataSources>(){
@Override
protected DataSources initialValue(){
return DataSources.stackoverflow;
}
};
public static DataSources get(){
return dataSourceTypes.get();
}
public static void set(DataSources dataSourceType){
dataSourceTypes.set(dataSourceType);
}
}

View File

@ -0,0 +1,6 @@
package com.ow2.rec.datasource;
public enum DataSources {
stackoverflow,ossean_production
}

View File

@ -0,0 +1,12 @@
package com.ow2.rec.datasource;
import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;
public class ThreadLocalRountingDataSource extends AbstractRoutingDataSource{
@Override
protected Object determineCurrentLookupKey() {
return DataSourceTypeManager.get();
}
}

View File

@ -0,0 +1,143 @@
package com.ow2.rec.lucene;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
import org.wltea.analyzer.lucene.IKAnalyzer;
public class IKTest {
/**
* @param args
* @throws IOException
*/
public static final String TEST_INDEX = "testIndex";
public static void main(String[] args) throws IOException {
//Lucene Document的域名
String fieldName = "text";
//检索内容
String text = "IK Analyzer是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切分算法。";
String str = "java-mybatis-spring";
//实例化IKAnalyzer分词器
Analyzer analyzer = new IKAnalyzer(true);
Directory directory = null;
IndexWriter iwriter = null;
IndexReader ireader = null;
IndexSearcher isearcher = null;
try {
//创建Directory关联源文件
directory = FSDirectory.open(Paths.get(TEST_INDEX));
//创建索引的配置信息
IndexWriterConfig iwConfig = new IndexWriterConfig(analyzer);
iwConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
iwriter = new IndexWriter(directory , iwConfig);
//写入索引
Document doc = new Document();
doc.add(new StringField("ID", "10000", Field.Store.YES));
doc.add(new TextField(fieldName, text, Field.Store.YES));
Document doc1 = new Document();
doc.add(new StringField("ID", "12000", Field.Store.YES));
doc.add(new TextField(fieldName, "bangbangbang", Field.Store.YES));
Document doc2 = new Document();
doc.add(new StringField("ID", "16000", Field.Store.YES));
doc.add(new TextField(fieldName, "把成绩好vkfdsjfps", Field.Store.YES));
iwriter.addDocument(doc);
iwriter.addDocument(doc1);
iwriter.addDocument(doc2);
iwriter.close();
//搜索过程**********************************
IndexReader indexReader = DirectoryReader.open(directory);//索引读取类
isearcher = new IndexSearcher(indexReader); //搜索入口工具类
String keyword = "中文分词工具包";
//使用QueryParser查询分析器构造Query对象
QueryParser qp = new QueryParser(fieldName,analyzer);//实例查询条件类
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
Query query = qp.parse(keyword);//解析输入字符串
System.out.println("Query = " + query);
TopDocs topDocs = isearcher.search(query,5);//搜索相似度最高的5条记录
//System.out.println("命中:" + topDocs.totalHits);
//输出结果
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
for (int i = 0; i < topDocs.totalHits; i++){
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
Explanation explanation = isearcher.explain(query, scoreDocs[i].doc);
System.out.println(explanation.toString());
System.out.println(scoreDocs[i].doc);//得到文档id
System.out.println(scoreDocs[i].score);//文档得分
String[] prjNames = targetDoc.getValues(fieldName);
for (int j = 0; j<prjNames.length;j++){
System.out.println(prjNames[j]);
}
System.out.println(targetDoc.get("ID")+" "+targetDoc.get("text"));
//System.out.println("内容:" + targetDoc.toString());
}
} catch (CorruptIndexException e) {
e.printStackTrace();
} catch (LockObtainFailedException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
} finally{
if(ireader != null){
try {
ireader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(directory != null){
try {
directory.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}

View File

@ -0,0 +1,54 @@
package com.ow2.rec.lucene;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
public class IKTest2 {
private final static Logger log = LoggerFactory.getLogger(IKTest2.class);;
public static void main(String[] args) {
String text = "IK Analyzer 2012 是一个结合词典分词和文法分词的中文分词开源工具包。它使用了全新的正向迭代最细粒度切分算法。";
//String text ="java-mybatis-spring-sqlserver-connection";
long ct = System.currentTimeMillis();
for(String word : splitKeywords(text)){
System.out.println(word);
}
System.out.printf("TIME %f\n s",(float) (System.currentTimeMillis() - ct)/1000);
}
/**
* 关键字切分
* @param sentence 要分词的句子
* @return 返回分词结果
*/
public static List<String> splitKeywords(String sentence) {
List<String> keys = new ArrayList<String>();
StringReader reader = new StringReader(sentence);
IKSegmenter ikseg = new IKSegmenter(reader, true);
try{
do{
Lexeme me = ikseg.next();
if(me == null)
break;
String term = me.getLexemeText();
keys.add(term);
}while(true);
}catch(IOException e){
log.error("Unable to split keywords", e);
}
return keys;
}
}

View File

@ -0,0 +1,176 @@
package com.ow2.rec.lucene;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.List;
import javax.annotation.Resource;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import org.wltea.analyzer.lucene.IKAnalyzer;
import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.dao.UserTagDao;
import com.ow2.rec.model.Project;
import com.ow2.rec.util.Normalizer;
@Component("luceneindex")
public class LuceneIndex {
private Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
public static final String PRJ_INDEX_PATH = "prjLuceneIndex";
public static final String TEST_INDEX = "testIndex";
public static String tagFieldName = "userTags";
public static String userIdFieldName = "userId";
public static String prjIdFieldName = "prjId";
public static String prjNameFieldName = "prjName";
public static String prjDescFieldName = "prjDesc";
private int step = 1000;
@Resource
private UserTagDao userTagDao;
@Resource
private ProjectDao projectDao;
public void run() {
logger.info("start creating index!!!");
long start = System.currentTimeMillis();
Analyzer ikanalyzer = new IKAnalyzer(true);//智能分词
try {
createPrjIndex(ikanalyzer);
} catch (IOException e) {
logger.error("IOException: " + e);
}
long end = System.currentTimeMillis();
logger.info("createIndex time: " + (end - start) / 1000 + "s");
}
//创建索引文件
public IndexWriter createIndexWriter(String indexPath, Analyzer analyzer)
throws IOException {
Directory dire = FSDirectory.open(Paths.get(indexPath));
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(dire, iwc);
return iw;
}
public void createPrjIndex(Analyzer analyzer)
throws IOException {
int startId = 0;
int endId = startId + step;//step=100000;
IndexWriter writer = createIndexWriter(PRJ_INDEX_PATH, analyzer);
int maxPrjId = projectDao.getNewLast();
while (startId < maxPrjId) {
try {
List<Project> projects = projectDao.getBatchPrjs(startId, endId);
for(Project project : projects) {
if (project.getName() == null) {
continue;
}
Document doc = new Document();
doc.add(new StringField(prjIdFieldName, String.valueOf(project.getId()), Store.YES));
//对Title和desc设定权重
String prjName = project.getName().replaceAll("[-]", " ");
Field prjNameField = new TextField(prjNameFieldName, prjName, Store.YES);
prjNameField.setBoost(2.0f);
doc.add(prjNameField);
String prjDescString = project.getDescription();
if (prjDescString == null) {
prjDescString = "";
}
Field prjDescField = new TextField(prjDescFieldName,prjDescString,Store.YES);
prjDescField.setBoost(0.8f);
doc.add(prjDescField);
writer.addDocument(doc);
}
logger.info("lucene indexed project: "+ startId + "-->"+ endId);
} catch (IOException e) {
logger.error("createPrjIndex IOException: " + e);
}
//(startId,endId]
if ((maxPrjId - endId) <= step) {
startId = endId;
endId = maxPrjId;
}
else {
startId = endId;
endId += step;
}
}
writer.commit();
writer.close();
}
public static void main(String []args) throws IOException, ParseException {
Directory directory = FSDirectory.open(Paths.get(TEST_INDEX));
IndexWriterConfig iwc = new IndexWriterConfig(new IKAnalyzer(false));
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
IndexWriter iw = new IndexWriter(directory, iwc);
//第一个doc
Document doc = new Document();
doc.add(new StringField("prjId", String.valueOf(11), Store.YES));
Field prjNameField = new TextField("name", "baidu books", Store.NO);
prjNameField.setBoost(60.0f);
doc.add(prjNameField);
iw.updateDocument(new Term("prjId",String.valueOf(11)),doc);
//第二个doc
Document doc1 = new Document();
doc1.add(new StringField("prjId", String.valueOf(22), Store.YES));
Field prjNameField1 = new TextField("name", "google books", Store.YES);
//prjNameField1.setBoost(0.5f);
doc1.add(prjNameField1);
List<String> synonymsList = Normalizer.tagsSegmentation("<Android SDK>,<Android UI>");
for(String synonym : synonymsList){
TextField textField = new TextField("prjsynonyms", synonym, Store.YES);
textField.setBoost(2f);
doc1.add(textField);
}
iw.updateDocument(new Term("prjId",String.valueOf(22)), doc1);
iw.commit();
iw.close();
QueryParser parser = new QueryParser("prjsynonyms", new IKAnalyzer());
IndexReader ir = DirectoryReader.open(directory);
IndexSearcher is = new IndexSearcher(ir);
parser.setDefaultOperator(QueryParser.Operator.AND);
Query query = parser.parse("android UI");
TopDocs tds = is.search(query, 10);
ScoreDoc[] sds = tds.scoreDocs;
for(ScoreDoc sd : sds){
Document d = is.doc(sd.doc);
String[] aString = d.getValues("prjsynonyms");
Explanation explanation = is.explain(query, sd.doc);
System.out.println(explanation.toString());
System.out.println(d.getField("prjId"));
System.out.println(d.getField("name"));
System.out.println(sd.score);
}
}
}

View File

@ -0,0 +1,140 @@
package com.ow2.rec.lucene;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Resource;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
import org.wltea.analyzer.lucene.IKAnalyzer;
import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.dao.TagDao;
import com.ow2.rec.model.Project;
@Component
public class LuceneSearch {
@Resource
private ProjectDao projectDao;
@Resource
private TagDao tagDao;
private static Logger logger = LoggerFactory.getLogger(LuceneSearch.class);
public LinkedHashMap<Integer, Double> userTagToPrjMatchByLucene(int uid,String keyWords,
List<String> keyWordsList, String [] searchField,
LinkedHashMap<Integer, Double> matchMap, IndexReader prjIndexReader) throws ParseException {
int tagAllCount = tagDao.getTagAllCounts();
int tagCount;
int tagCountForUser;
int tagAllCountForUser;
float termFrec;
String tableName = getTargetTable(uid);
try {
IndexSearcher is = new IndexSearcher(prjIndexReader);
BooleanQuery.setMaxClauseCount(1500);
BooleanQuery query = new BooleanQuery();
//Similariy是计算Lucene打分的最主要的类
Similarity similarity = new DefaultSimilarity(){
//document(项目)包含的tag
@Override
public float coord(int overlap, int maxOverlap) {
//添加doc weight
return overlap * overlap * overlap / (float)maxOverlap;
}
};
is.setSimilarity(similarity);
Map<String, Float> boosts = new HashMap<String, Float>();
tagAllCountForUser = tagDao.getTagTf2(tableName, uid);
for(String keyWordsTerm : keyWordsList){
//tag tf
tagCountForUser = tagDao.getTagTf1(tableName, uid, keyWordsTerm);
float tf_tag = (float)tagCountForUser/tagAllCountForUser;
//tag idf
tagCount = tagDao.getTagCounts(keyWordsTerm);
float idf_tag =(float) (Math.log(tagAllCount/(double)(tagCount+1)) + 1.0);
termFrec = tf_tag*idf_tag*100;
//String termFrec_str = String.valueOf(termFrec);
//查询索引文档中是否包含了指定的Term,经过分词以后要自行处理小写
for(int i =0;i<searchField.length;i++){
Term term = new Term(searchField[i], keyWordsTerm);
TermQuery tq = new TermQuery(term);
tq.setBoost(termFrec);
query.add(tq, BooleanClause.Occur.SHOULD);
}
//QueryParser会经过分词器会使用分词器把我们的queryString(用户输入的查询关键字)进行分词
//QueryParser multiFieldQuery = new MultiFieldQueryParser(searchField, new IKAnalyzer(true));
//keyWordsTerm = keyWordsTerm+"^"+termFrec_str;
//多条件组合查询 should:
}
TopDocs td = is.search(query,100);//TopDocs(int totalHits, ScoreDoc[] scoreDocs, float maxScore)
System.out.println("共检索出 " + td.totalHits + " 条记录");
ScoreDoc[] sds = td.scoreDocs;
for (ScoreDoc sd : sds) {
Document d = is.doc(sd.doc);
Explanation explanation = is.explain(query,sd.doc);
String prjId = d.get(LuceneIndex.prjIdFieldName);
//计算最终得分
int maxPrjWeight = 43461; //一个项目最多的watcher数目
int nowPrjWeight = projectDao.getWatchers(Integer.parseInt(prjId));
float docWeight = (float) (nowPrjWeight * 1.0 / maxPrjWeight);
//打印得分详情
//logger.info(explanation.toString());
int pId = Integer.parseInt(prjId);
if (matchMap.containsKey(pId))
matchMap.put(pId, matchMap.get(pId) + sd.score * docWeight);
else
matchMap.put(pId, Double.parseDouble(String.valueOf(sd.score * docWeight)));
}
} catch (IOException e) {
logger.error("tagToPrjMatchByLucene IOException: " + e);
}
return matchMap;
}
public String getTargetTable(int uid){
int table_num = uid/350000 + 1;
String table_name = "usertags_"+String.valueOf(table_num);
return table_name;
}
}

View File

@ -0,0 +1,97 @@
package com.ow2.rec.main;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import javax.annotation.Resource;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
import com.ow2.rec.dao.MatchResultDao;
import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.model.MatchItem;
@Component
public class GetGreatPrj {
@Resource
private ProjectDao projectDao;
@Resource
private MatchResultDao matchResultDao;
public static void main(String [] args){
@SuppressWarnings("resource")
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext_mybatis.xml");
GetGreatPrj mainClass = applicationContext.getBean(GetGreatPrj.class);
mainClass.getPrjList(99);
}
public List<Integer> getPrjList(int uid){
int prjId;
int watcherNum;
Map<Integer,Integer> priMap = new HashMap();
List<Integer> prjList = matchResultDao.GetMatchPrjList(uid);
for(int i=0;i<prjList.size();i++){
prjId = prjList.get(i);
if(projectDao.getWatchers(prjId) == null)
continue;
watcherNum = projectDao.getWatchers(prjId);
priMap.put(prjId, watcherNum);
}
List<Integer> TopPrjList = new LinkedList();
Map<Integer, Integer> afterSortMap = sortMapByValue(priMap);
Iterator i = afterSortMap.entrySet().iterator();
int count=0;
while(i.hasNext()){
Map.Entry e = (Map.Entry)i.next();
count++;
System.out.println((Integer)e.getKey()+" num: "+(Integer)e.getValue());
TopPrjList.add((Integer)e.getKey());
if(count==5)
break;
}
return TopPrjList;
}
public Map<Integer, Integer> sortMapByValue(Map<Integer, Integer> oriMap) {
Map<Integer, Integer> sortedMap = new LinkedHashMap<Integer, Integer>();
if (oriMap != null && !oriMap.isEmpty()) {
List<Map.Entry<Integer, Integer>> entryList = new ArrayList<Map.Entry<Integer, Integer>>(oriMap.entrySet());
Collections.sort(entryList,
new Comparator<Map.Entry<Integer, Integer>>() {
public int compare(Entry<Integer, Integer> entry1,
Entry<Integer, Integer> entry2) {
int value1 = 0, value2 = 0;
try {
value1 = entry1.getValue();
value2 = entry2.getValue();
} catch (NumberFormatException e) {
value1 = 0;
value2 = 0;
}
return value2 - value1;
}
});
Iterator<Map.Entry<Integer, Integer>> iter = entryList.iterator();
Map.Entry<Integer, Integer> tmpEntry = null;
while (iter.hasNext()) {
tmpEntry = iter.next();
sortedMap.put(tmpEntry.getKey(), tmpEntry.getValue());
}
}
return sortedMap;
}
}

View File

@ -0,0 +1,27 @@
package com.ow2.rec.main;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
import com.ow2.rec.lucene.LuceneIndex;
@Component
public class IndexMain {
@Autowired
private LuceneIndex luceneIndex;
public static void main(String[] args) {
@SuppressWarnings("resource")
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext_mybatis.xml");
IndexMain mainClass = applicationContext.getBean(IndexMain.class);
mainClass.start();
}
public void start(){
luceneIndex.run();
}
}

View File

@ -0,0 +1,72 @@
package com.ow2.rec.main;
import java.text.DecimalFormat;
import java.util.LinkedList;
import java.util.List;
import org.apache.lucene.queryparser.classic.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import com.ow2.rec.dao.UserTagDao;
import com.ow2.rec.lucene.LuceneIndex;
import com.ow2.rec.model.UserTag;
@Component
public class Main {
@Autowired
private Match match;
@Resource
private UserTagDao userTagDao;
private Logger logger = LoggerFactory.getLogger(getClass());
private int step = 5000;
public static void main(String[] args) throws InterruptedException, ParseException {
@SuppressWarnings("resource")
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext_mybatis.xml");
Main mainClass = applicationContext.getBean(Main.class);
mainClass.start();
}
public void start(){
int startId = 0;
int stopId = startId + step;
int maxId = userTagDao.getMaxId();
//int maxId = 50000;
long start = System.currentTimeMillis();
List<UserTag> userTagList = new LinkedList();
while(startId < maxId){
long batchStart = System.currentTimeMillis();
userTagList = userTagDao.getUserTag(startId, stopId);
match.userTagToPrjMatch(userTagList);
if(maxId <= stopId+step){
startId = stopId;
stopId = maxId;
}
else{
startId = stopId;
stopId = stopId+step;
}
long batchEnd = System.currentTimeMillis();
DecimalFormat df = new DecimalFormat( "0.00");
double totalTime = (double)(batchEnd - batchStart)/60000;
double rate = (double)(stopId - startId)/((batchEnd - batchStart)/1000);
logger.info("current Batch tag-->prj total time: " + df.format(totalTime) + "min; "
+ " average " + df.format(rate) + " 条/s");
}
long end = System.currentTimeMillis();
DecimalFormat df = new DecimalFormat( "0.00");
double totalTime = (double)(end - start)/60000;
logger.info("tag-->prj total time: " + df.format(totalTime) + "min");
}
}

View File

@ -0,0 +1,110 @@
package com.ow2.rec.main;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.dao.MatchResultDao;
import com.ow2.rec.lucene.LuceneIndex;
import com.ow2.rec.lucene.LuceneSearch;
import com.ow2.rec.model.MatchItem;
import com.ow2.rec.model.UserTag;
import com.ow2.rec.util.Normalizer;
import javax.annotation.Resource;
@Component("match")
public class Match {
private Logger logger = LoggerFactory.getLogger(getClass());
// private double tagToPrjNameWeight = 1.0;
// private double tagToPrjDescWeight = 0.8;
// private double threshholdWeight = 1.3;
@Resource
private MatchResultDao matchResultDao;
@Resource
private ProjectDao projectDao;
@Autowired
private LuceneSearch luceneSearch;
private String [] searchFieldList = {LuceneIndex.prjNameFieldName,LuceneIndex.prjDescFieldName};
public void userTagToPrjMatch(List<UserTag> userTagList) {
LinkedHashMap<Integer, Double> matchMap = new LinkedHashMap<Integer, Double>();
try {
Directory dire = FSDirectory.open(Paths
.get(LuceneIndex.PRJ_INDEX_PATH));
if (!DirectoryReader.indexExists(dire)) {
return;
}
IndexReader indexReader = DirectoryReader.open(dire);
List<String> tagList = new LinkedList();
for(UserTag userTag :userTagList) {
int userId = userTag.getUserId();
String tags = userTag.getAllTags();
//分词时采用的是TermQuery自行处理小写
if (tags != null && tags.length() > 0) {
tags = tags.toLowerCase();
}
tagList = Normalizer.getTagList(tags);
if(tagList==null)
continue;
if (tagList.size() > 0) {
//用户帖子匹配github项目名称
matchMap = luceneSearch.userTagToPrjMatchByLucene(userId,tags,
tagList,searchFieldList,matchMap, indexReader);
// //用户帖子匹配github项目描述
// matchMap = luceneSearch.userTagToPrjMatchByLucene(userId,tags,
// tagList, LuceneIndex.prjDescFieldName,matchMap, indexReader);
}
if (matchMap.size() > 0)
insertToMatchResult(userId, matchMap);
matchMap.clear();
logger.info("current UserId: " + userId);
}
indexReader.close();
dire.close();
} catch (Exception e) {
logger.error("tagToPrjMatch error in Match: " + e);
}
}
@Transactional(propagation=Propagation.REQUIRES_NEW)
public void insertToMatchResult(int userId, LinkedHashMap<Integer, Double> matchMap) {
//matchMap : prjId matchScore
for (Map.Entry<Integer, Double> entry : matchMap.entrySet()) {
try {
int prjId = entry.getKey();
double matchScore = entry.getValue();
MatchItem matchResult = new MatchItem();
matchResult.setUserId(userId);
matchResult.setProjectId(prjId);
matchResult.setMatchScore(matchScore);
matchResultDao.insertMatchResult(matchResult);
} catch (Exception e) {
logger.error("insertMatchResult error: " + e);
}
}
}
}

View File

@ -0,0 +1,54 @@
package com.ow2.rec.main;
import java.util.LinkedList;
import java.util.List;
import javax.annotation.Resource;
import org.apache.lucene.document.Field.Store;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Component;
import com.ow2.rec.dao.UserTagDao;
import com.ow2.rec.dao.ProjectDao;
import com.ow2.rec.model.Project;
@Component
public class TestMysql {
@Resource
private ProjectDao projectDao;
@Resource
private UserTagDao userTagDao;
public static void main(String args[]){
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(
"applicationContext_mybatis.xml");
TestMysql a = (TestMysql)applicationContext.getBean(TestMysql.class);
String text="<>";
String tagstr = text.replaceAll("[<>]", "");
List<String> aa = new LinkedList();
System.out.println(tagstr.isEmpty());
if(tagstr.isEmpty()){
System.out.println("aaaa");
aa = null;
}
System.out.println(aa==null);
//System.out.println("project max id :"+a.getMaxId());
//System.out.println("userTag max id :"+a.getMaxIdFromUserTag());
}
public int getMaxId(){
List<Project> list = projectDao.getBatchPrjs(0, 10);
System.out.println(list.size());
for(int i=0;i<list.size();i++)
System.out.println(list.get(i).getDescription());
return projectDao.getNewLast();
}
public int getMaxIdFromUserTag(){
return userTagDao.getMaxId();
}
}

View File

@ -0,0 +1,31 @@
package com.ow2.rec.model;
public class MatchItem {
private int userId;
private int projectId;
private double matchScore;
public int getUserId() {
return userId;
}
public void setUserId(int userId) {
this.userId = userId;
}
public int getProjectId() {
return projectId;
}
public void setProjectId(int projectId) {
this.projectId = projectId;
}
public double getMatchScore() {
return matchScore;
}
public void setMatchScore(double matchScore) {
this.matchScore = matchScore;
}
}

View File

@ -0,0 +1,31 @@
package com.ow2.rec.model;
public class Project {
private int id;
private String name;
private String description;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

View File

@ -0,0 +1,29 @@
package com.ow2.rec.model;
public class Tag {
private int id;
private String name;
private int count;
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getCount() {
return count;
}
public void setCount(int count) {
this.count = count;
}
}

View File

@ -0,0 +1,23 @@
package com.ow2.rec.model;
public class UserTag {
private int userId;
private String allTags;
public int getUserId() {
return userId;
}
public void setUserId(int userId) {
this.userId = userId;
}
public String getAllTags() {
return allTags;
}
public void setAllTags(String allTags) {
this.allTags = allTags;
}
}

View File

@ -0,0 +1,83 @@
package com.ow2.rec.util;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
public class Normalizer {
public static String removeVersion(String str){
//现在对于2.45还不能处理也就是点后边两个连续的后缀
return str.replaceAll("\\-*\\d+(\\.[\\d|x|X])*$","");
}
//对字符串进行规整去除版本号后缀和连接符等
public static String normalize(String str) {
//目前可能会删除掉一些确实以数字结尾的项目在停用词里添加-
return str.replaceAll("[\\-]+"," ").replaceAll("'s"," ").replaceAll("[^0-9a-zA-Z\u4E00-\u9FA5]"," ").toLowerCase().trim();
}
public static List<String> getTagList(String text) {
List<String> tmpTagList = new ArrayList<String>();
//String tagstr = text.substring(1, text.length()-1);
String tagstr = text.replaceAll("[<>]", "");
if(tagstr.isEmpty())
return null;
String [] tmpArray = tagstr.split(",");
Collections.addAll(tmpTagList, tmpArray);
return tmpTagList;
}
public static void main(String []args) {
String content = "大声道Corvette123 *&%9&^&%&^%(*_asd's_12 L'Aur--ore, 1766-1775";
int a = 1 + 11000/11000;
String targetTableName = "relative_memo_to_open_source_projects_" + a;
String result = content.replaceAll("[^0-9a-zA-Z]"," ");
String str = "reverse-proxy,osx,iterator,freetext,text,laptop,userscripts,rest,datetime,keyboard-shortcuts,voting,unicode,mysql,internationalization,greasemonkey,row,noscript,java-ee,monitor,parameters,macros,flash,exception-handling,controls,api-design,environment,etag,condition,model-view-controller,rounded-corners,clr,wmd,count,crash,string,error-handling,design-patterns,python,internet-explorer,c++,load-balancing,ms-word,csrf,stateserver,tagging,xss,silverlight,fonts,median,yield,qa,visual-studio,release,entropy,security,windows-server-2008,pcre,hashtable,.net,memory-leaks,twitter,release-management,web-services,add-in,operating-system,table,seo,escaping,web-applications,gmail,windows-server-2003,hpricot,numberformat,updatecheck,kindle-kdk,sql-server-2005,language-features,friendly-url,formatting,hashbang,build-process,hash,tags,windows-vista,javascript,delegates,mobile-website,google-search-api,statistics,windows,crc32,css-border-radius,configuration,slug,tuning,comparison,search,dotnetopenauth,ssl,url-rewrite-module,vb.net,modularity,vbscript,comet,language-design,estimation,com,benchmarking,file-io,outlook,markdown,firefox,datediff,web-standards,web-config,cocoa,apache,version-control,licensing,linux,diff,validation,regex,asp.net-mvc,java,compression,spam,outlook-vba,shebang,perl,visual-studio-2008,iphone,maintenance,memory,visual-studio-2003,aggregate-functions,framebusting,memorystream,iis,google-chrome,c#,cross-browser,database-tools,precision,linq,automatic-properties,fragment-identifier,overflow,editing,ruby,serializable,tfs,jquery,floating-point,exception,phpmailer,arrays,require,voice-recording,excel-vba,database-design,mobile,optimization,openstv,iframe,cocoa-touch,kindle,gaps-and-islands,deadlock,powershell,web-api,browser,build-automation,project-management,conferences,scan,process,hardware,captcha,full-text-search,ergonomics,locking,mouse,objective-c,tuplizer,incoming-mail,https,asp.net,sql-server,excel,sinatra,lamp,html,split,.net-3.5,performance,nhibernate,string.format,css,templates,openid,http,internet-explorer-8,transactions,url-rewriting,indexing,categories,deflate,gzip,xcode,iis-7,sql,date,php,tsql,audio,reflection,algorithm,database,url,spam-prevention,http-status-code-404,visual-studio-team-system,html-sanitizing,caching,language-agnostic,lambda,interface,cpu";
String str1 = "<cocoa,build-automation,string.format,language-agnostic,pcre,laptop,serializable,require,java-ee,updatecheck,spam,ms-word,process,exception,maintenance,voice-recording,vb.net,memorystream,performance,deadlock,linq,benchmarking,sql,iis,comet,file-io,openstv,regex,monitor,iframe,entropy,statistics,security,environment,flash,release,windows-vista,language-design,configuration,asp.net-mvc,.net,compression,rounded-corners,google-chrome,search,fonts,tuning,project-management,tuplizer,automatic-properties,database,freetext,parameters,web-config,split,php,web-api,conferences,tagging,css,.net-3.5,crc32,database-design,cross-browser,incoming-mail,precision,lambda,language-features,kindle-kdk,browser,editing,gmail,fragment-identifier,scan,javascript,apache,powershell,hardware,ruby,visual-studio,userscripts,gzip,seo,dotnetopenauth,excel-vba,design-patterns,gaps-and-islands,escaping,algorithm,internet-explorer,modularity,voting,date,ergonomics,sinatra,audio,sql-server-2005,web-applications,internationalization,html,keyboard-shortcuts,controls,clr,macros,framebusting,firefox,http-status-code-404,c#,python,outlook-vba,web-services,csrf,https,median,unicode,url-rewrite-module,noscript,api-design,datediff,hashbang,greasemonkey,sql-server,datetime,memory,tsql,rest,nhibernate,tags,visual-studio-2003,delegates,vbscript,com,stateserver,css-border-radius,database-tools,arrays,transactions,text,perl,url,categories,crash,interface,overflow,ssl,row,http,captcha,deflate,google-search-api,mobile,markdown,memory-leaks,internet-explorer-8,reflection,phpmailer,xss,twitter,objective-c,licensing,string,url-rewriting,floating-point,operating-system,version-control,optimization,tfs,wmd,caching,hash,friendly-url,formatting,count,java,yield,visual-studio-team-system,cocoa-touch,model-view-controller,jquery,diff,mobile-website,iis-7,silverlight,c++,mysql,estimation,mouse,slug,locking,outlook,kindle,exception-handling,qa,html-sanitizing,build-process,lamp,spam-prevention,validation,error-handling,osx,hpricot,numberformat,hashtable,cpu,aggregate-functions,load-balancing,windows-server-2003,table,iterator,etag,asp.net,full-text-search,reverse-proxy,openid,comparison,excel,linux,iphone,shebang,add-in,visual-studio-2008,windows,templates,condition,indexing,release-management,xcode,web-standards,windows-server-2008>";
System.out.println(getTagList(str).size());
System.out.println(getTagList(str1).size());
// for(int i = 0;i< getTagList(str).size();i++){
// System.out.println(getTagList(str).get(i));
// }
}
public static boolean isAllNumber(String checkStr) {
Pattern pattern = Pattern.compile("[0-9]*");
Matcher isNum = pattern.matcher(checkStr);
return isNum.matches();
}
// 标签分离函数
public static List<String> tagsSegmentation(String tags) {
List<String> tag = new ArrayList<String>();
if (tags != null) {
String regex = "<[^<>]*>";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(tags);
while (matcher.find()) {
String t = matcher.group();
t = t.substring(1, t.length() - 1);
tag.add(t);
}
}
return tag;
}
}

View File

@ -0,0 +1,74 @@
package com.ow2.rec.util;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
public class SimilarityCounter {
//两个文本结合 以及各自个各个分量的权重的倒数
public static float countSimilarity(List<String>prj,int[]prjWeight,List<String>tag,int[]tagWeight){
//如果没有共同的元素就直接返回0
List<String> prjTmp = new LinkedList<String>(prj);
prjTmp.retainAll(tag);
if(prjTmp.size() == 0){
return 0;
}
//如果完全匹配就直接返回1
if(prj.containsAll(tag) && tag.containsAll(prj))
return 1;
//如果项目名称为单个字母直接返回0
if(prj.size() == 1 && prj.get(0).length() == 1){
return 0;
}
Set<String> union= new HashSet<String>();//此集合用于存储项目和标签匹配时他们的原子性元素的并集
union.addAll(prj);
union.addAll(tag);
//生成两个向量
float prjVector[] = new float[union.size()];
float tagVector[] = new float[union.size()];
int i = 0;
for(String item : union){
if(prj.contains(item))
prjVector[i] = 1.0F / prjWeight[prj.indexOf(item)];
if(tag.contains(item))
tagVector[i] = 1.0F / tagWeight[tag.indexOf(item)];
i++;
}
//两个向量的点积
float dotMulti = 0,absPrjUnSqrt = 0, absTagUnSqrt =0;
for(i = 0; i< prjVector.length; i++){
dotMulti += prjVector[i] * tagVector[i];
absPrjUnSqrt += prjVector[i] * prjVector[i];
absTagUnSqrt += tagVector[i] * tagVector[i];
}
float absPrj = 0, absTag =0;
absPrj = (float) Math.sqrt(absPrjUnSqrt );
absTag = (float) Math.sqrt(absTagUnSqrt );
return formatFloat(dotMulti / ( absPrj * absTag),3);
}
private static float formatFloat(float f,int num){
int base = (int) Math.pow(10,num);
return (float) (Math.round(f * base))/base;
}
}

View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>IK Analyzer 扩展配置</comment>
<!--用户可以在这里配置自己的扩展字典 -->
<entry key="ext_dict">ext.dic;</entry>
<!--用户可以在这里配置自己的扩展停止词字典-->
<entry key="ext_stopwords">stopword_1.dic;code_keywords.dic</entry>
</properties>

2
src/main/resources/README.txt Executable file
View File

@ -0,0 +1,2 @@
配置文件和main2012, quantitifier要在ik文件夹下
停用词要在src文件夹下

View File

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" />
<property name="password" value="123456" />
<property name="initialSize" value="5" />
<property name="maxActive" value="100" />
</bean>
<bean id="sqlSessionFactoryTwo" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSourceTwo" />
</bean>
<bean id="dataSourceOne" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url"
value="jdbc:mysql://localhost:3306/stackoverflow?characterEncoding=UTF-8" />
<property name="username" value="root" />
<property name="password" value="123456" />
<property name="initialSize" value="5" />
<property name="maxActive" value="100" />
</bean>
<bean id="sqlSessionFactoryOne" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dataSourceOne" />
</bean>
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="com.ow2.rec.dao" />
<property name="sqlSessionFactory" ref="sqlSessionFactoryOne"></property>
</bean>
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="com.ow2.rec.dao2" />
<property name="sqlSessionFactory" ref="sqlSessionFactoryTwo"></property>
</bean>
<context:annotation-config/>
<context:component-scan base-package="com.ow2.rec"/>
</beans>

View File

@ -0,0 +1,58 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:mvc="http://www.springframework.org/schema/mvc"
xmlns:aop="http://www.springframework.org/schema/aop"
xsi:schemaLocation="http://www.springframework.org/schema/mvc
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.0.xsd
http://www.springframework.org/schema/aop
http://www.springframework.org/schema/aop/spring-aop.xsd">
<aop:aspectj-autoproxy/>
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url"
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull&amp;autoReconnect=true" />
<property name="username" value="root" />
<property name="password" value="123456" />
<property name="initialSize" value="5" />
<property name="maxActive" value="500" />
</bean>
<bean id="dataSourceOne" class="org.apache.commons.dbcp.BasicDataSource"
destroy-method="close">
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
<property name="url"
value="jdbc:mysql://localhost:3306/stackoverflow?characterEncoding=UTF-8" />
<property name="username" value="root" />
<property name="password" value="123456" />
<property name="initialSize" value="5" />
<property name="maxActive" value="500" />
</bean>
<bean id="dynamicDataSource" class="com.ow2.rec.datasource.ThreadLocalRountingDataSource">
<property name="defaultTargetDataSource" ref="dataSourceOne" />
<property name="targetDataSources">
<map key-type="com.ow2.rec.datasource.DataSources">
<entry key="stackoverflow" value-ref="dataSourceOne"/>
<entry key="ossean_production" value-ref="dataSourceTwo"/>
</map>
</property>
</bean>
<bean id="sqlSessionFactory" class="org.mybatis.spring.SqlSessionFactoryBean">
<property name="dataSource" ref="dynamicDataSource" />
</bean>
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
<property name="basePackage" value="com.ow2.rec" />
<property name="sqlSessionFactory" ref="sqlSessionFactory"></property>
</bean>
<context:annotation-config/>
<context:component-scan base-package="com.ow2.rec"/>
</beans>

View File

5
src/main/resources/ext.dic Executable file
View File

@ -0,0 +1,5 @@
sql server
web server
http server
linux kernel
face++

28
src/main/resources/log4j.xml Executable file
View File

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
<param name="File" value="./log/info.log" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<root>
<level value="info" />
<appender-ref ref="file" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>

275713
src/main/resources/main2012.dic Executable file

File diff suppressed because it is too large Load Diff

316
src/main/resources/quantifier.dic Executable file
View File

@ -0,0 +1,316 @@
丈
世纪
位数
像素
克拉
公亩
公克
公分
公升
公尺
公担
公斤
公里
公顷
分钟
分米
加仑
千克
千米
厘米
周年
小时
平方
平方公尺
平方公里
平方分米
平方厘米
平方码
平方米
平方英寸
平方英尺
平方英里
平米
年代
年级
月份
毫升
毫米
毫克
海里
点钟
盎司
秒钟
立方公尺
立方分米
立方厘米
立方码
立方米
立方英寸
立方英尺
英亩
英寸
英尺
英里
阶段

735
src/main/resources/stopword.dic Executable file
View File

@ -0,0 +1,735 @@
a
an
and
are
as
at
be
but
by
for
if
in
into
is
it
no
not
of
on
or
such
that
the
their
then
there
these
they
this
to
was
will
with
一个
codekeywords:
private
protected
public
abstract
class
extends
final
implements
interface
native
new
static
strictfp
synchronized
transient
volatile
break
continue
return
do
while
if
else
for
instanceof
switch
case
defult
catch
finally
throw
throws
try
import
package
boolean
byte
char
double
float
int
long
short
null
true
false
super
this
void
form
do
sourceforge
com
cn
www
http
js
script
a
test
error
exception
about
website
able
abstract
console
sql
windows
exe
txt
doc
xls
local
net
web
server
from
using
error
can
date
file
xml
can
system
url
value
create
text
set
get
list
table
select
distinct
object
open
close
clear
all
time
have
org
main
start
end
version
private
public
index
api
method
source
root
content
write
read
view
one
page
run
log
win
bin
first
next
src
app
request
println
namespace
printf
button
title
local
define
lib
make
files
help
article
bool
boolean
config
load
args
date
thread
cpu
more
document
util
info
home
style
body
print
hello
world
find
left
std
debug
etc
like
top
now
map
context
other
post
format
client
encoding
session
program
datebase
control
language
base
process
want
values
used
status
project
color
array
loaclhost
please
click
integer
example
archive
response
command
event
param
what
core
build
link
display
copy
stdio
release
mode
after
check
cpp
c
port
target
should
library
lib
tcp
udp
state
sum
free
last
connect
configuration
none
download
software
buffer
query
bit
temp
word
block
two
send
device
layout
sdk
cache
alert
see
change
work
kernel
filter
handel
save
tools
min
count
header
level
framework
sys
email
push
pull
heap
stack
user
must
demo
just
serach
res
tmp
down
option
img
network
model
stop
problem
position
node
space
login
font
todo
background
resource
mac
bytes
ios
empty
note
self
tag
column
studio
red
done
baidu
sleep
convert
global
field
reference
way
settings
simple
ctrl
wait
meta
edit
runtime
store
equals
services
bean
ftp
exec
non
enter
lock
admin
configure
png
jpg
abc
loop
vector
setup
mail
flag
machine
sample
share
image
ref
join
where
parameter
users
contain
know
template
class
math
pdf
parse
timeout
cat
through
basic
invoke
resources
media
uri
paltform
err
sudo
range
menu
mapping
tables
master
memset
reset
environment
pop
commit
report
task
active
domain
good
step
setting
day
hash
move
collections
phone
engine
callback
datetime
objects
mobile
profile
always
storage
alt
trim
applications
via
design
nothing
makefile
clean
small
solution
custom
people
projects
family
attributes
enum
own
works
messages
low
signal
browser
give
aaa
event
assert
look
company
person
either
case
desktop
touch
book
unit
docs
params
dump
pool
feature
hashmap
seconds
owner
goto
keys
editor
env
month
year
loading
changes
black
fetch
guide
enterprise
multi
foundation
reflect
unique
upload
play
idea
exist
elements
account
numbers
examples
plain
solid
game
hide
apps
symbol
layer
least
weight
beta
jpeg
attr
refresh
apply
focus
fun
cfg
threads
receive
recieved
easy
hard
mode
logs
password
passwd
packages
ignore
scripts
cell
later
plus
his
pub
going
logging
channel
think
serial
things
loader
player
libs
team
backup
conector
side
utf8
speed
doing
price
timestamp
repository
init
HelloWorld
tesing
scan
components
speed
unlock
products
mid
said
token
optional
today
rest
pack
coding
contain
points
days
safe
typeof
community
others
manual
sets
closed
fine
groups
specify
sources
workder
problems
perform
servers
articles
skip
blocks
connections
related
notify
onload
known
assembly
useful
monitor
thing
forum
come
notification
rules
developers
cross
notice
soft
progress
fork
hosts
logger
role
everything
future
references
jni
locale
follow
words
updates
necessary
writer
mouse
sign
repoter
containing
route
analysis
four
comments
programs
maps
turn
versions
drive
parser
track
money
utils
reload
decimal
resume
behavior
hand
board
light
regex
terminal
reduce
requests
fff
0xfffff
strong
edge
schedule
pair
swf
tom
dependency
rule
escape
hook
temporary
issues
exchange
review
various
life
friend
face
complex
readme
prop
expert
extend
often
automatic
capture
difference
scheme
develop
together
steps
reply
beginning
understand
identified
wall
along
market
minute
hour
university
plan
templates
codes
ccc
diff
ppt
study
abcd
lost
choice
explain
likely
guid
taken
period
FAQ
symbols
thus
tips
compute
school
languages
describe
resolution
deal
detect
yellow
hope
visit
xyz
iii
dog
jump
knowledge
publish
inserted
levels
chip
preference
done
adjust
earth
expressions
house
central
rich
chat
sessions
deep
when
where
how
who
仅供参考

8176
src/main/resources/stopword_1.dic Executable file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,38 @@
package com.ow2.rec;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* Unit test for simple App.
*/
public class AppTest
extends TestCase
{
/**
* Create the test case
*
* @param testName name of the test case
*/
public AppTest( String testName )
{
super( testName );
}
/**
* @return the suite of tests being tested
*/
public static Test suite()
{
return new TestSuite( AppTest.class );
}
/**
* Rigourous Test :-)
*/
public void testApp()
{
assertTrue( true );
}
}