add Architecture pic and some test files

2016-09-19 12:12:18 +08:00 · 2016-09-19 12:12:18 +08:00 · af2164e2ee
parent e69a0ef55a
commit af2164e2ee
10 changed files with 185 additions and 284945 deletions
--- a/ARCHITECTURE.png
+++ b/ARCHITECTURE.png
--- a/src/main/java/com/ow2/rec/main/CreateIndexTest.java
+++ b/src/main/java/com/ow2/rec/main/CreateIndexTest.java
@ -0,0 +1,87 @@
+package com.ow2.rec.main;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+
+import javax.annotation.Resource;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.stereotype.Component;
+import org.wltea.analyzer.lucene.IKAnalyzer;
+
+import com.ow2.rec.model.Project;
+import com.ow2.rec.sourceDao.ProjectDao;
+
+
+@Component
+public class CreateIndexTest {
+
+	@Resource
+	private ProjectDao projectDao;
+	
+	public void start(){
+		Analyzer ikanalyzer = new IKAnalyzer(true);//智能分词
+		int step = 1000;
+		int startId = 0;
+		int endId = startId + step;//step=1000;
+		int maxPrjId = projectDao.getNewLast();
+		try {
+			IndexWriter writer = createIndexWriter("testIndex", ikanalyzer);
+			while (startId < maxPrjId) {
+				List<Project> projects = projectDao.getBatchPrjs_nofilter(startId, endId);
+				System.out.println("project " + startId + " to project " + endId);
+				for(Project project:projects){
+					Document doc = new Document();
+					String prjDescString = project.getDescription();
+					if(prjDescString == null)
+						prjDescString = "null";
+					Field prjDescField = new TextField("prjDesc",prjDescString,Store.YES);
+					doc.add(prjDescField);
+					writer.addDocument(doc);
+				}
+				if ((maxPrjId - endId) <= step) {
+					startId = endId;
+					endId = maxPrjId;
+				}
+				else {
+					startId = endId;
+					endId += step;
+				}
+			}
+			writer.commit();
+			writer.close();
+			System.out.println("finish");
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+	
+	public static void main(String[] args){
+		ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
+		CreateIndexTest mainClass = applicationContext.getBean(CreateIndexTest.class);
+		mainClass.start();
+	}
+	
+	//创建索引文件
+	public IndexWriter createIndexWriter(String indexPath, Analyzer analyzer)
+			throws IOException {
+		Directory dire = FSDirectory.open(Paths.get(indexPath));
+		IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
+		iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+		IndexWriter iw = new IndexWriter(dire, iwc);
+		return iw;
+	}
+	
+}
--- a/src/main/java/com/ow2/rec/main/SpeedTest.java
+++ b/src/main/java/com/ow2/rec/main/SpeedTest.java
@ -0,0 +1,84 @@
+package com.ow2.rec.main;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+
+import javax.annotation.Resource;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.stereotype.Component;
+
+import com.ow2.rec.lucene.LuceneIndex;
+import com.ow2.rec.sourceDao.ProjectDao;
+
+@Component
+public class SpeedTest {
+
+	@Resource
+	private ProjectDao sourceDao;
+	
+	public void start(){
+		long time1 = System.currentTimeMillis();
+//		List<String> result = sourceDao.getLikeResults("https://github.com/llvm-project/llvm-project");
+		long time2 = System.currentTimeMillis();
+//		System.out.println("共检索出 " + result.size() + " 条记录");
+		System.out.println("没有索引运行时间为:" + (time2 - time1));
+		
+		long time3 = System.currentTimeMillis();
+		Directory dire;
+		try {
+			dire = FSDirectory.open(Paths
+					.get("testIndex"));
+			if (!DirectoryReader.indexExists(dire)) { 
+				return;
+			}
+			IndexReader indexReader = DirectoryReader.open(dire);
+			IndexSearcher is = new IndexSearcher(indexReader);
+			BooleanQuery.setMaxClauseCount(9000);
+			BooleanQuery query = new BooleanQuery();
+			//Similariy是计算Lucene打分的最主要的类
+			Similarity similarity = new DefaultSimilarity(){
+				//document(项目)包含的tag
+				@Override
+				public float coord(int overlap, int maxOverlap) {
+					//添加doc weight
+					return overlap * overlap * overlap / (float)maxOverlap;
+				}
+
+			};
+			is.setSimilarity(similarity);
+			Term term = new Term("prjDesc", "https://github.com/llvm-project/llvm-project");
+			TermQuery tq = new TermQuery(term);
+			query.add(tq, BooleanClause.Occur.SHOULD);
+			TopDocs td = is.search(query,10000);
+			System.out.println("共检索出 " + td.totalHits + " 条记录");
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+		
+		long time4 = System.currentTimeMillis();
+		System.out.println("有索引运行时间为:" + (time4 - time3));
+	}
+	
+	public static void main(String[] args){
+		ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
+		SpeedTest mainClass = applicationContext.getBean(SpeedTest.class);
+		mainClass.start();
+	}
+	
+}
--- a/src/main/java/com/ow2/rec/sourceDao/ProjectDao.java
+++ b/src/main/java/com/ow2/rec/sourceDao/ProjectDao.java
@ -15,14 +15,28 @@ public interface ProjectDao {
 	public List<Project> getBatchPrjs(@Param("startId") int startId,
 			@Param("endId") int endId);
 	
+	
+	//只是取出数据 没有筛选
+	@Select("select id,name,description from github_projects where id>#{startId} AND id<=#{endId}")
+	public List<Project> getBatchPrjs_nofilter(@Param("startId") int startId, @Param("endId") int endId);
+	
 	//项目原创，没有fork且没有删除的
 	@Select("select MAX(id) from github_projects where forked_from is null and deleted = 0")
 	public Integer getNewLast();
 	
+	//搜索最大的id
+	@Select("select max(id) from github_projects")
+	public Integer getMaxId();
+	
 	@Select("select id,name,description from github_projects where id = #{prjId}")
 	public Project getPrjById(@Param("prjId") int prjId);
 	

 	@Select("select count(user_id) from github_watchers where repo_id=#{prjId}")
 	public Integer getWatchers(@Param("prjId") int prjId);
+	
+	
+	//sql语句查询
+	@Select("select description from github_projects where description like \'%${desc}%\'")
+	public List<String> getLikeResults(@Param("desc") String desc);
 }
--- a/src/main/resources/code_keywords.dic
+++ b/src/main/resources/code_keywords.dic
--- a/src/main/resources/ext.dic
+++ b/src/main/resources/ext.dic
@ -1,5 +0,0 @@
-sql server
-web server
-http server
-linux kernel
-face++
--- a/src/main/resources/main2012.dic
+++ b/src/main/resources/main2012.dic
--- a/src/main/resources/quantifier.dic
+++ b/src/main/resources/quantifier.dic
@ -1,316 +0,0 @@
-丈
-下
-世
-世纪
-两
-个
-中
-串
-亩
-人
-介
-付
-代
-件
-任
-份
-伏
-伙
-位
-位数
-例
-倍
-像素
-元
-克
-克拉
-公亩
-公克
-公分
-公升
-公尺
-公担
-公斤
-公里
-公顷
-具
-册
-出
-刀
-分
-分钟
-分米
-划
-列
-则
-刻
-剂
-剑
-副
-加仑
-勺
-包
-匙
-匹
-区
-千克
-千米
-升
-卷
-厅
-厘
-厘米
-双
-发
-口
-句
-只
-台
-叶
-号
-名
-吨
-听
-员
-周
-周年
-品
-回
-团
-圆
-圈
-地
-场
-块
-坪
-堆
-声
-壶
-处
-夜
-大
-天
-头
-套
-女
-孔
-字
-宗
-室
-家
-寸
-对
-封
-尊
-小时
-尺
-尾
-局
-层
-届
-岁
-师
-帧
-幅
-幕
-幢
-平方
-平方公尺
-平方公里
-平方分米
-平方厘米
-平方码
-平方米
-平方英寸
-平方英尺
-平方英里
-平米
-年
-年代
-年级
-度
-座
-式
-引
-张
-成
-战
-截
-户
-房
-所
-扇
-手
-打
-批
-把
-折
-担
-拍
-招
-拨
-拳
-指
-掌
-排
-撮
-支
-文
-斗
-斤
-方
-族
-日
-时
-曲
-月
-月份
-期
-本
-朵
-村
-束
-条
-来
-杯
-枚
-枝
-枪
-架
-柄
-柜
-栋
-栏
-株
-样
-根
-格
-案
-桌
-档
-桩
-桶
-梯
-棵
-楼
-次
-款
-步
-段
-毛
-毫
-毫升
-毫米
-毫克
-池
-洲
-派
-海里
-滴
-炮
-点
-点钟
-片
-版
-环
-班
-瓣
-瓶
-生
-男
-画
-界
-盆
-盎司
-盏
-盒
-盘
-相
-眼
-石
-码
-碗
-碟
-磅
-种
-科
-秒
-秒钟
-窝
-立方公尺
-立方分米
-立方厘米
-立方码
-立方米
-立方英寸
-立方英尺
-站
-章
-笔
-等
-筐
-筒
-箱
-篇
-篓
-篮
-簇
-米
-类
-粒
-级
-组
-维
-缕
-缸
-罐
-网
-群
-股
-脚
-船
-艇
-艘
-色
-节
-英亩
-英寸
-英尺
-英里
-行
-袋
-角
-言
-课
-起
-趟
-路
-车
-转
-轮
-辆
-辈
-连
-通
-遍
-部
-里
-重
-针
-钟
-钱
-锅
-门
-间
-队
-阶段
-隅
-集
-页
-顶
-顷
-项
-顿
-颗
-餐
-首
--- a/src/main/resources/stopword.dic
+++ b/src/main/resources/stopword.dic
@ -1,735 +0,0 @@
-a
-an
-and
-are
-as
-at
-be
-but
-by
-for
-if
-in
-into
-is
-it
-no
-not
-of
-on
-or
-such
-that
-the
-their
-then
-there
-these
-they
-this
-to
-was
-will
-with
-小
-一个
-的
-
-codekeywords:
-private
-protected
-public
-abstract
-class
-extends
-final
-implements
-interface
-native
-new
-static
-strictfp
-synchronized
-transient
-volatile
-break
-continue
-return
-do
-while
-if
-else
-for
-instanceof
-switch
-case
-defult
-catch
-finally
-throw
-throws
-try
-import
-package
-boolean
-byte
-char
-double
-float
-int
-long
-short
-null
-true
-false
-super
-this
-void
-form
-do
-sourceforge
-com
-cn
-www
-http
-js
-script
-a
-test
-error
-exception
-about
-website
-able
-abstract
-console
-sql
-windows
-exe
-txt
-doc
-xls
-local
-net
-web
-server
-from
-using
-error
-can
-date
-file
-xml
-can
-system
-url
-value
-create
-text
-set
-get
-list
-table
-select
-distinct
-object
-open
-close
-clear
-all
-time
-have
-org
-main
-start
-end
-version
-private
-public
-index
-api
-method
-source
-root
-content
-write
-read
-view
-one
-page
-run
-log
-win
-bin
-first
-next
-src
-app
-request
-println
-namespace
-printf
-button
-title
-local
-define
-lib
-make
-files
-help
-article
-bool
-boolean
-config
-load
-args
-date
-thread
-cpu
-more
-document
-util
-info
-home
-style
-body
-print
-hello
-world
-find
-left
-std
-debug
-etc
-like
-top
-now
-map
-context
-other
-post
-format
-client
-encoding
-session
-program
-datebase
-control
-language
-base
-process
-want
-values
-used
-status
-project
-color
-array
-loaclhost
-please
-click
-integer
-example
-archive
-response
-command
-event
-param
-what
-core
-build
-link
-display
-copy
-stdio
-release
-mode
-after
-check
-cpp
-c
-port
-target
-should
-library
-lib
-tcp
-udp
-state
-sum
-free
-last
-connect
-configuration
-none
-download
-software
-buffer
-query
-bit
-temp
-word
-block
-two
-send
-device
-layout
-sdk
-cache
-alert
-see
-change
-work
-kernel
-filter
-handel
-save
-tools
-min
-count
-header
-level
-framework
-sys
-email
-push
-pull
-heap
-stack
-user
-must
-demo
-just
-serach
-res
-tmp
-down
-option
-img
-network
-model
-stop
-problem
-position
-node
-space
-login
-font
-todo
-background
-resource
-mac
-bytes
-ios
-empty
-note
-self
-tag
-column
-studio
-red
-done
-baidu
-sleep
-convert
-global
-field
-reference
-way
-settings
-simple
-ctrl
-wait
-meta
-edit
-runtime
-store
-equals
-services
-bean
-ftp
-exec
-non
-enter
-lock
-admin
-configure
-png
-jpg
-abc
-loop
-vector
-setup
-mail
-flag
-machine
-sample
-share
-image
-ref
-join
-where
-parameter
-users
-contain
-know
-template
-class
-math
-pdf
-parse
-timeout
-cat
-through
-basic
-invoke
-resources
-media
-uri
-paltform
-err
-sudo
-range
-menu
-mapping
-tables
-master
-memset
-reset
-environment
-pop
-commit
-report
-task
-active
-domain
-good
-step
-setting
-day
-hash
-move
-collections
-phone
-engine
-callback
-datetime
-objects
-mobile
-profile
-always
-storage
-alt
-trim
-applications
-via
-design
-nothing
-makefile
-clean
-small
-solution
-custom
-people
-projects
-family
-attributes
-enum
-own
-works
-messages
-low
-signal
-browser
-give
-aaa
-event
-assert
-look
-company
-person
-either
-case
-desktop
-touch
-book
-unit
-docs
-params
-dump
-pool
-feature
-hashmap
-seconds
-owner
-goto
-keys
-editor
-env
-month
-year
-loading
-changes
-black
-fetch
-guide
-enterprise
-multi
-foundation
-reflect
-unique
-upload
-play
-idea
-exist
-elements
-account
-numbers
-examples
-plain
-solid
-game
-hide
-apps
-symbol
-layer
-least
-weight
-beta
-jpeg
-attr
-refresh
-apply
-focus
-fun
-cfg
-threads
-receive
-recieved
-easy
-hard
-mode
-logs
-password
-passwd
-packages
-ignore
-scripts
-cell
-later
-plus
-his
-pub
-going
-logging
-channel
-think
-serial
-things
-loader
-player
-libs
-team
-backup
-conector
-side
-utf8
-speed
-doing
-price
-timestamp
-repository
-init
-HelloWorld
-tesing
-scan
-components
-speed
-unlock
-products
-mid
-said
-token
-optional
-today
-rest
-pack
-coding
-contain
-points
-days
-safe
-typeof
-community
-others
-manual
-sets
-closed
-fine
-groups
-specify
-sources
-workder
-problems
-perform
-servers
-articles
-skip
-blocks
-connections
-related
-notify
-onload
-known
-assembly
-useful
-monitor
-thing
-forum
-come
-notification
-rules
-developers
-cross
-notice
-soft
-progress
-fork
-hosts
-logger
-role
-everything
-future
-references
-jni
-locale
-follow
-words
-updates
-necessary
-writer
-mouse
-sign
-repoter
-containing
-route
-analysis
-four
-comments
-programs
-maps
-turn
-versions
-drive
-parser
-track
-money
-utils
-reload
-decimal
-resume
-behavior
-hand
-board
-light
-regex
-terminal
-reduce
-requests
-fff
-0xfffff
-strong
-edge
-schedule
-pair
-swf
-tom
-dependency
-rule
-escape
-hook
-temporary
-issues
-exchange
-review
-various
-life
-friend
-face
-complex
-readme
-prop
-expert
-extend
-often
-automatic
-capture
-difference
-scheme
-develop
-together
-steps
-reply
-beginning
-understand
-identified
-wall
-along
-market
-minute
-hour
-university
-plan
-templates
-codes
-ccc
-diff
-ppt
-study
-abcd
-lost
-choice
-explain
-likely
-guid
-taken
-period
-FAQ
-symbols
-thus
-tips
-compute
-school
-languages
-describe
-resolution
-deal
-detect
-yellow
-hope
-visit
-xyz
-iii
-dog
-jump
-knowledge
-publish
-inserted
-levels
-chip
-preference
-done
-adjust
-earth
-expressions
-house
-central
-rich
-chat
-sessions
-deep
-when
-where
-how
-who
-仅供参考
--- a/src/main/resources/stopword_1.dic
+++ b/src/main/resources/stopword_1.dic