85 lines
7.0 KiB
Java
Executable File
85 lines
7.0 KiB
Java
Executable File
//Copyright (c) 2016 Yarong Zeng, NUDT.
|
||
package com.ow2.rec.util;
|
||
|
||
import java.io.IOException;
|
||
import java.io.Reader;
|
||
import java.io.StringReader;
|
||
import java.util.ArrayList;
|
||
import java.util.Collections;
|
||
import java.util.List;
|
||
import java.util.regex.Matcher;
|
||
import java.util.regex.Pattern;
|
||
import java.util.regex.PatternSyntaxException;
|
||
|
||
import org.wltea.analyzer.core.IKSegmenter;
|
||
import org.wltea.analyzer.core.Lexeme;
|
||
|
||
public class Normalizer {
|
||
|
||
public static String removeVersion(String str){
|
||
|
||
//现在对于2.45还不能处理，也就是点后边两个连续的后缀
|
||
return str.replaceAll("\\-*\\d+(\\.[\\d|x|X])*$","");
|
||
}
|
||
|
||
//对字符串进行规整，去除版本号后缀和连接符等
|
||
public static String normalize(String str) {
|
||
//目前可能会删除掉一些确实以数字结尾的项目；在停用词里添加‘-’
|
||
return str.replaceAll("[\\-]+"," ").replaceAll("'s"," ").replaceAll("[^0-9a-zA-Z\u4E00-\u9FA5]"," ").toLowerCase().trim();
|
||
}
|
||
|
||
|
||
public static List<String> getTagList(String text) {
|
||
List<String> tmpTagList = new ArrayList<String>();
|
||
//String tagstr = text.substring(1, text.length()-1);
|
||
String tagstr = text.replaceAll("[<>]", "");
|
||
if(tagstr.isEmpty())
|
||
return null;
|
||
String [] tmpArray = tagstr.split(",");
|
||
Collections.addAll(tmpTagList, tmpArray);
|
||
return tmpTagList;
|
||
}
|
||
|
||
public static void main(String []args) {
|
||
String content = "大声道Corvette123 *&%9&^&%&^%(*_asd's_12 L'Aur--ore, 1766-1775";
|
||
int a = 1 + 11000/11000;
|
||
String targetTableName = "relative_memo_to_open_source_projects_" + a;
|
||
String result = content.replaceAll("[^0-9a-zA-Z]"," ");
|
||
String str = "reverse-proxy,osx,iterator,freetext,text,laptop,userscripts,rest,datetime,keyboard-shortcuts,voting,unicode,mysql,internationalization,greasemonkey,row,noscript,java-ee,monitor,parameters,macros,flash,exception-handling,controls,api-design,environment,etag,condition,model-view-controller,rounded-corners,clr,wmd,count,crash,string,error-handling,design-patterns,python,internet-explorer,c++,load-balancing,ms-word,csrf,stateserver,tagging,xss,silverlight,fonts,median,yield,qa,visual-studio,release,entropy,security,windows-server-2008,pcre,hashtable,.net,memory-leaks,twitter,release-management,web-services,add-in,operating-system,table,seo,escaping,web-applications,gmail,windows-server-2003,hpricot,numberformat,updatecheck,kindle-kdk,sql-server-2005,language-features,friendly-url,formatting,hashbang,build-process,hash,tags,windows-vista,javascript,delegates,mobile-website,google-search-api,statistics,windows,crc32,css-border-radius,configuration,slug,tuning,comparison,search,dotnetopenauth,ssl,url-rewrite-module,vb.net,modularity,vbscript,comet,language-design,estimation,com,benchmarking,file-io,outlook,markdown,firefox,datediff,web-standards,web-config,cocoa,apache,version-control,licensing,linux,diff,validation,regex,asp.net-mvc,java,compression,spam,outlook-vba,shebang,perl,visual-studio-2008,iphone,maintenance,memory,visual-studio-2003,aggregate-functions,framebusting,memorystream,iis,google-chrome,c#,cross-browser,database-tools,precision,linq,automatic-properties,fragment-identifier,overflow,editing,ruby,serializable,tfs,jquery,floating-point,exception,phpmailer,arrays,require,voice-recording,excel-vba,database-design,mobile,optimization,openstv,iframe,cocoa-touch,kindle,gaps-and-islands,deadlock,powershell,web-api,browser,build-automation,project-management,conferences,scan,process,hardware,captcha,full-text-search,ergonomics,locking,mouse,objective-c,tuplizer,incoming-mail,https,asp.net,sql-server,excel,sinatra,lamp,html,split,.net-3.5,performance,nhibernate,string.format,css,templates,openid,http,internet-explorer-8,transactions,url-rewriting,indexing,categories,deflate,gzip,xcode,iis-7,sql,date,php,tsql,audio,reflection,algorithm,database,url,spam-prevention,http-status-code-404,visual-studio-team-system,html-sanitizing,caching,language-agnostic,lambda,interface,cpu";
|
||
String str1 = "<cocoa,build-automation,string.format,language-agnostic,pcre,laptop,serializable,require,java-ee,updatecheck,spam,ms-word,process,exception,maintenance,voice-recording,vb.net,memorystream,performance,deadlock,linq,benchmarking,sql,iis,comet,file-io,openstv,regex,monitor,iframe,entropy,statistics,security,environment,flash,release,windows-vista,language-design,configuration,asp.net-mvc,.net,compression,rounded-corners,google-chrome,search,fonts,tuning,project-management,tuplizer,automatic-properties,database,freetext,parameters,web-config,split,php,web-api,conferences,tagging,css,.net-3.5,crc32,database-design,cross-browser,incoming-mail,precision,lambda,language-features,kindle-kdk,browser,editing,gmail,fragment-identifier,scan,javascript,apache,powershell,hardware,ruby,visual-studio,userscripts,gzip,seo,dotnetopenauth,excel-vba,design-patterns,gaps-and-islands,escaping,algorithm,internet-explorer,modularity,voting,date,ergonomics,sinatra,audio,sql-server-2005,web-applications,internationalization,html,keyboard-shortcuts,controls,clr,macros,framebusting,firefox,http-status-code-404,c#,python,outlook-vba,web-services,csrf,https,median,unicode,url-rewrite-module,noscript,api-design,datediff,hashbang,greasemonkey,sql-server,datetime,memory,tsql,rest,nhibernate,tags,visual-studio-2003,delegates,vbscript,com,stateserver,css-border-radius,database-tools,arrays,transactions,text,perl,url,categories,crash,interface,overflow,ssl,row,http,captcha,deflate,google-search-api,mobile,markdown,memory-leaks,internet-explorer-8,reflection,phpmailer,xss,twitter,objective-c,licensing,string,url-rewriting,floating-point,operating-system,version-control,optimization,tfs,wmd,caching,hash,friendly-url,formatting,count,java,yield,visual-studio-team-system,cocoa-touch,model-view-controller,jquery,diff,mobile-website,iis-7,silverlight,c++,mysql,estimation,mouse,slug,locking,outlook,kindle,exception-handling,qa,html-sanitizing,build-process,lamp,spam-prevention,validation,error-handling,osx,hpricot,numberformat,hashtable,cpu,aggregate-functions,load-balancing,windows-server-2003,table,iterator,etag,asp.net,full-text-search,reverse-proxy,openid,comparison,excel,linux,iphone,shebang,add-in,visual-studio-2008,windows,templates,condition,indexing,release-management,xcode,web-standards,windows-server-2008>";
|
||
System.out.println(getTagList(str).size());
|
||
System.out.println(getTagList(str1).size());
|
||
// for(int i = 0;i< getTagList(str).size();i++){
|
||
// System.out.println(getTagList(str).get(i));
|
||
// }
|
||
|
||
|
||
}
|
||
|
||
public static boolean isAllNumber(String checkStr) {
|
||
Pattern pattern = Pattern.compile("[0-9]*");
|
||
Matcher isNum = pattern.matcher(checkStr);
|
||
return isNum.matches();
|
||
}
|
||
|
||
// 标签分离函数
|
||
public static List<String> tagsSegmentation(String tags) {
|
||
List<String> tag = new ArrayList<String>();
|
||
|
||
if (tags != null) {
|
||
String regex = "<[^<>]*>";
|
||
Pattern pattern = Pattern.compile(regex);
|
||
Matcher matcher = pattern.matcher(tags);
|
||
|
||
while (matcher.find()) {
|
||
String t = matcher.group();
|
||
t = t.substring(1, t.length() - 1);
|
||
|
||
tag.add(t);
|
||
}
|
||
}
|
||
return tag;
|
||
}
|
||
|
||
}
|