From 192bff31fee9dd2edebf0daca6c3ed9b3617c291 Mon Sep 17 00:00:00 2001 From: wangtao Date: Mon, 23 Nov 2015 17:13:40 +0800 Subject: [PATCH] add ossean_tag to the branch --- ossean_tag | 1 - ossean_tag/.classpath | 18 + ossean_tag/.gitignore | 6 + ossean_tag/.project | 23 ++ .../.settings/org.eclipse.jdt.core.prefs | 5 + .../.settings/org.eclipse.m2e.core.prefs | 4 + .../bin/resources/applicationContext.xml | 36 ++ ossean_tag/bin/resources/log4j.xml | 39 +++ ossean_tag/bin/start.sh | 21 ++ ossean_tag/pom.xml | 98 ++++++ ossean_tag/procedure_for_tag.iml | 15 + ossean_tag/src/main/assembly/assembly.xml | 19 + .../com/alan/myfunction/GenerateTags.java | 223 ++++++++++++ .../com/alan/myfunction/ProcedureTag.java | 30 ++ .../com/alan/myfunction/SQLConnection.java | 60 ++++ .../java/com/alan/myfunction/TagMatch.java | 325 ++++++++++++++++++ .../src/main/resources/applicationContext.xml | 36 ++ ossean_tag/src/main/resources/log4j.xml | 39 +++ .../ApplicationContextTest.java | 18 + .../test/java/procedure_for_tag/Sl4jTest.java | 20 ++ 20 files changed, 1035 insertions(+), 1 deletion(-) delete mode 160000 ossean_tag create mode 100644 ossean_tag/.classpath create mode 100644 ossean_tag/.gitignore create mode 100644 ossean_tag/.project create mode 100644 ossean_tag/.settings/org.eclipse.jdt.core.prefs create mode 100644 ossean_tag/.settings/org.eclipse.m2e.core.prefs create mode 100644 ossean_tag/bin/resources/applicationContext.xml create mode 100644 ossean_tag/bin/resources/log4j.xml create mode 100644 ossean_tag/bin/start.sh create mode 100644 ossean_tag/pom.xml create mode 100644 ossean_tag/procedure_for_tag.iml create mode 100644 ossean_tag/src/main/assembly/assembly.xml create mode 100644 ossean_tag/src/main/java/com/alan/myfunction/GenerateTags.java create mode 100644 ossean_tag/src/main/java/com/alan/myfunction/ProcedureTag.java create mode 100644 ossean_tag/src/main/java/com/alan/myfunction/SQLConnection.java create mode 100644 ossean_tag/src/main/java/com/alan/myfunction/TagMatch.java create mode 100644 ossean_tag/src/main/resources/applicationContext.xml create mode 100644 ossean_tag/src/main/resources/log4j.xml create mode 100644 ossean_tag/src/test/java/procedure_for_tag/ApplicationContextTest.java create mode 100644 ossean_tag/src/test/java/procedure_for_tag/Sl4jTest.java diff --git a/ossean_tag b/ossean_tag deleted file mode 160000 index cd55a2a..0000000 --- a/ossean_tag +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cd55a2a26844916e8882517932f995ba78e80050 diff --git a/ossean_tag/.classpath b/ossean_tag/.classpath new file mode 100644 index 0000000..a7112dd --- /dev/null +++ b/ossean_tag/.classpath @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/ossean_tag/.gitignore b/ossean_tag/.gitignore new file mode 100644 index 0000000..52c96b7 --- /dev/null +++ b/ossean_tag/.gitignore @@ -0,0 +1,6 @@ +/target/ +#.project +#/.settings +/lib/ +/procedure_for_tag +/log/* \ No newline at end of file diff --git a/ossean_tag/.project b/ossean_tag/.project new file mode 100644 index 0000000..938f411 --- /dev/null +++ b/ossean_tag/.project @@ -0,0 +1,23 @@ + + + procedure_for_tag + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.m2e.core.maven2Builder + + + + + + org.eclipse.jdt.core.javanature + org.eclipse.m2e.core.maven2Nature + + diff --git a/ossean_tag/.settings/org.eclipse.jdt.core.prefs b/ossean_tag/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..abec6ca --- /dev/null +++ b/ossean_tag/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,5 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 +org.eclipse.jdt.core.compiler.compliance=1.5 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.5 diff --git a/ossean_tag/.settings/org.eclipse.m2e.core.prefs b/ossean_tag/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 0000000..f897a7f --- /dev/null +++ b/ossean_tag/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/ossean_tag/bin/resources/applicationContext.xml b/ossean_tag/bin/resources/applicationContext.xml new file mode 100644 index 0000000..1d6f8b2 --- /dev/null +++ b/ossean_tag/bin/resources/applicationContext.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ossean_tag/bin/resources/log4j.xml b/ossean_tag/bin/resources/log4j.xml new file mode 100644 index 0000000..189e938 --- /dev/null +++ b/ossean_tag/bin/resources/log4j.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ossean_tag/bin/start.sh b/ossean_tag/bin/start.sh new file mode 100644 index 0000000..f088b8e --- /dev/null +++ b/ossean_tag/bin/start.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +find ./target/classes -name "*.properties"|xargs rm -f +find ./target/classes -name "*.xml"|xargs rm -f +find ./target/classes -name "*.dic"|xargs rm -f + +#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib + +tmp='./bin/resources' +tmp='./target/classes':$tmp +tmp='./target/procedure_for_tag-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources.jar':$tmp + +CLASSPATH=$tmp:$CLASSPATH + + +echo $CLASSPATH +#JVM_ARGS="-Xms:512m -XX:MinPermSize=128m" +#echo JVM_ARGS=$JVM_ARGS +#ulimit -n 400000 +#echo "" > nohup.out +java -classpath $CLASSPATH com.alan.myfunction.ProcedureTag>>info.log 2>&1 & \ No newline at end of file diff --git a/ossean_tag/pom.xml b/ossean_tag/pom.xml new file mode 100644 index 0000000..060c739 --- /dev/null +++ b/ossean_tag/pom.xml @@ -0,0 +1,98 @@ + + + UTF-8 + + 4.0.0 + org.kg + procedure_for_tag + 0.0.1-SNAPSHOT + procedure_for_tag + + + ${basedir}/src/main/java + + + ${basedir}/src/main/resources + + *.xml + *.properties + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.7 + 1.7 + + + + maven-assembly-plugin + 2.5.1 + + + + com.alan.myfunction.ProcedureTag + + + + src/main/assembly/assembly.xml + + + + + + + + + + org.springframework + spring-context + 4.1.2.RELEASE + + + + c3p0 + c3p0 + 0.9.1.2 + + + + junit + junit + 4.0 + jar + test + true + + + + org.quartz-scheduler + quartz + 1.8.4 + + + + mysql + mysql-connector-java + 5.1.30 + + + + commons-lang + commons-lang + 2.6 + + + + + org.slf4j + slf4j-log4j12 + 1.7.7 + + + + \ No newline at end of file diff --git a/ossean_tag/procedure_for_tag.iml b/ossean_tag/procedure_for_tag.iml new file mode 100644 index 0000000..73f608b --- /dev/null +++ b/ossean_tag/procedure_for_tag.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ossean_tag/src/main/assembly/assembly.xml b/ossean_tag/src/main/assembly/assembly.xml new file mode 100644 index 0000000..99332bf --- /dev/null +++ b/ossean_tag/src/main/assembly/assembly.xml @@ -0,0 +1,19 @@ + + + jar-with-dependencies-without-resources + + jar + + false + + + / + false + true + runtime + + + \ No newline at end of file diff --git a/ossean_tag/src/main/java/com/alan/myfunction/GenerateTags.java b/ossean_tag/src/main/java/com/alan/myfunction/GenerateTags.java new file mode 100644 index 0000000..4226a6c --- /dev/null +++ b/ossean_tag/src/main/java/com/alan/myfunction/GenerateTags.java @@ -0,0 +1,223 @@ +package com.alan.myfunction; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GenerateTags { + private Logger logger = LoggerFactory.getLogger(getClass()); + public final static int batchSize = 5000; + SQLConnection sqlconn = null; + String tableName = ""; + String colName = ""; + + public GenerateTags(String tableName, String colName) { + this.sqlconn = SQLConnection.getInstance(); + this.tableName = tableName; + this.colName = colName; + + this.initRecord(this.tableName); + + // this.initRecord(this.ospHlpTable); + } + + public void extract() { + // tagMatchRelativeMemo(); + extractFromTKnowledge(); + // tagMatchOSP(); + } + + // 从t_knowledge中抽取 + private void extractFromTKnowledge() { + Connection conn = this.sqlconn.getConnection(); + + try { + conn.setAutoCommit(false); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + int maxId = getMaxId(conn, this.tableName); + int minId = getMinId(conn, this.tableName); + // int offset = 0; + ResultSet rs = null; + int lower = minId; + int upper = minId + GenerateTags.batchSize; + while (lower < maxId) { + rs = getBatch(conn, this.tableName, this.colName, lower, upper); + lower = lower + TagMatch.batchSize; + upper = upper + TagMatch.batchSize; + + if (rs != null) { + batchExtract(rs, conn, this.tableName); + } + logger.info("extracted tags from t_knowledge: " + lower + "-->" + + upper); + } + + // type: 0-->osp, 1-->memo + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + // e.printStackTrace(); + } + } + + // 取得要匹配的最大id + private int getMaxId(Connection conn, String tableName) { + int rt = 0; + String sql = "SELECT MAX(id) AS num FROM " + tableName; + try { + PreparedStatement pst = conn.prepareStatement(sql); + // pst.setString(1, type); + // pst.set + ResultSet rs = pst.executeQuery(); + rs.next(); + rt = rs.getInt("num"); + pst.close(); + rs.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + return rt; + } + + // 取得需要匹配的最小id + private int getMinId(Connection conn, String tableName) { + int rt = 0; + String recordName = tableName + "_for_tag"; + String sql = "SELECT insert_id from t_records WHERE insert_table = ?"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + pst.setString(1, recordName); + ResultSet rs = pst.executeQuery(); + rs.next(); + rt = rs.getInt("insert_id"); + pst.close(); + rs.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + return rt; + } + + // 取得一批需要匹配的条目 + private ResultSet getBatch(Connection conn, String tableName, + String colName, int lower, int upper) { + ResultSet rt = null; + String sql = "SELECT id, " + colName + " AS tags from " + tableName + + " WHERE id > ? AND id <= ?"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + // pst.setString(1, col); + // pst.setString(2, table); + pst.setInt(1, lower); + pst.setInt(2, upper); + rt = pst.executeQuery(); + // pst.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + return rt; + } + + // 匹配一批 条目 + private void batchExtract(ResultSet rs, Connection conn, String tableName) { + String rawTags = ""; + int cursor = 0; + int itemId = 0; + String sql = "INSERT IGNORE INTO tags (name) VALUES " + "(?)"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + while (rs.next()) { + cursor = rs.getInt("id"); + rawTags = rs.getString("tags"); + // System.out.println(rawTags); + + String regx = "<(.*?)>"; + Pattern pattern = Pattern.compile(regx); + Matcher matcher; + + if ((rawTags != null) + && !("".equals(rawTags == null ? null : rawTags.trim()))) { + // resultTags = resultTags.replace(" ", ""); + matcher = pattern.matcher(rawTags); + while (matcher.find()) { + if (matcher.group(1) != "") { + pst.setString(1, + matcher.group(1).trim().replace(" ", "") + .toLowerCase()); + + pst.addBatch(); + } + } + // System.out.println(rs.getString(1)); + } + } + pst.executeBatch(); + conn.commit(); + + String recordName = tableName + "_for_tag"; + String updateSql = "UPDATE t_records SET insert_id = ? WHERE insert_table = ?"; + pst = conn.prepareStatement(updateSql); + pst.setInt(1, cursor); + pst.setString(2, recordName); + pst.executeUpdate(); + conn.commit(); + pst.close(); + System.out.println(recordName + ": " + cursor); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + } + + // 初始化记录指针表 + private void initRecord(String table) { + Connection conn = this.sqlconn.getConnection(); + String recordName = table + "_for_tag"; + ResultSet rs = null; + String sql = "SELECT * from t_records where insert_table = ?"; + try { + conn.setAutoCommit(false); + PreparedStatement pst = conn.prepareStatement(sql); + pst.setString(1, recordName); + rs = pst.executeQuery(); + if (!rs.next()) { + String initSql = "INSERT INTO t_records (insert_table, insert_id) VALUES (?, ?)"; + pst = conn.prepareStatement(initSql); + pst.setString(1, recordName); + pst.setInt(2, 0); + pst.executeUpdate(); + } + conn.commit(); + pst.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + +} diff --git a/ossean_tag/src/main/java/com/alan/myfunction/ProcedureTag.java b/ossean_tag/src/main/java/com/alan/myfunction/ProcedureTag.java new file mode 100644 index 0000000..f0b4857 --- /dev/null +++ b/ossean_tag/src/main/java/com/alan/myfunction/ProcedureTag.java @@ -0,0 +1,30 @@ +package com.alan.myfunction; + +import java.sql.SQLException; + + +public class ProcedureTag { + public static void main(String[] args) throws SQLException { + + + while (true) { + //MyJDBC.myexcute("t_knowledge"); + //GenerateTags gt = new GenerateTags("relative_memo_info_help", "tags"); + GenerateTags gt = new GenerateTags("t_knowledge", "tags"); + //从t_knowledge中分离中tag来存到tags中 + gt.extract(); + + + TagMatch tm = new TagMatch("relative_memo_info_help", "rm_id", "open_source_project_help", "osp_id"); +// 从relative_memo_info_help中取出记录与tags中 的记录匹配 + tm.match(); + + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } +} diff --git a/ossean_tag/src/main/java/com/alan/myfunction/SQLConnection.java b/ossean_tag/src/main/java/com/alan/myfunction/SQLConnection.java new file mode 100644 index 0000000..b74786e --- /dev/null +++ b/ossean_tag/src/main/java/com/alan/myfunction/SQLConnection.java @@ -0,0 +1,60 @@ +package com.alan.myfunction; + +import java.beans.PropertyVetoException; +import java.sql.Connection; +import java.sql.SQLException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.context.support.FileSystemXmlApplicationContext; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; + +import com.mchange.v2.c3p0.ComboPooledDataSource; + +public class SQLConnection { + private Logger logger = LoggerFactory.getLogger(getClass()); + private static SQLConnection dbcputils = null; + private ComboPooledDataSource cpds = null; + + private SQLConnection() { + if (cpds == null) { + // ApplicationContext applicationContext = new + // ClassPathXmlApplicationContext("applicationContext.xml"); + ApplicationContext applicationContext = new ClassPathXmlApplicationContext( + "classpath:applicationContext.xml"); + this.cpds = (ComboPooledDataSource) applicationContext + .getBean("dataSource"); + } + } + + public synchronized static SQLConnection getInstance() { + if (dbcputils == null) + dbcputils = new SQLConnection(); + return dbcputils; + } + + public Connection getConnection() { + Connection con = null; + try { + con = cpds.getConnection(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + return con; + } + + public static void main(String[] args) throws SQLException { + Connection con = null; + long begin = System.currentTimeMillis(); + for (int i = 0; i < 1000000; i++) { + con = SQLConnection.getInstance().getConnection(); + con.close(); + } + long end = System.currentTimeMillis(); + System.out.println("��ʱΪ:" + (end - begin) + "ms"); + } +} diff --git a/ossean_tag/src/main/java/com/alan/myfunction/TagMatch.java b/ossean_tag/src/main/java/com/alan/myfunction/TagMatch.java new file mode 100644 index 0000000..7249196 --- /dev/null +++ b/ossean_tag/src/main/java/com/alan/myfunction/TagMatch.java @@ -0,0 +1,325 @@ +package com.alan.myfunction; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TagMatch { + private Logger logger = LoggerFactory.getLogger(getClass()); + public final static int batchSize = 5000; + SQLConnection sqlconn = null; + String memoHlpTable = ""; + String memoHlpColName = ""; + String ospHlpTable = ""; + String ospHlpColName = ""; + Map tags = new HashMap(); + + public TagMatch(String memoTable, String memoColName, String ospTable, + String ospColName) { + this.sqlconn = SQLConnection.getInstance(); + this.memoHlpTable = memoTable; + this.memoHlpColName = memoColName; + this.ospHlpTable = ospTable; + this.ospHlpColName = ospColName; + this.initTags(); + this.initRecord(this.memoHlpTable); + // this.initRecord(this.ospHlpTable); + } + + public void match() { + tagMatchRelativeMemo(); + // tagMatchOSP(); + } + + // 分离帖子标签 + private void tagMatchRelativeMemo() { + Connection conn = this.sqlconn.getConnection(); + + try { + conn.setAutoCommit(false); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + int maxId = getMaxId(conn, this.memoHlpTable); + int minId = getMinId(conn, this.memoHlpTable); + // int offset = 0; + ResultSet rs = null; + int lower = minId; + int upper = minId + TagMatch.batchSize; + while (lower < maxId) { + rs = getBatch(conn, this.memoHlpTable, this.memoHlpColName, lower, + upper); + lower = lower + TagMatch.batchSize; + upper = upper + TagMatch.batchSize; + + // type: 0-->osp, 1-->memo + if (rs != null) { + batchMatch(rs, conn, this.memoHlpTable); + + } + logger.info("matched tags from relative_memo_help: " + lower + + "-->" + upper); + } + + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + } + + // 分离项目标签 + private void tagMatchOSP() { + Connection conn = this.sqlconn.getConnection(); + + try { + conn.setAutoCommit(false); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + int maxId = getMaxId(conn, this.ospHlpTable); + int minId = getMinId(conn, this.ospHlpTable); + int offset = 0; + ResultSet rs = null; + int lower = minId; + int upper = minId + TagMatch.batchSize; + while (lower < maxId) { + rs = getBatch(conn, this.ospHlpTable, this.ospHlpColName, lower, + upper); + lower = lower + TagMatch.batchSize; + upper = upper + TagMatch.batchSize; + + if (rs != null) { + batchMatch(rs, conn, this.ospHlpTable); + } + } + + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + } + + // 取得要匹配的最大id + private int getMaxId(Connection conn, String type) { + int rt = 0; + String sql = "SELECT MAX(id) AS num FROM " + type; + try { + PreparedStatement pst = conn.prepareStatement(sql); + // pst.setString(1, type); + // pst.set + ResultSet rs = pst.executeQuery(); + rs.next(); + rt = rs.getInt("num"); + pst.close(); + rs.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + return rt; + } + + // 取得需要匹配的最小id + private int getMinId(Connection conn, String type) { + int rt = 0; + String sql = "SELECT insert_id from t_records WHERE insert_table = ?"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + pst.setString(1, type); + ResultSet rs = pst.executeQuery(); + rs.next(); + rt = rs.getInt("insert_id"); + pst.close(); + rs.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + return rt; + } + + // 取得一批需要匹配的条目 + private ResultSet getBatch(Connection conn, String table, String col, + int lower, int upper) { + ResultSet rt = null; + String sql = "SELECT id, " + col + " AS item_id, tags from " + table + + " WHERE id > ? AND id <= ?"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + // pst.setString(1, col); + // pst.setString(2, table); + pst.setInt(1, lower); + pst.setInt(2, upper); + rt = pst.executeQuery(); + // pst.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + return rt; + } + + // 匹配一批 条目 + private void batchMatch(ResultSet rs, Connection conn, String table) { + String rawTag = ""; + int cursor = 0; + int itemId = 0; + String sql = "INSERT IGNORE INTO taggings (tag_id, taggable_id, taggable_type, context, created_at) VALUES " + + "(?, ?, ?, 'tags', ?)"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + // pst.setString(4, "tags"); + pst.setDate(4, new java.sql.Date(new java.util.Date().getTime())); + if ("relative_memo_info_help".equals(table)) { + pst.setString(3, "RelativeMemo"); + } else if ("open_source_project".equals(table)) { + pst.setString(3, "OpenSourceProject"); + } else { + pst.setString(3, "Other"); + } + + while (rs.next()) { + cursor = rs.getInt("id"); + rawTag = rs.getString("tags"); + if (rawTag != null) { + rawTag = rawTag.toLowerCase(); + } else { + continue; + } + itemId = rs.getInt("item_id"); + pst.setInt(2, itemId); + List tagList = toList(rawTag); + Iterator it = tagList.iterator(); + while (it.hasNext()) { + String tag = it.next(); + tag = tag.replace("'", "\\'"); + int tagId = 0; + if (this.tags.containsKey(tag)) { + tagId = this.tags.get(tag); + } + pst.setInt(1, tagId); + pst.addBatch(); + } + } + + pst.executeBatch(); + conn.commit(); + + String updateSql = "UPDATE t_records SET insert_id = ? WHERE insert_table = ?"; + pst = conn.prepareStatement(updateSql); + pst.setInt(1, cursor); + pst.setString(2, table); + pst.executeUpdate(); + conn.commit(); + pst.close(); + System.out.println(table + ": " + cursor); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + } + + // 字符串标签变链表 + private List toList(String tags) { + List rt = new ArrayList(); + + String regx = "<(.*?)>"; + Pattern pattern = Pattern.compile(regx); + Matcher matcher; + + // resultTags = resultTags.replace(" ", ""); + matcher = pattern.matcher(tags); + while (matcher.find()) { + if (matcher.group(1) != "") { + rt.add(matcher.group(1).trim().replace(" ", "")); + } + } + // System.out.println(rs.getString(1)); + + return rt; + } + + // 初始化标签，将标签读到内存中 + private void initTags() { + Connection conn = this.sqlconn.getConnection(); + ResultSet rs = null; + String sql = "SELECT id, name from tags"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + rs = pst.executeQuery(); + while (rs.next()) { + String name = rs.getString("name"); + Integer id = rs.getInt("id"); + this.tags.put(name, id); + } + pst.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + } + + // 初始化记录指针表 + private void initRecord(String table) { + Connection conn = this.sqlconn.getConnection(); + ResultSet rs = null; + String sql = "SELECT * from t_records where insert_table = ?"; + try { + PreparedStatement pst = conn.prepareStatement(sql); + pst.setString(1, table); + rs = pst.executeQuery(); + if (!rs.next()) { + String initSql = "INSERT INTO t_records (insert_table, insert_id) VALUES (?, ?)"; + pst = conn.prepareStatement(initSql); + pst.setString(1, table); + pst.setInt(2, 0); + pst.executeUpdate(); + } + pst.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + + try { + conn.close(); + } catch (SQLException e) { + // TODO Auto-generated catch block + logger.info(e.getStackTrace().toString()); + } + } + +} diff --git a/ossean_tag/src/main/resources/applicationContext.xml b/ossean_tag/src/main/resources/applicationContext.xml new file mode 100644 index 0000000..49c0bc7 --- /dev/null +++ b/ossean_tag/src/main/resources/applicationContext.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ossean_tag/src/main/resources/log4j.xml b/ossean_tag/src/main/resources/log4j.xml new file mode 100644 index 0000000..189e938 --- /dev/null +++ b/ossean_tag/src/main/resources/log4j.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ossean_tag/src/test/java/procedure_for_tag/ApplicationContextTest.java b/ossean_tag/src/test/java/procedure_for_tag/ApplicationContextTest.java new file mode 100644 index 0000000..bae592c --- /dev/null +++ b/ossean_tag/src/test/java/procedure_for_tag/ApplicationContextTest.java @@ -0,0 +1,18 @@ +package procedure_for_tag; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import com.mchange.v2.c3p0.ComboPooledDataSource; + +public class ApplicationContextTest { + + public static void main(String[] args) { + // TODO Auto-generated method stub + ApplicationContext applicationContext = new ClassPathXmlApplicationContext( + "applicationContext.xml"); + ComboPooledDataSource cpds = (ComboPooledDataSource) applicationContext + .getBean("dataSource"); + } + +} diff --git a/ossean_tag/src/test/java/procedure_for_tag/Sl4jTest.java b/ossean_tag/src/test/java/procedure_for_tag/Sl4jTest.java new file mode 100644 index 0000000..99c925d --- /dev/null +++ b/ossean_tag/src/test/java/procedure_for_tag/Sl4jTest.java @@ -0,0 +1,20 @@ +package procedure_for_tag; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +public class Sl4jTest { + //private final Log logger = LogFactory.getLog(getClass()); + private Logger logger = LoggerFactory.getLogger(getClass()); + + public void test(){ + logger.info("info2"); + logger.error("error2"); + } + public static void main(String[] args) { + // TODO Auto-generated method stub + Sl4jTest st = new Sl4jTest(); + st.test(); + } + +}