modify some extractor and generaterawpages

config/configuration.txt

@@ -1,25 +1,4 @@
 51cto_blog 51cto_blog_html_detail
 apache apache_html_detail
-cnblog_q_solve cnblogs_q_solved_html_detail
 cnblog_q_unsolve cnblogs_q_unsolved_html_detail
 codeproject codeproject_html_detail
-csdn_ask csdn_ask_html_detail
-csdn_blogs csdn_blog_html_detail
-csdn_topic csdn_topic_html_detail
-deWen_question dewen_question_html_detail
-freecode freecode_html_detail
-gna gna_html_detail
-iteye_ask iteye_ask_html_detail
-iteye_Blog iteye_blog_html_detail
-lagou lagou_html_detail 
-linuxtone linuxtone_html_detail
-lupaworld lupaworld_html_detail
-neitui neitui_html_detail
-OpenHubProject openhub_html_detail 
-oschina_project oschina_project_html_detail
-oschina_question oschina_question_html_detail
-phpChina_post phpchina_html_detail
-SFProject softpedia_html_detail
-slashdot_news slashdot_html_detail
-softpedia softpedia_html_detail
-stackoverflow_q stackoverflow_html_detail

src/main/java/extension/OneExtractor.java

@@ -12,6 +12,7 @@ import net.trustie.one.Configuration;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
 
 import core.ConsolePipeline;
 import core.ModelPageProcessor;
@@ -20,6 +21,7 @@ import core.PageModelPipeline;
 import core.Pipeline;
 import core.Site;
 
+@Component
 public class OneExtractor {
 	private Site site;
 
@@ -189,8 +191,7 @@ public class OneExtractor {
 	
 	public void start(){
 		int recordId ;
-		//int tableId = pagedao.getMaxId(tables.get(uuid));
-		int tableId = 24;
+		int tableId = this.downloader.getMaxId();
 		init();
 		
 		if(readCursor.getValue(uuid) != null){
@@ -201,6 +202,7 @@ public class OneExtractor {
 		
 		while(recordId < tableId){			
 			recordId = Integer.parseInt(readCursor.getValue(uuid).trim());
+			tableId = this.downloader.getMaxId();
 			System.out.println(recordId);
 			logger.info(this.uuid + " Extractor Started!");
 			this.run();

src/main/java/extension/PageDownloader.java

@@ -8,6 +8,8 @@ public interface PageDownloader {
 
 	public int getMinId();
 	
+    public int getMaxId();
+	
 //	public int getBeginId(int begin_id , int interval);
 	
 }

src/main/java/extension/ReadCursor.java

@@ -28,13 +28,13 @@ public class ReadCursor {
 		}
 	}
 
-	public String getValue(String key) {
+	public  String getValue(String key) {
 		if (this.properties.containsKey(key))
 			return this.properties.getProperty(key);
 		return null;
 	}
 
-	public void setValue(String key, String value) {
+	public synchronized void setValue(String key, String value) {
 		try {
 			FileOutputStream out_file = new FileOutputStream(
 					new File(file_path));			

src/main/java/net/trustie/downloader/GenerateRawPage.java

@@ -35,6 +35,11 @@ public class GenerateRawPage implements PageDownloader{
 		return this;
 	}
 
+	@Override
+	public int getMaxId() {
+		return pagedao.getMaxId(this.tableName);
+	}
+
 	
 
 

src/main/java/net/trustie/extractor/ApacheProject_Extractor.java

@@ -2,8 +2,11 @@ package net.trustie.extractor;
 
 import java.sql.SQLException;
 
+import javax.annotation.Resource;
+
 import net.trustie.downloader.DataBasePageErrorOutPut;
 import net.trustie.downloader.GenerateRawPage;
+import net.trustie.downloader.PageDao;
 import net.trustie.model.ApacheProject_Model;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,12 +15,17 @@ import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.stereotype.Component;
 
+import core.ExtractorBegin;
 import core.PageModelPipeline;
 import core.Site;
+import extension.OneExtractor;
 import extension.OsseanExtractor;
+import extension.ReadCursor;
 
-@Component
-public class ApacheProject_Extractor {
+@Component("apache")
+public class ApacheProject_Extractor implements ExtractorBegin {
+	
+	private ReadCursor readCursor = new ReadCursor("./cursor/apache.txt");
 	
 	@SuppressWarnings("rawtypes")
 	@Qualifier("apacheProjectPipeline")
@@ -31,6 +39,9 @@ public class ApacheProject_Extractor {
 	@Qualifier("errorPageToDB")
 	@Autowired
 	private DataBasePageErrorOutPut dbPageErrorOutPut;
+	
+	@Resource
+	private PageDao pagedao;
 
 	public void begin() {
 		generateRawPage.setTable("apache_html_detail");
@@ -43,14 +54,39 @@ public class ApacheProject_Extractor {
 				.setPageErrorOutPut(dbPageErrorOutPut).start();
 	}
 
-	public static void main(String[] args) throws SQLException {
-		ApplicationContext aContext = new ClassPathXmlApplicationContext(
-				"classpath:/spring/applicationContext*.xml");
+//	public static void main(String[] args) throws SQLException {
+//		ApplicationContext aContext = new ClassPathXmlApplicationContext(
+//				"classpath:/spring/applicationContext*.xml");
+//
+//		final ApacheProject_Extractor extractor = aContext
+//				.getBean(ApacheProject_Extractor.class);
+//
+//		extractor.begin();
+//	}
 
-		final ApacheProject_Extractor extractor = aContext
-				.getBean(ApacheProject_Extractor.class);
+	@Override
+	public void threadRun() {
+		// TODO Auto-generated method stub
+		generateRawPage.setTable("apache_html_detail");
+		dbPageErrorOutPut.setTableName("apache_error_page");
+		System.out.println("apache----> begin! ");
+		OneExtractor
+				.create(Site.me().setResultNum(100), modelPipeline,
+						ApacheProject_Model.class).setUUID("apache")
+				.setDownloader(generateRawPage)
+				.setPageErrorOutPut(dbPageErrorOutPut).start();
+	}
 
-		extractor.begin();
+	@Override
+	public boolean check() {
+		// TODO Auto-generated method stub
+		int cursor = Integer.parseInt(readCursor.getValue("cnblog_q_solve"));
+		int maxId = pagedao.getMaxId("cnblogs_q_solved_html_detail");
+		
+		if(cursor >= maxId){
+			return true ; 
+		}
+		return false;
 	}
 
 }

src/main/java/net/trustie/extractor/CNblogQSolve_Extractor.java

@@ -23,7 +23,7 @@ import extension.OneExtractor;
 import extension.OsseanExtractor;
 import extension.ReadCursor;
 
-@Component
+@Component("cnblog_q_solve")
 public class CNblogQSolve_Extractor implements ExtractorBegin {
 	
 	private ReadCursor readCursor = new ReadCursor("./cursor/cnblog_q_solve.txt");
@@ -65,7 +65,7 @@ public class CNblogQSolve_Extractor implements ExtractorBegin {
 		// TODO Auto-generated method stub
 		generateRawPage.setTable("cnblogs_q_solved_html_detail");
 		dbPageErrorOutPut.setTableName("cnblogs_q_solved_error_page");
-
+		System.out.println("cnblog_q_solve----> begin! ");
 		OneExtractor
 				.create(Site.me().setResultNum(100), modelPipeline,
 						CNblogsQ_Model.class).setUUID("cnblog_q_solve")
@@ -87,17 +87,17 @@ public class CNblogQSolve_Extractor implements ExtractorBegin {
 	
 	
 	
-	public static void main(String[] args) throws SQLException {
-//		ApplicationContext aContext = new ClassPathXmlApplicationContext(
-//				"classpath:/spring/applicationContext*.xml");
+//	public static void main(String[] args) throws SQLException {
+////		ApplicationContext aContext = new ClassPathXmlApplicationContext(
+////				"classpath:/spring/applicationContext*.xml");
+////
+////		final CNblogQSolve_Extractor extractor = aContext
+////				.getBean(CNblogQSolve_Extractor.class);
 //
-//		final CNblogQSolve_Extractor extractor = aContext
-//				.getBean(CNblogQSolve_Extractor.class);
-
-		Configuration config = new Configuration();
-		
-		CNblogQSolve_Extractor extractor = Configuration.aContext.getBean(CNblogQSolve_Extractor.class);
-		//extractor.begin();
-		extractor.threadRun();
-	}
+//		Configuration config = new Configuration();
+//		
+//		CNblogQSolve_Extractor extractor = Configuration.aContext.getBean(CNblogQSolve_Extractor.class);
+//		//extractor.begin();
+//		extractor.threadRun();
+//	}
 }

src/main/java/net/trustie/extractor/CNblogQUnSolve_Extractor.java

@@ -2,20 +2,30 @@ package net.trustie.extractor;
 
 import java.sql.SQLException;
 
+import javax.annotation.Resource;
+
 import net.trustie.downloader.DataBasePageErrorOutPut;
 import net.trustie.downloader.GenerateRawPage;
+import net.trustie.downloader.PageDao;
 import net.trustie.model.CNblogsQ_Model;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.stereotype.Component;
+
+import core.ExtractorBegin;
 import core.PageModelPipeline;
 import core.Site;
+import extension.OneExtractor;
 import extension.OsseanExtractor;
+import extension.ReadCursor;
 
-@Component
-public class CNblogQUnSolve_Extractor {
+@Component("cnblog_q_unsolve")
+public class CNblogQUnSolve_Extractor implements ExtractorBegin{
+	private ReadCursor readCursor = new ReadCursor("./cursor/cnblog_q_unsolve.txt");
+	
 	@SuppressWarnings("rawtypes")
 	@Qualifier("cnblogsQPipeline")
 	@Autowired
@@ -28,7 +38,10 @@ public class CNblogQUnSolve_Extractor {
 	@Qualifier("errorPageToDB")
 	@Autowired
 	private DataBasePageErrorOutPut dbPageErrorOutPut;
-
+	
+	@Resource
+	private PageDao pagedao;
+	
 	public void begin() {
 		generateRawPage.setTable("cnblogs_q_unsolved_html_detail");
 		dbPageErrorOutPut.setTableName("cnblogs_q_unsolved_error_page");
@@ -49,4 +62,27 @@ public class CNblogQUnSolve_Extractor {
 
 		extractor.begin();
 	}
+
+	@Override
+	public void threadRun() {
+		generateRawPage.setTable("cnblogs_q_unsolved_html_detail");
+		dbPageErrorOutPut.setTableName("cnblogs_q_unsolved_error_page");
+		System.out.println("cnblog_q_unsolve----> begin! ");
+		OneExtractor
+				.create(Site.me().setResultNum(100), modelPipeline,
+						CNblogsQ_Model.class).setUUID("cnblog_q_unsolve")
+				.setDownloader(generateRawPage)
+				.setPageErrorOutPut(dbPageErrorOutPut).start();
+	}
+
+	@Override
+	public boolean check() {
+		int cursor = Integer.parseInt(readCursor.getValue("cnblog_q_solve"));
+		int maxId = pagedao.getMaxId("cnblogs_q_solved_html_detail");
+		
+		if(cursor >= maxId){
+			return true ; 
+		}
+		return false;
+	}
 }

src/main/java/net/trustie/extractor/CTO51Blogs_Extractor.java

@@ -2,8 +2,11 @@ package net.trustie.extractor;
 
 import java.sql.SQLException;
 
+import javax.annotation.Resource;
+
 import net.trustie.downloader.DataBasePageErrorOutPut;
 import net.trustie.downloader.GenerateRawPage;
+import net.trustie.downloader.PageDao;
 import net.trustie.model.CTO51Blogs_Model;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -12,14 +15,18 @@ import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.stereotype.Component;
 
+import core.ExtractorBegin;
 import core.PageModelPipeline;
 import core.Site;
+import extension.OneExtractor;
 import extension.OsseanExtractor;
+import extension.ReadCursor;
 
-@Component
-public class CTO51Blogs_Extractor {
+@Component("51cto_blog")
+public class CTO51Blogs_Extractor implements ExtractorBegin{
+	
+	private ReadCursor readCursor = new ReadCursor("./cursor/51cto_blog.txt");
 	
-
 	@SuppressWarnings("rawtypes")
 	@Qualifier("CTO51BlogsPipeline")
 	@Autowired
@@ -33,6 +40,9 @@ public class CTO51Blogs_Extractor {
 	@Autowired
 	private DataBasePageErrorOutPut dbPageErrorOutPut;
 
+	@Resource 
+	private PageDao pagedao;
+	
 	public void begin() {
 		generateRawPage.setTable("51cto_blog_html_detail");
 		dbPageErrorOutPut.setTableName("51cto_blog_error_page");
@@ -44,14 +54,37 @@ public class CTO51Blogs_Extractor {
 				.setPageErrorOutPut(dbPageErrorOutPut).start();
 	}
 
-	public static void main(String[] args) throws SQLException {
-		ApplicationContext aContext = new ClassPathXmlApplicationContext(
-				"classpath:/spring/applicationContext*.xml");
+//	public static void main(String[] args) throws SQLException {
+//		ApplicationContext aContext = new ClassPathXmlApplicationContext(
+//				"classpath:/spring/applicationContext*.xml");
+//
+//		final CTO51Blogs_Extractor extractor = aContext
+//				.getBean(CTO51Blogs_Extractor.class);
+//
+//		extractor.begin();
+//	}
 
-		final CTO51Blogs_Extractor extractor = aContext
-				.getBean(CTO51Blogs_Extractor.class);
+	@Override
+	public void threadRun() {
+		generateRawPage.setTable("51cto_blog_html_detail");
+		dbPageErrorOutPut.setTableName("51cto_blog_error_page");
+		System.out.println("51cto_blog----> begin! ");
+		OneExtractor
+				.create(Site.me().setResultNum(100), modelPipeline,
+						CTO51Blogs_Model.class).setUUID("51cto_blog")
+				.setDownloader(generateRawPage)
+				.setPageErrorOutPut(dbPageErrorOutPut).start();
+	}
 
-		extractor.begin();
+	@Override
+	public boolean check() {
+		int cursor = Integer.parseInt(readCursor.getValue("cnblog_q_solve"));
+		int maxId = pagedao.getMaxId("cnblogs_q_solved_html_detail");
+		
+		if(cursor >= maxId){
+			return true ; 
+		}
+		return false;
 	}
 
 }

src/main/java/net/trustie/extractor/Codeproject_Extractor.java

@@ -1,8 +1,11 @@
 package net.trustie.extractor;
 import java.sql.SQLException;
 
+import javax.annotation.Resource;
+
 import net.trustie.downloader.DataBasePageErrorOutPut;
 import net.trustie.downloader.GenerateRawPage;
+import net.trustie.downloader.PageDao;
 import net.trustie.model.Codeproject_Model;
 import net.trustie.model.FreeCode_Model;
 
@@ -12,12 +15,18 @@ import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 import org.springframework.stereotype.Component;
 
+import core.ExtractorBegin;
 import core.PageModelPipeline;
 import core.Site;
+import extension.OneExtractor;
 import extension.OsseanExtractor;
+import extension.ReadCursor;
 
-@Component
-public class Codeproject_Extractor {
+@Component("codeproject")
+public class Codeproject_Extractor implements ExtractorBegin{
+	
+	private ReadCursor readCursor = new ReadCursor("./cursor/codeproject.txt");
+	
 	@SuppressWarnings("rawtypes")
 	@Qualifier("Codeproject_pipeline")
 	@Autowired
@@ -30,7 +39,10 @@ public class Codeproject_Extractor {
 	@Qualifier("errorPageToDB")
 	@Autowired
 	private DataBasePageErrorOutPut dbPageErrorOutPut;	
-
+	
+	@Resource
+	private PageDao pagedao;
+	
 	public void begin() {
 		generateRawPage.setTable("codeproject_html_detail");
 		dbPageErrorOutPut.setTableName("codeproject_error_page");		
@@ -49,4 +61,26 @@ public class Codeproject_Extractor {
 
 		extractor.begin();
 	}
+
+	@Override
+	public void threadRun() {
+		generateRawPage.setTable("codeproject_html_detail");
+		dbPageErrorOutPut.setTableName("codeproject_error_page");	
+		System.out.println("codeproject----> begin! ");
+		OneExtractor
+				.create(Site.me().setResultNum(100), modelPipeline,
+						Codeproject_Model.class).setUUID("codeproject")
+				.setDownloader(generateRawPage).start();
+	}
+
+	@Override
+	public boolean check() {
+		int cursor = Integer.parseInt(readCursor.getValue("cnblog_q_solve"));
+		int maxId = pagedao.getMaxId("cnblogs_q_solved_html_detail");
+		
+		if(cursor >= maxId){
+			return true ; 
+		}
+		return false;
+	}
 }

src/main/java/net/trustie/one/MyThread.java

@@ -21,16 +21,20 @@ public class MyThread extends Thread {
 	public void run() {
 		ExtractorBegin eb = (ExtractorBegin) Configuration.aContext
 				.getBean(site);
+		
+		System.out.println("begin!  "+site );
 
 		try {
 			while (!Thread.currentThread().isInterrupted()
 					&& Configuration.flags.get(site) == true) {
 				eb.threadRun();
+				logger.info("begin!  "+site );
 			}
 		} catch (Exception e) {
 			e.printStackTrace();
 		} finally {
 			Configuration.flags.put(site, false);
+			logger.info(site+" is over !");
 		}
 	}
 }

src/main/java/net/trustie/one/OneEx.java

@@ -13,28 +13,32 @@ import net.trustie.extractor.CNblogQSolve_Extractor;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.LineIterator;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.stereotype.Component;
 
 import extension.ReadCursor;
 
-
+@Component
 public class OneEx extends TimerTask{
 	private ReadCursor readCursor = null ;	
 	private ExecutorService pool = null ;	
-	private Configuration config ; 
+	private static Configuration config ; 
+	private Timer timer = null;
 	
 	@Resource 
 	private PageDao pagedao;
 	
 	public OneEx(){
 		//初始化一些配置
-	    config = new Configuration();		
+	    config = new Configuration();	
+	    timer = new Timer();
 	}
 	
 	public void begin(){
-		
 		initPool();
 		//logger.info(this.uuid + " Extractor Started!");
-		//timer.schedule(this, site.getFirstTime(), site.getPeriod());
+		timer.schedule(this, 1000, 20000);//1秒以后启动任务，以后主进程每隔20秒轮询数据库
 		
 		
 	}
@@ -43,28 +47,17 @@ public class OneEx extends TimerTask{
 		pool = Executors.newFixedThreadPool(20);	
 	}
 	
+	public void test(){
+		int i = pagedao.getMaxId("codeproject_html_detail");
+		System.out.println("<"+i+">");
+	}
 	public static void main(String[] args) {
-		//System.out.println("hello ");
-		
-//		ExecutorService pool = Executors.newCachedThreadPool();
-//		Thread t1 = new MyThread();
-//		Thread t2 = new MyThread();
-//		Thread t3 = new MyThread();
-//		Thread t4 = new MyThread();
-//		Thread t5 = new MyThread();
-//		Thread t6 = new MyThread();
-//		
-//		pool.execute(t1);
-//		pool.execute(t2);
-//		pool.execute(t3);
-//		pool.execute(t4);
-//		pool.execute(t5);
-//		pool.execute(t6);
-//		
-//		pool.shutdown();
-		
-		//Configuration ccf = new Configuration();	
+//		ApplicationContext aContext = new ClassPathXmlApplicationContext(
+//				"classpath:/spring/applicationContext*.xml");
+
+		final OneEx oneEx = config.aContext.getBean(OneEx.class);
 		
+		oneEx.begin();
 	}
 
 	@Override
@@ -74,11 +67,15 @@ public class OneEx extends TimerTask{
 		for(String site : sites){
 			readCursor = new ReadCursor("./cursor/"+site+".txt");
 			int recordId = Integer.parseInt(readCursor.getValue(site));
+			System.out.println(recordId+"");
+			System.out.println(pagedao.getMaxId(Configuration.tables.get(site))+"");
+			
 			int maxId =pagedao.getMaxId(Configuration.tables.get(site));
 			
-			if(recordId <= maxId && Configuration.flags.get(site) == false ){
+			if(recordId < maxId && Configuration.flags.get(site) == false ){
 				Thread t = new MyThread(site);
 				pool.execute(t);
+				Configuration.flags.put(site, true);
 			}
 		}