add extract model of lagou and modify the Dao files of some sites

bin/lagou.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+SITE='lagou'
+
+find ./target/classes -name "*.xml"|xargs rm -f
+
+tmp='./bin/resources'
+tmp='./target/classes':$tmp
+tmp='./target/osseanextractor-0.0.1-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+JAVA_OPTS="-Xms256m -Xmx512m -Xmn128m"
+
+echo $CLASSPATH
+
+java $JAVA_OPTS -DlogFilePath=${SITE} -classpath $CLASSPATH net.trustie.extractor.lagou_Extractor >>log/${SITE}.log 2>&1 &

bin/resources/Copy of lagou.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+SITE='lagou'
+
+find ./target/classes -name "*.xml"|xargs rm -f
+
+tmp='./bin/resources'
+tmp='./target/classes':$tmp
+tmp='./target/osseanextractor-0.0.1-jar-with-dependencies-without-resources/*':$tmp
+
+CLASSPATH=$tmp:$CLASSPATH
+
+JAVA_OPTS="-Xms256m -Xmx512m -Xmn128m"
+
+echo $CLASSPATH
+
+java $JAVA_OPTS -DlogFilePath=${SITE} -classpath $CLASSPATH net.trustie.extractor.lagou_Extractor >>log/${SITE}.log 2>&1 &

src/main/java/net/trustie/dao/CNblogNews_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface CNblogNews_Dao {
 
 	@Insert("insert into cnblog_news"
-			+ "(`newsId`,`newsUrl`,`newsTitle`,`tag`,`newsCategorie`,`newsContent`,`comeFrom`,`originFrom`,`extractTime`,`newsAuthor`,`newsAuthorUrl`,`pageMD5`,`history`)"
+			+ "(`newsId`,`Url`,`newsTitle`,`tag`,`newsCategorie`,`newsContent`,`comeFrom`,`originFrom`,`extractTime`,`newsAuthor`,`newsAuthorUrl`,`pageMD5`,`history`)"
 			+ " values (#{newsId},#{newsUrl},#{newsTitle},#{tag},#{newsCategorie},#{newsContent},#{comeFrom},#{originFrom},#{extractTime},#{newsAuthor},#{newsAuthorUrl},#{pageMD5},#{history})")
 	public int add(CNblogsNews_Model cNews_Model);
 }

src/main/java/net/trustie/dao/CNblogsQ_Dao.java

@@ -7,7 +7,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface CNblogsQ_Dao {
 	
 	@Insert("insert into cnblog_question"
-			+ "(`questionId`,`questionUrl`,`questionTitle`,`tag`,`questionContent`,`voteNum`,`viewNum`,`scoreBean`,`answerNum`,`questionType`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`questionId`,`Url`,`questionTitle`,`tag`,`questionContent`,`voteNum`,`viewNum`,`scoreBean`,`answerNum`,`questionType`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{questionUrl},#{questionTitle},#{tag},#{questionContent},#{voteNum},#{viewNum},#{scoreBean},#{answerNum},#{questionType},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(CNblogsQ_Model cNblogsQ_Model);
 	

src/main/java/net/trustie/dao/CTO51Blogs_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface CTO51Blogs_Dao {
 
 	@Insert("insert into 51cto_blog"
-			+ "(`post_id`,`post_url`,`post_title`,`post_tags`,`post_content`,`post_type`,`view_num`,`reply_num`,`vote_up`,`author_name`,`author_url`,`extractTime`,`post_time`,`history`,`pageMD5`,`post_urlMD5`)"
+			+ "(`post_id`,`Url`,`post_title`,`post_tags`,`post_content`,`post_type`,`view_num`,`reply_num`,`vote_up`,`author_name`,`author_url`,`extractTime`,`post_time`,`history`,`pageMD5`,`post_urlMD5`)"
 			+ "values (#{post_id},#{post_url},#{post_title},#{post_tags},#{post_content},#{post_type},#{view_num},#{reply_num},#{vote_up},#{author_name},#{author_url},#{extractTime},#{post_time},#{history},#{pageMD5},#{post_urlMD5})")
 	public int add(CTO51Blogs_Model ct051Model);
 	

src/main/java/net/trustie/dao/Codeproject_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface Codeproject_Dao {
 	@Insert("insert into codeproject"
-			+ "(`blog_id`,`blog_url`,`blog_title`,`blog_tags`,`blog_type`,`blog_content`,`favorite_num`,`view_num`,`stars`,`answer_num`,`download_num`,`create_time`,`extractTime`,`revision_num`,`author_name`,`author_url`,`pageMD5`,`history`)"
+			+ "(`blog_id`,`Url`,`blog_title`,`blog_tags`,`blog_type`,`blog_content`,`favorite_num`,`view_num`,`stars`,`answer_num`,`download_num`,`create_time`,`extractTime`,`revision_num`,`author_name`,`author_url`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{questionUrl},#{questionTitle},#{tag},#{categories},#{questionContent},#{favorite_num},#{view_num},#{stars},#{answer_num},#{download_num},#{createTime},#{extractTime},#{revision_num},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(Codeproject_Model codeproject_Model);
 }

src/main/java/net/trustie/dao/CsdnAsk_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface CsdnAsk_Dao {
 	@Insert("insert into csdn_ask"
-			+ "(`issueId`,`issueUrl`,`issueTitle`,`tag`,`issueContent`,`voteNum`,`answerNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`issueId`,`Url`,`issueTitle`,`tag`,`issueContent`,`voteNum`,`answerNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{issueId},#{issueUrl},#{issueTitle},#{tag},#{issueContent},#{voteNum},#{answerNum},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(CsdnAsk_Model ask_Model);
 }

src/main/java/net/trustie/dao/CsdnBlogs_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface CsdnBlogs_Dao {
 	@Insert("insert into csdn_blogs"
-			+ "(`blogId`,`blogUrl`,`blogTitle`,`blogTag`,`blogCategory`,`blogContent`,`readNum`,`commentNum`,`supportNum`,`opposeNum`,`postTime`,`extractTime`,`blogPageMD5`,`author`,`authorUrl`,`history`)"
+			+ "(`blogId`,`Url`,`blogTitle`,`blogTag`,`blogCategory`,`blogContent`,`readNum`,`commentNum`,`supportNum`,`opposeNum`,`postTime`,`extractTime`,`blogPageMD5`,`author`,`authorUrl`,`history`)"
 			+ "values (#{blogId},#{blogUrl},#{blogTitle},#{blogTag},#{blogCategory},#{blogContent},#{readNum},#{commentNum},#{supportNum},#{opposeNum},#{postTime},#{extractTime},#{blogPageMD5},#{author},#{authorUrl},#{history})")
 	public int add(CsdnBlogs_Model csdnBlogsModel); 
 }

src/main/java/net/trustie/dao/CsdnTopic_Dao.java

@@ -7,7 +7,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface CsdnTopic_Dao {
 
 	@Insert("insert into csdn_topics"
-			+ "(`topicId`,`topicTitle`,`topicUrl`,`topicContent`,`tags`,`topicScore`,`replyNum`,`postTime`,`extractTime`,`pageMD5`,`history`,`author`,`author_url`)"
+			+ "(`topicId`,`topicTitle`,`Url`,`topicContent`,`tags`,`topicScore`,`replyNum`,`postTime`,`extractTime`,`pageMD5`,`history`,`author`,`author_url`)"
 			+ "values (#{topicId},#{topicTitle},#{topicUrl},#{topicContent},#{tag},#{topicScore},#{replyNum},#{postTime},#{extractTime},#{pageMD5},#{history},#{author},#{author_url})")
 	public int add(CsdnTopic_Model cModel);
 }

src/main/java/net/trustie/dao/DeWenQ_Dao.java

@@ -7,7 +7,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface DeWenQ_Dao {
 	
 	@Insert("insert into dewen_question"
-			+ "(`issueId`,`issueUrl`,`issueTitle`,`tag`,`issueDetail`,`scanerNum`,`attentionNum`,`commentNum`,`answerNum`,`extractTime`,`postTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`issueId`,`Url`,`issueTitle`,`tag`,`issueDetail`,`scanerNum`,`attentionNum`,`commentNum`,`answerNum`,`extractTime`,`postTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{issueId},#{issueUrl},#{issueTitle},#{tag},#{issueDetail},#{scanerNum},#{attentionNum},#{commentNum},#{answerNum},#{extractTime},#{postTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(DeWenQ_Model deWenQ_Model);
 

src/main/java/net/trustie/dao/IteyeAsk_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface IteyeAsk_Dao {
 	@Insert("insert into iteye_ask"
-			+ "(`questionId`,`questionUrl`,`questionTitle`,`tag`,`categories`,`questionContent`,`voteUp`,`voteDown`,`questionScore`,`answerNum`,`interestNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`questionId`,`Url`,`questionTitle`,`tag`,`categories`,`questionContent`,`voteUp`,`voteDown`,`questionScore`,`answerNum`,`interestNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{questionUrl},#{questionTitle},#{tag},#{categories},#{questionContent},#{voteUp},#{voteDown},#{questionScore},#{answerNum},#{interestNum},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(IteyeAsk_Model ask_Model);
 }

src/main/java/net/trustie/dao/IteyeBlog_Dao.java

@@ -7,7 +7,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface IteyeBlog_Dao {
 	@Insert("insert into iteye_blog"
-			+ "(`blog_id`,`blog_url`,`blog_title`,`blog_tags`,`blog_type`,`blog_content`,`vote_up`,`vote_down`,`answer_num`,`view_num`,`post_time`,`extractTime`,`author_name`,`author_url`,`pageMD5`,`history`)"
+			+ "(`blog_id`,`Url`,`blog_title`,`blog_tags`,`blog_type`,`blog_content`,`vote_up`,`vote_down`,`answer_num`,`view_num`,`post_time`,`extractTime`,`author_name`,`author_url`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{questionUrl},#{questionTitle},#{tag},#{categories},#{questionContent},#{voteUp},#{voteDown},#{answerNum},#{view_num},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(IteyeBlog_Model blog_Model);
 }

src/main/java/net/trustie/dao/LinuxtoneDao.java

@@ -5,7 +5,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface LinuxtoneDao {
 	@Insert("insert into linuxtone"
-			+ "(`post_url`,`post_title`,`post_content`,`post_type`,`view_num`,`reply_num`,`vote_up`,`vote_down`,`favorited_num`,`shared_num`,`post_time`,`extractTime`,`crawled_time`,`author_name`,`author_url`,`history`)"
+			+ "(`Url`,`post_title`,`post_content`,`post_type`,`view_num`,`reply_num`,`vote_up`,`vote_down`,`favorited_num`,`shared_num`,`post_time`,`extractTime`,`crawled_time`,`author_name`,`author_url`,`history`)"
 			+ " values (#{postUrl},#{postTitle},#{postContent},#{postType},#{viewNum},#{replyNum},#{voteUp},#{voteDown},#{favoritedNum},#{sharedNum},#{postTime},#{extractTime},#{crawledTime},#{authorName},#{authorUrl},#{history})")
 	public int add(Linuxtone_Model linuxtone_Model);
 }

src/main/java/net/trustie/dao/Lupaworld_Dao.java

@@ -8,7 +8,7 @@ import org.apache.ibatis.annotations.Param;
 public interface Lupaworld_Dao {
 	
 	
-	@Insert("insert into lupaworld (id,post_id,post_url,post_title,post_tags,post_content,post_type,view_num,reply_num,vote_up,vote_down,crawled_time,extractTime,author_name,author_url,history,created_time) values (null,#{model.post_id},#{model.post_url},#{model.post_title},#{model.post_tags},#{model.post_content},#{model.post_type},#{model.view_num},#{model.reply_num},#{model.vote_up},#{model.vote_down},#{model.crawled_time},#{model.extractTime},#{model.author_name},#{model.author_url},#{model.history},#{model.created_time})")
+	@Insert("insert into lupaworld (id,post_id,Url,post_title,post_tags,post_content,post_type,view_num,reply_num,vote_up,vote_down,crawled_time,extractTime,author_name,author_url,history,created_time) values (null,#{model.post_id},#{model.post_url},#{model.post_title},#{model.post_tags},#{model.post_content},#{model.post_type},#{model.view_num},#{model.reply_num},#{model.vote_up},#{model.vote_down},#{model.crawled_time},#{model.extractTime},#{model.author_name},#{model.author_url},#{model.history},#{model.created_time})")
 	public void add(@Param("model") Lupaworld_Model model);
 
 }

src/main/java/net/trustie/dao/OSChinaQuestion_Dao.java

@@ -6,7 +6,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface OSChinaQuestion_Dao {
 	@Insert("insert into oschina_question"
-			+ "(`questionId`,`questionUrl`,`questionTitle`,`tag`,`questionType`,`questionContent`,`housedNum`,`laudNum`,`replyNum`,`viewNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`questionId`,`Url`,`questionTitle`,`tag`,`questionType`,`questionContent`,`housedNum`,`laudNum`,`replyNum`,`viewNum`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{questionUrl},#{questionTitle},#{tag},#{questionType},#{questionContent},#{housedNum},#{laudNum},#{replyNum},#{viewNum},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(OSChinaQuestion_Model chinaQuestion_Model);
 }

src/main/java/net/trustie/dao/PHPChina_Dao.java

@@ -7,7 +7,7 @@ import org.apache.ibatis.annotations.Insert;
 public interface PHPChina_Dao {
 
 	@Insert("insert into phpchina"
-			+ "(`post_id`,`post_url`,`post_title`,`post_content`,`view_num`,`reply_num`,`vote_up`,`vote_down`,`favorited_num`,`post_time`,`shared_num`,`extractTime`,`author_name`,`author_url`,`pageMD5`,`history`)"
+			+ "(`post_id`,`Url`,`post_title`,`post_content`,`view_num`,`reply_num`,`vote_up`,`vote_down`,`favorited_num`,`post_time`,`shared_num`,`extractTime`,`author_name`,`author_url`,`pageMD5`,`history`)"
 			+ " values (#{postId},#{postUrl},#{postTitle},#{postContent},#{viewNum},#{replyNum},#{voteUp},#{voteDown},#{favoritedNum},#{postTime},#{sharedNum},#{extractTime},#{postAuthor},#{postAuthorUrl},#{pageMD5},#{history})")
 	public int add(PHPChina_Model phpChina_Model);
 }

src/main/java/net/trustie/dao/SlashdotNews_Dao.java

@@ -4,7 +4,7 @@ import net.trustie.model.SlashdotNews_Model;
 import org.apache.ibatis.annotations.Insert;
 public interface SlashdotNews_Dao {
 	@Insert("insert into slashdot"
-			+ "(`post_id`,`post_url`,`post_title`,`post_content`,`author`,`author_url`,`reply_num`,`extractTime`,`history`,`post_time`)"
+			+ "(`post_id`,`Url`,`post_title`,`post_content`,`author`,`author_url`,`reply_num`,`extractTime`,`history`,`post_time`)"
 			+ "values (#{post_id},#{post_url},#{post_title},#{post_content},#{author},#{author_url},#{reply_num},#{extractTime},#{history},#{post_time})")
 	public int add(SlashdotNews_Model slashdotnewsModel);
 

src/main/java/net/trustie/dao/StackOverflow_Dao.java

@@ -5,7 +5,7 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface StackOverflow_Dao {	
 	@Insert("insert into stackoverflow_q"
-			+ "(`questionId`,`url`,`questionTitle`,`tag`,`questionContent`,`voteNum`,`likeNum`,`answerNum`,`viewNum`,`activeTime`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
+			+ "(`questionId`,`Url`,`questionTitle`,`tag`,`questionContent`,`voteNum`,`likeNum`,`answerNum`,`viewNum`,`activeTime`,`postTime`,`extractTime`,`author`,`authorUrl`,`pageMD5`,`history`)"
 			+ " values (#{questionId},#{url},#{questionTitle},#{tag},#{questionContent},#{voteNum},#{likeNum},#{answerNum},#{viewNum},#{activeTime},#{postTime},#{extractTime},#{author},#{authorUrl},#{pageMD5},#{history})")
 	public int add(StackOverflow_Model overflow_Model);
 }

src/main/java/net/trustie/dao/lagou_Dao.java

@@ -0,0 +1,12 @@
+package net.trustie.dao;
+
+import net.trustie.model.lagou_Model;
+import org.apache.ibatis.annotations.Insert;
+public interface lagou_Dao {
+	@Insert("insert into lagou"
+			+ "(`Url`,`post_time`,`post_title`,`post_tags`,`post_content`,`salary`,`experience`,`comp_name`,`comp_url`,`addr`,`comp_scale`,`comp_field`,`financing`,`post_id`,`extractTime`,`history`,`work_type`,`degree`)"
+			+ "values (#{post_url},#{post_time},#{post_title},#{post_tags},#{post_content},#{salary},#{experience},#{comp_name},#{comp_url},#{addr},#{comp_scale},#{comp_field},#{financing},#{post_id},#{extractTime},#{history},#{work_type},#{degree})")
+	public int add(lagou_Model lgModel);
+
+
+}

src/main/java/net/trustie/dao/neitui_Dao.java

@@ -5,8 +5,8 @@ import org.apache.ibatis.annotations.Insert;
 
 public interface neitui_Dao {
 	@Insert("insert into neitui"
-			+ "(`author_name`,`author_url`,`post_url`,`post_time`,`post_title`,`tags`,`post_content`,`salary`,`experience`,`comp_name`,`comp_url`,`addr`,`comp_scale`,`comp_field`,`financing`,`comp_vision`,`extractTime`,`history`)"
-			+ "values (#{author_name},#{author_url},#{post_url},#{post_time},#{post_title},#{tags},#{post_content},#{salary},#{experience},#{comp_name},#{comp_url},#{addr},#{comp_scale},#{comp_field},#{financing},#{comp_vision},#{extractTime},#{history})")
+			+ "(`author_name`,`author_url`,`Url`,`post_time`,`post_title`,`post_tags`,`post_content`,`salary`,`experience`,`comp_name`,`comp_url`,`addr`,`comp_scale`,`comp_field`,`financing`,`comp_vision`,`extractTime`,`history`)"
+			+ "values (#{author_name},#{author_url},#{post_url},#{post_time},#{post_title},#{post_tags},#{post_content},#{salary},#{experience},#{comp_name},#{comp_url},#{addr},#{comp_scale},#{comp_field},#{financing},#{comp_vision},#{extractTime},#{history})")
 	public int add(neitui_Model ntModel);
 
 }

src/main/java/net/trustie/extractor/lagou_Extractor.java

@@ -0,0 +1,56 @@
+package net.trustie.extractor;
+
+import java.sql.SQLException;
+
+import net.trustie.downloader.DataBasePageErrorOutPut;
+import net.trustie.downloader.GenerateRawPage;
+import net.trustie.model.lagou_Model;
+import net.trustie.model.neitui_Model;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.stereotype.Component;
+
+import core.PageModelPipeline;
+import core.Site;
+import extension.OsseanExtractor;
+
+@Component
+public class lagou_Extractor {
+	@SuppressWarnings("rawtypes")
+	@Qualifier("lagouPipeline")
+	@Autowired
+	private PageModelPipeline modelPipeline;
+
+	@Qualifier("generateRawPage")
+	@Autowired
+	private GenerateRawPage generateRawPage;
+
+	@Qualifier("errorPageToDB")
+	@Autowired
+	private DataBasePageErrorOutPut dbPageErrorOutPut;
+
+	public void begin() {
+		generateRawPage.setTable("lagou_html_detail");
+		dbPageErrorOutPut.setTableName("lagou_error_page");
+
+		OsseanExtractor
+				.create(Site.me().setResultNum(100), modelPipeline,
+						lagou_Model.class).setUUID("lagou")
+				.setDownloader(generateRawPage)
+				.setPageErrorOutPut(dbPageErrorOutPut).start();
+	}
+
+	public static void main(String[] args) throws SQLException {
+		ApplicationContext aContext = new ClassPathXmlApplicationContext(
+				"classpath:/spring/applicationContext*.xml");
+
+		final lagou_Extractor extractor = aContext
+				.getBean(lagou_Extractor.class);
+
+		extractor.begin();
+	}
+
+}

src/main/java/net/trustie/model/lagou_Model.java

@@ -0,0 +1,234 @@
+package net.trustie.model;
+
+import java.util.Date;
+import net.trustie.utils.DateHandler;
+import net.trustie.utils.StringHandler;
+import us.codecraft.webmagic.model.annotation.ExtractBy;
+import core.AfterExtractor;
+import core.Page;
+import core.ValidateExtractor;
+
+public class lagou_Model implements AfterExtractor, ValidateExtractor {
+
+	private String post_url = "";
+	private String post_id = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/div/text()")
+	private String post_time = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dt/h1/allText()")
+	private String post_title = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/text()")
+	private String post_tags = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[2]/allText()")
+	private String post_content = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/span[5]/text()")
+	public String work_type = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/span[4]/text()")
+	private String degree = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/span[1]/text()")
+	private String salary = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[1]/dl/dd[1]/span[3]/text()")
+	private String experience = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dt/a/img/@alt")
+	private String comp_name = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dd/ul[1]/li[3]/a/text()")
+	private String comp_url = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dd/div[1]/text()")
+	private String addr = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dd/ul[1]/li[2]/allText()")
+	private String comp_scale = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dd/ul[1]/li[1]/allText()")
+	private String comp_field = "";
+	@ExtractBy("//*[@id='container']/div[1]/div[2]/dl/dd/ul[2]/li/allText()")
+	private String financing = "";
+	private String extractTime = "";
+	private int history = 0;
+
+	public void afterProcess(Page page) {
+		// 处理page_url
+		this.post_url = page.getPageUrl();
+		// 处理post_id
+		this.post_id = StringHandler
+				.matchRightString(page.getPageUrl(), "\\d+");
+		// 处理post_tags
+		this.post_tags = "<" + post_tags.replaceAll("职位诱惑 : ", "").trim() + ">";
+		// 处理extractTime
+		this.extractTime = DateHandler.getExtractTime();
+		// 处理post_content
+		this.post_content = post_content.replaceAll("职位描述", "");
+		// 处理comp_scale
+		this.comp_scale = comp_scale.replaceAll("规模 ", "");
+		// 处理comp_field
+		this.comp_field = comp_field.replaceAll("领域 ", "");
+		// 处理financing
+		this.financing = financing.replaceAll("目前阶段 ", "");
+		// 处理post_time
+		this.post_time = post_time.replaceAll("发布时间：", "").replaceAll("发布", "");
+		Date date = new Date();
+
+		if (StringHandler.canMatchRightString(post_time, "\\d+:\\d+"))
+			this.post_time = extractTime.replaceAll("\\d+:\\d+:\\d+",
+					this.post_time + ":00");
+		/*
+		 * else if(StringHandler.canMatchRightString(post_time, "\\d+\\S+"))
+		 * this.post_time=DateHandler.handlerDefaultDate(post_time, date);
+		 */
+		else
+			this.post_time = DateHandler.formatAllTypeDate(post_time, date);
+	}
+
+	public void validate(Page page) {
+		if (StringHandler.isAtLeastOneBlank(this.post_url, this.post_title,
+				this.extractTime)) {
+			page.setResultSkip(this, true);
+			return;
+		}
+		if (!DateHandler.canFormatToDate(this.post_time, this.extractTime)) {
+			page.setResultSkip(this, true);
+		}	
+	}
+
+	public String getPost_id() {
+		return post_id;
+	}
+
+	public void setPost_id(String post_id) {
+		this.post_id = post_id;
+	}
+
+	public String getPost_url() {
+		return post_url;
+	}
+
+	public void setPost_url(String post_url) {
+		this.post_url = post_url;
+	}
+
+	public String getPost_time() {
+		return post_time;
+	}
+
+	public void setPost_time(String post_time) {
+		this.post_time = post_time;
+	}
+
+	public String getPost_title() {
+		return post_title;
+	}
+
+	public void setPost_title(String post_title) {
+		this.post_title = post_title;
+	}
+
+	public String getPost_tags() {
+		return post_tags;
+	}
+
+	public void setPost_tags(String post_tags) {
+		this.post_tags = post_tags;
+	}
+
+	public String getPost_content() {
+		return post_content;
+	}
+
+	public void setPost_content(String post_content) {
+		this.post_content = post_content;
+	}
+
+	public String getWork_type() {
+		return work_type;
+	}
+
+	public void setWork_type(String work_type) {
+		this.work_type = work_type;
+	}
+
+	public String getDegree() {
+		return degree;
+	}
+
+	public void setDegree(String degree) {
+		this.degree = degree;
+	}
+
+	public String getSalary() {
+		return salary;
+	}
+
+	public void setSalary(String salary) {
+		this.salary = salary;
+	}
+
+	public String getExperience() {
+		return experience;
+	}
+
+	public void setExperience(String experience) {
+		this.experience = experience;
+	}
+
+	public String getComp_name() {
+		return comp_name;
+	}
+
+	public void setComp_name(String comp_name) {
+		this.comp_name = comp_name;
+	}
+
+	public String getComp_url() {
+		return comp_url;
+	}
+
+	public void setComp_url(String comp_url) {
+		this.comp_url = comp_url;
+	}
+
+	public String getAddr() {
+		return addr;
+	}
+
+	public void setAddr(String addr) {
+		this.addr = addr;
+	}
+
+	public String getComp_scale() {
+		return comp_scale;
+	}
+
+	public void setComp_scale(String comp_scale) {
+		this.comp_scale = comp_scale;
+	}
+
+	public String getComp_field() {
+		return comp_field;
+	}
+
+	public void setComp_field(String comp_field) {
+		this.comp_field = comp_field;
+	}
+
+	public String getFinancing() {
+		return financing;
+	}
+
+	public void setFinancing(String financing) {
+		this.financing = financing;
+	}
+
+	public String getExtractTime() {
+		return extractTime;
+	}
+
+	public void setExtractTime(String extractTime) {
+		this.extractTime = extractTime;
+	}
+
+	public int getHistory() {
+		return history;
+	}
+
+	public void setHistory(int history) {
+		this.history = history;
+	}
+
+}

src/main/java/net/trustie/pipeline/lagou_pipeline.java

@@ -0,0 +1,17 @@
+package net.trustie.pipeline;
+import javax.annotation.Resource;
+import org.springframework.stereotype.Component;
+import us.codecraft.webmagic.Task;
+import net.trustie.dao.lagou_Dao;
+import net.trustie.model.lagou_Model;
+import core.PageModelPipeline;
+@Component("lagouPipeline")
+public class lagou_pipeline implements PageModelPipeline<lagou_Model>{
+	
+	@Resource
+	private lagou_Dao lg_Dao;
+
+	public void process(lagou_Model lg_Model, Task arg1) {		
+		lg_Dao.add(lg_Model);
+}
+}

src/main/java/net/trustie/utils/DateHandler.java

@@ -114,7 +114,7 @@ public class DateHandler {
 	 * @param string
 	 * @return
 	 */
-	private static String handlerDefaultDate(String string, Date date) {
+	public static String handlerDefaultDate(String string, Date date) {
 		if (StringHandler.canMatchRightStrings(string, "\\d+", "年", "前")
 				|| StringHandler.canMatchRightStrings(string, "\\d+", "year")) {
 			int years = Integer.parseInt(StringHandler.matchRightString(string,