From 401e4a9d5d6838422886fe754628c4decc36f1db Mon Sep 17 00:00:00 2001 From: rails -v Date: Wed, 26 Aug 2015 13:40:20 +0800 Subject: [PATCH] multiple thread mode --- .classpath | 11 +- .idea/.name | 1 + .idea/compiler.xml | 32 ++ .idea/copyright/profiles_settings.xml | 3 + .idea/encodings.xml | 6 + .../Maven__aopalliance_aopalliance_1_0.xml | 13 + .../Maven__com_alibaba_fastjson_1_1_37.xml | 13 + .../Maven__com_google_guava_guava_15_0.xml | 13 + ...n__com_jayway_jsonpath_json_path_0_8_1.xml | 13 + ...Maven__commons_codec_commons_codec_1_6.xml | 13 + ..._collections_commons_collections_3_2_1.xml | 13 + .../Maven__commons_dbcp_commons_dbcp_1_4.xml | 13 + .../Maven__commons_io_commons_io_1_3_2.xml | 13 + .../Maven__commons_lang_commons_lang_2_6.xml | 13 + ..._commons_logging_commons_logging_1_1_3.xml | 13 + ...Maven__commons_pool_commons_pool_1_5_4.xml | 13 + .../libraries/Maven__junit_junit_dep_4_10.xml | 13 + .idea/libraries/Maven__log4j_log4j_1_2_17.xml | 13 + ...ven__mysql_mysql_connector_java_5_1_18.xml | 13 + .../Maven__net_minidev_json_smart_1_1_1.xml | 13 + ...__org_apache_commons_commons_lang3_3_1.xml | 13 + ...apache_httpcomponents_httpclient_4_3_3.xml | 13 + ...g_apache_httpcomponents_httpcore_4_3_2.xml | 13 + .../Maven__org_assertj_assertj_core_1_5_0.xml | 13 + .../Maven__org_hamcrest_hamcrest_core_1_1.xml | 13 + .../Maven__org_jsoup_jsoup_1_7_2.xml | 13 + .../Maven__org_mybatis_mybatis_3_1_1.xml | 13 + ...aven__org_mybatis_mybatis_spring_1_1_1.xml | 13 + .../Maven__org_slf4j_slf4j_api_1_7_6.xml | 13 + .../Maven__org_slf4j_slf4j_log4j12_1_7_6.xml | 13 + ...ringframework_spring_aop_3_1_1_RELEASE.xml | 13 + ...ringframework_spring_asm_3_1_1_RELEASE.xml | 13 + ...ngframework_spring_beans_3_1_1_RELEASE.xml | 13 + ...framework_spring_context_3_1_1_RELEASE.xml | 13 + ...ingframework_spring_core_3_1_1_RELEASE.xml | 13 + ...mework_spring_expression_3_1_1_RELEASE.xml | 13 + ...ingframework_spring_jdbc_3_1_1_RELEASE.xml | 13 + ...pringframework_spring_tx_3_1_1_RELEASE.xml | 13 + .../Maven__redis_clients_jedis_2_0_0.xml | 13 + ...aven__us_codecraft_webmagic_core_0_5_2.xml | 13 + ..._us_codecraft_webmagic_extension_0_5_2.xml | 13 + .../Maven__us_codecraft_xsoup_0_2_4.xml | 13 + .idea/misc.xml | 42 ++ .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 400 ++++++++++++++++++ config/configuration.txt | 25 ++ osseanextractor.iml | 52 +++ pom.xml | 8 +- resources/applicationContext-myBatis.xml | 12 +- src/main/java/core/ExtractorBegin.java | 8 + src/main/java/extension/OneExtractor.java | 243 +++++++++++ src/main/java/extension/OsseanExtractor.java | 1 + .../comment/model/CsdnAsk_Comment_Model.java | 9 + .../java/net/trustie/downloader/PageDao.java | 3 + .../extractor/CNblogNews_Extractor.java | 3 +- .../extractor/CNblogQSolve_Extractor.java | 58 ++- .../trustie/extractor/CsdnAsk_Extractor.java | 1 + .../java/net/trustie/one/Configuration.java | 50 +++ src/main/java/net/trustie/one/MyThread.java | 36 ++ src/main/java/net/trustie/one/OneEx.java | 86 ++++ 61 files changed, 1566 insertions(+), 19 deletions(-) create mode 100644 .idea/.name create mode 100644 .idea/compiler.xml create mode 100644 .idea/copyright/profiles_settings.xml create mode 100644 .idea/encodings.xml create mode 100644 .idea/libraries/Maven__aopalliance_aopalliance_1_0.xml create mode 100644 .idea/libraries/Maven__com_alibaba_fastjson_1_1_37.xml create mode 100644 .idea/libraries/Maven__com_google_guava_guava_15_0.xml create mode 100644 .idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml create mode 100644 .idea/libraries/Maven__commons_codec_commons_codec_1_6.xml create mode 100644 .idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml create mode 100644 .idea/libraries/Maven__commons_dbcp_commons_dbcp_1_4.xml create mode 100644 .idea/libraries/Maven__commons_io_commons_io_1_3_2.xml create mode 100644 .idea/libraries/Maven__commons_lang_commons_lang_2_6.xml create mode 100644 .idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml create mode 100644 .idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml create mode 100644 .idea/libraries/Maven__junit_junit_dep_4_10.xml create mode 100644 .idea/libraries/Maven__log4j_log4j_1_2_17.xml create mode 100644 .idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml create mode 100644 .idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml create mode 100644 .idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml create mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_3.xml create mode 100644 .idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml create mode 100644 .idea/libraries/Maven__org_assertj_assertj_core_1_5_0.xml create mode 100644 .idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml create mode 100644 .idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml create mode 100644 .idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml create mode 100644 .idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_api_1_7_6.xml create mode 100644 .idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_6.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml create mode 100644 .idea/libraries/Maven__redis_clients_jedis_2_0_0.xml create mode 100644 .idea/libraries/Maven__us_codecraft_webmagic_core_0_5_2.xml create mode 100644 .idea/libraries/Maven__us_codecraft_webmagic_extension_0_5_2.xml create mode 100644 .idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 config/configuration.txt create mode 100644 osseanextractor.iml create mode 100644 src/main/java/core/ExtractorBegin.java create mode 100644 src/main/java/extension/OneExtractor.java create mode 100644 src/main/java/net/trustie/one/Configuration.java create mode 100644 src/main/java/net/trustie/one/MyThread.java create mode 100644 src/main/java/net/trustie/one/OneEx.java diff --git a/.classpath b/.classpath index eebbbee..662675b 100644 --- a/.classpath +++ b/.classpath @@ -19,6 +19,15 @@ - + + + + + + + + + + diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..ccfbe3f --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +osseanextractor \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml new file mode 100644 index 0000000..2e68044 --- /dev/null +++ b/.idea/compiler.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/copyright/profiles_settings.xml b/.idea/copyright/profiles_settings.xml new file mode 100644 index 0000000..e7bedf3 --- /dev/null +++ b/.idea/copyright/profiles_settings.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..b26911b --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml b/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml new file mode 100644 index 0000000..37e92eb --- /dev/null +++ b/.idea/libraries/Maven__aopalliance_aopalliance_1_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_alibaba_fastjson_1_1_37.xml b/.idea/libraries/Maven__com_alibaba_fastjson_1_1_37.xml new file mode 100644 index 0000000..4d58f91 --- /dev/null +++ b/.idea/libraries/Maven__com_alibaba_fastjson_1_1_37.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_google_guava_guava_15_0.xml b/.idea/libraries/Maven__com_google_guava_guava_15_0.xml new file mode 100644 index 0000000..53d491b --- /dev/null +++ b/.idea/libraries/Maven__com_google_guava_guava_15_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml b/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml new file mode 100644 index 0000000..bede375 --- /dev/null +++ b/.idea/libraries/Maven__com_jayway_jsonpath_json_path_0_8_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml b/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml new file mode 100644 index 0000000..3c99fc5 --- /dev/null +++ b/.idea/libraries/Maven__commons_codec_commons_codec_1_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml new file mode 100644 index 0000000..ba838cd --- /dev/null +++ b/.idea/libraries/Maven__commons_collections_commons_collections_3_2_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_4.xml b/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_4.xml new file mode 100644 index 0000000..43a884d --- /dev/null +++ b/.idea/libraries/Maven__commons_dbcp_commons_dbcp_1_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml b/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml new file mode 100644 index 0000000..ec9d11d --- /dev/null +++ b/.idea/libraries/Maven__commons_io_commons_io_1_3_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml new file mode 100644 index 0000000..efdbb31 --- /dev/null +++ b/.idea/libraries/Maven__commons_lang_commons_lang_2_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml b/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml new file mode 100644 index 0000000..b4f6654 --- /dev/null +++ b/.idea/libraries/Maven__commons_logging_commons_logging_1_1_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml b/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml new file mode 100644 index 0000000..32f2237 --- /dev/null +++ b/.idea/libraries/Maven__commons_pool_commons_pool_1_5_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__junit_junit_dep_4_10.xml b/.idea/libraries/Maven__junit_junit_dep_4_10.xml new file mode 100644 index 0000000..acf9279 --- /dev/null +++ b/.idea/libraries/Maven__junit_junit_dep_4_10.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__log4j_log4j_1_2_17.xml b/.idea/libraries/Maven__log4j_log4j_1_2_17.xml new file mode 100644 index 0000000..abf1ff9 --- /dev/null +++ b/.idea/libraries/Maven__log4j_log4j_1_2_17.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml b/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml new file mode 100644 index 0000000..312ed7e --- /dev/null +++ b/.idea/libraries/Maven__mysql_mysql_connector_java_5_1_18.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml b/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml new file mode 100644 index 0000000..a272cd1 --- /dev/null +++ b/.idea/libraries/Maven__net_minidev_json_smart_1_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml new file mode 100644 index 0000000..6a7f858 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_3.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_3.xml new file mode 100644 index 0000000..d558486 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpclient_4_3_3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml new file mode 100644 index 0000000..795f5f2 --- /dev/null +++ b/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_3_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_assertj_assertj_core_1_5_0.xml b/.idea/libraries/Maven__org_assertj_assertj_core_1_5_0.xml new file mode 100644 index 0000000..36c715f --- /dev/null +++ b/.idea/libraries/Maven__org_assertj_assertj_core_1_5_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml new file mode 100644 index 0000000..82c9dff --- /dev/null +++ b/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml b/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml new file mode 100644 index 0000000..9a4c0fa --- /dev/null +++ b/.idea/libraries/Maven__org_jsoup_jsoup_1_7_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml b/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml new file mode 100644 index 0000000..cc0e646 --- /dev/null +++ b/.idea/libraries/Maven__org_mybatis_mybatis_3_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml b/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml new file mode 100644 index 0000000..514b084 --- /dev/null +++ b/.idea/libraries/Maven__org_mybatis_mybatis_spring_1_1_1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_6.xml b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_6.xml new file mode 100644 index 0000000..b5119bd --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_api_1_7_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_6.xml b/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_6.xml new file mode 100644 index 0000000..5d7cc88 --- /dev/null +++ b/.idea/libraries/Maven__org_slf4j_slf4j_log4j12_1_7_6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml new file mode 100644 index 0000000..775411e --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_aop_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml new file mode 100644 index 0000000..8f37f12 --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_asm_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml new file mode 100644 index 0000000..c36490b --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_beans_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml new file mode 100644 index 0000000..ffe3ef5 --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_context_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml new file mode 100644 index 0000000..3f8bcd7 --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_core_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml new file mode 100644 index 0000000..8728e57 --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_expression_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml new file mode 100644 index 0000000..fb8aabd --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_jdbc_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml b/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml new file mode 100644 index 0000000..ac31ecd --- /dev/null +++ b/.idea/libraries/Maven__org_springframework_spring_tx_3_1_1_RELEASE.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__redis_clients_jedis_2_0_0.xml b/.idea/libraries/Maven__redis_clients_jedis_2_0_0.xml new file mode 100644 index 0000000..9372119 --- /dev/null +++ b/.idea/libraries/Maven__redis_clients_jedis_2_0_0.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__us_codecraft_webmagic_core_0_5_2.xml b/.idea/libraries/Maven__us_codecraft_webmagic_core_0_5_2.xml new file mode 100644 index 0000000..6606ac5 --- /dev/null +++ b/.idea/libraries/Maven__us_codecraft_webmagic_core_0_5_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__us_codecraft_webmagic_extension_0_5_2.xml b/.idea/libraries/Maven__us_codecraft_webmagic_extension_0_5_2.xml new file mode 100644 index 0000000..7c3bc1c --- /dev/null +++ b/.idea/libraries/Maven__us_codecraft_webmagic_extension_0_5_2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml b/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml new file mode 100644 index 0000000..26619d2 --- /dev/null +++ b/.idea/libraries/Maven__us_codecraft_xsoup_0_2_4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..dfca8c4 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + 1.8 + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..dad1015 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..469269b --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,400 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + 1432444136539 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/configuration.txt b/config/configuration.txt new file mode 100644 index 0000000..82b0934 --- /dev/null +++ b/config/configuration.txt @@ -0,0 +1,25 @@ +51cto_blog 51cto_blog_html_detail +apache apache_html_detail +cnblog_q_solve cnblogs_q_solved_html_detail +cnblog_q_unsolve cnblogs_q_unsolved_html_detail +codeproject codeproject_html_detail +csdn_ask csdn_ask_html_detail +csdn_blogs csdn_blog_html_detail +csdn_topic csdn_topic_html_detail +deWen_question dewen_question_html_detail +freecode freecode_html_detail +gna gna_html_detail +iteye_ask iteye_ask_html_detail +iteye_Blog iteye_blog_html_detail +lagou lagou_html_detail +linuxtone linuxtone_html_detail +lupaworld lupaworld_html_detail +neitui neitui_html_detail +OpenHubProject openhub_html_detail +oschina_project oschina_project_html_detail +oschina_question oschina_question_html_detail +phpChina_post phpchina_html_detail +SFProject softpedia_html_detail +slashdot_news slashdot_html_detail +softpedia softpedia_html_detail +stackoverflow_q stackoverflow_html_detail diff --git a/osseanextractor.iml b/osseanextractor.iml new file mode 100644 index 0000000..1a24369 --- /dev/null +++ b/osseanextractor.iml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9052e54..dd0db06 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ UTF-8 - + @@ -77,7 +77,11 @@ mysql-connector-java 5.1.18 - + + commons-io + commons-io + 2.4 + diff --git a/resources/applicationContext-myBatis.xml b/resources/applicationContext-myBatis.xml index 6942397..1dc9bba 100644 --- a/resources/applicationContext-myBatis.xml +++ b/resources/applicationContext-myBatis.xml @@ -8,9 +8,9 @@ destroy-method="close"> - - + value="jdbc:mysql://localhost:3306/pages?characterEncoding=UTF-8" /> + + @@ -25,9 +25,9 @@ destroy-method="close"> - - + value="jdbc:mysql://localhost:3306/extract_result?characterEncoding=UTF-8" /> + + diff --git a/src/main/java/core/ExtractorBegin.java b/src/main/java/core/ExtractorBegin.java new file mode 100644 index 0000000..af5d1db --- /dev/null +++ b/src/main/java/core/ExtractorBegin.java @@ -0,0 +1,8 @@ +package core; + +public interface ExtractorBegin { + + public void threadRun(); + + public boolean check(); +} diff --git a/src/main/java/extension/OneExtractor.java b/src/main/java/extension/OneExtractor.java new file mode 100644 index 0000000..eb06f7f --- /dev/null +++ b/src/main/java/extension/OneExtractor.java @@ -0,0 +1,243 @@ +package extension; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Timer; + +import javax.annotation.Resource; + +import net.trustie.downloader.PageDao; +import net.trustie.one.Configuration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import core.ConsolePipeline; +import core.ModelPageProcessor; +import core.ModelPipeline; +import core.PageModelPipeline; +import core.Pipeline; +import core.Site; + +public class OneExtractor { + private Site site; + + protected Logger logger = LoggerFactory.getLogger(getClass()); + + private String uuid; + + private PageDownloader downloader; + + private Timer timer; + + private ReadCursor readCursor; + + private ModelPipeline modelPipeline; + + protected List pipelines = new ArrayList(); + + private ModelPageProcessor modelPageProcessor; + + private String cursorPath; + + public String defaultPath = "./cursor/"; + + public List modelName = new ArrayList(); + + private PageErrorOutPut pageErrorOutPut; + + private int recordId = 0; + + /** + * add by houxiang + */ + private Configuration config = new Configuration(); + + @Resource + private PageDao pagedao; + + private Map tables = null ; + + protected OneExtractor(ModelPageProcessor modelPageProcessor) { + this.modelPageProcessor = modelPageProcessor; + } + + public OneExtractor(Site site, PageModelPipeline pageModelPipeline, + Class... pageModels) { + this(ModelPageProcessor.create(pageModels)); + this.site = site; + this.modelPipeline = new ModelPipeline(); + this.addPipeline(modelPipeline); + for (Class pageModel : pageModels) { + modelName.add(pageModel.getCanonicalName()); + if (pageModelPipeline != null) { + this.modelPipeline.put(pageModel, pageModelPipeline); + } + } + } + + public static OneExtractor create(Site site, + PageModelPipeline modelPipeline, Class... pageModels) { + return new OneExtractor(site, modelPipeline, pageModels); + } + + public OneExtractor setDownloader(PageDownloader downloader) { + this.downloader = downloader; + return this; + } + + public OneExtractor setUUID(String uuid) { + this.uuid = uuid; + return this; + } + + public OneExtractor setPageErrorOutPut(PageErrorOutPut errorOutPut) { + this.pageErrorOutPut = errorOutPut; + return this; + } + + public OneExtractor setCursorPath(String cursorPath) { + this.cursorPath = cursorPath; + return this; + } + + public OneExtractor addPipeline(Pipeline pipeline) { + this.pipelines.add(pipeline); + return this; + } + + public OneExtractor setPipelines(List pipelines) { + this.pipelines = pipelines; + return this; + } + + public OneExtractor clearPipeline() { + pipelines = new ArrayList(); + return this; + } + + + public void run() { + logger.info("One Extraction Task Started!"); + + if (readCursor.getValue(uuid) != null) + recordId = Integer.parseInt(readCursor.getValue(uuid).trim()); + else + recordId = downloader.getMinId(); + + List rawList = this.downloader.downloadPages(recordId, + this.site.getResultNum()); + + createPageList(rawList); + for (RawPage rawPage : rawList) { + if (rawPage.getPage() != null + && !rawPage.getPage().getResultItems().isSkip()) { + try { + modelPageProcessor.process(rawPage.getPage()); + rawPage.setExtracted(true); + + if (!rawPage.getPage().getResultItems().isSkip()) { + + for (Pipeline pipeline : pipelines) + pipeline.process( + rawPage.getPage().getResultItems(), null); + + if (!rawPage.getPage() + .isAllResultSkip( + modelName.toArray(new String[modelName + .size()]))) { + logger.info(rawPage.toString() + + " Extracted And Stored Successed!"); + rawPage.setStored(true); + } + } + if (!rawPage.isExtracted() || !rawPage.isStored()) + pageErrorOutPut.returnErrorPage(rawPage, + "May Caused By Model Problem! Or Page Error!"); + + } catch (Exception e) { + pageErrorOutPut.returnErrorPage(rawPage, e); + } + } + } + +// if(rawList.size()==this.site.getResultNum()){ +// recordId = this.downloader.getBeginId(recordId, +// this.site.getResultNum()); +// readCursor.setValue(uuid, String.valueOf(recordId)); +// }else{ +// +// } + + if (rawList.size() > 0) { + // 每次在所选取的最大的limit的表中找到最大的id号 + int num = rawList.size()-1; + recordId = rawList.get(num).getId()+1; +// recordId = this.downloader.getBeginId(recordId, +// this.site.getResultNum())+1; + readCursor.setValue(uuid, String.valueOf(recordId)); + } + + } + +// public void start() { +// init(); +// logger.info(this.uuid + " Extractor Started!"); +// timer.schedule(this, site.getFirstTime(), site.getPeriod()); +// } + + public void start(){ + int recordId ; + //int tableId = pagedao.getMaxId(tables.get(uuid)); + int tableId = 24; + init(); + + if(readCursor.getValue(uuid) != null){ + recordId = Integer.parseInt(readCursor.getValue(uuid).trim()); + }else{ + recordId = pagedao.getMinId(tables.get(uuid)); + } + + while(recordId < tableId){ + recordId = Integer.parseInt(readCursor.getValue(uuid).trim()); + System.out.println(recordId); + logger.info(this.uuid + " Extractor Started!"); + this.run(); + } + } + + protected void init() { + if (downloader == null) { + logger.error("No Page Source!"); + System.exit(0); + } + if (timer == null) { + timer = new Timer(); + } + if (cursorPath == null) { + this.cursorPath = defaultPath + uuid + ".txt"; + } + if (readCursor == null) { + readCursor = new ReadCursor(cursorPath); + } + + if (pipelines.isEmpty()) { + pipelines.add((Pipeline) new ConsolePipeline()); + } + + if (pageErrorOutPut == null) { + pageErrorOutPut = new DefaultPageErrorOutPut(); + } + + if (tables == null){ + tables = Configuration.tables; + } + } + + protected void createPageList(List rawPages) { + for (RawPage rawPage : rawPages) { + rawPage.generatPage(); + } + } +} diff --git a/src/main/java/extension/OsseanExtractor.java b/src/main/java/extension/OsseanExtractor.java index 6dc1f69..660aff6 100644 --- a/src/main/java/extension/OsseanExtractor.java +++ b/src/main/java/extension/OsseanExtractor.java @@ -41,6 +41,7 @@ public class OsseanExtractor extends TimerTask { private PageErrorOutPut pageErrorOutPut; private int recordId = 0; + protected OsseanExtractor(ModelPageProcessor modelPageProcessor) { this.modelPageProcessor = modelPageProcessor; diff --git a/src/main/java/net/trustie/comment/model/CsdnAsk_Comment_Model.java b/src/main/java/net/trustie/comment/model/CsdnAsk_Comment_Model.java index 70c6cdc..8cbf6fc 100644 --- a/src/main/java/net/trustie/comment/model/CsdnAsk_Comment_Model.java +++ b/src/main/java/net/trustie/comment/model/CsdnAsk_Comment_Model.java @@ -1,5 +1,14 @@ package net.trustie.comment.model; +import net.trustie.utils.StringHandler; + public class CsdnAsk_Comment_Model { + + public static void main(String[] args) { + String str = "<]ljhjihid89898fsdf>dfsfs"; + String regix = "<.[^>]+>"; + String ans = StringHandler.matchRightString(str,regix); + System.out.println(ans); + } } diff --git a/src/main/java/net/trustie/downloader/PageDao.java b/src/main/java/net/trustie/downloader/PageDao.java index 0766efd..2fcdfeb 100644 --- a/src/main/java/net/trustie/downloader/PageDao.java +++ b/src/main/java/net/trustie/downloader/PageDao.java @@ -17,6 +17,9 @@ public interface PageDao { @Select("select Min(id) from ${table_name}") public int getMinId(@Param("table_name") String table_name); + @Select("select Max(id) from ${table_name}") + public int getMaxId(@Param("table_name") String table_name); + // @Select("select MAX(id) FROM ${table_name} where id in (SELECT t.id FROM (SELECT id FROM ${table_name} WHERE id >${begin_id} limit #{interval}) AS t);") // public int getMaxBeginId(@Param("table_name")String table_name,@Param("begin_id") int begin_id, @Param("interval") int interval); diff --git a/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java b/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java index 5017fc2..f2004f8 100644 --- a/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java +++ b/src/main/java/net/trustie/extractor/CNblogNews_Extractor.java @@ -5,6 +5,7 @@ import java.sql.SQLException; import net.trustie.downloader.DataBasePageErrorOutPut; import net.trustie.downloader.GenerateRawPage; import net.trustie.model.CNblogsNews_Model; +import net.trustie.one.Configuration; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -48,7 +49,7 @@ public class CNblogNews_Extractor { ApplicationContext aContext = new ClassPathXmlApplicationContext( "classpath:/spring/applicationContext*.xml"); - final CNblogNews_Extractor extractor = aContext + final CNblogNews_Extractor extractor = Configuration.aContext .getBean(CNblogNews_Extractor.class); extractor.begin(); diff --git a/src/main/java/net/trustie/extractor/CNblogQSolve_Extractor.java b/src/main/java/net/trustie/extractor/CNblogQSolve_Extractor.java index f7af031..d1456f7 100644 --- a/src/main/java/net/trustie/extractor/CNblogQSolve_Extractor.java +++ b/src/main/java/net/trustie/extractor/CNblogQSolve_Extractor.java @@ -8,6 +8,7 @@ import net.trustie.downloader.DataBasePageErrorOutPut; import net.trustie.downloader.GenerateRawPage; import net.trustie.downloader.PageDao; import net.trustie.model.CNblogsQ_Model; +import net.trustie.one.Configuration; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,12 +16,18 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.stereotype.Component; +import core.ExtractorBegin; import core.PageModelPipeline; import core.Site; +import extension.OneExtractor; import extension.OsseanExtractor; +import extension.ReadCursor; @Component -public class CNblogQSolve_Extractor { +public class CNblogQSolve_Extractor implements ExtractorBegin { + + private ReadCursor readCursor = new ReadCursor("./cursor/cnblog_q_solve.txt"); + @SuppressWarnings("rawtypes") @Qualifier("cnblogsQPipeline") @Autowired @@ -49,15 +56,48 @@ public class CNblogQSolve_Extractor { .setPageErrorOutPut(dbPageErrorOutPut).start(); } - public static void main(String[] args) throws SQLException { - ApplicationContext aContext = new ClassPathXmlApplicationContext( - "classpath:/spring/applicationContext*.xml"); + /** + * 为多线程任务调用准备 + */ + + @Override + public void threadRun() { + // TODO Auto-generated method stub + generateRawPage.setTable("cnblogs_q_solved_html_detail"); + dbPageErrorOutPut.setTableName("cnblogs_q_solved_error_page"); - final CNblogQSolve_Extractor extractor = aContext - .getBean(CNblogQSolve_Extractor.class); - - extractor.begin(); - + OneExtractor + .create(Site.me().setResultNum(100), modelPipeline, + CNblogsQ_Model.class).setUUID("cnblog_q_solve") + .setDownloader(generateRawPage) + .setPageErrorOutPut(dbPageErrorOutPut).start(); } + @Override + public boolean check() { + // TODO Auto-generated method stub + int cursor = Integer.parseInt(readCursor.getValue("cnblog_q_solve")); + int maxId = pagedao.getMaxId("cnblogs_q_solved_html_detail"); + + if(cursor >= maxId){ + return true ; + } + return false; + } + + + + public static void main(String[] args) throws SQLException { +// ApplicationContext aContext = new ClassPathXmlApplicationContext( +// "classpath:/spring/applicationContext*.xml"); +// +// final CNblogQSolve_Extractor extractor = aContext +// .getBean(CNblogQSolve_Extractor.class); + + Configuration config = new Configuration(); + + CNblogQSolve_Extractor extractor = Configuration.aContext.getBean(CNblogQSolve_Extractor.class); + //extractor.begin(); + extractor.threadRun(); + } } diff --git a/src/main/java/net/trustie/extractor/CsdnAsk_Extractor.java b/src/main/java/net/trustie/extractor/CsdnAsk_Extractor.java index 78336eb..df8f6c4 100644 --- a/src/main/java/net/trustie/extractor/CsdnAsk_Extractor.java +++ b/src/main/java/net/trustie/extractor/CsdnAsk_Extractor.java @@ -29,6 +29,7 @@ public class CsdnAsk_Extractor { private DataBasePageErrorOutPut dbPageErrorOutPut; public void begin() { + generateRawPage.setTable("csdn_ask_html_detail"); dbPageErrorOutPut.setTableName("csdn_ask_error_page"); diff --git a/src/main/java/net/trustie/one/Configuration.java b/src/main/java/net/trustie/one/Configuration.java new file mode 100644 index 0000000..4b887ba --- /dev/null +++ b/src/main/java/net/trustie/one/Configuration.java @@ -0,0 +1,50 @@ +package net.trustie.one; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +public class Configuration { + + public static ApplicationContext aContext = null; + + public static Map tables = new HashMap(); + + //hashmap不是线程安全的，要用Collections.synchronizedMap来进行封装 + @SuppressWarnings("unchecked") + public static Map flags = Collections + .synchronizedMap(new HashMap()); + + private File file = null; + + static { + aContext = new ClassPathXmlApplicationContext( + "classpath:/spring/applicationContext*.xml"); + } + + public Configuration() { + file = new File("./config/configuration.txt"); + + try { + LineIterator litr = FileUtils.lineIterator(file); + while (litr.hasNext()) { + String[] strs = litr.next().split(" "); + String site = strs[0]; + String tableName = strs[1]; + flags.put(site, false); + tables.put(site, tableName); + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + +} diff --git a/src/main/java/net/trustie/one/MyThread.java b/src/main/java/net/trustie/one/MyThread.java new file mode 100644 index 0000000..101a617 --- /dev/null +++ b/src/main/java/net/trustie/one/MyThread.java @@ -0,0 +1,36 @@ +package net.trustie.one; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; + +import core.ExtractorBegin; + +public class MyThread extends Thread { + private String site; + + protected Logger logger = LoggerFactory.getLogger(getClass()); + + public MyThread() { + + } + public MyThread(String site) { + this.site = site; + } + public void run() { + ExtractorBegin eb = (ExtractorBegin) Configuration.aContext + .getBean(site); + + try { + while (!Thread.currentThread().isInterrupted() + && Configuration.flags.get(site) == true) { + eb.threadRun(); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + Configuration.flags.put(site, false); + } + } +} diff --git a/src/main/java/net/trustie/one/OneEx.java b/src/main/java/net/trustie/one/OneEx.java new file mode 100644 index 0000000..37234f2 --- /dev/null +++ b/src/main/java/net/trustie/one/OneEx.java @@ -0,0 +1,86 @@ +package net.trustie.one; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import javax.annotation.Resource; + +import net.trustie.downloader.PageDao; +import net.trustie.extractor.CNblogQSolve_Extractor; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.LineIterator; + +import extension.ReadCursor; + + +public class OneEx extends TimerTask{ + private ReadCursor readCursor = null ; + private ExecutorService pool = null ; + private Configuration config ; + + @Resource + private PageDao pagedao; + + public OneEx(){ + //初始化一些配置 + config = new Configuration(); + } + + public void begin(){ + + initPool(); + //logger.info(this.uuid + " Extractor Started!"); + //timer.schedule(this, site.getFirstTime(), site.getPeriod()); + + + } + + public void initPool(){ + pool = Executors.newFixedThreadPool(20); + } + + public static void main(String[] args) { + //System.out.println("hello "); + +// ExecutorService pool = Executors.newCachedThreadPool(); +// Thread t1 = new MyThread(); +// Thread t2 = new MyThread(); +// Thread t3 = new MyThread(); +// Thread t4 = new MyThread(); +// Thread t5 = new MyThread(); +// Thread t6 = new MyThread(); +// +// pool.execute(t1); +// pool.execute(t2); +// pool.execute(t3); +// pool.execute(t4); +// pool.execute(t5); +// pool.execute(t6); +// +// pool.shutdown(); + + //Configuration ccf = new Configuration(); + + } + + @Override + public void run() { + // TODO Auto-generated method stub + Set sites = config.flags.keySet(); + for(String site : sites){ + readCursor = new ReadCursor("./cursor/"+site+".txt"); + int recordId = Integer.parseInt(readCursor.getValue(site)); + int maxId =pagedao.getMaxId(Configuration.tables.get(site)); + + if(recordId <= maxId && Configuration.flags.get(site) == false ){ + Thread t = new MyThread(site); + pool.execute(t); + } + } + + } +}