diff --git a/.gitignore b/.gitignore index 809aa5e..e0a3863 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.log +*DS_Store *nohup* diff --git a/new_crawler/pr_diff.py b/new_crawler/pr_diff.py index 5eb656c..07264ba 100644 --- a/new_crawler/pr_diff.py +++ b/new_crawler/pr_diff.py @@ -16,7 +16,7 @@ formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.NOTSET) - +old import urllib2 import redis from lxml import etree diff --git a/new_crawler/sorted_prjs.txt b/new_crawler/sorted_prjs.txt index 4e2948f..6ec4152 100644 --- a/new_crawler/sorted_prjs.txt +++ b/new_crawler/sorted_prjs.txt @@ -1,4 +1,3 @@ -33253535 sauron-demo sauron 2198 789 brianchandotcom liferay-portal 1980 20170979 everypolitician everypolitician-data 1481 6866209 Homebrew homebrew 1345 diff --git a/new_crawler/test.py b/new_crawler/test.py index d48289f..9ea5e1a 100644 --- a/new_crawler/test.py +++ b/new_crawler/test.py @@ -1,40 +1,8 @@ #coding:utf-8 -import urllib2 -import redis -import re -from lxml import etree -import logging -logger = logging.getLogger() -hdlr = logging.FileHandler("push.log") -formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') -hdlr.setFormatter(formatter) -logger.addHandler(hdlr) -logger.setLevel(logging.NOTSET) - -page_url = " https://github.com/sauron-demo/sauron/pulls?page=28&q=is%3Apr" - -# 先下载页面 -req = urllib2.Request(page_url) -ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8') - -pr_lis=etree.HTML(ini_html).xpath('//*/ul[@class="js-navigation-container js-active-navigation-container"]/li') -print "len prl: %d"%(len(pr_lis),) -for pr_li in pr_lis: - pr_number = pr_li.xpath('div/div[2]/div/span[1]') - print pr_number - pr_number = pr_number[0].text - print ("pr text : %s"%(len(pr_number),)) - pr_number = re.findall(r"#(\d+)",pr_number)[0] - - diff_url = "https://patch-diff.githubusercontent.com/raw/%s/%s/pull/%s.diff"%('sauron-demo','sauron',pr_number) - print diff_url - req = urllib2.Request(diff_url) - ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8') - # print ini_html - # conn.ping(True) - # cursor.execute("insert into pr_diff(project_id,pr_id,diff) values(%s,%s,%s)",(project_id,pr_number,ini_html)) - # conn.commit() - \ No newline at end of file +with open("sorted_prjs.txt") as fp: + for line in fp: + ls = line.split("\t") + print line, "\t https://github.com/%s/%s/pulls"%(ls[1],ls[2]) \ No newline at end of file diff --git a/get_max_pr_number.py b/old_crawler/get_max_pr_number.py similarity index 100% rename from get_max_pr_number.py rename to old_crawler/get_max_pr_number.py diff --git a/pr_diff.py b/old_crawler/pr_diff.py similarity index 100% rename from pr_diff.py rename to old_crawler/pr_diff.py diff --git a/pr_diff.py.old b/old_crawler/pr_diff.py.old similarity index 100% rename from pr_diff.py.old rename to old_crawler/pr_diff.py.old diff --git a/prj_selection.py b/old_crawler/prj_selection.py similarity index 100% rename from prj_selection.py rename to old_crawler/prj_selection.py diff --git a/prjs_with_nubmer.txt b/old_crawler/prjs_with_nubmer.txt similarity index 100% rename from prjs_with_nubmer.txt rename to old_crawler/prjs_with_nubmer.txt diff --git a/project.txt b/old_crawler/project.txt similarity index 100% rename from project.txt rename to old_crawler/project.txt diff --git a/push_pr_dif.py b/old_crawler/push_pr_dif.py similarity index 100% rename from push_pr_dif.py rename to old_crawler/push_pr_dif.py diff --git a/readme b/old_crawler/readme similarity index 100% rename from readme rename to old_crawler/readme diff --git a/sorted_prjs.txt b/old_crawler/sorted_prjs.txt similarity index 100% rename from sorted_prjs.txt rename to old_crawler/sorted_prjs.txt diff --git a/prj_sample/prj_cadidate.txt b/prj_sample/prj_cadidate.txt new file mode 100644 index 0000000..59699a9 --- /dev/null +++ b/prj_sample/prj_cadidate.txt @@ -0,0 +1,121 @@ +202 mozilla-b2g gaia 1264 +22091370 kubernetes kubernetes 1119 （强推，标签好） +1142 saltstack salt 1034 +10313229 rust-lang rust 855 +1334 rails rails 765 +37443782 tgstation tgstation 727 +3149844 cms-sw cmssw 723 （强推，标签好） +10998645 docker docker 701 （强推，标签好） +10252 dimagi commcare-hq 659 +11703554 code-dot-org code-dot-org 628 +17694 ceph ceph 623 （强推，标签好） +13721 guardian frontend 621 +20201298 everypolitician viewer-sinatra 616 +3929178 edx edx-platform 604 +634 symfony symfony 580 （强推，标签好） +10070 symfony symfony-docs 249 +6 cocos2d cocos2d-x 565 +1486 ansible ansible 551 （强推，标签好） +3928853 CleverRaven Cataclysm-DDA 544 +10322399 ManageIQ manageiq 528 （强推，标签好） +13378441 dotnet corefx 496 （强推，标签好） +524804 owncloud core 479 +3085 xbmc xbmc 476 （强推，标签好） +17302594 elastic elasticsearch 463 （强推，标签好） +2184 joomla joomla-cms 460 +3275 Baystation12 Baystation12 430 +9706049 odoo odoo 428 +1992097 laravel framework 425 +6093 JuliaLang julia 413 +10952404 servo servo 408 +7452331 highfidelity hifi 387 (强推，标签好） +29067220 Automattic wp-calypso 377 (强推，标签好） +15022227 dotnet roslyn 367 +11372864 openshift origin 354 +8149324 cockroachdb cockroach 347 (慎选，title prefix) +1226 django django 345 +20078281 nodejs node 325 (强推，标签好） +1528221 PrestaShop PrestaShop 317 +15889688 dotnet coreclr 309 +4708601 twbs bootstrap 300 +1366147 neo4j neo4j 300 (强推，标签好） +37 angular angular.js 300 +864 bitcoin bitcoin 295 +10019334 juju juju 295 +11065419 hashicorp terraform 289 +3931825 d3athrow vgstation13 284 +16827100 grpc grpc 282(强推，标签好） +23995 cdnjs cdnjs 275 +853 rapid7 metasploit-framework 272 +20796 hazelcast hazelcast 265(强推，标签好） +12159636 angular angular 263 (强推，标签好） +312 Katello katello 261 +540 emberjs ember.js 259 +27329 pydata pandas 259(强推，标签好） +9316 cakephp cakephp 256 +1347553 openshift origin-server 254 +29 Itseez opencv 254 +10045 akka akka 242 +10739107 chapel-lang chapel 242 +6260617 facebook presto 242 +54076 OpenRA OpenRA 236 +4135 scala scala 235 +27261269 ampproject amphtml 233 +9793 sympy sympy 230 +2265342 RIOT-OS RIOT 228 +183 zendframework zf2 226 +1545591 opf openproject 225 +4594792 dolphin-emu dolphin 224 +17991525 facebook react-native 224(强推，标签好） +4456243 kaltura server 222 +7655968 DynamoDS Dynamo 221 +20096 yiisoft yii2 220 +5590 D-Programming-Language phobos 218 +17296591 elastic kibana 218(强推，标签好） +174642 Sylius Sylius 211 +961 ipython ipython 209 +34896252 home-assistant home-assistant 207(强推，标签好） +14364 Dolibarr dolibarr 213 +3905191 facebook react 213 +67874 openmicroscopy openmicroscopy 212 +8407619 crate crate 212 +1350237 hrydgard ppsspp 205 +13139830 KSP-CKAN NetKAN 205 +4524570 CartoDB cartodb 205 +15762 woothemes woocommerce 204 +392 adobe brackets 204 +5219 scikit-learn scikit-learn 202 +13393876 ParadiseSS13 Paradise 202 + +*********候选 +26245476 mapsme omim 220 +3242732 amplab tachyon 221 +32567918 Alluxio alluxio 221 +3276602 livecode livecode 224 +18617258 google ggrc-core 231 +5482310 akeneo pim-community-dev 234 +847 puppetlabs puppet 239 +2185 appcelerator titanium_mobile 366 +3281281 wildfly wildfly 405 +8196280 apache spark 730 +29261651 apple swift 406 +6866209 Homebrew homebrew 1345 +3407 hpcc-systems HPCC-Platform 354 +11793080 caskdata cdap 364 +3777 NixOS nixpkgs 801 +1360482 Wikia app 523 +424371 ets-berkeley-edu calcentral 268 +2924028 civicrm civicrm-core 419 +11760007 udacity create-your-own-adventure 303 +11250 wbond package_control_channel 251 + + + + +********不要的 +1958 infinispan infinispan 208 +20170979 everypolitician everypolitician-data 1481 都是一个用户提交，没有评论 +9570147 caskroom homebrew-cask 1262 title 好多都是 update 版本 +5805826 ocaml opam-repository 352 +6761827 EFForg https-everywhere 328 +3087647 w3c web-platform-tests 216 \ No newline at end of file diff --git a/prj_sample/prj_sample.txt b/prj_sample/prj_sample.txt new file mode 100644 index 0000000..964c0d8 --- /dev/null +++ b/prj_sample/prj_sample.txt @@ -0,0 +1,88 @@ +202 mozilla-b2g gaia 1264 +22091370 kubernetes kubernetes 1119 标签好 +1142 saltstack salt 1034 +10313229 rust-lang rust 855 +1334 rails rails 765 +37443782 tgstation tgstation 727 +3149844 cms-sw cmssw 723 标签好 +10998645 docker docker 701 标签好 +10252 dimagi commcare-hq 659 +11703554 code-dot-org code-dot-org 628 +17694 ceph ceph 623 标签好 +13721 guardian frontend 621 +20201298 everypolitician viewer-sinatra 616 +3929178 edx edx-platform 604 +634 symfony symfony 580 标签好 +10070 symfony symfony-docs 249 +6 cocos2d cocos2d-x 565 +1486 ansible ansible 551 标签好 +3928853 CleverRaven Cataclysm-DDA 544 +10322399 ManageIQ manageiq 528 标签好 +13378441 dotnet corefx 496 标签好 +524804 owncloud core 479 +3085 xbmc xbmc 476 标签好 +17302594 elastic elasticsearch 463 标签好 +2184 joomla joomla-cms 460 +3275 Baystation12 Baystation12 430 +9706049 odoo odoo 428 +1992097 laravel framework 425 +6093 JuliaLang julia 413 +10952404 servo servo 408 +7452331 highfidelity hifi 387 标签好 +29067220 Automattic wp-calypso 377 标签好 +15022227 dotnet roslyn 367 +11372864 openshift origin 354 +8149324 cockroachdb cockroach 347 +1226 django django 345 +20078281 nodejs node 325 标签好 +1528221 PrestaShop PrestaShop 317 +15889688 dotnet coreclr 309 +4708601 twbs bootstrap 300 +1366147 neo4j neo4j 300 标签好 +37 angular angular.js 300 +864 bitcoin bitcoin 295 +10019334 juju juju 295 +11065419 hashicorp terraform 289 +3931825 d3athrow vgstation13 284 +16827100 grpc grpc 282 标签好 +23995 cdnjs cdnjs 275 +853 rapid7 metasploit-framework 272 +20796 hazelcast hazelcast 265 标签好 +12159636 angular angular 263 标签好 +312 Katello katello 261 +540 emberjs ember.js 259 +27329 pydata pandas 259 标签好 +9316 cakephp cakephp 256 +1347553 openshift origin-server 254 +29 Itseez opencv 254 +10045 akka akka 242 +10739107 chapel-lang chapel 242 +6260617 facebook presto 242 +54076 OpenRA OpenRA 236 +4135 scala scala 235 +27261269 ampproject amphtml 233 +9793 sympy sympy 230 +2265342 RIOT-OS RIOT 228 +183 zendframework zf2 226 +1545591 opf openproject 225 +4594792 dolphin-emu dolphin 224 +17991525 facebook react-native 224 标签好 +4456243 kaltura server 222 +7655968 DynamoDS Dynamo 221 +20096 yiisoft yii2 220 +5590 D-Programming-Language phobos 218 +17296591 elastic kibana 218 标签好 +174642 Sylius Sylius 211 +961 ipython ipython 209 +34896252 home-assistant home-assistant 207 标签好 +14364 Dolibarr dolibarr 213 +3905191 facebook react 213 +67874 openmicroscopy openmicroscopy 212 +8407619 crate crate 212 +1350237 hrydgard ppsspp 205 +13139830 KSP-CKAN NetKAN 205 +4524570 CartoDB cartodb 205 +15762 woothemes woocommerce 204 +392 adobe brackets 204 +5219 scikit-learn scikit-learn 202 +13393876 ParadiseSS13 Paradise 202 \ No newline at end of file diff --git a/prj_sample/prj_status.py b/prj_sample/prj_status.py new file mode 100644 index 0000000..8533253 --- /dev/null +++ b/prj_sample/prj_status.py @@ -0,0 +1,46 @@ +#coding:utf-8 +''' +Created on 2017年06月09日 +获取prj的基本状态信息 +star fork watch 等 +@author: StarLee +''' + +import urllib2 +from lxml import etree + + +with open("prj_sample.txt","r") as fp: + for line in fp.readlines()[:1]: + ls = line.split("\t") + rid = ls[0] + un = ls[1] + rn = ls[2] + prn = int(ls[3].strip())*25 + + # 下载首页 + url ='https://github.com/%s/%s'%(un,rn) + req = urllib2.Request(url) + ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8') + + # watch,star,fork + # //*[@id="js-repo-pjax-container"]/div[1]/div[1]/ul/li[1]/a[2] + lis=etree.HTML(ini_html).xpath('//*/ul[@class="pagehead-actions"]/li/a[2]') + wn = lis[0].text.strip() + sn = lis[1].text.strip() + fn = lis[2].text.strip() + + #open pr + # //*/span[@class="counter"] + opn = etree.HTML(ini_html).xpath('//*/span[@class="counter"]')[0].text.strip() + print opn + + + #contributors + # //*[@id="js-repo-pjax-container"]/div[2]/div[1]/div[2]/div/div/ul/li[4]/a/span + cntn = etree.HTML(ini_html).xpath('//*/ul[@class="numbers-summary"]/li[4]/a/span')[0].text.strip() + print cntn + + with open("prj_status.txt","a") as fp: + fp.write("%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s", + (rid,un,rn,prn,wn,sn,fn,opn,cntn))