minor
This commit is contained in:
parent
96ef16d2c4
commit
84bc9d8eee
|
|
@ -1,2 +1,3 @@
|
|||
*.log
|
||||
*DS_Store
|
||||
*nohup*
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
|||
hdlr.setFormatter(formatter)
|
||||
logger.addHandler(hdlr)
|
||||
logger.setLevel(logging.NOTSET)
|
||||
|
||||
old
|
||||
import urllib2
|
||||
import redis
|
||||
from lxml import etree
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
33253535 sauron-demo sauron 2198
|
||||
789 brianchandotcom liferay-portal 1980
|
||||
20170979 everypolitician everypolitician-data 1481
|
||||
6866209 Homebrew homebrew 1345
|
||||
|
|
|
|||
|
|
@ -1,40 +1,8 @@
|
|||
#coding:utf-8
|
||||
|
||||
import urllib2
|
||||
import redis
|
||||
import re
|
||||
from lxml import etree
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger()
|
||||
hdlr = logging.FileHandler("push.log")
|
||||
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
|
||||
hdlr.setFormatter(formatter)
|
||||
logger.addHandler(hdlr)
|
||||
logger.setLevel(logging.NOTSET)
|
||||
|
||||
page_url = " https://github.com/sauron-demo/sauron/pulls?page=28&q=is%3Apr"
|
||||
|
||||
# 先下载页面
|
||||
req = urllib2.Request(page_url)
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
|
||||
pr_lis=etree.HTML(ini_html).xpath('//*/ul[@class="js-navigation-container js-active-navigation-container"]/li')
|
||||
print "len prl: %d"%(len(pr_lis),)
|
||||
for pr_li in pr_lis:
|
||||
pr_number = pr_li.xpath('div/div[2]/div/span[1]')
|
||||
print pr_number
|
||||
pr_number = pr_number[0].text
|
||||
print ("pr text : %s"%(len(pr_number),))
|
||||
pr_number = re.findall(r"#(\d+)",pr_number)[0]
|
||||
|
||||
diff_url = "https://patch-diff.githubusercontent.com/raw/%s/%s/pull/%s.diff"%('sauron-demo','sauron',pr_number)
|
||||
print diff_url
|
||||
req = urllib2.Request(diff_url)
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
# print ini_html
|
||||
# conn.ping(True)
|
||||
# cursor.execute("insert into pr_diff(project_id,pr_id,diff) values(%s,%s,%s)",(project_id,pr_number,ini_html))
|
||||
# conn.commit()
|
||||
|
||||
with open("sorted_prjs.txt") as fp:
|
||||
for line in fp:
|
||||
ls = line.split("\t")
|
||||
print line, "\t https://github.com/%s/%s/pulls"%(ls[1],ls[2])
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
202 mozilla-b2g gaia 1264
|
||||
22091370 kubernetes kubernetes 1119 （强推，标签好）
|
||||
1142 saltstack salt 1034
|
||||
10313229 rust-lang rust 855
|
||||
1334 rails rails 765
|
||||
37443782 tgstation tgstation 727
|
||||
3149844 cms-sw cmssw 723 （强推，标签好）
|
||||
10998645 docker docker 701 （强推，标签好）
|
||||
10252 dimagi commcare-hq 659
|
||||
11703554 code-dot-org code-dot-org 628
|
||||
17694 ceph ceph 623 （强推，标签好）
|
||||
13721 guardian frontend 621
|
||||
20201298 everypolitician viewer-sinatra 616
|
||||
3929178 edx edx-platform 604
|
||||
634 symfony symfony 580 （强推，标签好）
|
||||
10070 symfony symfony-docs 249
|
||||
6 cocos2d cocos2d-x 565
|
||||
1486 ansible ansible 551 （强推，标签好）
|
||||
3928853 CleverRaven Cataclysm-DDA 544
|
||||
10322399 ManageIQ manageiq 528 （强推，标签好）
|
||||
13378441 dotnet corefx 496 （强推，标签好）
|
||||
524804 owncloud core 479
|
||||
3085 xbmc xbmc 476 （强推，标签好）
|
||||
17302594 elastic elasticsearch 463 （强推，标签好）
|
||||
2184 joomla joomla-cms 460
|
||||
3275 Baystation12 Baystation12 430
|
||||
9706049 odoo odoo 428
|
||||
1992097 laravel framework 425
|
||||
6093 JuliaLang julia 413
|
||||
10952404 servo servo 408
|
||||
7452331 highfidelity hifi 387 (强推，标签好）
|
||||
29067220 Automattic wp-calypso 377 (强推，标签好）
|
||||
15022227 dotnet roslyn 367
|
||||
11372864 openshift origin 354
|
||||
8149324 cockroachdb cockroach 347 (慎选，title prefix)
|
||||
1226 django django 345
|
||||
20078281 nodejs node 325 (强推，标签好）
|
||||
1528221 PrestaShop PrestaShop 317
|
||||
15889688 dotnet coreclr 309
|
||||
4708601 twbs bootstrap 300
|
||||
1366147 neo4j neo4j 300 (强推，标签好）
|
||||
37 angular angular.js 300
|
||||
864 bitcoin bitcoin 295
|
||||
10019334 juju juju 295
|
||||
11065419 hashicorp terraform 289
|
||||
3931825 d3athrow vgstation13 284
|
||||
16827100 grpc grpc 282(强推，标签好）
|
||||
23995 cdnjs cdnjs 275
|
||||
853 rapid7 metasploit-framework 272
|
||||
20796 hazelcast hazelcast 265(强推，标签好）
|
||||
12159636 angular angular 263 (强推，标签好）
|
||||
312 Katello katello 261
|
||||
540 emberjs ember.js 259
|
||||
27329 pydata pandas 259(强推，标签好）
|
||||
9316 cakephp cakephp 256
|
||||
1347553 openshift origin-server 254
|
||||
29 Itseez opencv 254
|
||||
10045 akka akka 242
|
||||
10739107 chapel-lang chapel 242
|
||||
6260617 facebook presto 242
|
||||
54076 OpenRA OpenRA 236
|
||||
4135 scala scala 235
|
||||
27261269 ampproject amphtml 233
|
||||
9793 sympy sympy 230
|
||||
2265342 RIOT-OS RIOT 228
|
||||
183 zendframework zf2 226
|
||||
1545591 opf openproject 225
|
||||
4594792 dolphin-emu dolphin 224
|
||||
17991525 facebook react-native 224(强推，标签好）
|
||||
4456243 kaltura server 222
|
||||
7655968 DynamoDS Dynamo 221
|
||||
20096 yiisoft yii2 220
|
||||
5590 D-Programming-Language phobos 218
|
||||
17296591 elastic kibana 218(强推，标签好）
|
||||
174642 Sylius Sylius 211
|
||||
961 ipython ipython 209
|
||||
34896252 home-assistant home-assistant 207(强推，标签好）
|
||||
14364 Dolibarr dolibarr 213
|
||||
3905191 facebook react 213
|
||||
67874 openmicroscopy openmicroscopy 212
|
||||
8407619 crate crate 212
|
||||
1350237 hrydgard ppsspp 205
|
||||
13139830 KSP-CKAN NetKAN 205
|
||||
4524570 CartoDB cartodb 205
|
||||
15762 woothemes woocommerce 204
|
||||
392 adobe brackets 204
|
||||
5219 scikit-learn scikit-learn 202
|
||||
13393876 ParadiseSS13 Paradise 202
|
||||
|
||||
*********候选
|
||||
26245476 mapsme omim 220
|
||||
3242732 amplab tachyon 221
|
||||
32567918 Alluxio alluxio 221
|
||||
3276602 livecode livecode 224
|
||||
18617258 google ggrc-core 231
|
||||
5482310 akeneo pim-community-dev 234
|
||||
847 puppetlabs puppet 239
|
||||
2185 appcelerator titanium_mobile 366
|
||||
3281281 wildfly wildfly 405
|
||||
8196280 apache spark 730
|
||||
29261651 apple swift 406
|
||||
6866209 Homebrew homebrew 1345
|
||||
3407 hpcc-systems HPCC-Platform 354
|
||||
11793080 caskdata cdap 364
|
||||
3777 NixOS nixpkgs 801
|
||||
1360482 Wikia app 523
|
||||
424371 ets-berkeley-edu calcentral 268
|
||||
2924028 civicrm civicrm-core 419
|
||||
11760007 udacity create-your-own-adventure 303
|
||||
11250 wbond package_control_channel 251
|
||||
|
||||
|
||||
|
||||
|
||||
********不要的
|
||||
1958 infinispan infinispan 208
|
||||
20170979 everypolitician everypolitician-data 1481 都是一个用户提交，没有评论
|
||||
9570147 caskroom homebrew-cask 1262 title 好多都是 update 版本
|
||||
5805826 ocaml opam-repository 352
|
||||
6761827 EFForg https-everywhere 328
|
||||
3087647 w3c web-platform-tests 216
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
202 mozilla-b2g gaia 1264
|
||||
22091370 kubernetes kubernetes 1119 标签好
|
||||
1142 saltstack salt 1034
|
||||
10313229 rust-lang rust 855
|
||||
1334 rails rails 765
|
||||
37443782 tgstation tgstation 727
|
||||
3149844 cms-sw cmssw 723 标签好
|
||||
10998645 docker docker 701 标签好
|
||||
10252 dimagi commcare-hq 659
|
||||
11703554 code-dot-org code-dot-org 628
|
||||
17694 ceph ceph 623 标签好
|
||||
13721 guardian frontend 621
|
||||
20201298 everypolitician viewer-sinatra 616
|
||||
3929178 edx edx-platform 604
|
||||
634 symfony symfony 580 标签好
|
||||
10070 symfony symfony-docs 249
|
||||
6 cocos2d cocos2d-x 565
|
||||
1486 ansible ansible 551 标签好
|
||||
3928853 CleverRaven Cataclysm-DDA 544
|
||||
10322399 ManageIQ manageiq 528 标签好
|
||||
13378441 dotnet corefx 496 标签好
|
||||
524804 owncloud core 479
|
||||
3085 xbmc xbmc 476 标签好
|
||||
17302594 elastic elasticsearch 463 标签好
|
||||
2184 joomla joomla-cms 460
|
||||
3275 Baystation12 Baystation12 430
|
||||
9706049 odoo odoo 428
|
||||
1992097 laravel framework 425
|
||||
6093 JuliaLang julia 413
|
||||
10952404 servo servo 408
|
||||
7452331 highfidelity hifi 387 标签好
|
||||
29067220 Automattic wp-calypso 377 标签好
|
||||
15022227 dotnet roslyn 367
|
||||
11372864 openshift origin 354
|
||||
8149324 cockroachdb cockroach 347
|
||||
1226 django django 345
|
||||
20078281 nodejs node 325 标签好
|
||||
1528221 PrestaShop PrestaShop 317
|
||||
15889688 dotnet coreclr 309
|
||||
4708601 twbs bootstrap 300
|
||||
1366147 neo4j neo4j 300 标签好
|
||||
37 angular angular.js 300
|
||||
864 bitcoin bitcoin 295
|
||||
10019334 juju juju 295
|
||||
11065419 hashicorp terraform 289
|
||||
3931825 d3athrow vgstation13 284
|
||||
16827100 grpc grpc 282 标签好
|
||||
23995 cdnjs cdnjs 275
|
||||
853 rapid7 metasploit-framework 272
|
||||
20796 hazelcast hazelcast 265 标签好
|
||||
12159636 angular angular 263 标签好
|
||||
312 Katello katello 261
|
||||
540 emberjs ember.js 259
|
||||
27329 pydata pandas 259 标签好
|
||||
9316 cakephp cakephp 256
|
||||
1347553 openshift origin-server 254
|
||||
29 Itseez opencv 254
|
||||
10045 akka akka 242
|
||||
10739107 chapel-lang chapel 242
|
||||
6260617 facebook presto 242
|
||||
54076 OpenRA OpenRA 236
|
||||
4135 scala scala 235
|
||||
27261269 ampproject amphtml 233
|
||||
9793 sympy sympy 230
|
||||
2265342 RIOT-OS RIOT 228
|
||||
183 zendframework zf2 226
|
||||
1545591 opf openproject 225
|
||||
4594792 dolphin-emu dolphin 224
|
||||
17991525 facebook react-native 224 标签好
|
||||
4456243 kaltura server 222
|
||||
7655968 DynamoDS Dynamo 221
|
||||
20096 yiisoft yii2 220
|
||||
5590 D-Programming-Language phobos 218
|
||||
17296591 elastic kibana 218 标签好
|
||||
174642 Sylius Sylius 211
|
||||
961 ipython ipython 209
|
||||
34896252 home-assistant home-assistant 207 标签好
|
||||
14364 Dolibarr dolibarr 213
|
||||
3905191 facebook react 213
|
||||
67874 openmicroscopy openmicroscopy 212
|
||||
8407619 crate crate 212
|
||||
1350237 hrydgard ppsspp 205
|
||||
13139830 KSP-CKAN NetKAN 205
|
||||
4524570 CartoDB cartodb 205
|
||||
15762 woothemes woocommerce 204
|
||||
392 adobe brackets 204
|
||||
5219 scikit-learn scikit-learn 202
|
||||
13393876 ParadiseSS13 Paradise 202
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
#coding:utf-8
|
||||
'''
|
||||
Created on 2017年06月09日
|
||||
获取prj的基本状态信息
|
||||
star fork watch 等
|
||||
@author: StarLee
|
||||
'''
|
||||
|
||||
import urllib2
|
||||
from lxml import etree
|
||||
|
||||
|
||||
with open("prj_sample.txt","r") as fp:
|
||||
for line in fp.readlines()[:1]:
|
||||
ls = line.split("\t")
|
||||
rid = ls[0]
|
||||
un = ls[1]
|
||||
rn = ls[2]
|
||||
prn = int(ls[3].strip())*25
|
||||
|
||||
# 下载首页
|
||||
url ='https://github.com/%s/%s'%(un,rn)
|
||||
req = urllib2.Request(url)
|
||||
ini_html = urllib2.urlopen(req,timeout=20).read().lower().decode('utf-8')
|
||||
|
||||
# watch,star,fork
|
||||
# //*[@id="js-repo-pjax-container"]/div[1]/div[1]/ul/li[1]/a[2]
|
||||
lis=etree.HTML(ini_html).xpath('//*/ul[@class="pagehead-actions"]/li/a[2]')
|
||||
wn = lis[0].text.strip()
|
||||
sn = lis[1].text.strip()
|
||||
fn = lis[2].text.strip()
|
||||
|
||||
#open pr
|
||||
# //*/span[@class="counter"]
|
||||
opn = etree.HTML(ini_html).xpath('//*/span[@class="counter"]')[0].text.strip()
|
||||
print opn
|
||||
|
||||
|
||||
#contributors
|
||||
# //*[@id="js-repo-pjax-container"]/div[2]/div[1]/div[2]/div/div/ul/li[4]/a/span
|
||||
cntn = etree.HTML(ini_html).xpath('//*/ul[@class="numbers-summary"]/li[4]/a/span')[0].text.strip()
|
||||
print cntn
|
||||
|
||||
with open("prj_status.txt","a") as fp:
|
||||
fp.write("%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s",
|
||||
(rid,un,rn,prn,wn,sn,fn,opn,cntn))
|
||||
Loading…
Reference in New Issue