2018/2/4

access_token.txt

@@ -1,13 +1,20 @@
-add768048e897033ca815bce9d8995460415f3ff
-8018c253968eb99307dc6fec2f0d490680413915
-5e44f44d9683b4acf51dfa1bfca202c82e2b75be
-c14df78db9bdeabc210d11631c1d97d0cd276afc
-07bb4205c29c3636bfdc52e87f6ea540aa51af8f
-3e3a67d79fe62e01297bd9fd60bb323abc1f16f0
-55fc0571c8e3af95470a026010b5b74eadbd2a1a
-976a123d0a4aae963eb92ebf816dcf1a2a0dcde2
-d40e70c14da2276b346d93b0f907c69be548089a
-6984dcb6316e5ffddd56e067aebd126d5922c125
-aff28dda38447f1b39da0263e246a93dbd517dcd
-9dfb33f69564e45d59e7d812d2a8bb26b8255daf
-ae326c1e155416b36335a42f81e2ae98cc210dda
+46f8bc047714acc56eb6ce202daf31e7085d7529
+0e24322d58975bebf6c99d698f9610fe831ee6e5
+386f5e20793e90fee8e1016e4e8960c95b33dc9c
+85b065f505ac0a15cb68d16c9e297daa417fe569
+d95c55bdfb227a09b249c9ae52f90e3ccaf9c3c3
+c4731bb5e6f8142e9f07d3e40fe2d0aef806231f
+0adde2938aebb589a77ea908d4a21dddc352aedb
+21b3763b5991c55c65b6f3c3f537e8126489a861
+22b9980eddd59ff3a212be6a63d2e2bc8db16fbd
+ef615fa3e33d9217a959d270871f0c37cfd81391
+752601a9d0fddc902c36a9451860e59b7f4aa719
+9e8c23c7640ee5ab6008b2ab14a8667176cc5485
+300485fd759823ac022a15f6476ebb5179a4fa0b
+8f092cc0cf7fabe5f7afce096f477f63b7ddf0de
+1d8aa277fa9fa04409c511ec6765ec21d3751d35
+6666996f59f9f70104b95bf5070a3a80bee1383d
+ba03eb5ec1133cc4dfd7b46c62d09857996a8c9e
+a93f7f43654bc60f08df1bd49d310f6bb4290dd3
+f231d6a5094eada9a185fdfa1deb0d090c0f4832
+02312165a3746e67549306b33c1661b45b230764

clearTime.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+while ((1)); do
+  > nohup.out
+  sleep 60
+done
+

issues_get.py

@@ -26,6 +26,9 @@ format_issues="%s/milestones?page=%s&&access_token=%s&&per_page=100&&state=all"
 # number_null = 0
 def issues_get():
     name=r.rpop("ruby_projects_1")
+    id = name.split(" ")[1]
+    print id 
+    print name
     if(name==None):
         pass
     else:
@@ -53,7 +56,7 @@ def issues_get():
             it_c.setopt(it_c.SSL_VERIFYPEER, 0)
             it_c.setopt(it_c.SSL_VERIFYHOST, 0)
             it_c.setopt(it_c.FOLLOWLOCATION, 5)
-            # print(it_url)
+            #print it_url
             try:
                 it_c.perform()
             except Exception as ee:
@@ -67,13 +70,13 @@ def issues_get():
 
                 it_b.close()
                 print "doit"
-                print it_hhh
+                #print it_hhh
                 if(type(it_hhh)==dict):
-                    print it_hhh
+                    #print it_hhh
                     if (it_hhh['message'] == 'Not Found'):
                         # ver.execute(updated_repos,(user_url,repo_url))
                         issues_tag=0
-                    print "123321"
+                   # print "123321"
                 elif(type(it_hhh)==list):
                     if(len(it_hhh)==0):
                         # number_null = number_null+1
@@ -102,9 +105,8 @@ def issues_get():
                     page_num += 1
                 if(issues_tag==0):
                     break
-
-
-        ver.execute("update ruby_projects_1 set tag=1 where `url`=%s ",url)
+       
+        ver.execute("update ruby_projects_1 set tag=1 where id=%s",(id,))
 
 if __name__ == "__main__":
 
@@ -113,4 +115,4 @@ if __name__ == "__main__":
 
         issues_get()
         time.sleep(3)
-    # print number_null
+    # print number_null

issues_producer.py

@@ -17,7 +17,7 @@ conn = MySQLdb.connect(host="127.0.0.1",user="influx",passwd="influx1234",db="gi
 # conn = MySQLdb.connect(host="127.0.0.1",user="root",passwd="a123456",db="ruby_projects",charset='utf8' )
 conn.autocommit(1)
 cursor = conn.cursor()
-sql_select = "select `url`,tag from ruby_projects_1 WHERE tag=0"
+sql_select = "select `url`,id from ruby_projects_1 WHERE tag=0"
 
 # peo_name='zyr1031'
 patch_size = 10000 #how many urls push to redis every time
@@ -30,9 +30,9 @@ def start():
     urls_added = cursor.fetchall()
     # print urls_added
     for url in urls_added:
-        # print url
-        name = "%s %s"%(url[0],url[1])
-        # print name
+        #print url
+        name = "%s %s"%(url[0],str(url[1]))
+        #print name
 
         if name in tem_list:
             pass
@@ -45,4 +45,4 @@ def start():
 if __name__ == "__main__":
     while True:
         start()
-        time.sleep(100)
+        time.sleep(100)

produce.log

@@ -0,0 +1 @@
+2018-02-03 03:42:44,996 INFO producer begains to work

token_producer.py

@@ -7,7 +7,7 @@ r = redis.Redis(host="127.0.0.1",port=6379,db=0)
 
 #conn = MySQLdb.connect(host="127.0.0.1",user="root",passwd="a123456",db="gitlab",charset='utf8' )
 patch_size = 10000
-r.delete("access_token_zlb")
+r.delete("access_token")
 def start():
     while True:
         fopen = open("access_token.txt", "r")
@@ -16,13 +16,13 @@ def start():
             token.append(i.replace("\n", ""))
         token_len = len(token)
         fopen.close()
-        num_redis = r.llen("access_token_zlb")
+        num_redis = r.llen("access_token")
         if(num_redis<patch_size):
             for jj in range(num_redis,patch_size+1000):
-                r.lpush("access_token_zlb","%s"%(token[jj%token_len]))
+                r.lpush("access_token","%s"%(token[jj%token_len]))
         time.sleep(2)
 
 
 
 if __name__ == "__main__":
-    start()
+    start()