init commit

.gitignore

@@ -0,0 +1,3 @@
+log
+nohup.out
+*.swp

list_repos.py

@@ -0,0 +1,80 @@
+import urllib2
+import json
+import MySQLdb
+import time
+import logging
+
+logger = logging.getLogger()
+hdlr = logging.FileHandler("log")
+formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+hdlr.setFormatter(formatter)
+logger.addHandler(hdlr)
+logger.setLevel(logging.NOTSET)
+
+conn = MySQLdb.connect(host="192.168.80.104",user="influx",passwd="influx1234",db="pages",charset='utf8' )
+
+cursor = conn.cursor()
+sql_insert = "insert into repo_urls(link,url,full_name,state) values(%s,%s,%s,%s)"
+sql_error = "insert into errors(url,message,time) values(%s,%s,%s)"
+
+send_headers = {"Content-Type":"application/json","Authorization":"token e41a69a15db215bd7959320360e77a988c4e19a3"}
+url = "https://api.github.com/repositories?since=330001"
+
+while True:  
+    try:
+        req = urllib2.Request(url,headers = send_headers) 
+        logger.info("process link :%s"%url)
+        result = urllib2.urlopen(req)
+	logger.info("downloading is done")
+        prjs = json.loads(result.read())    
+	logger.info("done parsing json")
+        for prj in prjs:
+            try:
+                cursor.execute(sql_insert,(url,prj.get("url"),prj.get("full_name"),0))
+            except Exception,e:
+                logger.error(e.message)
+	logger.info("done storing to db")
+        conn.commit()
+        links = result.info().get("Link")
+        if "next" in links:
+            url = links[1:links.index(">")]
+            logger.info("next url:%s"%url)
+        else:
+            logger.info("no next link any more")
+            num_repos = len(prjs)
+            while True:
+                time.sleep(10*60)
+                logger.info("check it again")
+                result = urllib2.urlopen(req)
+                prjs = json.loads(result.read())
+                new_num_repos = len(prjs)
+                if new_num_repos > num_repos:
+                    for prj in prjs[num_repos:new_num_repos]:
+                        cursor.execute(sql_insert,(url,prj.get("url"),prj.get("full_name"),0))
+                    conn.commit()
+                    num_repos = new_num_repos
+                links = result.info().get("Link")
+                if "next" in links:
+                    url = links[0:links.index(">")]
+                    logger.info("yeah, next link is here!!")
+                    break
+            
+    except urllib2.URLError,e:
+        if hasattr(e,"reason"):
+            logger.error("Failed to reach the server")
+            logger.error("The reason: "%e.reason)
+            cursor.execute(sql_error,(url,e.reason,time.strftime('%Y-%m-%d %H:%M:%S')))
+        elif hasattr(e,"code"):
+            logger.error("The server couldn't fulfill the request")
+            logger.error("Error code: %s"%str(e.code))
+            logger.error("Return content: %s"%e.read())
+            cursor.execute(sql_error,(url,e.read(),time.strftime('%Y-%m-%d %H:%M:%S')))
+        else:
+            pass     
+        conn.commit()
+    except Exception,e:
+        logger.error(e.message)
+        pass
+
+cursor.close()
+conn.close()

producer.py

@@ -0,0 +1,2 @@
+import redis
+print redis.__doc__

text.py

@@ -0,0 +1,32 @@
+import urllib2
+import json
+import commands
+import time
+url = "https://api.github.com/repositories"
+send_headers = {"Content-Type":"application/json"}
+
+def get_ip_address(ifname):
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    return socket.inet_ntoa(fcntl.ioctl(
+        s.fileno(),
+        0x8915,  # SIOCGIFADDR
+        struct.pack('256s', ifname[:15])
+    )[20:24])
+
+while True:
+    req = urllib2.Request(url,headers = send_headers)
+    try :
+	result = urllib2.urlopen(req)
+	ratelimit_remaining = result.info().get("X-RateLimit-Remaining")
+        print "ratelimit_remaining: %s"%ratelimit_remaining
+        if ratelimit_remaining == 0:
+	    print get_ip_address("eth0")
+            commands.getoutput("pppoe-stop")
+	    time.sleep(1)
+	    commands.getoutput("pppoe-start")
+	    print get_ip_address("eth0")
+	next_url = result.info().get("Link")
+	print next_url.index(">")
+	url = next_url[1,next_url.index(">")]
+    except Exception, e:
+	print e