feat(data): better multiprocessing for gfibot.data.update
This commit is contained in:
parent
9cd7fb55ac
commit
c48a4782fa
|
|
@ -1,3 +1,6 @@
|
|||
# Ignore the default stdout file
|
||||
nohup.out
|
||||
|
||||
# Ignore IDEA related files
|
||||
*.idea/
|
||||
|
||||
|
|
@ -145,4 +148,4 @@ dmypy.json
|
|||
gfibot/backend/set_tokens_private.py
|
||||
|
||||
# trained models
|
||||
gfi-bot
|
||||
gfi-bot
|
||||
|
|
|
|||
|
|
@ -1,16 +1,31 @@
|
|||
import psutil
|
||||
import logging
|
||||
|
||||
from datetime import datetime
|
||||
from mongoengine import *
|
||||
|
||||
|
||||
class GitHubFetchLog(Document):
|
||||
"""A log describing a dataset fetch procedure"""
|
||||
|
||||
class Log(Document):
|
||||
owner: str = StringField(required=True)
|
||||
name: str = StringField(required=True)
|
||||
pid: int = IntField(required=True)
|
||||
user_github_login: str = StringField(null=True)
|
||||
update_begin: datetime = DateTimeField(required=True)
|
||||
update_end: datetime = DateTimeField(null=True)
|
||||
|
||||
meta = {
|
||||
"allow_inheritance": True,
|
||||
"indexes": [
|
||||
{"fields": ["owner", "name"]},
|
||||
{"fields": ["user_github_login"]},
|
||||
{"fields": ["update_end"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class GitHubFetchLog(Log):
|
||||
"""A log describing a dataset fetch procedure"""
|
||||
|
||||
updated_stars: int = IntField(null=True)
|
||||
updated_commits: int = IntField(null=True)
|
||||
updated_issues: int = IntField(null=True)
|
||||
|
|
@ -24,23 +39,25 @@ class GitHubFetchLog(Document):
|
|||
rate_open_issue: int = IntField(null=True)
|
||||
rate_user: int = IntField(null=True)
|
||||
|
||||
meta = {
|
||||
"collection": "github_fetch_log",
|
||||
"indexes": [
|
||||
{"fields": ["owner", "name"]},
|
||||
{"fields": ["user_github_login"]},
|
||||
{"fields": ["update_end"]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class DatasetBuildLog(Document):
|
||||
class DatasetBuildLog(Log):
|
||||
"""A log describing a dataset build procedure"""
|
||||
|
||||
owner: str = StringField(required=True)
|
||||
name: str = StringField(required=True)
|
||||
user_github_login: str = StringField(null=True)
|
||||
update_begin: datetime = DateTimeField(null=True)
|
||||
update_end: datetime = DateTimeField(null=True)
|
||||
updated_open_issues: int = IntField(null=True)
|
||||
updated_resolved_issues: int = IntField(null=True)
|
||||
|
||||
|
||||
def log_exists(owner, name, log_type):
|
||||
existing_log: Log = log_type.objects(owner=owner, name=name, update_end=None)
|
||||
if existing_log.count() > 0:
|
||||
existing_log: Log = existing_log.first()
|
||||
if psutil.pid_exists(existing_log.pid):
|
||||
return True
|
||||
else:
|
||||
logging.warning(
|
||||
"%s/%s is being updated but its process is not running anymore",
|
||||
owner,
|
||||
name,
|
||||
)
|
||||
existing_log.delete()
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import os
|
||||
import re
|
||||
import nltk
|
||||
import logging
|
||||
|
|
@ -363,9 +364,14 @@ def get_dataset_for_repo(
|
|||
"""
|
||||
Update the Dataset collection with latest resolved and open issues for a single repo.
|
||||
"""
|
||||
if log_exists(owner, name, DatasetBuildLog):
|
||||
logger.info("%s/%s is already being updated, skipping", owner, name)
|
||||
return
|
||||
|
||||
log = DatasetBuildLog(
|
||||
owner=owner,
|
||||
name=name,
|
||||
pid=os.getpid(),
|
||||
github_user_login=github_login,
|
||||
update_begin=datetime.utcnow(),
|
||||
)
|
||||
|
|
@ -389,7 +395,13 @@ def get_dataset_all(since: datetime = None):
|
|||
since (datetime, optional): Only consider issues updated after this time.
|
||||
Defaults to None, which means to consider all issues.
|
||||
"""
|
||||
log = DatasetBuildLog(owner="", name="", update_begin=datetime.utcnow())
|
||||
if log_exists("", "", DatasetBuildLog):
|
||||
logger.info("A global dataset update is already being performed")
|
||||
return
|
||||
|
||||
log = DatasetBuildLog(
|
||||
owner="", name="", pid=os.getpid(), update_begin=datetime.utcnow()
|
||||
)
|
||||
log.save()
|
||||
|
||||
if since is None:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import os
|
||||
import re
|
||||
import psutil
|
||||
import logging
|
||||
import argparse
|
||||
import numpy as np
|
||||
import multiprocessing as mp
|
||||
|
||||
from typing import List, Dict, Set, Any, Optional
|
||||
from collections import Counter, defaultdict
|
||||
|
|
@ -493,13 +495,14 @@ def update_repo(
|
|||
If this function is called from backend, indicate which user intiated this update.
|
||||
Defaults to None.
|
||||
"""
|
||||
if GitHubFetchLog.objects(owner=owner, name=name, update_end=None).count() > 0:
|
||||
if log_exists(owner, name, GitHubFetchLog):
|
||||
logger.info("%s/%s is already being updated, skipping", owner, name)
|
||||
return None
|
||||
return
|
||||
|
||||
log = GitHubFetchLog(
|
||||
owner=owner,
|
||||
name=name,
|
||||
pid=os.getpid(),
|
||||
update_begin=datetime.now(timezone.utc),
|
||||
user_github_login=user_github_login,
|
||||
)
|
||||
|
|
@ -562,6 +565,13 @@ def update_repo(
|
|||
log.save()
|
||||
|
||||
|
||||
def update_under_one_token(token: str, repos=List[str]) -> None:
|
||||
logging.info("token = %s, repos = %s", token[0:6], repos)
|
||||
for repo in repos:
|
||||
owner, name = repo.split("/")
|
||||
update_repo(token, owner, name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--debug", action="store_true")
|
||||
|
|
@ -576,9 +586,11 @@ def main():
|
|||
|
||||
logger.info("Data update started at {}".format(datetime.now()))
|
||||
|
||||
params = defaultdict(list)
|
||||
for i, project in enumerate(CONFIG["gfibot"]["projects"]):
|
||||
owner, name = project.split("/")
|
||||
update_repo(valid_tokens[i % len(valid_tokens)], owner, name)
|
||||
params[valid_tokens[i % len(valid_tokens)]].append(project)
|
||||
with mp.Pool(len(valid_tokens)) as pool:
|
||||
pool.map(update_under_one_token, params.items())
|
||||
|
||||
logger.info("Data update finished at {}".format(datetime.now()))
|
||||
|
||||
|
|
|
|||
|
|
@ -143,14 +143,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
|||
|
||||
[[package]]
|
||||
name = "coverage"
|
||||
version = "6.3.3"
|
||||
version = "6.4.1"
|
||||
description = "Code coverage measurement for Python"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
tomli = {version = "*", optional = true, markers = "extra == \"toml\""}
|
||||
tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""}
|
||||
|
||||
[package.extras]
|
||||
toml = ["tomli"]
|
||||
|
|
@ -647,6 +647,17 @@ requests = "*"
|
|||
dev = ["tox", "twine", "therapist", "black", "flake8", "wheel"]
|
||||
test = ["nose", "mock"]
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
version = "5.9.1"
|
||||
description = "Cross-platform lib for process and system monitoring in Python."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
|
||||
|
||||
[package.extras]
|
||||
test = ["ipaddress", "mock", "enum34", "pywin32", "wmi"]
|
||||
|
||||
[[package]]
|
||||
name = "py"
|
||||
version = "1.11.0"
|
||||
|
|
@ -1194,7 +1205,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "fa1c12319140a962c6535e1867e59c578bb7c063b7af7bd4a809f84651a033fc"
|
||||
content-hash = "6c7265af4a7a90503b08f4dd2f609e95183156dffdf3b2114c91101a82b2fc0e"
|
||||
|
||||
[metadata.files]
|
||||
apscheduler = [
|
||||
|
|
@ -1315,47 +1326,47 @@ colorama = [
|
|||
{file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
|
||||
]
|
||||
coverage = [
|
||||
{file = "coverage-6.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df32ee0f4935a101e4b9a5f07b617d884a531ed5666671ff6ac66d2e8e8246d8"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75b5dbffc334e0beb4f6c503fb95e6d422770fd2d1b40a64898ea26d6c02742d"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:114944e6061b68a801c5da5427b9173a0dd9d32cd5fcc18a13de90352843737d"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab88a01cd180b5640ccc9c47232e31924d5f9967ab7edd7e5c91c68eee47a69"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad8f9068f5972a46d50fe5f32c09d6ee11da69c560fcb1b4c3baea246ca4109b"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4cd696aa712e6cd16898d63cf66139dc70d998f8121ab558f0e1936396dbc579"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c1a9942e282cc9d3ed522cd3e3cab081149b27ea3bda72d6f61f84eaf88c1a63"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c06455121a089252b5943ea682187a4e0a5cf0a3fb980eb8e7ce394b144430a9"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-win32.whl", hash = "sha256:cb5311d6ccbd22578c80028c5e292a7ab9adb91bd62c1982087fad75abe2e63d"},
|
||||
{file = "coverage-6.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:6d4a6f30f611e657495cc81a07ff7aa8cd949144e7667c5d3e680d73ba7a70e4"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:79bf405432428e989cad7b8bc60581963238f7645ae8a404f5dce90236cc0293"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:338c417613f15596af9eb7a39353b60abec9d8ce1080aedba5ecee6a5d85f8d3"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db094a6a4ae6329ed322a8973f83630b12715654c197dd392410400a5bfa1a73"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1414e8b124611bf4df8d77215bd32cba6e3425da8ce9c1f1046149615e3a9a31"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:93b16b08f94c92cab88073ffd185070cdcb29f1b98df8b28e6649145b7f2c90d"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fbc86ae8cc129c801e7baaafe3addf3c8d49c9c1597c44bdf2d78139707c3c62"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5ba058610e8289a07db2a57bce45a1793ec0d3d11db28c047aae2aa1a832572"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-win32.whl", hash = "sha256:8329635c0781927a2c6ae068461e19674c564e05b86736ab8eb29c420ee7dc20"},
|
||||
{file = "coverage-6.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:e5af1feee71099ae2e3b086ec04f57f9950e1be9ecf6c420696fea7977b84738"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e814a4a5a1d95223b08cdb0f4f57029e8eab22ffdbae2f97107aeef28554517e"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f4fbf3633cb0713437291b8848634ea97f89c7e849c2be17a665611e433f53"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3401b0d2ed9f726fadbfa35102e00d1b3547b73772a1de5508ef3bdbcb36afe7"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8586b177b4407f988731eb7f41967415b2197f35e2a6ee1a9b9b561f6323c8e9"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892e7fe32191960da559a14536768a62e83e87bbb867e1b9c643e7e0fbce2579"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afb03f981fadb5aed1ac6e3dd34f0488e1a0875623d557b6fad09b97a942b38a"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cbe91bc84be4e5ef0b1480d15c7b18e29c73bdfa33e07d3725da7d18e1b0aff2"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:91502bf27cbd5c83c95cfea291ef387469f2387508645602e1ca0fd8a4ba7548"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-win32.whl", hash = "sha256:c488db059848702aff30aa1d90ef87928d4e72e4f00717343800546fdbff0a94"},
|
||||
{file = "coverage-6.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6534fcdfb5c503affb6b1130db7b5bfc8a0f77fa34880146f7a5c117987d0"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc692c9ee18f0dd3214843779ba6b275ee4bb9b9a5745ba64265bce911aefd1a"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:462105283de203df8de58a68c1bb4ba2a8a164097c2379f664fa81d6baf94b81"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc972d829ad5ef4d4c5fcabd2bbe2add84ce8236f64ba1c0c72185da3a273130"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06f54765cdbce99901871d50fe9f41d58213f18e98b170a30ca34f47de7dd5e8"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7835f76a081787f0ca62a53504361b3869840a1620049b56d803a8cb3a9eeea3"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6f5fee77ec3384b934797f1873758f796dfb4f167e1296dc00f8b2e023ce6ee9"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:baa8be8aba3dd1e976e68677be68a960a633a6d44c325757aefaa4d66175050f"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d06380e777dd6b35ee936f333d55b53dc4a8271036ff884c909cf6e94be8b6c"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-win32.whl", hash = "sha256:f8cabc5fd0091976ab7b020f5708335033e422de25e20ddf9416bdce2b7e07d8"},
|
||||
{file = "coverage-6.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c9441d57b0963cf8340268ad62fc83de61f1613034b79c2b1053046af0c5284"},
|
||||
{file = "coverage-6.3.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:d522f1dc49127eab0bfbba4e90fa068ecff0899bbf61bf4065c790ddd6c177fe"},
|
||||
{file = "coverage-6.3.3.tar.gz", hash = "sha256:2781c43bffbbec2b8867376d4d61916f5e9c4cc168232528562a61d1b4b01879"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"},
|
||||
{file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"},
|
||||
{file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"},
|
||||
{file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"},
|
||||
{file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"},
|
||||
{file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"},
|
||||
{file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"},
|
||||
]
|
||||
cssselect = [
|
||||
{file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
|
||||
|
|
@ -1807,6 +1818,40 @@ premailer = [
|
|||
{file = "premailer-3.10.0-py2.py3-none-any.whl", hash = "sha256:021b8196364d7df96d04f9ade51b794d0b77bcc19e998321c515633a2273be1a"},
|
||||
{file = "premailer-3.10.0.tar.gz", hash = "sha256:d1875a8411f5dc92b53ef9f193db6c0f879dc378d618e0ad292723e388bfe4c2"},
|
||||
]
|
||||
psutil = [
|
||||
{file = "psutil-5.9.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:799759d809c31aab5fe4579e50addf84565e71c1dc9f1c31258f159ff70d3f87"},
|
||||
{file = "psutil-5.9.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9272167b5f5fbfe16945be3db475b3ce8d792386907e673a209da686176552af"},
|
||||
{file = "psutil-5.9.1-cp27-cp27m-win32.whl", hash = "sha256:0904727e0b0a038830b019551cf3204dd48ef5c6868adc776e06e93d615fc5fc"},
|
||||
{file = "psutil-5.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e7e10454cb1ab62cc6ce776e1c135a64045a11ec4c6d254d3f7689c16eb3efd2"},
|
||||
{file = "psutil-5.9.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:56960b9e8edcca1456f8c86a196f0c3d8e3e361320071c93378d41445ffd28b0"},
|
||||
{file = "psutil-5.9.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:44d1826150d49ffd62035785a9e2c56afcea66e55b43b8b630d7706276e87f22"},
|
||||
{file = "psutil-5.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c7be9d7f5b0d206f0bbc3794b8e16fb7dbc53ec9e40bbe8787c6f2d38efcf6c9"},
|
||||
{file = "psutil-5.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd9246e4cdd5b554a2ddd97c157e292ac11ef3e7af25ac56b08b455c829dca8"},
|
||||
{file = "psutil-5.9.1-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29a442e25fab1f4d05e2655bb1b8ab6887981838d22effa2396d584b740194de"},
|
||||
{file = "psutil-5.9.1-cp310-cp310-win32.whl", hash = "sha256:20b27771b077dcaa0de1de3ad52d22538fe101f9946d6dc7869e6f694f079329"},
|
||||
{file = "psutil-5.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:58678bbadae12e0db55186dc58f2888839228ac9f41cc7848853539b70490021"},
|
||||
{file = "psutil-5.9.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3a76ad658641172d9c6e593de6fe248ddde825b5866464c3b2ee26c35da9d237"},
|
||||
{file = "psutil-5.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6a11e48cb93a5fa606306493f439b4aa7c56cb03fc9ace7f6bfa21aaf07c453"},
|
||||
{file = "psutil-5.9.1-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:068935df39055bf27a29824b95c801c7a5130f118b806eee663cad28dca97685"},
|
||||
{file = "psutil-5.9.1-cp36-cp36m-win32.whl", hash = "sha256:0f15a19a05f39a09327345bc279c1ba4a8cfb0172cc0d3c7f7d16c813b2e7d36"},
|
||||
{file = "psutil-5.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:db417f0865f90bdc07fa30e1aadc69b6f4cad7f86324b02aa842034efe8d8c4d"},
|
||||
{file = "psutil-5.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:91c7ff2a40c373d0cc9121d54bc5f31c4fa09c346528e6a08d1845bce5771ffc"},
|
||||
{file = "psutil-5.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fea896b54f3a4ae6f790ac1d017101252c93f6fe075d0e7571543510f11d2676"},
|
||||
{file = "psutil-5.9.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3054e923204b8e9c23a55b23b6df73a8089ae1d075cb0bf711d3e9da1724ded4"},
|
||||
{file = "psutil-5.9.1-cp37-cp37m-win32.whl", hash = "sha256:d2d006286fbcb60f0b391741f520862e9b69f4019b4d738a2a45728c7e952f1b"},
|
||||
{file = "psutil-5.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b14ee12da9338f5e5b3a3ef7ca58b3cba30f5b66f7662159762932e6d0b8f680"},
|
||||
{file = "psutil-5.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:19f36c16012ba9cfc742604df189f2f28d2720e23ff7d1e81602dbe066be9fd1"},
|
||||
{file = "psutil-5.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:944c4b4b82dc4a1b805329c980f270f170fdc9945464223f2ec8e57563139cf4"},
|
||||
{file = "psutil-5.9.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b6750a73a9c4a4e689490ccb862d53c7b976a2a35c4e1846d049dcc3f17d83b"},
|
||||
{file = "psutil-5.9.1-cp38-cp38-win32.whl", hash = "sha256:a8746bfe4e8f659528c5c7e9af5090c5a7d252f32b2e859c584ef7d8efb1e689"},
|
||||
{file = "psutil-5.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:79c9108d9aa7fa6fba6e668b61b82facc067a6b81517cab34d07a84aa89f3df0"},
|
||||
{file = "psutil-5.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28976df6c64ddd6320d281128817f32c29b539a52bdae5e192537bc338a9ec81"},
|
||||
{file = "psutil-5.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b88f75005586131276634027f4219d06e0561292be8bd6bc7f2f00bdabd63c4e"},
|
||||
{file = "psutil-5.9.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:645bd4f7bb5b8633803e0b6746ff1628724668681a434482546887d22c7a9537"},
|
||||
{file = "psutil-5.9.1-cp39-cp39-win32.whl", hash = "sha256:32c52611756096ae91f5d1499fe6c53b86f4a9ada147ee42db4991ba1520e574"},
|
||||
{file = "psutil-5.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:f65f9a46d984b8cd9b3750c2bdb419b2996895b005aefa6cbaba9a143b1ce2c5"},
|
||||
{file = "psutil-5.9.1.tar.gz", hash = "sha256:57f1819b5d9e95cdfb0c881a8a5b7d542ed0b7c522d575706a80bedc848c8954"},
|
||||
]
|
||||
py = [
|
||||
{file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
|
||||
{file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
|
||||
|
|
|
|||
101
pyproject.toml
101
pyproject.toml
|
|
@ -1,11 +1,105 @@
|
|||
[gfibot]
|
||||
projects = [
|
||||
"Mihara/RasterPropMonitor",
|
||||
"Revolutionary-Games/Thrive",
|
||||
projects = [ # This list of 100 projects comes from RecGFI paper
|
||||
"osmlab/name-suggestion-index",
|
||||
"scikit-learn/scikit-learn",
|
||||
"pandas-dev/pandas",
|
||||
"mui-org/material-ui",
|
||||
"OpenMined/PySyft",
|
||||
"WordPress/gutenberg",
|
||||
"elastic/elasticsearch",
|
||||
"saltstack/salt",
|
||||
"Revolutionary-Games/Thrive",
|
||||
"sympy/sympy",
|
||||
"DotNetAnalyzers/StyleCopAnalyzers",
|
||||
"CleverRaven/Cataclysm-DDA",
|
||||
"rubberduck-vba/Rubberduck",
|
||||
"dotnet/runtime",
|
||||
"Leaflet/Leaflet",
|
||||
"nodejs/node",
|
||||
"elastic/kibana",
|
||||
"strongbox/strongbox",
|
||||
"microsoft/vscode",
|
||||
"duckduckgo/zeroclickinfo-spice",
|
||||
"jainaman224/Algo_Ds_Notes",
|
||||
"symfony/symfony",
|
||||
"AnalyticalGraphicsInc/cesium",
|
||||
"dotnet/roslyn",
|
||||
"gatsbyjs/gatsby",
|
||||
"zulip/zulip",
|
||||
"xamarin/Xamarin.Forms",
|
||||
"sindresorhus/refined-github",
|
||||
"symfony/symfony-docs",
|
||||
"appleseedhq/appleseed",
|
||||
"zeit/next.js",
|
||||
"FreezingMoon/AncientBeast",
|
||||
"OpenRA/OpenRA",
|
||||
"freeCodeCamp/freeCodeCamp",
|
||||
"OpenRCT2/OpenRCT2",
|
||||
"akkadotnet/akka.net",
|
||||
"facebook/react",
|
||||
"phpmyadmin/phpmyadmin",
|
||||
"JabRef/jabref",
|
||||
"prestodb/presto",
|
||||
"serverless/serverless",
|
||||
"microsoft/TypeScript",
|
||||
"Elgg/Elgg",
|
||||
"Swati4star/Images-to-PDF",
|
||||
"checkstyle/checkstyle",
|
||||
"dotnet/coreclr",
|
||||
"internetarchive/openlibrary",
|
||||
"oppia/oppia",
|
||||
"facebook/react-native",
|
||||
"RaRe-Technologies/gensim",
|
||||
"the-tale/the-tale",
|
||||
"Sylius/Sylius",
|
||||
"mozilla/fxa-content-server",
|
||||
"Trustroots/trustroots",
|
||||
"TryGhost/Ghost",
|
||||
"Qiskit/qiskit-terra",
|
||||
"mozilla/thimble.mozilla.org",
|
||||
"borgbackup/borg",
|
||||
"x64dbg/x64dbg",
|
||||
"PowerShell/PowerShell",
|
||||
"aws/aws-cdk",
|
||||
"matplotlib/matplotlib",
|
||||
"nextcloud/server",
|
||||
"aframevr/aframe",
|
||||
"osquery/osquery",
|
||||
"facebook/docusaurus",
|
||||
"WeblateOrg/weblate",
|
||||
"tomato42/tlsfuzzer",
|
||||
"dotnet/corert",
|
||||
"pnp/office365-cli",
|
||||
"Yoast/wordpress-seo",
|
||||
"gitextensions/gitextensions",
|
||||
"Showndarya/Hacktoberfest",
|
||||
"Mihara/RasterPropMonitor",
|
||||
"jupyterlab/jupyterlab",
|
||||
"elastic/eui",
|
||||
"netlify/netlify-cms",
|
||||
"pypa/warehouse",
|
||||
"bitcoin/bitcoin",
|
||||
"cBioPortal/cbioportal",
|
||||
"magento/magento2",
|
||||
"dotnet/machinelearning",
|
||||
"nilearn/nilearn",
|
||||
"yarnpkg/yarn",
|
||||
"stylelint/stylelint",
|
||||
"sendgrid/docs",
|
||||
"MvvmCross/MvvmCross",
|
||||
"composer/composer",
|
||||
"SimpleMachines/SMF2.1",
|
||||
"Submitty/Submitty",
|
||||
"adobe/brackets",
|
||||
"qTox/qTox",
|
||||
"moment/moment",
|
||||
"mne-tools/mne-python",
|
||||
"eslint/eslint",
|
||||
"facebook/jest",
|
||||
"chocolatey/choco",
|
||||
"octokit/octokit.net",
|
||||
"eclipse/omr",
|
||||
"badges/shields",
|
||||
]
|
||||
github_graphql_schema_path = "gfibot/data/github_graphql_schema.graphql"
|
||||
|
||||
|
|
@ -56,6 +150,7 @@ gensim = "^4.1.2"
|
|||
gunicorn = "^20.1.0"
|
||||
gevent = "^21.12.0"
|
||||
APScheduler = "^3.9.1"
|
||||
psutil = "^5.9.1"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
pytest = "^6.2"
|
||||
|
|
|
|||
Loading…
Reference in New Issue