590 lines
23 KiB
Plaintext
590 lines
23 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "1345f1c8",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"from tqdm import tqdm\n",
|
||
"from pathlib import Path\n",
|
||
"import pandas as pd\n",
|
||
"import numpy as np\n",
|
||
"from pymongo import MongoClient"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "46d49dd9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# 软件生态名\n",
|
||
"ECO_NAMES = [\n",
|
||
" # \"Apache\",\n",
|
||
" # \"Jira\",\n",
|
||
" # \"Mojang\",\n",
|
||
" # \"MongoDB\",\n",
|
||
" # \"Qt\",\n",
|
||
" \"RedHat\",\n",
|
||
"]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "03cf285e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"ISSUE_DIR = Path(\"../data/raw/issues\")\n",
|
||
"LINK_DIR = Path(\"../data/raw/links\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "61e8f96a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"PRO_ISSUE_DIR = Path(\"../data/processed/issues\")\n",
|
||
"PRO_ISSUE_DIR.mkdir(parents=True, exist_ok=True)\n",
|
||
"PRO_LINK_DIR = Path(\"../data/processed/links\")\n",
|
||
"PRO_LINK_DIR.mkdir(parents=True, exist_ok=True)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "40885839",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def load_issues(eco_name: str):\n",
|
||
" # 加载Issue数据DataFrame\n",
|
||
"\n",
|
||
" filename = ISSUE_DIR / (eco_name + \".csv\")\n",
|
||
" issue_df = pd.read_csv(\n",
|
||
" filename, sep=\";\", encoding=\"utf-8\", low_memory=False, index_col=[\"key\"]\n",
|
||
" )\n",
|
||
" return issue_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "1981c072",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def load_links(eco_name: str):\n",
|
||
" # 加载链接数据DataFrame\n",
|
||
"\n",
|
||
" filename = LINK_DIR / (eco_name + \".csv\")\n",
|
||
" link_df = pd.read_csv(filename, sep=\";\", encoding=\"utf-8\", low_memory=False)\n",
|
||
" return link_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "865088c5",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def clean_issues(issue_df: pd.DataFrame):\n",
|
||
" # 对Issue数据进行清洗\n",
|
||
"\n",
|
||
" # 把时间数据转换为统一格式\n",
|
||
" issue_df[\"created_time\"] = pd.to_datetime(\n",
|
||
" issue_df[\"created_time\"], errors=\"coerce\"\n",
|
||
" ).apply(lambda x: x.strftime(\"%Y-%m-%d %H:%M:%S\") if pd.notna(x) else np.nan)\n",
|
||
"\n",
|
||
" return issue_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 8,
|
||
"id": "4447a7d2",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def clean_links(link_df: pd.DataFrame, issue_df: pd.DataFrame):\n",
|
||
" # 对链接数据进行清洗\n",
|
||
"\n",
|
||
" def column_transform(row):\n",
|
||
" return str(sorted(set([row[\"in_issue_key\"], row[\"out_issue_key\"]])))\n",
|
||
"\n",
|
||
" # 一条（一般类型）链接会在两个Issue的字段中存在，需要清除其中一个副本\n",
|
||
" link_df.drop_duplicates(inplace=True)\n",
|
||
" print(f\"Left with {len(link_df)} links after removing link duplication\")\n",
|
||
"\n",
|
||
" # 清除Issue是私有的、无权访问的链接\n",
|
||
" condition = (\n",
|
||
" link_df[[\"in_issue_key\", \"out_issue_key\"]]\n",
|
||
" .isin(issue_df.index.values)\n",
|
||
" .all(axis=1)\n",
|
||
" )\n",
|
||
" link_df = link_df[condition]\n",
|
||
" print(f\"Left with {len(link_df)} links after removing half-private issues\")\n",
|
||
"\n",
|
||
" # 一对Issue间只允许存在一条链接，需要删除含有多条链接的Issue对\n",
|
||
" # 首先基于'link_key'字段删除重复的Issue对\n",
|
||
" # !注意：相同的'link_key'的Issue对之间是可能存在多种类型的链接，这会混淆关联关系，所以全部清除\n",
|
||
" link_df.drop_duplicates(subset=[\"link_key\"], keep=False, inplace=True)\n",
|
||
"\n",
|
||
" # 其次，以防'link_key'是反过来的，比如issue1_issue2和issue2_issue1\n",
|
||
" # 所以添加'sorted_issue_keys'字段，由链接的两个Issue的key升序组成\n",
|
||
" link_df[\"sorted_issue_keys\"] = link_df.apply(column_transform, axis=1)\n",
|
||
" # 找出链接的两端Issue的keys相同的行对应的'sorted_issue_keys'字段值\n",
|
||
" doublelinks = (\n",
|
||
" (link_df[\"sorted_issue_keys\"].value_counts() > 1)\n",
|
||
" .rename_axis(\"doubles\")\n",
|
||
" .reset_index(name=\"valid\")\n",
|
||
" )\n",
|
||
" valid_double_keys = set(doublelinks[doublelinks[\"valid\"] == True][\"doubles\"])\n",
|
||
"\n",
|
||
" # 把重复的'sorted_issue_keys'字段对应的链接类型取出来检查，若类型数大于1，则清除这些Issue对\n",
|
||
" for i in tqdm(valid_double_keys):\n",
|
||
" if len(set(link_df[link_df[\"sorted_issue_keys\"] == i][\"link_type\"])) > 1:\n",
|
||
" condition = link_df[\"sorted_issue_keys\"] != i\n",
|
||
" link_df = link_df[condition]\n",
|
||
" print(\n",
|
||
" f\"Left with {len(link_df)} links after removing issue-pairs with multiple types of links between them\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 最后，留下来的链接中仍然可能有重复链接类型的Issue对（通过Issue的key对调的方式实现的），清除其中一个\n",
|
||
" link_df.drop_duplicates(subset=[\"sorted_issue_keys\"], inplace=True)\n",
|
||
" print(\n",
|
||
" f\"Left with {len(link_df)} links after removing issue-pairs with duplicate same type of links\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" link_df.reset_index(inplace=True, drop=True)\n",
|
||
"\n",
|
||
" return link_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "ab8df842",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def joined_links(link_df: pd.DataFrame, issue_df: pd.DataFrame):\n",
|
||
" # 联合Issue和链接数据\n",
|
||
"\n",
|
||
" joined_df = link_df.join(issue_df.add_suffix(\"_in\"), on=\"in_issue_key\").join(\n",
|
||
" issue_df.add_suffix(\"_out\"), on=\"out_issue_key\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # !注意：补充Subtask类型链接创建时间\n",
|
||
" joined_df.loc[\n",
|
||
" (joined_df[\"link_type\"] == \"Subtask\") & (joined_df[\"link_created_time\"].isna()),\n",
|
||
" \"link_created_time\",\n",
|
||
" ] = joined_df[\"created_time_out\"]\n",
|
||
"\n",
|
||
" return joined_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "7a496e37",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def query_issue_closed_time(eco_name: str, issue_df: pd.DataFrame):\n",
|
||
" # 查询Issue的history，获取Issue关闭时间\n",
|
||
"\n",
|
||
" # 定义一个函数，用于处理每个分组\n",
|
||
" def handle_group(group):\n",
|
||
" # 如果分组内的closed_time全为NaN，则保留该分组的第一行\n",
|
||
" if group[\"closed_time\"].isna().all():\n",
|
||
" return group.iloc[0:1]\n",
|
||
" # 否则，返回closed_time最大值对应的行\n",
|
||
" else:\n",
|
||
" return group.loc[[group[\"closed_time\"].idxmax()]]\n",
|
||
"\n",
|
||
" # 把索引列转换为普通列，列名为key\n",
|
||
" issue_df = issue_df.reset_index().rename(columns={\"index\": \"key\"})\n",
|
||
" # 创建Issue关闭时间列\n",
|
||
" issue_df[\"closed_time\"] = None\n",
|
||
"\n",
|
||
" with MongoClient() as client:\n",
|
||
" # 链接数据库\n",
|
||
" db = client[\"JiraEcos\"]\n",
|
||
" histories_collection = db[eco_name + \"Histories\"]\n",
|
||
"\n",
|
||
" # 首先取出需要查询的Issue keys并移除重复项\n",
|
||
" issue_keys = issue_df[\"key\"].unique().tolist()\n",
|
||
"\n",
|
||
" # 构造聚合查询管道\n",
|
||
" pipeline = [\n",
|
||
" {\n",
|
||
" # 第一步：筛选key在issue_keys列表中的文档\n",
|
||
" \"$match\": {\"key\": {\"$in\": issue_keys}}\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第二步：展开history.items数组\n",
|
||
" # 进而返回每个具体更改事件\n",
|
||
" \"$unwind\": \"$history.items\"\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第三步：再次筛选满足特定field值的展开后的文档\n",
|
||
" # field字段为status保证更改事件是修改Issue状态\n",
|
||
" # toString字段为Closed保证是关闭Issue\n",
|
||
" \"$match\": {\n",
|
||
" \"history.items.field\": \"status\",\n",
|
||
" \"history.items.toString\": \"Closed\",\n",
|
||
" }\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第四步：指定返回文档的字段\n",
|
||
" \"$project\": {\n",
|
||
" \"_id\": 0,\n",
|
||
" \"query_key\": \"$key\",\n",
|
||
" \"created\": \"$history.created\",\n",
|
||
" \"field\": \"$history.items.field\",\n",
|
||
" \"to\": \"$history.items.to\",\n",
|
||
" \"toString\": \"$history.items.toString\",\n",
|
||
" }\n",
|
||
" },\n",
|
||
" ]\n",
|
||
"\n",
|
||
" # 查询数据库\n",
|
||
" query = list(histories_collection.aggregate(pipeline))\n",
|
||
" # 转换为DataFrame\n",
|
||
" query_df = pd.DataFrame(query)\n",
|
||
" # print(query_df.head())\n",
|
||
"\n",
|
||
" print(\n",
|
||
" f\"❕ Test print: {len(issue_df)} issues before merged with query DataFrame\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 合并DataFrame，基于key与query_key匹配\n",
|
||
" merged_df = pd.merge(\n",
|
||
" issue_df,\n",
|
||
" query_df,\n",
|
||
" left_on=\"key\",\n",
|
||
" right_on=\"query_key\",\n",
|
||
" how=\"left\",\n",
|
||
" )\n",
|
||
"\n",
|
||
" print(\n",
|
||
" f\"❕ Test print: {len(merged_df)} issues after merged with query DataFrame\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 将merged_df中的created值赋给合并后DataFrame的closed_time字段\n",
|
||
" merged_df[\"closed_time\"] = pd.to_datetime(merged_df[\"created\"], errors=\"coerce\")\n",
|
||
"\n",
|
||
" # 裁切出需要的字段\n",
|
||
" issue_df = merged_df[list(issue_df.columns)]\n",
|
||
"\n",
|
||
" # 最后，由于Issue可能会被多次开启与关闭，所以，保留最后一次关闭时间\n",
|
||
" # 按照key进行分组，使用groupby和apply处理每个分组\n",
|
||
" result_df = (\n",
|
||
" issue_df.groupby(\"key\", as_index=False)\n",
|
||
" .apply(handle_group)\n",
|
||
" .reset_index(drop=True)\n",
|
||
" )\n",
|
||
" # 统一时间格式\n",
|
||
" result_df[\"closed_time\"] = result_df[\"closed_time\"].apply(\n",
|
||
" lambda x: x.strftime(\"%Y-%m-%d %H:%M:%S\") if pd.notna(x) else np.nan\n",
|
||
" )\n",
|
||
" # 把key列重新设置为索引列\n",
|
||
" result_df = result_df.set_index(\"key\")\n",
|
||
"\n",
|
||
" print(f\"❕ Test print: {len(result_df)} issues after processed done\")\n",
|
||
"\n",
|
||
" return result_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 11,
|
||
"id": "3401742e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def query_link_created_time(eco_name: str, link_df: pd.DataFrame):\n",
|
||
" # 查询Issue的history，获取链接创建时间\n",
|
||
"\n",
|
||
" # 定义一个函数，用于处理每个分组\n",
|
||
" def handle_group(group):\n",
|
||
" # 如果分组内的link_created_time全为NaN，则保留该分组的第一行\n",
|
||
" if group[\"link_created_time\"].isna().all():\n",
|
||
" return group.iloc[0:1]\n",
|
||
" # 否则，返回link_created_time最大值对应的行\n",
|
||
" else:\n",
|
||
" return group.loc[[group[\"link_created_time\"].idxmax()]]\n",
|
||
"\n",
|
||
" # 裁切出需要的字段\n",
|
||
" link_df = link_df[[\"link_type\", \"in_issue_key\", \"out_issue_key\"]]\n",
|
||
" # 创建链接创建时间列\n",
|
||
" link_df[\"link_created_time\"] = None\n",
|
||
"\n",
|
||
" with MongoClient() as client:\n",
|
||
" # 链接数据库\n",
|
||
" db = client[\"JiraEcos\"]\n",
|
||
" histories_collection = db[eco_name + \"Histories\"]\n",
|
||
"\n",
|
||
" # 首先取出需要查询的Issue keys并移除重复项\n",
|
||
" out_issue_keys = link_df[\"out_issue_key\"].tolist()\n",
|
||
" out_issue_keys = list(set(out_issue_keys))\n",
|
||
"\n",
|
||
" # 构造聚合查询管道\n",
|
||
" pipeline = [\n",
|
||
" {\n",
|
||
" # 第一步：筛选key在out_issue_keys列表中的文档\n",
|
||
" # 从而保证只取出有链接的Issue的更改事件\n",
|
||
" \"$match\": {\"key\": {\"$in\": out_issue_keys}}\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第二步：展开history.items数组\n",
|
||
" # 进而返回每个具体更改事件\n",
|
||
" \"$unwind\": \"$history.items\"\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第三步：再次筛选满足特定field值的展开后的文档\n",
|
||
" # field保证更改事件是链接创建或删除\n",
|
||
" # to或toString字段不为空保证是创建链接的事件而不是删除\n",
|
||
" \"$match\": {\n",
|
||
" \"history.items.field\": {\n",
|
||
" \"$in\": [\"Link\", \"Epic Child\", \"Parent\", \"Parent Issue\"]\n",
|
||
" },\n",
|
||
" \"$or\": [\n",
|
||
" {\"history.items.to\": {\"$ne\": None}},\n",
|
||
" {\"history.items.toString\": {\"$ne\": None}},\n",
|
||
" ],\n",
|
||
" }\n",
|
||
" },\n",
|
||
" {\n",
|
||
" # 第四步：指定返回文档的格式\n",
|
||
" # target_key根据链接类型获取to或toString字段信息\n",
|
||
" \"$project\": {\n",
|
||
" \"_id\": 0,\n",
|
||
" \"key\": 1,\n",
|
||
" \"created\": \"$history.created\",\n",
|
||
" \"field\": \"$history.items.field\",\n",
|
||
" \"to\": \"$history.items.to\",\n",
|
||
" \"toString\": \"$history.items.toString\",\n",
|
||
" \"target_key\": {\n",
|
||
" \"$cond\": {\n",
|
||
" \"if\": {\"$eq\": [\"$history.items.field\", \"Link\"]},\n",
|
||
" \"then\": \"$history.items.to\",\n",
|
||
" \"else\": \"$history.items.toString\",\n",
|
||
" }\n",
|
||
" },\n",
|
||
" }\n",
|
||
" },\n",
|
||
" ]\n",
|
||
"\n",
|
||
" # 查询数据库\n",
|
||
" query = list(histories_collection.aggregate(pipeline))\n",
|
||
" # 转换为DataFrame\n",
|
||
" query_df = pd.DataFrame(query)\n",
|
||
" # print(query_df.head())\n",
|
||
"\n",
|
||
" # 合并DataFrame，基于out_issue_key和key匹配，in_issue_key和target_key匹配\n",
|
||
" merged_df = pd.merge(\n",
|
||
" link_df,\n",
|
||
" query_df,\n",
|
||
" left_on=[\"out_issue_key\", \"in_issue_key\"],\n",
|
||
" right_on=[\"key\", \"target_key\"],\n",
|
||
" how=\"left\",\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 将merged_df中的created值赋给合并后DataFrame的link_created_time字段\n",
|
||
" merged_df[\"link_created_time\"] = merged_df[\"created\"]\n",
|
||
"\n",
|
||
" # 裁切出需要的字段\n",
|
||
" link_df = merged_df[\n",
|
||
" [\"link_type\", \"in_issue_key\", \"out_issue_key\", \"link_created_time\"]\n",
|
||
" ]\n",
|
||
"\n",
|
||
" # 最后，由于in_issue和out_issue之间可能会发生相同类型链接的多次创建活动\n",
|
||
" # 所以，保留最后一次链接创建时间\n",
|
||
" # 转换link_created_time为datetime以确保比较的准确性\n",
|
||
" link_df[\"link_created_time\"] = pd.to_datetime(\n",
|
||
" link_df[\"link_created_time\"], errors=\"coerce\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 按照除link_created_time以外的所有字段进行分组，使用groupby和apply处理每个分组\n",
|
||
" result_df = (\n",
|
||
" link_df.groupby(\n",
|
||
" [\"link_type\", \"in_issue_key\", \"out_issue_key\"], as_index=False\n",
|
||
" )\n",
|
||
" .apply(handle_group)\n",
|
||
" .reset_index(drop=True)\n",
|
||
" )\n",
|
||
" # 统一时间格式\n",
|
||
" result_df[\"link_created_time\"] = result_df[\"link_created_time\"].apply(\n",
|
||
" lambda x: x.strftime(\"%Y-%m-%d %H:%M:%S\") if pd.notna(x) else np.nan\n",
|
||
" )\n",
|
||
"\n",
|
||
" return result_df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 12,
|
||
"id": "b42827a1",
|
||
"metadata": {
|
||
"scrolled": true
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"✔ Loaded 502297 raw issues and 405070 raw links for RedHat\n",
|
||
"Left with 268935 links after removing link duplication\n",
|
||
"Left with 249733 links after removing half-private issues\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/tmp/ipykernel_523178/4239802332.py:23: SettingWithCopyWarning: \n",
|
||
"A value is trying to be set on a copy of a slice from a DataFrame\n",
|
||
"\n",
|
||
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
|
||
" link_df.drop_duplicates(subset=[\"link_key\"], keep=False, inplace=True)\n",
|
||
"/tmp/ipykernel_523178/4239802332.py:27: SettingWithCopyWarning: \n",
|
||
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
"\n",
|
||
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
|
||
" link_df[\"sorted_issue_keys\"] = link_df.apply(column_transform, axis=1)\n",
|
||
"100%|██████████| 4004/4004 [01:48<00:00, 36.81it/s]\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Left with 238349 links after removing issue-pairs with multiple types of links between them\n",
|
||
"Left with 238053 links after removing issue-pairs with duplicate same type of links\n",
|
||
"✔ Cleaned 502297 issues for RedHat\n",
|
||
"✔ Cleaned 238053 links for RedHat\n",
|
||
"✔ Link type distribution:\n",
|
||
"link_type\n",
|
||
"Epic 67799\n",
|
||
"Subtask 45020\n",
|
||
"Related 44222\n",
|
||
"Cloners 29629\n",
|
||
"Blocks 21106\n",
|
||
"Incorporates 12847\n",
|
||
"Duplicate 7080\n",
|
||
"Causality 4122\n",
|
||
"Depend 2849\n",
|
||
"Document 1652\n",
|
||
"Issue split 694\n",
|
||
"Account 568\n",
|
||
"Triggers 465\n",
|
||
"Name: count, dtype: int64\n",
|
||
"❕ Test print: 502297 issues before merged with query DataFrame\n",
|
||
"❕ Test print: 561112 issues after merged with query DataFrame\n",
|
||
"❕ Test print: 502297 issues after processed done\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/tmp/ipykernel_523178/834440382.py:16: SettingWithCopyWarning: \n",
|
||
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
"\n",
|
||
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
|
||
" link_df[\"link_created_time\"] = None\n",
|
||
"/tmp/ipykernel_523178/834440382.py:100: SettingWithCopyWarning: \n",
|
||
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
|
||
"Try using .loc[row_indexer,col_indexer] = value instead\n",
|
||
"\n",
|
||
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
|
||
" link_df[\"link_created_time\"] = pd.to_datetime(\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"✅ ----------------------------\n",
|
||
"\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"for eco_name in ECO_NAMES:\n",
|
||
" # 加载Issue和链接数据DataFrame\n",
|
||
" issue_df = load_issues(eco_name)\n",
|
||
" link_df = load_links(eco_name)\n",
|
||
" print(\n",
|
||
" f\"✔ Loaded {len(issue_df)} raw issues and {len(link_df)} raw links for {eco_name}\"\n",
|
||
" )\n",
|
||
"\n",
|
||
" # 对Issue和链接数据进行清理\n",
|
||
" issue_df = clean_issues(issue_df)\n",
|
||
" link_df = clean_links(link_df, issue_df)\n",
|
||
" print(f\"✔ Cleaned {len(issue_df)} issues for {eco_name}\")\n",
|
||
" print(f\"✔ Cleaned {len(link_df)} links for {eco_name}\")\n",
|
||
"\n",
|
||
" # 打印不同链接类型分布\n",
|
||
" print(\"✔ Link type distribution:\")\n",
|
||
" print(link_df[\"link_type\"].value_counts())\n",
|
||
"\n",
|
||
" # 添加Issue关闭时间\n",
|
||
" issue_df = query_issue_closed_time(eco_name, issue_df)\n",
|
||
"\n",
|
||
" # 添加链接创建时间\n",
|
||
" link_df = query_link_created_time(eco_name, link_df)\n",
|
||
"\n",
|
||
" # 联合Issue和链接数据\n",
|
||
" link_df = joined_links(link_df, issue_df)\n",
|
||
"\n",
|
||
" # 保存清理后的Issue和链接数据\n",
|
||
" issue_df.to_csv(\n",
|
||
" PRO_ISSUE_DIR / (eco_name + \".csv\"),\n",
|
||
" sep=\";\",\n",
|
||
" index=True, #! issue_df的key被设置为了索引列，所以这里需要保存\n",
|
||
" )\n",
|
||
" link_df.to_csv(\n",
|
||
" PRO_LINK_DIR / (eco_name + \".csv\"),\n",
|
||
" sep=\";\",\n",
|
||
" index=False,\n",
|
||
" )\n",
|
||
"\n",
|
||
" print(\"✅ ----------------------------\\n\")"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.9.18"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|