forked from Trustie/forgeplus
490 lines
14 KiB
Ruby
490 lines
14 KiB
Ruby
namespace :batch_projects_to_es do
|
|
desc "batc projects to es"
|
|
task git_demo_raw: :environment do
|
|
project_name = ENV['name'] || "mindspore"
|
|
puts "project_id=================#{project_name}"
|
|
projects = Project.where("name like ?", "%#{project_name}%")
|
|
if ENV['count'].present?
|
|
projects = projects.limit(ENV['count'].to_i)
|
|
end
|
|
projects = Project.where(:name => 'opensource0311')
|
|
@client = OpenSearch::Client.new(
|
|
url: "http://localhost:9200",
|
|
retry_on_failure: 5,
|
|
request_timeout: 120,
|
|
log: true
|
|
)
|
|
@client.cluster.health
|
|
@index_count = 0
|
|
total_count = 0
|
|
projects.each do |project|
|
|
result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 5, token: project.owner.gitea_token)
|
|
total_count = result[:total_count]
|
|
puts "total_count==========#{total_count}"
|
|
if total_count > 50
|
|
total_page = (total_count / 50) + 1
|
|
total_page.times do |i|
|
|
add_commit_to_index(project, i + 1)
|
|
end
|
|
else
|
|
add_commit_to_index(project, 1)
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
# Time.now
|
|
# Wed Mar 15 14:12:09 2023 +0800
|
|
# Time.now.strftime("%a %b %d %H:%M:%S %Y")
|
|
# Time.now.strftime("%a %b %d %H:%M:%S %Y +0800")
|
|
Time.parse("2023-03-15 14:12:09").strftime("%a %b %d %H:%M:%S %Y +0800")
|
|
|
|
end
|
|
|
|
def add_commit_to_index(project, page)
|
|
# Gitea::Repository::Commits::ListSliceService.call(project.owner.login,project.identifier,sha: "", page: 1, limit: 1000, token: "a9244ecac647dd33fee3b480c5898baab1d3fe7d")
|
|
result = Gitea::Repository::Commits::ListService.call(project.owner.login,project.identifier,sha: "", page: page, limit: 50, token: project.owner.gitea_token)
|
|
total_count = result[:total_count]
|
|
# puts "result==========#{result}"
|
|
# puts "result[:body]==========#{result[:body].size}"
|
|
result[:body].each do |commit|
|
|
# puts "commit==========#{commit}"
|
|
commiter = commit['commit']['committer']
|
|
# "luoyuan <luoyuan7@huawei.com>"
|
|
commit_author = "#{commiter['name']} <#{commiter['email']}>"
|
|
commit_sha = commit['sha']
|
|
commit_message = commit['commit']['message']
|
|
|
|
commit_date = Time.parse(commit['commit']['author']['date'])
|
|
commit_date_str = commit_date.strftime("%a %b %d %H:%M:%S %Y +0800")
|
|
commit_date_fmt = commit_date.strftime("%Y-%m-%dT%H:%M:%S+00:00")
|
|
commit_date_timestamp = commit_date.strftime("%Y-%m-%dT%H:%M:%S.187594+00:00")
|
|
|
|
|
|
commit_files = Gitea::Repository::Commits::GetService.call(project.owner.login, project.identifier, commit_sha, project.owner.gitea_token, { diff: true })
|
|
# puts "commit_files========#{commit_files}"
|
|
# puts "commit_files========#{commit_files['Files']}"
|
|
mapping_files = []
|
|
commit_files['Files'].each do |file|
|
|
file_tpl = {
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"fa986ba5ed",
|
|
"b1f5f450fc"
|
|
],
|
|
"action": "M",
|
|
"file": "#{file['Name']}",
|
|
"added": "#{file['Addition']}",
|
|
"removed": "#{file['Deletion']}"
|
|
}
|
|
mapping_files.push(file_tpl)
|
|
end
|
|
|
|
|
|
document = {
|
|
"backend_name": "Git",
|
|
"backend_version": "0.12.1",
|
|
"perceval_version": "0.18.0",
|
|
"timestamp": commit_date.to_i,
|
|
"origin": "https://gitee.com/mindspore/mindspore.git",
|
|
"uuid": "#{SecureRandom.uuid.gsub("-", "")}",
|
|
"updated_on": commit_date.to_i,
|
|
"classified_fields_filtered": "",
|
|
"category": "commit",
|
|
"search_fields": {
|
|
"item_id": "#{SecureRandom.uuid.gsub("-", "")}"
|
|
},
|
|
"tag": "https://gitee.com/mindspore/mindspore.git",
|
|
"data": {
|
|
"commit": commit_sha,
|
|
"parents": [
|
|
"4dbb57e00488ae20381dddc7a20421aa14cf7c5c"
|
|
],
|
|
"refs": [],
|
|
"Author": commit_author,
|
|
"AuthorDate": commit_date_str,
|
|
"Commit": commit_author,
|
|
"CommitDate": commit_date_str,
|
|
"message": commit_message,
|
|
"files": mapping_files
|
|
},
|
|
"metadata__updated_on": commit_date_fmt,
|
|
"metadata__timestamp": commit_date_timestamp
|
|
}
|
|
|
|
puts "index_count=======#{@index_count}"
|
|
### 增加索引
|
|
@client.index(
|
|
index: 'git_demo_raw',
|
|
body: document,
|
|
# id: 100,
|
|
refresh: true
|
|
)
|
|
|
|
end
|
|
end
|
|
|
|
def document_tpl
|
|
|
|
{
|
|
"backend_name": "Git",
|
|
"backend_version": "0.12.1",
|
|
"perceval_version": "0.18.0",
|
|
"timestamp": 1.678935077187594E9,
|
|
"origin": "https://gitee.com/mindspore/mindspore.git",
|
|
"uuid": "#{SecureRandom.uuid.gsub("-", "")}",
|
|
"updated_on": 1.678860729E9,
|
|
"classified_fields_filtered": "",
|
|
"category": "commit",
|
|
"search_fields": {
|
|
"item_id": "7a6b59beff08e0b57bc5efa84c7a05d776f7515a"
|
|
},
|
|
"tag": "https://gitee.com/mindspore/mindspore.git",
|
|
"data": {
|
|
"commit": "7a6b59beff08e0b57bc5efa84c7a05d776f7515a",
|
|
"parents": [
|
|
"4dbb57e00488ae20381dddc7a20421aa14cf7c5c"
|
|
],
|
|
"refs": [],
|
|
"Author": "luoyuan <luoyuan7@huawei.com>",
|
|
"AuthorDate": "Wed Mar 15 11:20:04 2023 +0800",
|
|
"Commit": "luoyuan <luoyuan7@huawei.com>",
|
|
"CommitDate": "Wed Mar 15 14:12:09 2023 +0800",
|
|
"message": "[MS][OPS][r2.0] add MaxPoolGradWithArgmaxV2 op",
|
|
"files": [
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"fa986ba5ed",
|
|
"b1f5f450fc"
|
|
],
|
|
"action": "M",
|
|
"file": "config/super_bar_config.json",
|
|
"added": "6",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"21d56146bf",
|
|
"2922e34781"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/kernel/oplib/op_info_utils.cc",
|
|
"added": "2",
|
|
"removed": "2"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"878a190f3b",
|
|
"018b5aa779"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/kernel/oplib/oplib.cc",
|
|
"added": "1",
|
|
"removed": "1"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"016eed9c27",
|
|
"b68e4d1181"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/plugin/device/ascend/kernel/tbe/tbe_json/tbe_json_creator.cc",
|
|
"added": "3",
|
|
"removed": "2"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"d55722e2e6"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/cpu/kernel/max_pool_grad_with_argmax_v2_cpu_kernel.cc",
|
|
"added": "218",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"9fe35f2100"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/cpu/kernel/max_pool_grad_with_argmax_v2_cpu_kernel.h",
|
|
"added": "92",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"312a3fa5b4"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/gpu/kernel/cuda_impl/cuda_ops/maxpool_grad_with_argmax_v2_impl.cu",
|
|
"added": "154",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"20e50eca76"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/gpu/kernel/cuda_impl/cuda_ops/maxpool_grad_with_argmax_v2_impl.cuh",
|
|
"added": "27",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"d16071d706"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/gpu/kernel/nn/maxpool_grad_with_argmax_v2_gpu_kernel.cc",
|
|
"added": "250",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"55aeacac40"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/ccsrc/plugin/device/gpu/kernel/nn/maxpool_grad_with_argmax_v2_gpu_kernel.h",
|
|
"added": "74",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"8f483d5c2d",
|
|
"1c993bf431"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/transform/graph_ir/op_adapter_map.h",
|
|
"added": "1",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"f40e6843a8",
|
|
"7f5edd8d29"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/transform/graph_ir/op_declare/nn_pooling_ops_declare.cc",
|
|
"added": "12",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"87270c07b7",
|
|
"990e55e16b"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/ccsrc/transform/graph_ir/op_declare/nn_pooling_ops_declare.h",
|
|
"added": "3",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"f6310e7f5f",
|
|
"e8d917a4ae"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/core/ops/core_ops.h",
|
|
"added": "1",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"abb19d5d44"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/core/ops/grad/max_pool_grad_with_argmax_v2.cc",
|
|
"added": "180",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"f5380cd37d"
|
|
],
|
|
"action": "A",
|
|
"file": "mindspore/core/ops/grad/max_pool_grad_with_argmax_v2.h",
|
|
"added": "76",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"d9d5e8cef1",
|
|
"7c256fddcc"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/core/ops/op_name.h",
|
|
"added": "2",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"f51a6052fb",
|
|
"460d782db1"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/python/mindspore/ops/_grad_experimental/grad_nn_ops.py",
|
|
"added": "19",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"648f042eca",
|
|
"a0f0b5d4f5"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/python/mindspore/ops/_vmap/vmap_grad_nn_ops.py",
|
|
"added": "2",
|
|
"removed": "1"
|
|
},
|
|
{
|
|
"modes": [
|
|
"100644",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"e4b6688a5a",
|
|
"26ccff4aaf"
|
|
],
|
|
"action": "M",
|
|
"file": "mindspore/python/mindspore/ops/operations/_grad_ops.py",
|
|
"added": "23",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"1d0a838c4e"
|
|
],
|
|
"action": "A",
|
|
"file": "tests/st/ops/ascend/test_maxpool_grad_with_argmax_v2.py",
|
|
"added": "62",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"3d43a1d547"
|
|
],
|
|
"action": "A",
|
|
"file": "tests/st/ops/cpu/test_max_pool_grad_with_argmax_v2_cpu_op.py",
|
|
"added": "185",
|
|
"removed": "0"
|
|
},
|
|
{
|
|
"modes": [
|
|
"000000",
|
|
"100644"
|
|
],
|
|
"indexes": [
|
|
"0000000000",
|
|
"de97ee72e0"
|
|
],
|
|
"action": "A",
|
|
"file": "tests/st/ops/gpu/test_maxpool_grad_with_argmax_v2_gpu_op.py",
|
|
"added": "185",
|
|
"removed": "0"
|
|
}
|
|
]
|
|
},
|
|
"metadata__updated_on": "2023-03-15T06:12:09+00:00",
|
|
"metadata__timestamp": "2023-03-16T02:51:17.187594+00:00"
|
|
}
|
|
|
|
end
|
|
|
|
end |