FIX 完善devops流程

This commit is contained in:
Jasder 2020-07-14 15:06:09 +08:00
parent e74bba9092
commit 6527603e24
21 changed files with 1080 additions and 2619 deletions

2731
README.md

File diff suppressed because it is too large Load Diff

View File

@ -7,16 +7,16 @@ class DevOps::CloudAccountsController < ApplicationController
DevOps::CreateCloudAccountForm.new(devops_params).validate!
logger.info "######### devops_params: #{devops_params}"
logger.info "######### ......: #{(IPAddr.new devops_params[:ip_num]).to_i}"
logger.info "######### ......: #{DevopsCloudAccount.encrypted_secret(devops_params[:secret])}"
logger.info "######### ......: #{DevOps::CloudAccount.encrypted_secret(devops_params[:secret])}"
# 1. 保存华为云服务器帐号
logger.info "######### ......ff: #{devops_params.merge(ip_num: IPAddr.new(devops_params[:ip_num]).to_i, secret: DevopsCloudAccount.encrypted_secret(devops_params[:secret]))}"
create_params = devops_params.merge(ip_num: IPAddr.new(devops_params[:ip_num]).to_i, secret: DevopsCloudAccount.encrypted_secret(devops_params[:secret]))
logger.info "######### ......ff: #{devops_params.merge(ip_num: IPAddr.new(devops_params[:ip_num]).to_i, secret: DevOps::CloudAccount.encrypted_secret(devops_params[:secret]))}"
create_params = devops_params.merge(ip_num: IPAddr.new(devops_params[:ip_num]).to_i, secret: DevOps::CloudAccount.encrypted_secret(devops_params[:secret]))
logger.info "######### create_params: #{create_params}"
cloud_account = DevopsCloudAccount.new(create_params)
cloud_account = DevOps::CloudAccount.new(create_params)
cloud_account.user = current_user
cloud_account.save
# 2. 生成oauth2应用程序的client_id和client_secrete
gitea_oauth = Gitea::Oauth2::CreateService.call(current_user.gitea_token, {name: "pipeline", redirect_uris: [cloud_account.drone_url]})
gitea_oauth = Gitea::Oauth2::CreateService.call(current_user.gitea_token, {name: "pipeline", redirect_uris: ["#{cloud_account.drone_url}/login"]})
logger.info "######### gitea_oauth: #{gitea_oauth}"
oauth = Oauth.new(client_id: gitea_oauth['client_id'],
client_secret: gitea_oauth['client_secret'],
@ -42,6 +42,7 @@ class DevOps::CloudAccountsController < ApplicationController
redirect_url = "#{Gitea.gitea_config[:domain]}/login/oauth/authorize?client_id=#{oauth.client_id}&redirect_uri=#{cloud_account.drone_url}/login&response_type=code"
logger.info "######### redirect_url: #{redirect_url}"
if result
render_ok(redirect_url: redirect_url)
else

View File

@ -0,0 +1,20 @@
class DevOps::LanguagesController < ApplicationController
before_action :require_login
before_action :find_langugae, only: :show
def index
@languages = DevOps::Language.by_usage_amount_desc
end
def show
end
def common
@languages = DevOps::Language.six_common
end
private
def find_langugae
@language = DevOps::Language.find params[:id]
end
end

View File

@ -436,4 +436,9 @@ module ApplicationHelper
def find_user_by_login(login)
User.find_by_login login
end
def render_base64_decoded(str)
return nil if str.blank?
Base64.decode64 str
end
end

View File

@ -0,0 +1,2 @@
module DevOps::LanguagesHelper
end

View File

@ -1,23 +1,25 @@
class DevOps::Drone::Ci
attr_reader :host, :username, :password
attr_reader :host, :username, :password, :gitea_username
# host: drone server's ip
# username: drone server's account
# password: drone server's password
# eq:
# DevOps::Drone::Ci.new(@cloud_account.drone_ip, @cloud_account.account, @cloud_account.visible_secret).get_token
def initialize(host, username, password)
@host = host
@username = username
@password = password
def initialize(host, username, password, gitea_username)
@host = host
@username = username
@password = password
@gitea_username = gitea_username
end
def get_token
puts "--------- sshpass -p #{password} ssh -o 'StrictHostKeyChecking no' #{username}@#{host} '#{cmd}'"
`sshpass -p #{password} ssh -o "StrictHostKeyChecking no" #{username}@#{host} "#{cmd}"`
end
private
def cmd
"cd ..; cd var/lib/drone/; sqlite3 database.sqlite; .dump; select user_hash from users where user_login=#{username} "
"cd ..; cd var/lib/drone/; sqlite3 database.sqlite; .dump; select user_hash from users where user_login=#{gitea_username} "
end
end

View File

@ -33,7 +33,6 @@ class DevOps::Drone::Client
--restart always \
--name drone-agent--#{client_id} \
--net='bridge' \
drone/drone-runner-docker:1
"
drone/drone-runner-docker:1"
end
end

View File

@ -0,0 +1,110 @@
# @private
class DevOps::Drone::Request
format :json
headers 'Accept' => 'application/json'
parser Proc.new { |body, _| parse(body) }
attr_accessor :private_token
# Converts the response body to an ObjectifiedHash.
def self.parse(body)
body = decode(body)
if body.is_a? Hash
ObjectifiedHash.new body
elsif body.is_a? Array
body.collect! { |e| ObjectifiedHash.new(e) }
elsif body == true
body
else
raise Error::Parsing.new "Couldn't parse a response body"
end
end
# Decodes a JSON response into Ruby object.
def self.decode(response)
begin
JSON.load response
rescue JSON::ParserError
raise Error::Parsing.new "The response is not a valid JSON"
end
end
def get(path, options={})
set_httparty_config(options)
set_private_token_header(options)
validate self.class.get(path, options)
end
def post(path, options={})
set_httparty_config(options)
set_private_token_header(options, path)
validate self.class.post(path, options)
end
def put(path, options={})
set_httparty_config(options)
set_private_token_header(options)
validate self.class.put(path, options)
end
def delete(path, options={})
set_httparty_config(options)
set_private_token_header(options)
validate self.class.delete(path, options)
end
# Checks the response code for common errors.
# Returns parsed response for successful requests.
def validate(response)
# case response.code
# when 400; raise Error::BadRequest.new error_message(response)
# when 401; raise Error::Unauthorized.new error_message(response)
# when 403; raise Error::Forbidden.new error_message(response)
# when 404; raise Error::NotFound.new error_message(response)
# when 405; raise Error::MethodNotAllowed.new error_message(response)
# when 406; raise Error::DataNotAccepted.new error_message(response)
# when 409; raise Error::Conflict.new error_message(response)
# when 500; raise Error::InternalServerError.new error_message(response)
# when 502; raise Error::BadGateway.new error_message(response)
# when 503; raise Error::ServiceUnavailable.new error_message(response)
# end
response.parsed_response
end
# Sets a base_uri and default_params for requests.
# @raise [Error::MissingCredentials] if endpoint not set.
def set_request_defaults(endpoint, private_token, sudo=nil)
raise Error::MissingCredentials.new("Please set an endpoint to API") unless endpoint
@private_token = private_token
self.class.base_uri endpoint
self.class.default_params :sudo => sudo
self.class.default_params.delete(:sudo) if sudo.nil?
end
private
# Sets a PRIVATE-TOKEN header for requests.
# @raise [Error::MissingCredentials] if private_token not set.
def set_private_token_header(options, path=nil)
unless path == '/session'
raise Error::MissingCredentials.new("Please set a private_token for user") unless @private_token
options[:headers] = {'PRIVATE-TOKEN' => @private_token}
end
end
# Set HTTParty configuration
# @see https://github.com/jnunemaker/httparty
def set_httparty_config(options)
if self.httparty
options.merge!(self.httparty)
end
end
def error_message(response)
"Server responded with code #{response.code}, message: #{response.parsed_response.message}. " \
"Request URI: #{response.request.base_uri}#{response.request.path}"
end
end

View File

@ -33,9 +33,12 @@ class DevOps::Drone::Server
`
end
# TODO 一下代码方便测试,正式环境请移除
# docker rm -f `docker ps -qa`
def generate_cmd
"service docker start; docker run \
"service docker start; docker rm -f `docker ps -qa`; docker run \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /var/lib/drone:/data \
-e DRONE_GITEA_SERVER=#{gitea_url} \
-e DRONE_GITEA_CLIENT_ID=#{client_id} \
-e DRONE_GITEA_CLIENT_SECRET=#{client_secret} \
@ -48,8 +51,7 @@ class DevOps::Drone::Server
--detach=true \
--name=drone-server-#{client_id} \
--net='bridge' \
drone/drone:1
"
drone/drone:1"
end
private

5
app/models/dev_ops.rb Normal file
View File

@ -0,0 +1,5 @@
module DevOps
def self.table_name_prefix
'dev_ops_'
end
end

View File

@ -1,8 +1,7 @@
class DevopsCloudAccount < ApplicationRecord
class DevOps::CloudAccount < ApplicationRecord
belongs_to :project
belongs_to :user
def drone_host
[drone_ip, ":80"].join
end
@ -22,4 +21,5 @@ class DevopsCloudAccount < ApplicationRecord
def self.encrypted_secret(str)
Base64.encode64(str.strip).gsub(/\n/, '')
end
end

View File

@ -0,0 +1,15 @@
class DevOps::Language < ApplicationRecord
# before_save :encode_content
belongs_to :cover, class_name: "Attachment", foreign_key: :cover_id, optional: true
scope :six_common, -> { limit(6).by_usage_amount_desc }
scope :without_content, -> { select(column_names - ['content']) }
scope :by_usage_amount_desc, -> { order(usage_amount: :desc) }
private
def encode_content
self.content = Base64.encode64 content
end
end

View File

@ -0,0 +1,6 @@
json.array! @languages do |lang|
json.id lang.id
json.name lang.name
json.content render_base64_decoded lang.content
json.cover_url lang.cover_id.present? ? download_url(lang.cover) : nil
end

View File

@ -0,0 +1,6 @@
json.array! @languages do |lang|
json.id lang.id
json.name lang.name
json.content render_base64_decoded lang.content
json.cover_url lang.cover_id.present? ? download_url(lang.cover) : nil
end

View File

@ -0,0 +1,4 @@
json.id @language.id
json.name @language.name
json.content render_base64_decoded @language.content
json.cover_url @language.cover_id.present? ? download_url(@language.cover) : nil

View File

@ -18,6 +18,11 @@ Rails.application.routes.draw do
scope '/api' do
namespace :dev_ops do
resources :cloud_accounts, only: [:create]
resources :languages, only: [:index, :create, :show] do
collection do
get :common
end
end
end
resources :composes do

View File

@ -0,0 +1,11 @@
class CreateDevOpsLanguages < ActiveRecord::Migration[5.2]
def change
create_table :dev_ops_languages do |t|
t.string :name, null: false, comment: 'The name of project language.'
t.text :content, null: false, comment: 'The content of project language.'
t.integer :usage_amount, default: 0, comment: 'number of people Using the language'
t.timestamps
end
end
end

View File

@ -0,0 +1,14 @@
class CreateDevOpsCloudAccounts < ActiveRecord::Migration[5.2]
def change
create_table :dev_ops_cloud_accounts do |t|
t.integer :project_id, null: false
t.integer :user_id, null: false
t.integer :ip_num
t.string :account
t.string :secret
t.timestamps
end
add_index :dev_ops_cloud_accounts, [:project_id, :user_id, :ip_num], name: 'dev_ops_cloud_accounts_p_u_ip'
end
end

View File

@ -0,0 +1,5 @@
class AddCoverIdToDevOpsLanguages < ActiveRecord::Migration[5.2]
def change
add_column :dev_ops_languages, :cover_id, :integer
end
end

View File

@ -0,0 +1,5 @@
class DropDevopsCloundAccountsTable < ActiveRecord::Migration[5.2]
def change
drop_table(:devops_cloud_accounts) if ActiveRecord::Base.connection.tables.include?('devops_cloud_accounts')
end
end

View File

@ -0,0 +1,715 @@
desc "Initialize the data for dev ops languages"
namespace :dev_ops_languages do
task init: :environment do
# DevOps::Language.bulk_insert do |worker|
# languages.each do |attrs|
# worker.add(attrs)
# end
# end
languages.each do |l|
content = Base64.encode64 l[:content]
DevOps::Language.create!(name: l[:name], content: content)
end
end
def languages
[
{
name: "C",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: gcc
commands:
- ./configure
- make
- make test"
},
{
name: "C++",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: gcc
commands:
- ./configure
- make
- make test"
},
{
name: "Docker",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: docker:dind
volumes:
- name: dockersock
path: /var/run/docker.sock
commands:
- docker ps -a
volumes:
- name: dockersock
host:
path: /var/run/docker.sock"
},
{
name: "Java",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: maven:3-jdk-10
commands:
- mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
- mvn test -B"
},
{
name: "R",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: r-base
commands:
- R -e 'install.packages(c('package1','package2'))'
- R CMD build ."
},
{
name: "Ruby",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: ruby
commands:
- bundle install --jobs=3 --retry=3
- rake"
},
{
name: "PHP",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: install
image: composer
commands:
- composer install
- name: test
image: php:7
commands:
- vendor/bin/phpunit --configuration config.xml"
},
{
name: "Python",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: python
commands:
- pip install -r requirements.txt
- pytest"
},
{
name: "MySQL",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: mysql
commands:
- sleep 15
- mysql -u root -h database --execute='SELECT VERSION();'
services:
- name: database
image: mysql
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
MYSQL_DATABASE: test"
},
{
name: "MongoDB",
content: 'kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: ping
image: mongo:4
commands:
- sleep 5
- mongo --host mongo --eval "db.version()"
services:
- name: mongo
image: mongo:4
command: [ --smallfiles ]'
},
{
name: "Clojure",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: clojure
commands:
- lein test"
},
{
name: "CouchDB",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: couchdb:2.2
commands:
- sleep 15
- curl http://database:5984
services:
- name: database
image: couchdb:2.2"
},
{
name: "Crystal",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: crystallang/crystal
commands:
- shards install
- crystal spec.2"
},
{
name: "D",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: dlanguage/dmd
commands:
- dub test"
},
{
name: "Dart",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: google/dart
commands:
- pub get
- pub run test"
},
{
name: "Docker (dind)",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: docker:dind
volumes:
- name: dockersock
path: /var/run
commands:
- sleep 5 # give docker enough time to start
- docker ps -a
services:
- name: docker
image: docker:dind
privileged: true
volumes:
- name: dockersock
path: /var/run
volumes:
- name: dockersock
temp: {}"
},
{
name: "Elasticsearch",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: alpine:3.8
commands:
- apk add curl
- sleep 45
- curl http://database:9200
services:
- name: database
image: elasticsearch:5-alpine"
},
{
name: "Elixir",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: elixir:1.5
commands:
- mix local.rebar --force
- mix local.hex --force
- mix deps.get
- mix test"
},
{
name: "Erlang",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: erlang:21
commands:
- rebar get-deps
- rebar compile
- rebar skip_deps=true eunit"
},
{
name: "20.Go (with Gopath)",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
workspace:
base: /go
path: src/hello-world
steps:
- name: test
image: golang
commands:
- go get
- go test"
},
{
name: "21.Go (with Modules)",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: golang
commands:
- go test
- go build"
},
{
name: "Gradle",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: gradle:jdk10
commands:
- gradle assemble
- gradle check"
},
{
name: "Groovy",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: gradle:2.5-jdk8
commands:
- ./gradlew assemble
- ./gradlew check"
},
{
name: "Haskell",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: haskell
commands:
- cabal install --only-dependencies --enable-tests
- cabal configure --enable-tests
- cabal build
- cabal test"
},
{
name: "Haxe",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: haxe
commands:
- haxelib install build.hxml
- haxe build.hxml"
},
{
name: "MariaDB",
content: 'kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: mariadb
commands:
- sleep 15
- mysql -u root -h database --execute="SELECT VERSION();"
services:
- name: database
image: mariadb
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
MYSQL_DATABASE: test'
},
{
name: "Maven",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: maven:3-jdk-10
commands:
- mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
- mvn test -B"
},
{
name: "Memcached",
content: 'kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: ubuntu
commands:
- apt-get update -qq
- apt-get install -y -qq telnet > /dev/null
- (sleep 1; echo "stats"; sleep 2; echo "quit";) | telnet cache 11211 || true
services:
- name: cache
image: memcached:alpine
command: [ -vv ]'
},
{
name: "Nats",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: ruby:2
commands:
- gem install nats
- nats-pub -s tcp://nats:4222 greeting 'hello'
- nats-pub -s tcp://nats:4222 greeting 'world'
services:
- name: nats
image: nats:1.3.0"
},
{
name: "Node",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: node
commands:
- npm install
- npm test"
},
{
name: "Perl",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: perl
commands:
- cpanm --quiet --installdeps --notest .
- perl Build.PL
- ./Build test"
},
{
name: "Postgres",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: postgres:9-alpine
commands:
- psql -U postgres -d test -h database
services:
- name: database
image: postgres:9-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_DB: test"
},
{
name: "Redis",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: redis
commands:
- sleep 5
- redis-cli -h redis ping
- redis-cli -h redis set FOO bar
- redis-cli -h redis get FOO
services:
- name: redis
image: redis"
},
{
name: "RethinkDB",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: node:9
commands:
- npm install -s -g recli
- recli -h database -j" + 'r.db("rethinkdb").table("stats")' +
"services:
- name: database
image: rethinkdb:2
command: [ rethinkdb, --bind, all ]"
},
{
name: "Rust",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: rust:1.30
commands:
- cargo build --verbose --all
- cargo test --verbose --all"
},
{
name: "Swift",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: swift:4
commands:
- swift build
- swift test"
},
{
name: "Vault",
content: "kind: pipeline
name: default
platform:
os: linux
arch: arm64
steps:
- name: test
image: vault:1.0.0-beta2
environment:
VAULT_ADDR: http://vault:8200
VAULT_TOKEN: dummy
commands:
- sleep 5
- vault kv put secret/my-secret my-value=s3cr3t
- vault kv get secret/my-secret
services:
- name: vault
image: vault:1.0.0-beta2
environment:
VAULT_DEV_ROOT_TOKEN_ID: dummy"
}
]
end
end