feat: conf file password supports ciphertext (#1207)

Co-authored-by: tanxiao <tanxiao@asiainfo.com>

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gin-contrib/pprof v1.3.0
 	github.com/gin-gonic/gin v1.7.4
 	github.com/go-ldap/ldap/v3 v3.4.1
-	github.com/go-redis/redis/v9 v9.0.0-beta.2
+	github.com/go-redis/redis/v9 v9.0.0-rc.1
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.2
@@ -73,8 +73,8 @@ require (
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	go.uber.org/automaxprocs v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
-	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/genproto v0.0.0-20211007155348-82e027067bd4 // indirect

go.sum

@@ -121,8 +121,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
-github.com/go-redis/redis/v9 v9.0.0-beta.2 h1:ZSr84TsnQyKMAg8gnV+oawuQezeJR11/09THcWCQzr4=
+github.com/go-redis/redis/v9 v9.0.0-rc.1 h1:/+bS+yeUnanqAbuD3QwlejzQZ+4eqgfUtFTG4b+QnXs=
-github.com/go-redis/redis/v9 v9.0.0-beta.2/go.mod h1:Bldcd/M/bm9HbnNPi/LUtYBSD8ttcZYBMupwMXhdU0o=
+github.com/go-redis/redis/v9 v9.0.0-rc.1/go.mod h1:8et+z03j0l8N+DvsVnclzjf3Dl/pFHgRk+2Ct1qw66A=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -296,7 +296,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
-github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q=
+github.com/onsi/gomega v1.21.1 h1:OB/euWYIExnPBohllTicTHmGTrMaqJ67nIu80j0/uEM=
 github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc h1:Ak86L+yDSOzKFa7WM5bf5itSOo1e3Xh8bm5YCMUXIjQ=
 github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -360,8 +360,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -476,8 +476,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -540,8 +540,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

src/main.go

@@ -34,6 +34,11 @@ func newWebapiCmd() *cli.Command {
 				Aliases: []string{"c"},
 				Usage:   "specify configuration file(.json,.yaml,.toml)",
 			},
+			&cli.StringFlag{
+				Name:    "key",
+				Aliases: []string{"k"},
+				Usage:   "specify the secret key for configuration file field encryption",
+			},
 		},
 		Action: func(c *cli.Context) error {
 			printEnv()
@@ -43,6 +48,9 @@ func newWebapiCmd() *cli.Command {
 				opts = append(opts, webapi.SetConfigFile(c.String("conf")))
 			}
 			opts = append(opts, webapi.SetVersion(version.VERSION))
+			if c.String("key") != "" {
+				opts = append(opts, webapi.SetKey(c.String("key")))
+			}
 
 			webapi.Run(opts...)
 			return nil
@@ -60,6 +68,11 @@ func newServerCmd() *cli.Command {
 				Aliases: []string{"c"},
 				Usage:   "specify configuration file(.json,.yaml,.toml)",
 			},
+			&cli.StringFlag{
+				Name:    "key",
+				Aliases: []string{"k"},
+				Usage:   "specify the secret key for configuration file field encryption",
+			},
 		},
 		Action: func(c *cli.Context) error {
 			printEnv()
@@ -69,6 +82,9 @@ func newServerCmd() *cli.Command {
 				opts = append(opts, server.SetConfigFile(c.String("conf")))
 			}
 			opts = append(opts, server.SetVersion(version.VERSION))
+			if c.String("key") != "" {
+				opts = append(opts, server.SetKey(c.String("key")))
+			}
 
 			server.Run(opts...)
 			return nil

src/pkg/secu/aes.go

@@ -0,0 +1,100 @@
+package secu
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"strings"
+)
+
+// BASE64StdEncode base64编码
+func BASE64StdEncode(src []byte) string {
+	return base64.StdEncoding.EncodeToString(src)
+}
+
+// BASE64StdDecode base64解码
+func BASE64StdDecode(src string) ([]byte, error) {
+	dst, err := base64.StdEncoding.DecodeString(src)
+	if err != nil {
+		return nil, err
+	}
+	return dst, nil
+}
+
+func PKCS7Padding(ciphertext []byte, blockSize int) []byte {
+	padding := blockSize - len(ciphertext)%blockSize
+	padtext := bytes.Repeat([]byte{byte(padding)}, padding)
+	return append(ciphertext, padtext...)
+}
+
+func PKCS7UnPadding(originData []byte) []byte {
+	length := len(originData)
+	unpadding := int(originData[length-1])
+	return originData[:(length - unpadding)]
+}
+
+//AES加密
+func AesEncrypt(origData, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	//加密块填充
+	blockSize := block.BlockSize()
+	padOrigData := PKCS7Padding(origData, blockSize)
+	//初始化CBC加密
+	blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
+	crypted := make([]byte, len(padOrigData))
+	//加密
+	blockMode.CryptBlocks(crypted, padOrigData)
+	return crypted, nil
+}
+
+//AES解密
+func AesDecrypt(crypted, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	blockSize := block.BlockSize()
+	blockMode := cipher.NewCBCDecrypter(block, key[:blockSize])
+	origData := make([]byte, len(crypted))
+	//解密
+	blockMode.CryptBlocks(origData, crypted)
+	//去除填充
+	origData = PKCS7UnPadding(origData)
+	return origData, nil
+}
+
+// 针对配置文件属性进行解密处理
+func DealWithDecrypt(src string, key string) (string, error) {
+	//如果是{{cipher}}前缀，则代表是加密过的属性，先解密
+	if strings.HasPrefix(src, "{{cipher}}") {
+		data := src[10:]
+		decodeData, err := BASE64StdDecode(data)
+		if err != nil {
+			return src, err
+		}
+		//解密
+		origin, err := AesDecrypt(decodeData, []byte(key))
+		if err != nil {
+			return src, err
+		}
+		//返回明文
+		return string(origin), nil
+	} else {
+		return src, nil
+	}
+}
+
+// 针对配置文件属性进行加密处理
+func DealWithEncrypt(src string, key string) (string, error) {
+	encrypted, err := AesEncrypt([]byte(src), []byte(key))
+	if err != nil {
+		return src, err
+	}
+
+	data := BASE64StdEncode(encrypted)
+	return "{{cipher}}" + data, nil
+}

src/server/config/config.go

@@ -19,6 +19,7 @@ import (
 	"github.com/didi/nightingale/v5/src/pkg/httpx"
 	"github.com/didi/nightingale/v5/src/pkg/logx"
 	"github.com/didi/nightingale/v5/src/pkg/ormx"
+	"github.com/didi/nightingale/v5/src/pkg/secu"
 	"github.com/didi/nightingale/v5/src/storage"
 )
 
@@ -27,7 +28,61 @@ var (
 	once sync.Once
 )
 
-func MustLoad(fpaths ...string) {
+func DealConfigCrypto(key string) {
+	decryptDsn, err := secu.DealWithDecrypt(C.DB.DSN, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the db dsn", err)
+		os.Exit(1)
+	}
+	C.DB.DSN = decryptDsn
+
+	decryptRedisPwd, err := secu.DealWithDecrypt(C.Redis.Password, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the redis password", err)
+		os.Exit(1)
+	}
+	C.Redis.Password = decryptRedisPwd
+
+	decryptSmtpPwd, err := secu.DealWithDecrypt(C.SMTP.Pass, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the smtp password", err)
+		os.Exit(1)
+	}
+	C.SMTP.Pass = decryptSmtpPwd
+
+	decryptHookPwd, err := secu.DealWithDecrypt(C.Alerting.Webhook.BasicAuthPass, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the alert webhook password", err)
+		os.Exit(1)
+	}
+	C.Alerting.Webhook.BasicAuthPass = decryptHookPwd
+
+	decryptIbexPwd, err := secu.DealWithDecrypt(C.Ibex.BasicAuthPass, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the ibex password", err)
+		os.Exit(1)
+	}
+	C.Ibex.BasicAuthPass = decryptIbexPwd
+
+	decryptReaderPwd, err := secu.DealWithDecrypt(C.Reader.BasicAuthPass, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the reader password", err)
+		os.Exit(1)
+	}
+	C.Reader.BasicAuthPass = decryptReaderPwd
+
+	for index, v := range C.Writers {
+		decryptWriterPwd, err := secu.DealWithDecrypt(v.BasicAuthPass, key)
+		if err != nil {
+			fmt.Printf("failed to decrypt the writer password: %s , error: %s", v.BasicAuthPass, err.Error())
+			os.Exit(1)
+		}
+		C.Writers[index].BasicAuthPass = decryptWriterPwd
+	}
+
+}
+
+func MustLoad(key string, fpaths ...string) {
 	once.Do(func() {
 		loaders := []multiconfig.Loader{
 			&multiconfig.TagLoader{},
@@ -66,6 +121,8 @@ func MustLoad(fpaths ...string) {
 		}
 		m.MustLoad(C)
 
+		DealConfigCrypto(key)
+
 		if C.EngineDelay == 0 {
 			C.EngineDelay = 120
 		}

src/server/server.go

@@ -28,6 +28,7 @@ import (
 type Server struct {
 	ConfigFile string
 	Version    string
+	Key        string
 }
 
 type ServerOption func(*Server)
@@ -44,6 +45,12 @@ func SetVersion(v string) ServerOption {
 	}
 }
 
+func SetKey(k string) ServerOption {
+	return func(s *Server) {
+		s.Key = k
+	}
+}
+
 // Run run server
 func Run(opts ...ServerOption) {
 	code := 1
@@ -92,7 +99,7 @@ func (s Server) initialize() (func(), error) {
 	fns.Add(cancel)
 
 	// parse config file
-	config.MustLoad(s.ConfigFile)
+	config.MustLoad(s.Key, s.ConfigFile)
 
 	// init i18n
 	i18n.Init()

src/webapi/config/config.go

@@ -14,6 +14,7 @@ import (
 	"github.com/didi/nightingale/v5/src/pkg/logx"
 	"github.com/didi/nightingale/v5/src/pkg/oidcc"
 	"github.com/didi/nightingale/v5/src/pkg/ormx"
+	"github.com/didi/nightingale/v5/src/pkg/secu"
 	"github.com/didi/nightingale/v5/src/pkg/tls"
 	"github.com/didi/nightingale/v5/src/storage"
 )
@@ -23,7 +24,40 @@ var (
 	once sync.Once
 )
 
-func MustLoad(fpaths ...string) {
+func DealConfigCrypto(key string) {
+	decryptDsn, err := secu.DealWithDecrypt(C.DB.DSN, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the db dsn", err)
+		os.Exit(1)
+	}
+	C.DB.DSN = decryptDsn
+
+	decryptRedisPwd, err := secu.DealWithDecrypt(C.Redis.Password, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the redis password", err)
+		os.Exit(1)
+	}
+	C.Redis.Password = decryptRedisPwd
+
+	decryptIbexPwd, err := secu.DealWithDecrypt(C.Ibex.BasicAuthPass, key)
+	if err != nil {
+		fmt.Println("failed to decrypt the ibex password", err)
+		os.Exit(1)
+	}
+	C.Ibex.BasicAuthPass = decryptIbexPwd
+
+	for index, v := range C.Clusters {
+		decryptClusterPwd, err := secu.DealWithDecrypt(v.BasicAuthPass, key)
+		if err != nil {
+			fmt.Printf("failed to decrypt the clusters password: %s , error: %s", v.BasicAuthPass, err.Error())
+			os.Exit(1)
+		}
+		C.Clusters[index].BasicAuthPass = decryptClusterPwd
+	}
+
+}
+
+func MustLoad(key string, fpaths ...string) {
 	once.Do(func() {
 		loaders := []multiconfig.Loader{
 			&multiconfig.TagLoader{},
@@ -63,6 +97,8 @@ func MustLoad(fpaths ...string) {
 
 		m.MustLoad(C)
 
+		DealConfigCrypto(key)
+
 		if !strings.HasPrefix(C.Ibex.Address, "http") {
 			C.Ibex.Address = "http://" + C.Ibex.Address
 		}

src/webapi/router/router.go

@@ -331,5 +331,8 @@ func configRoute(r *gin.Engine, version string) {
 		service.PUT("/configs", configsPut)
 		service.POST("/configs", configsPost)
 		service.DELETE("/configs", configsDel)
+
+		service.POST("/conf-prop/encrypt", confPropEncrypt)
+		service.POST("/conf-prop/decrypt", confPropDecrypt)
 	}
 }

src/webapi/router/router_crypto.go

@@ -0,0 +1,62 @@
+package router
+
+import (
+	"github.com/didi/nightingale/v5/src/pkg/secu"
+	"github.com/gin-gonic/gin"
+	"github.com/toolkits/pkg/ginx"
+)
+
+type confPropCrypto struct {
+	Data string `json:"data" binding:"required"`
+	Key  string `json:"key" binding:"required"`
+}
+
+func confPropEncrypt(c *gin.Context) {
+	var f confPropCrypto
+	ginx.BindJSON(c, &f)
+
+	k := len(f.Key)
+	switch k {
+	default:
+		c.String(400, "The key length should be 16, 24 or 32")
+		return
+	case 16, 24, 32:
+		break
+	}
+
+	s, err := secu.DealWithEncrypt(f.Data, f.Key)
+	if err != nil {
+		c.String(500, err.Error())
+	}
+
+	c.JSON(200, gin.H{
+		"src":     f.Data,
+		"key":     f.Key,
+		"encrypt": s,
+	})
+}
+
+func confPropDecrypt(c *gin.Context) {
+	var f confPropCrypto
+	ginx.BindJSON(c, &f)
+
+	k := len(f.Key)
+	switch k {
+	default:
+		c.String(400, "The key length should be 16, 24 or 32")
+		return
+	case 16, 24, 32:
+		break
+	}
+
+	s, err := secu.DealWithDecrypt(f.Data, f.Key)
+	if err != nil {
+		c.String(500, err.Error())
+	}
+
+	c.JSON(200, gin.H{
+		"src":     f.Data,
+		"key":     f.Key,
+		"decrypt": s,
+	})
+}

src/webapi/webapi.go

@@ -24,6 +24,7 @@ import (
 type Webapi struct {
 	ConfigFile string
 	Version    string
+	Key        string
 }
 
 type WebapiOption func(*Webapi)
@@ -40,6 +41,12 @@ func SetVersion(v string) WebapiOption {
 	}
 }
 
+func SetKey(k string) WebapiOption {
+	return func(s *Webapi) {
+		s.Key = k
+	}
+}
+
 // Run run webapi
 func Run(opts ...WebapiOption) {
 	code := 1
@@ -83,7 +90,7 @@ EXIT:
 
 func (a Webapi) initialize() (func(), error) {
 	// parse config file
-	config.MustLoad(a.ConfigFile)
+	config.MustLoad(a.Key, a.ConfigFile)
 
 	// init i18n
 	i18n.Init(config.C.I18N)