OSSInnovation
/
runc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #340 from dqminh/replace-env-netlink 

nsexec: replace usage of environment variable with netlink message
This commit is contained in:
Mrunal Patel 2015-12-09 14:21:45 -08:00
parent 39b80c46bc 7d423cb7a1
commit 0267ad05b0
5 changed files with 269 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
libcontainer/container_linux.go
View File

@ -3,8 +3,10 @@
package libcontainer
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
@ -19,6 +21,7 @@ import (
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/configs"
"github.com/opencontainers/runc/libcontainer/criurpc"
"github.com/vishvananda/netlink/nl"
)
const stdioFdCount = 3
@ -218,7 +221,7 @@ func (c *linuxContainer) newParentProcess(p *Process, doInit bool) (parentProces
return nil, newSystemError(err)
}
if !doInit {
return c.newSetnsProcess(p, cmd, parentPipe, childPipe), nil
return c.newSetnsProcess(p, cmd, parentPipe, childPipe)
}
return c.newInitProcess(p, cmd, parentPipe, childPipe)
}
@ -273,23 +276,24 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c
}, nil
}
func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) *setnsProcess {
cmd.Env = append(cmd.Env,
fmt.Sprintf("_LIBCONTAINER_INITPID=%d", c.initProcess.pid()),
"_LIBCONTAINER_INITTYPE=setns",
)
if p.consolePath != "" {
cmd.Env = append(cmd.Env, "_LIBCONTAINER_CONSOLE_PATH="+p.consolePath)
func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*setnsProcess, error) {
cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE=setns")
// for setns process, we dont have to set cloneflags as the process namespaces
// will only be set via setns syscall
data, err := c.bootstrapData(0, c.initProcess.pid(), p.consolePath)
if err != nil {
return nil, err
}
// TODO: set on container for process management
return &setnsProcess{
cmd: cmd,
cgroupPaths: c.cgroupManager.GetPaths(),
childPipe: childPipe,
parentPipe: parentPipe,
config: c.newInitConfig(p),
process: p,
}
cmd: cmd,
cgroupPaths: c.cgroupManager.GetPaths(),
childPipe: childPipe,
parentPipe: parentPipe,
config: c.newInitConfig(p),
process: p,
bootstrapData: data,
}, nil
}
func (c *linuxContainer) newInitConfig(process *Process) *initConfig {
@ -1021,3 +1025,25 @@ func (c *linuxContainer) currentState() (*State, error) {
}
return state, nil
}
// bootstrapData encodes the necessary data in netlink binary format as a io.Reader.
// Consumer can write the data to a bootstrap program such as one that uses
// nsenter package to bootstrap the container's init process correctly, i.e. with
// correct namespaces, uid/gid mapping etc.
func (c *linuxContainer) bootstrapData(cloneFlags uintptr, pid int, consolePath string) (io.Reader, error) {
// create the netlink message
r := nl.NewNetlinkRequest(int(InitMsg), 0)
// write pid
r.AddData(&Int32msg{
Type: PidAttr,
Value: uint32(pid),
})
// write console path
if consolePath != "" {
r.AddData(&Bytemsg{
Type: ConsolePathAttr,
Value: []byte(consolePath),
})
}
return bytes.NewReader(r.Serialize()), nil
}

60
libcontainer/message_linux.go Normal file
View File

@ -0,0 +1,60 @@
// +build linux
package libcontainer
import (
"syscall"
"github.com/vishvananda/netlink/nl"
)
// list of known message types we want to send to bootstrap program
// The number is randomly chosen to not conflict with known netlink types
const (
InitMsg uint16 = 62000
PidAttr uint16 = 27281
ConsolePathAttr uint16 = 27282
)
type Int32msg struct {
Type uint16
Value uint32
}
// int32msg has the following representation
// | nlattr len | nlattr type |
// | uint32 value |
func (msg *Int32msg) Serialize() []byte {
buf := make([]byte, msg.Len())
native := nl.NativeEndian()
native.PutUint16(buf[0:2], uint16(msg.Len()))
native.PutUint16(buf[2:4], msg.Type)
native.PutUint32(buf[4:8], msg.Value)
return buf
}
func (msg *Int32msg) Len() int {
return syscall.NLA_HDRLEN + 4
}
// bytemsg has the following representation
// | nlattr len | nlattr type |
// | value | pad |
type Bytemsg struct {
Type uint16
Value []byte
}
func (msg *Bytemsg) Serialize() []byte {
l := msg.Len()
buf := make([]byte, (l+syscall.NLA_ALIGNTO-1) & ^(syscall.NLA_ALIGNTO-1))
native := nl.NativeEndian()
native.PutUint16(buf[0:2], uint16(l))
native.PutUint16(buf[2:4], msg.Type)
copy(buf[4:], msg.Value)
return buf
}
func (msg *Bytemsg) Len() int {
return syscall.NLA_HDRLEN + len(msg.Value) + 1 // null-terminated
}

98
libcontainer/nsenter/nsenter_test.go
View File

@ -1,12 +1,17 @@
package nsenter
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"strings"
"syscall"
"testing"
"github.com/opencontainers/runc/libcontainer"
"github.com/vishvananda/netlink/nl"
)
type pid struct {
@ -15,7 +20,7 @@ type pid struct {
func TestNsenterAlivePid(t *testing.T) {
args := []string{"nsenter-exec"}
r, w, err := os.Pipe()
parent, child, err := newPipe()
if err != nil {
t.Fatalf("failed to create pipe %v", err)
}
@ -23,16 +28,22 @@ func TestNsenterAlivePid(t *testing.T) {
cmd := &exec.Cmd{
Path: os.Args[0],
Args: args,
ExtraFiles: []*os.File{w},
Env: []string{fmt.Sprintf("_LIBCONTAINER_INITPID=%d", os.Getpid()), "_LIBCONTAINER_INITPIPE=3"},
ExtraFiles: []*os.File{child},
Env: []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
}
if err := cmd.Start(); err != nil {
t.Fatalf("nsenter failed to start %v", err)
}
w.Close()
decoder := json.NewDecoder(r)
r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
r.AddData(&libcontainer.Int32msg{
Type: libcontainer.PidAttr,
Value: uint32(os.Getpid()),
})
if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
t.Fatal(err)
}
decoder := json.NewDecoder(parent)
var pid *pid
if err := decoder.Decode(&pid); err != nil {
@ -51,34 +62,67 @@ func TestNsenterAlivePid(t *testing.T) {
func TestNsenterInvalidPid(t *testing.T) {
args := []string{"nsenter-exec"}
cmd := &exec.Cmd{
Path: os.Args[0],
Args: args,
Env: []string{"_LIBCONTAINER_INITPID=-1"},
parent, child, err := newPipe()
if err != nil {
t.Fatalf("failed to create pipe %v", err)
}
err := cmd.Run()
if err == nil {
cmd := &exec.Cmd{
Path: os.Args[0],
Args: args,
ExtraFiles: []*os.File{child},
Env: []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
}
if err := cmd.Start(); err != nil {
t.Fatal("nsenter exits with a zero exit status")
}
r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
r.AddData(&libcontainer.Int32msg{
Type: libcontainer.PidAttr,
Value: 0,
})
if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
t.Fatal(err)
}
if err := cmd.Wait(); err == nil {
t.Fatal("nsenter exits with a zero exit status")
}
}
func TestNsenterDeadPid(t *testing.T) {
dead_cmd := exec.Command("true")
if err := dead_cmd.Run(); err != nil {
deadCmd := exec.Command("true")
if err := deadCmd.Run(); err != nil {
t.Fatal(err)
}
args := []string{"nsenter-exec"}
cmd := &exec.Cmd{
Path: os.Args[0],
Args: args,
Env: []string{fmt.Sprintf("_LIBCONTAINER_INITPID=%d", dead_cmd.Process.Pid)},
parent, child, err := newPipe()
if err != nil {
t.Fatalf("failed to create pipe %v", err)
}
err := cmd.Run()
if err == nil {
cmd := &exec.Cmd{
Path: os.Args[0],
Args: args,
ExtraFiles: []*os.File{child},
Env: []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
}
if err := cmd.Start(); err != nil {
t.Fatal("nsenter exits with a zero exit status")
}
r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
r.AddData(&libcontainer.Int32msg{
Type: libcontainer.PidAttr,
Value: uint32(deadCmd.Process.Pid),
})
if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
t.Fatal(err)
}
if err := cmd.Wait(); err == nil {
t.Fatal("nsenter exits with a zero exit status")
}
}
@ -89,3 +133,11 @@ func init() {
}
return
}
func newPipe() (parent *os.File, child *os.File, err error) {
fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0)
if err != nil {
return nil, nil, err
}
return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
}

96
libcontainer/nsenter/nsexec.c
View File

@ -17,6 +17,11 @@
#include <sched.h>
#include <signal.h>
#include <linux/netlink.h>
#include <linux/types.h>
#include <stdint.h>
#include <sys/socket.h>
/* All arguments should be above stack, because it grows down */
struct clone_arg {
/*
@ -63,24 +68,33 @@ static int clone_parent(jmp_buf * env)
return child;
}
static uint32_t readint32(char *buf)
{
return *(uint32_t *) buf;
}
// list of known message types we want to send to bootstrap program
// These are defined in libcontainer/message_linux.go
#define INIT_MSG 62000
#define PID_ATTR 27281
#define CONSOLE_PATH_ATTR 27282
void nsexec()
{
char *namespaces[] = { "ipc", "uts", "net", "pid", "mnt", "user" };
const int num = sizeof(namespaces) / sizeof(char *);
jmp_buf env;
char buf[PATH_MAX], *val;
int i, tfd, self_tfd, child, len, pipenum, consolefd = -1;
pid_t pid;
char *console;
int i, tfd, self_tfd, child, n, len, pipenum, consolefd = -1;
pid_t pid = 0;
val = getenv("_LIBCONTAINER_INITPID");
if (val == NULL)
// if we dont have INITTYPE or this is the init process, skip the bootstrap process
val = getenv("_LIBCONTAINER_INITTYPE");
if (val == NULL || strcmp(val, "standard") == 0) {
return;
pid = atoi(val);
snprintf(buf, sizeof(buf), "%d", pid);
if (strcmp(val, buf)) {
pr_perror("Unable to parse _LIBCONTAINER_INITPID");
}
if (strcmp(val, "setns") != 0) {
pr_perror("Invalid inittype %s", val);
exit(1);
}
@ -89,7 +103,6 @@ void nsexec()
pr_perror("Child pipe not found");
exit(1);
}
pipenum = atoi(val);
snprintf(buf, sizeof(buf), "%d", pipenum);
if (strcmp(val, buf)) {
@ -97,13 +110,56 @@ void nsexec()
exit(1);
}
console = getenv("_LIBCONTAINER_CONSOLE_PATH");
if (console != NULL) {
consolefd = open(console, O_RDWR);
if (consolefd < 0) {
pr_perror("Failed to open console %s", console);
exit(1);
char nlbuf[NLMSG_HDRLEN];
struct nlmsghdr *nh;
if ((n = read(pipenum, nlbuf, NLMSG_HDRLEN)) != NLMSG_HDRLEN) {
pr_perror("Failed to read netlink header, got %d", n);
exit(1);
}
nh = (struct nlmsghdr *)nlbuf;
if (nh->nlmsg_type == NLMSG_ERROR) {
pr_perror("Invalid netlink header message");
exit(1);
}
if (nh->nlmsg_type != INIT_MSG) {
pr_perror("Unexpected netlink message type %d", nh->nlmsg_type);
exit(1);
}
// read the netlink payload
len = NLMSG_PAYLOAD(nh, 0);
char data[len];
if ((n = read(pipenum, data, len)) != len) {
pr_perror("Failed to read netlink payload, got %d", n);
exit(1);
}
int start = 0;
struct nlattr *attr;
while (start < len) {
int payload_len;
attr = (struct nlattr *)((void *)data + start);
start += NLA_HDRLEN;
payload_len = attr->nla_len - NLA_HDRLEN;
switch (attr->nla_type) {
case PID_ATTR:
pid = (pid_t) readint32(data + start);
break;
case CONSOLE_PATH_ATTR:
consolefd = open((char *)data + start, O_RDWR);
if (consolefd < 0) {
pr_perror("Failed to open console %s", (char *)data + start);
exit(1);
}
break;
}
start += NLA_ALIGN(payload_len);
}
// required pid to be passed
if (pid == 0) {
pr_perror("missing pid");
exit(1);
}
/* Check that the specified process exists */
@ -133,15 +189,13 @@ void nsexec()
}
/* Skip namespaces we're already part of */
if (fstatat(self_tfd, namespaces[i], &self_st, 0) != -1 &&
st.st_ino == self_st.st_ino) {
if (fstatat(self_tfd, namespaces[i], &self_st, 0) != -1 && st.st_ino == self_st.st_ino) {
continue;
}
fd = openat(tfd, namespaces[i], O_RDONLY);
if (fd == -1) {
pr_perror("Failed to open ns file %s for ns %s", buf,
namespaces[i]);
pr_perror("Failed to open ns file %s for ns %s", buf, namespaces[i]);
exit(1);
}
// Set the namespace.

30
libcontainer/process_linux.go
View File

@ -41,13 +41,14 @@ type parentProcess interface {
}
type setnsProcess struct {
cmd *exec.Cmd
parentPipe *os.File
childPipe *os.File
cgroupPaths map[string]string
config *initConfig
fds []string
process *Process
cmd *exec.Cmd
parentPipe *os.File
childPipe *os.File
cgroupPaths map[string]string
config *initConfig
fds []string
process *Process
bootstrapData io.Reader
}
func (p *setnsProcess) startTime() (string, error) {
@ -64,6 +65,16 @@ func (p *setnsProcess) signal(sig os.Signal) error {
func (p *setnsProcess) start() (err error) {
defer p.parentPipe.Close()
err = p.cmd.Start()
p.childPipe.Close()
if err != nil {
return newSystemError(err)
}
if p.bootstrapData != nil {
if _, err := io.Copy(p.parentPipe, p.bootstrapData); err != nil {
return newSystemError(err)
}
}
if err = p.execSetns(); err != nil {
return newSystemError(err)
}
@ -96,11 +107,6 @@ func (p *setnsProcess) start() (err error) {
// before the go runtime boots, we wait on the process to die and receive the child's pid
// over the provided pipe.
func (p *setnsProcess) execSetns() error {
err := p.cmd.Start()
p.childPipe.Close()
if err != nil {
return newSystemError(err)
}
status, err := p.cmd.Process.Wait()
if err != nil {
p.cmd.Wait()