From 7fa13b27737650e846995512245bbf1b2bbf64ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulik?= Date: Wed, 8 Apr 2020 23:03:36 +0200 Subject: [PATCH 1/3] intelrdt: change parseCpuInfoFile to return struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Szulik --- libcontainer/intelrdt/intelrdt.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index b19580f1..d40ae644 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -191,8 +191,7 @@ type intelRdtData struct { // Check if Intel RDT sub-features are enabled in init() func init() { // 1. Check if hardware and kernel support Intel RDT sub-features - // "cat_l3" flag for CAT and "mba" flag for MBA - isCatFlagSet, isMbaFlagSet, err := parseCpuInfoFile("/proc/cpuinfo") + flagsSet, err := parseCpuInfoFile("/proc/cpuinfo") if err != nil { return } @@ -207,7 +206,7 @@ func init() { // "resource control" filesystem. Intel RDT sub-features can be // selectively disabled or enabled by kernel command line // (e.g., rdt=!l3cat,mba) in 4.14 and newer kernel - if isCatFlagSet { + if flagsSet.CAT { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3")); err == nil { isCatEnabled = true } @@ -217,7 +216,7 @@ func init() { // MBA should be enabled because MBA Software Controller // depends on MBA isMbaEnabled = true - } else if isMbaFlagSet { + } else if flagsSet.MBA { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "MB")); err == nil { isMbaEnabled = true } @@ -298,13 +297,17 @@ func isIntelRdtMounted() bool { return true } -func parseCpuInfoFile(path string) (bool, bool, error) { - isCatFlagSet := false - isMbaFlagSet := false +type cpuInfoFlags struct { + CAT bool // Cache Allocation Technology + MBA bool // Memory Bandwidth Allocation +} + +func parseCpuInfoFile(path string) (cpuInfoFlags, error) { + infoFlags := cpuInfoFlags{} f, err := os.Open(path) if err != nil { - return false, false, err + return infoFlags, err } defer f.Close() @@ -319,19 +322,19 @@ func parseCpuInfoFile(path string) (bool, bool, error) { for _, flag := range flags { switch flag { case "cat_l3": - isCatFlagSet = true + infoFlags.CAT = true case "mba": - isMbaFlagSet = true + infoFlags.MBA = true } } - return isCatFlagSet, isMbaFlagSet, nil + return infoFlags, nil } } if err := s.Err(); err != nil { - return false, false, err + return infoFlags, err } - return isCatFlagSet, isMbaFlagSet, nil + return infoFlags, nil } func parseUint(s string, base, bitSize int) (uint64, error) { From d1e4c7b803e5e1986fd8f9aa6b67014886d4a5db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulik?= Date: Wed, 8 Apr 2020 23:05:35 +0200 Subject: [PATCH 2/3] intelrdt: add mbm stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Szulik --- events.go | 3 + libcontainer/intelrdt/intelrdt.go | 34 +++++++- libcontainer/intelrdt/mbm.go | 102 ++++++++++++++++++++++++ libcontainer/intelrdt/mbm_test.go | 128 ++++++++++++++++++++++++++++++ libcontainer/intelrdt/stats.go | 11 +++ types/events.go | 5 ++ 6 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 libcontainer/intelrdt/mbm.go create mode 100644 libcontainer/intelrdt/mbm_test.go diff --git a/events.go b/events.go index fb3f6302..b2d3087a 100644 --- a/events.go +++ b/events.go @@ -161,6 +161,9 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot s.IntelRdt.MemBwSchema = is.MemBwSchema } + if intelrdt.IsMbmEnabled() { + s.IntelRdt.MBMStats = is.MBMStats + } } s.NetworkInterfaces = ls.Interfaces diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index d40ae644..6be7f898 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -55,6 +55,10 @@ import ( * | | |-- cbm_mask * | | |-- min_cbm_bits * | | |-- num_closids + * | |-- L3_MON + * | | |-- max_threshold_occupancy + * | | |-- mon_features + * | | |-- num_rmids * | |-- MB * | |-- bandwidth_gran * | |-- delay_linear @@ -221,6 +225,17 @@ func init() { isMbaEnabled = true } } + + if flagsSet.MBMTotal || flagsSet.MBMLocal { + if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil { + isMbmEnabled = true + } + + enabledMonFeatures, err = getMonFeatures(intelRdtRoot) + if err != nil { + return + } + } } // Return the mount point path of Intel RDT "resource control" filesysem @@ -300,6 +315,10 @@ func isIntelRdtMounted() bool { type cpuInfoFlags struct { CAT bool // Cache Allocation Technology MBA bool // Memory Bandwidth Allocation + + // Memory Bandwidth Monitoring related. + MBMTotal bool + MBMLocal bool } func parseCpuInfoFile(path string) (cpuInfoFlags, error) { @@ -325,6 +344,10 @@ func parseCpuInfoFile(path string) (cpuInfoFlags, error) { infoFlags.CAT = true case "mba": infoFlags.MBA = true + case "cqm_mbm_total": + infoFlags.MBMTotal = true + case "cqm_mbm_local": + infoFlags.MBMLocal = true } } return infoFlags, nil @@ -589,7 +612,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { schemaRootStrings := strings.Split(tmpRootStrings, "\n") // The L3 cache and memory bandwidth schemata in 'container_id' group - tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata") + containerPath := m.GetPath() + tmpStrings, err := getIntelRdtParamString(containerPath, "schemata") if err != nil { return nil, err } @@ -641,6 +665,14 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { } } + if IsMbmEnabled() { + mbmStats, err := getMBMStats(containerPath) + if err != nil { + return stats, err + } + stats.MBMStats = mbmStats + } + return stats, nil } diff --git a/libcontainer/intelrdt/mbm.go b/libcontainer/intelrdt/mbm.go new file mode 100644 index 00000000..c76f5155 --- /dev/null +++ b/libcontainer/intelrdt/mbm.go @@ -0,0 +1,102 @@ +// +build linux + +package intelrdt + +import ( + "bufio" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +var ( + // The flag to indicate if Intel RDT/MBM is enabled + isMbmEnabled bool + + enabledMonFeatures monFeatures +) + +type monFeatures struct { + mbmTotalBytes bool + mbmLocalBytes bool +} + +// Check if Intel RDT/MBM is enabled +func IsMbmEnabled() bool { + return isMbmEnabled +} + +func getMonFeatures(intelRdtRoot string) (monFeatures, error) { + file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) + defer file.Close() + if err != nil { + return monFeatures{}, err + } + return parseMonFeatures(file) +} + +func parseMonFeatures(reader io.Reader) (monFeatures, error) { + scanner := bufio.NewScanner(reader) + + monFeatures := monFeatures{} + + for scanner.Scan() { + + switch feature := scanner.Text(); feature { + + case "mbm_total_bytes": + monFeatures.mbmTotalBytes = true + case "mbm_local_bytes": + monFeatures.mbmLocalBytes = true + default: + logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) + } + } + + return monFeatures, scanner.Err() +} + +func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) { + var mbmStats []MBMNumaNodeStats + + numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) + if err != nil { + return &mbmStats, err + } + + for _, file := range numaFiles { + if file.IsDir() { + numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name())) + if err != nil { + return &mbmStats, nil + } + mbmStats = append(mbmStats, *numaStats) + } + } + + return &mbmStats, nil +} + +func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) { + stats := &MBMNumaNodeStats{} + if enabledMonFeatures.mbmTotalBytes { + mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes") + if err != nil { + return nil, err + } + stats.MBMTotalBytes = mbmTotalBytes + } + + if enabledMonFeatures.mbmLocalBytes { + mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes") + if err != nil { + return nil, err + } + stats.MBMLocalBytes = mbmLocalBytes + } + + return stats, nil +} diff --git a/libcontainer/intelrdt/mbm_test.go b/libcontainer/intelrdt/mbm_test.go new file mode 100644 index 00000000..9ddf8ba2 --- /dev/null +++ b/libcontainer/intelrdt/mbm_test.go @@ -0,0 +1,128 @@ +// +build linux + +package intelrdt + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestParseMonFeatures(t *testing.T) { + t.Run("All features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures( + strings.NewReader("mbm_total_bytes\nmbm_local_bytes")) + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{true, true} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Cannot gather all features!") + } + }) + + t.Run("No features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) + + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{false, false} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Expected no features available but there is any!") + } + }) +} + +func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) { + testDir, err := ioutil.TempDir("", "rdt_mbm_test") + if err != nil { + return "", err + } + monDataPath := filepath.Join(testDir, "mon_data") + + for _, numa := range NUMANodes { + numaPath := filepath.Join(monDataPath, numa) + err = os.MkdirAll(numaPath, os.ModePerm) + if err != nil { + return "", err + } + + for fileName, value := range mocks { + err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) + if err != nil { + return "", err + } + } + + } + + return testDir, nil +} + +func TestGetMbmStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + } + + mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedMBM) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + + stats, err := getMBMStats(mockedMBM) + if err != nil { + t.Fatal(err) + } + + if len(*stats) != len(mocksNUMANodesToCreate) { + t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v", + len(mocksNUMANodesToCreate), len(*stats)) + } + + checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { + if got.MBMTotalBytes != expected.MBMTotalBytes { + t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", + expected.MBMTotalBytes, + got.MBMTotalBytes) + } + + if got.MBMLocalBytes != expected.MBMLocalBytes { + t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", + expected.MBMLocalBytes, + got.MBMLocalBytes) + } + + } + + expectedStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + checkStatCorrection((*stats)[0], expectedStats, t) + checkStatCorrection((*stats)[1], expectedStats, t) + }) +} diff --git a/libcontainer/intelrdt/stats.go b/libcontainer/intelrdt/stats.go index df5686f3..f90293be 100644 --- a/libcontainer/intelrdt/stats.go +++ b/libcontainer/intelrdt/stats.go @@ -15,6 +15,14 @@ type MemBwInfo struct { NumClosids uint64 `json:"num_closids,omitempty"` } +type MBMNumaNodeStats struct { + // The 'mbm_total_bytes' in 'container_id' group + MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"` + + // The 'mbm_local_bytes' in 'container_id' group + MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"` +} + type Stats struct { // The read-only L3 cache information L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"` @@ -33,6 +41,9 @@ type Stats struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"` } func NewStats() *Stats { diff --git a/types/events.go b/types/events.go index c6f0e97a..bdc0a515 100644 --- a/types/events.go +++ b/types/events.go @@ -1,5 +1,7 @@ package types +import "github.com/opencontainers/runc/libcontainer/intelrdt" + // Event struct for encoding the event data to json. type Event struct { Type string `json:"type"` @@ -113,6 +115,9 @@ type IntelRdt struct { // The memory bandwidth schema in 'container_id' group MemBwSchema string `json:"mem_bw_schema,omitempty"` + + // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group + MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"` } type NetworkInterface struct { From 799d94818d0012df524d20349f5f179be6d97f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szulik?= Date: Wed, 15 Apr 2020 21:18:41 +0200 Subject: [PATCH 3/3] intelrdt: Add Cache Monitoring Technology stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paweł Szulik --- events.go | 5 +- libcontainer/intelrdt/cmt.go | 22 +++++ libcontainer/intelrdt/cmt_test.go | 56 +++++++++++ libcontainer/intelrdt/intelrdt.go | 12 +-- libcontainer/intelrdt/mbm.go | 76 +-------------- libcontainer/intelrdt/mbm_test.go | 112 +++++---------------- libcontainer/intelrdt/monitoring.go | 85 ++++++++++++++++ libcontainer/intelrdt/monitoring_test.go | 118 +++++++++++++++++++++++ libcontainer/intelrdt/stats.go | 14 ++- types/events.go | 5 +- 10 files changed, 335 insertions(+), 170 deletions(-) create mode 100644 libcontainer/intelrdt/cmt.go create mode 100644 libcontainer/intelrdt/cmt_test.go create mode 100644 libcontainer/intelrdt/monitoring.go create mode 100644 libcontainer/intelrdt/monitoring_test.go diff --git a/events.go b/events.go index b2d3087a..fcda3b17 100644 --- a/events.go +++ b/events.go @@ -161,9 +161,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot s.IntelRdt.MemBwSchema = is.MemBwSchema } - if intelrdt.IsMbmEnabled() { + if intelrdt.IsMBMEnabled() { s.IntelRdt.MBMStats = is.MBMStats } + if intelrdt.IsCMTEnabled() { + s.IntelRdt.CMTStats = is.CMTStats + } } s.NetworkInterfaces = ls.Interfaces diff --git a/libcontainer/intelrdt/cmt.go b/libcontainer/intelrdt/cmt.go new file mode 100644 index 00000000..5c406e10 --- /dev/null +++ b/libcontainer/intelrdt/cmt.go @@ -0,0 +1,22 @@ +package intelrdt + +var ( + cmtEnabled bool +) + +// Check if Intel RDT/CMT is enabled. +func IsCMTEnabled() bool { + return cmtEnabled +} + +func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) { + stats := &CMTNumaNodeStats{} + + llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy") + if err != nil { + return nil, err + } + stats.LLCOccupancy = llcOccupancy + + return stats, nil +} diff --git a/libcontainer/intelrdt/cmt_test.go b/libcontainer/intelrdt/cmt_test.go new file mode 100644 index 00000000..e061695e --- /dev/null +++ b/libcontainer/intelrdt/cmt_test.go @@ -0,0 +1,56 @@ +package intelrdt + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetCMTNumaNodeStats(t *testing.T) { + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "llc_occupancy": 9123911, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather mbm", func(t *testing.T) { + enabledMonFeatures.llcOccupancy = true + + stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) + } + stats = append(stats, *other) + } + + expectedStats := CMTNumaNodeStats{ + LLCOccupancy: mocksFilesToCreate["llc_occupancy"], + } + + checkCMTStatCorrection(stats[0], expectedStats, t) + checkCMTStatCorrection(stats[1], expectedStats, t) + }) +} + +func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) { + if got.LLCOccupancy != expected.LLCOccupancy { + t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v", + expected.LLCOccupancy, + got.LLCOccupancy) + } +} diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index 6be7f898..5b19d55a 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -228,7 +228,8 @@ func init() { if flagsSet.MBMTotal || flagsSet.MBMLocal { if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil { - isMbmEnabled = true + mbmEnabled = true + cmtEnabled = true } enabledMonFeatures, err = getMonFeatures(intelRdtRoot) @@ -665,12 +666,9 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) { } } - if IsMbmEnabled() { - mbmStats, err := getMBMStats(containerPath) - if err != nil { - return stats, err - } - stats.MBMStats = mbmStats + err = getMonitoringStats(containerPath, stats) + if err != nil { + return nil, err } return stats, nil diff --git a/libcontainer/intelrdt/mbm.go b/libcontainer/intelrdt/mbm.go index c76f5155..3730bab5 100644 --- a/libcontainer/intelrdt/mbm.go +++ b/libcontainer/intelrdt/mbm.go @@ -2,82 +2,14 @@ package intelrdt -import ( - "bufio" - "io" - "io/ioutil" - "os" - "path/filepath" - - "github.com/sirupsen/logrus" -) - var ( // The flag to indicate if Intel RDT/MBM is enabled - isMbmEnabled bool - - enabledMonFeatures monFeatures + mbmEnabled bool ) -type monFeatures struct { - mbmTotalBytes bool - mbmLocalBytes bool -} - -// Check if Intel RDT/MBM is enabled -func IsMbmEnabled() bool { - return isMbmEnabled -} - -func getMonFeatures(intelRdtRoot string) (monFeatures, error) { - file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) - defer file.Close() - if err != nil { - return monFeatures{}, err - } - return parseMonFeatures(file) -} - -func parseMonFeatures(reader io.Reader) (monFeatures, error) { - scanner := bufio.NewScanner(reader) - - monFeatures := monFeatures{} - - for scanner.Scan() { - - switch feature := scanner.Text(); feature { - - case "mbm_total_bytes": - monFeatures.mbmTotalBytes = true - case "mbm_local_bytes": - monFeatures.mbmLocalBytes = true - default: - logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) - } - } - - return monFeatures, scanner.Err() -} - -func getMBMStats(containerPath string) (*[]MBMNumaNodeStats, error) { - var mbmStats []MBMNumaNodeStats - - numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) - if err != nil { - return &mbmStats, err - } - - for _, file := range numaFiles { - if file.IsDir() { - numaStats, err := getMBMNumaNodeStats(filepath.Join(containerPath, "mon_data", file.Name())) - if err != nil { - return &mbmStats, nil - } - mbmStats = append(mbmStats, *numaStats) - } - } - - return &mbmStats, nil +// Check if Intel RDT/MBM is enabled. +func IsMBMEnabled() bool { + return mbmEnabled } func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) { diff --git a/libcontainer/intelrdt/mbm_test.go b/libcontainer/intelrdt/mbm_test.go index 9ddf8ba2..ae0f202c 100644 --- a/libcontainer/intelrdt/mbm_test.go +++ b/libcontainer/intelrdt/mbm_test.go @@ -3,71 +3,12 @@ package intelrdt import ( - "io/ioutil" "os" "path/filepath" - "strconv" - "strings" "testing" ) -func TestParseMonFeatures(t *testing.T) { - t.Run("All features available", func(t *testing.T) { - parsedMonFeatures, err := parseMonFeatures( - strings.NewReader("mbm_total_bytes\nmbm_local_bytes")) - if err != nil { - t.Errorf("Error while parsing mon features err = %v", err) - } - - expectedMonFeatures := monFeatures{true, true} - - if parsedMonFeatures != expectedMonFeatures { - t.Error("Cannot gather all features!") - } - }) - - t.Run("No features available", func(t *testing.T) { - parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) - - if err != nil { - t.Errorf("Error while parsing mon features err = %v", err) - } - - expectedMonFeatures := monFeatures{false, false} - - if parsedMonFeatures != expectedMonFeatures { - t.Error("Expected no features available but there is any!") - } - }) -} - -func mockMBM(NUMANodes []string, mocks map[string]uint64) (string, error) { - testDir, err := ioutil.TempDir("", "rdt_mbm_test") - if err != nil { - return "", err - } - monDataPath := filepath.Join(testDir, "mon_data") - - for _, numa := range NUMANodes { - numaPath := filepath.Join(monDataPath, numa) - err = os.MkdirAll(numaPath, os.ModePerm) - if err != nil { - return "", err - } - - for fileName, value := range mocks { - err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) - if err != nil { - return "", err - } - } - - } - - return testDir, nil -} - -func TestGetMbmStats(t *testing.T) { +func TestGetMBMNumaNodeStats(t *testing.T) { mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} mocksFilesToCreate := map[string]uint64{ @@ -75,10 +16,10 @@ func TestGetMbmStats(t *testing.T) { "mbm_local_bytes": 2361361, } - mockedMBM, err := mockMBM(mocksNUMANodesToCreate, mocksFilesToCreate) + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) defer func() { - err := os.RemoveAll(mockedMBM) + err := os.RemoveAll(mockedL3_MON) if err != nil { t.Fatal(err) } @@ -92,29 +33,13 @@ func TestGetMbmStats(t *testing.T) { enabledMonFeatures.mbmTotalBytes = true enabledMonFeatures.mbmLocalBytes = true - stats, err := getMBMStats(mockedMBM) - if err != nil { - t.Fatal(err) - } - - if len(*stats) != len(mocksNUMANodesToCreate) { - t.Fatalf("Wrong number of stats slices from NUMA nodes. Expected: %v but got: %v", - len(mocksNUMANodesToCreate), len(*stats)) - } - - checkStatCorrection := func(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { - if got.MBMTotalBytes != expected.MBMTotalBytes { - t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", - expected.MBMTotalBytes, - got.MBMTotalBytes) + stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate)) + for _, numa := range mocksNUMANodesToCreate { + other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa)) + if err != nil { + t.Fatal(err) } - - if got.MBMLocalBytes != expected.MBMLocalBytes { - t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", - expected.MBMLocalBytes, - got.MBMLocalBytes) - } - + stats = append(stats, *other) } expectedStats := MBMNumaNodeStats{ @@ -122,7 +47,22 @@ func TestGetMbmStats(t *testing.T) { MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], } - checkStatCorrection((*stats)[0], expectedStats, t) - checkStatCorrection((*stats)[1], expectedStats, t) + checkMBMStatCorrection(stats[0], expectedStats, t) + checkMBMStatCorrection(stats[1], expectedStats, t) }) } + +func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) { + if got.MBMTotalBytes != expected.MBMTotalBytes { + t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v", + expected.MBMTotalBytes, + got.MBMTotalBytes) + } + + if got.MBMLocalBytes != expected.MBMLocalBytes { + t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v", + expected.MBMLocalBytes, + got.MBMLocalBytes) + } + +} diff --git a/libcontainer/intelrdt/monitoring.go b/libcontainer/intelrdt/monitoring.go new file mode 100644 index 00000000..4ccc8eae --- /dev/null +++ b/libcontainer/intelrdt/monitoring.go @@ -0,0 +1,85 @@ +package intelrdt + +import ( + "bufio" + "github.com/sirupsen/logrus" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +var ( + enabledMonFeatures monFeatures +) + +type monFeatures struct { + mbmTotalBytes bool + mbmLocalBytes bool + llcOccupancy bool +} + +func getMonFeatures(intelRdtRoot string) (monFeatures, error) { + file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features")) + defer file.Close() + if err != nil { + return monFeatures{}, err + } + return parseMonFeatures(file) +} + +func parseMonFeatures(reader io.Reader) (monFeatures, error) { + scanner := bufio.NewScanner(reader) + + monFeatures := monFeatures{} + + for scanner.Scan() { + switch feature := scanner.Text(); feature { + case "mbm_total_bytes": + monFeatures.mbmTotalBytes = true + case "mbm_local_bytes": + monFeatures.mbmLocalBytes = true + case "llc_occupancy": + monFeatures.llcOccupancy = true + default: + logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature) + } + } + + return monFeatures, scanner.Err() +} + +func getMonitoringStats(containerPath string, stats *Stats) error { + numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data")) + if err != nil { + return err + } + + var mbmStats []MBMNumaNodeStats + var cmtStats []CMTNumaNodeStats + + for _, file := range numaFiles { + if file.IsDir() { + numaPath := filepath.Join(containerPath, "mon_data", file.Name()) + if IsMBMEnabled() { + numaMBMStats, err := getMBMNumaNodeStats(numaPath) + if err != nil { + return err + } + mbmStats = append(mbmStats, *numaMBMStats) + } + if IsCMTEnabled() { + numaCMTStats, err := getCMTNumaNodeStats(numaPath) + if err != nil { + return err + } + cmtStats = append(cmtStats, *numaCMTStats) + } + } + } + + stats.MBMStats = &mbmStats + stats.CMTStats = &cmtStats + + return err +} diff --git a/libcontainer/intelrdt/monitoring_test.go b/libcontainer/intelrdt/monitoring_test.go new file mode 100644 index 00000000..ec886a7e --- /dev/null +++ b/libcontainer/intelrdt/monitoring_test.go @@ -0,0 +1,118 @@ +package intelrdt + +import ( + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestParseMonFeatures(t *testing.T) { + t.Run("All features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures( + strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy")) + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{true, true, true} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Cannot gather all features!") + } + }) + + t.Run("No features available", func(t *testing.T) { + parsedMonFeatures, err := parseMonFeatures(strings.NewReader("")) + + if err != nil { + t.Errorf("Error while parsing mon features err = %v", err) + } + + expectedMonFeatures := monFeatures{false, false, false} + + if parsedMonFeatures != expectedMonFeatures { + t.Error("Expected no features available but there is any!") + } + }) +} + +func mockResctrlL3_MON(NUMANodes []string, mocks map[string]uint64) (string, error) { + testDir, err := ioutil.TempDir("", "rdt_mbm_test") + if err != nil { + return "", err + } + monDataPath := filepath.Join(testDir, "mon_data") + + for _, numa := range NUMANodes { + numaPath := filepath.Join(monDataPath, numa) + err = os.MkdirAll(numaPath, os.ModePerm) + if err != nil { + return "", err + } + + for fileName, value := range mocks { + err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777) + if err != nil { + return "", err + } + } + + } + + return testDir, nil +} + +func TestGetMonitoringStats(t *testing.T) { + enabledMonFeatures.mbmTotalBytes = true + enabledMonFeatures.mbmLocalBytes = true + enabledMonFeatures.llcOccupancy = true + mbmEnabled = true + cmtEnabled = true + + mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"} + + mocksFilesToCreate := map[string]uint64{ + "mbm_total_bytes": 9123911, + "mbm_local_bytes": 2361361, + "llc_occupancy": 123331, + } + + mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate) + + defer func() { + err := os.RemoveAll(mockedL3_MON) + if err != nil { + t.Fatal(err) + } + }() + + if err != nil { + t.Fatal(err) + } + + t.Run("Gather monitoring stats", func(t *testing.T) { + var stats Stats + err := getMonitoringStats(mockedL3_MON, &stats) + if err != nil { + t.Fatal(err) + } + + expectedMBMStats := MBMNumaNodeStats{ + MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"], + MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"], + } + + expectedCMTStats := CMTNumaNodeStats{LLCOccupancy: mocksFilesToCreate["llc_occupancy"]} + + for _, gotMBMStat := range *stats.MBMStats { + checkMBMStatCorrection(gotMBMStat, expectedMBMStats, t) + } + + for _, gotCMTStat := range *stats.CMTStats { + checkCMTStatCorrection(gotCMTStat, expectedCMTStats, t) + } + }) +} diff --git a/libcontainer/intelrdt/stats.go b/libcontainer/intelrdt/stats.go index f90293be..eeb0ee9f 100644 --- a/libcontainer/intelrdt/stats.go +++ b/libcontainer/intelrdt/stats.go @@ -16,13 +16,18 @@ type MemBwInfo struct { } type MBMNumaNodeStats struct { - // The 'mbm_total_bytes' in 'container_id' group + // The 'mbm_total_bytes' in 'container_id' group. MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"` - // The 'mbm_local_bytes' in 'container_id' group + // The 'mbm_local_bytes' in 'container_id' group. MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"` } +type CMTNumaNodeStats struct { + // The 'llc_occupancy' in 'container_id' group. + LLCOccupancy uint64 `json:"llc_occupancy,omitempty"` +} + type Stats struct { // The read-only L3 cache information L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"` @@ -43,7 +48,10 @@ type Stats struct { MemBwSchema string `json:"mem_bw_schema,omitempty"` // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group - MBMStats *[]MBMNumaNodeStats `json:"mbm_statistics,omitempty"` + MBMStats *[]MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]CMTNumaNodeStats `json:"cmt_stats,omitempty"` } func NewStats() *Stats { diff --git a/types/events.go b/types/events.go index bdc0a515..34d170c8 100644 --- a/types/events.go +++ b/types/events.go @@ -117,7 +117,10 @@ type IntelRdt struct { MemBwSchema string `json:"mem_bw_schema,omitempty"` // The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group - MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_statistics,omitempty"` + MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_stats,omitempty"` + + // The cache monitoring technology statistics from NUMA nodes in 'container_id' group + CMTStats *[]intelrdt.CMTNumaNodeStats `json:"cmt_stats,omitempty"` } type NetworkInterface struct {