gitlinkuser1
/
xiuos
forked from xuos/xiuos
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: kernel selector in Kconfig

This commit is contained in:
Forsworns 2022-02-16 14:49:54 +08:00
parent aa27dea6e9
commit 60cd74e296
13 changed files with 621 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
Ubiquitous/Kselector_features Normal file
View File

@ -0,0 +1,65 @@
menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh)
menuconfig FS
bool "require file system"
default n
if FS
config FS_VFS_FATFS
bool "Using FATFS file system"
default n
config FS_CH376
bool "Using CH376 file system"
default n
config FS_LWEXT4
bool "Using LWEXT4 file system"
default n
endif
menuconfig INDUSTRIAL
bool "Require industrial protocols support"
default n
if INDUSTRIAL
config INDUSTRIAL_OPCUA
bool "Support OPCUA"
default n
config INDUSTRIAL_SNAP7
bool "Support Siemens Snap7"
default n
config INDUSTRIAL_MODBUS
bool "Support Modbus"
default n
config INDUSTRIAL_RS232
bool "Support RS232"
default n
config INDUSTRIAL_RS485
bool "Support RS485"
default n
endif
menuconfig BUS
bool "require bus"
default y
if BUS
menu "Required BUS"
config USB
bool "require USB bus"
default n
config SERIAL
bool "require serial bus"
default n
endmenu
endif
endmenu

65
Ubiquitous/Kselector_features_meta Normal file
View File

@ -0,0 +1,65 @@
menu "Required Feature" # do not modify this, otherwise, (see `egrep '^# Required'` in kernel_selector.sh)
menuconfig FS
bool "require file system"
default n
if FS
config FS_VFS_FATFS
bool "Using FATFS file system"
default n
config FS_CH376
bool "Using CH376 file system"
default n
config FS_LWEXT4
bool "Using LWEXT4 file system"
default n
endif
menuconfig INDUSTRIAL
bool "Require industrial protocols support"
default n
if INDUSTRIAL
config INDUSTRIAL_OPCUA
bool "Support OPCUA"
default n
config INDUSTRIAL_SNAP7
bool "Support Siemens Snap7"
default n
config INDUSTRIAL_MODBUS
bool "Support Modbus"
default n
config INDUSTRIAL_RS232
bool "Support RS232"
default n
config INDUSTRIAL_RS485
bool "Support RS485"
default n
endif
menuconfig BUS
bool "require bus"
default y
if BUS
menu "Required BUS"
config USB
bool "require USB bus"
default n
config SERIAL
bool "require serial bus"
default n
endmenu
endif
endmenu

26
Ubiquitous/Kselector_params Normal file
View File

@ -0,0 +1,26 @@
menu "Required Parameter"
config RAM_LESS_THAN
int "RAM footprint limit (MB) upper bound"
default 10000
help
Kernel RAM footprint should be lower than this limit
config RAM_GREATER_THAN
int "RAM footprint limit (MB) lower bound"
default 0
help
Kernel RAM footprint should be higher than this limit
config ROM_LESS_THAN
int "ROM limit (MB) upper bound"
default 10000
help
Kernel ROM requirement should be lower than this limit
config ROM_GREATER_THAN
int "ROM limit (MB) lower bound"
default 0
help
Kernel ROM requirement should be higher than this limit
endmenu

26
Ubiquitous/Kselector_params_meta Normal file
View File

@ -0,0 +1,26 @@
menu "Required Parameter"
config RAM_LESS_THAN
int "RAM footprint limit (MB) upper bound"
default 10000
help
Kernel RAM footprint should be lower than this limit
config RAM_GREATER_THAN
int "RAM footprint limit (MB) lower bound"
default 0
help
Kernel RAM footprint should be higher than this limit
config ROM_LESS_THAN
int "ROM limit (MB) upper bound"
default 10000
help
Kernel ROM requirement should be lower than this limit
config ROM_GREATER_THAN
int "ROM limit (MB) lower bound"
default 0
help
Kernel ROM requirement should be higher than this limit
endmenu

4
Ubiquitous/XiUOS/.gitignore vendored
View File

@ -6,3 +6,7 @@ build
XiUOS.*
*.swp
.vscode
# for kernel_selector
.selector
.selector.old
requirement.yaml

27
Ubiquitous/XiUOS/Makefile
View File

@ -9,6 +9,8 @@ support :=kd233 stm32f407-st-discovery maix-go stm32f407zgt6 aiit-riscv64-board
SRC_DIR:=
export BOARD ?=kd233
# This is the environment variable for kconfig-mconf
export KCONFIG_CONFIG ?= .config
ifeq ($(filter $(BOARD),$(support)),)
$(warning "You should choose board like this:make BOARD=kd233")
@ -25,11 +27,15 @@ MAKEFILES =$(KERNEL_ROOT)/.config
-include $(KERNEL_ROOT)/.config
export BSP_ROOT ?= $(KERNEL_ROOT)/board/$(BOARD)
export UBIQUITOUS_ROOT ?= ..
include board/$(BOARD)/config.mk
export BSP_BUILD_DIR := board/$(BOARD)
export HOSTTOOLS_DIR ?= $(KERNEL_ROOT)/tool/hosttools
export CONFIG2H_EXE ?= $(HOSTTOOLS_DIR)/xsconfig.sh
export GEN_KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/generate_kselector.py
export FEATURE2YAML_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.sh
export KSELECTOR_EXE ?= $(HOSTTOOLS_DIR)/kernel_selector.py
export CPPPATHS
export SRC_APP_DIR := ../../APP_Framework
export SRC_KERNEL_DIR := arch board lib fs kernel resources tool
@ -74,6 +80,9 @@ COMPILE_ALL:
show_info:
@echo "CONFIG_COMPILER_APP is :" $(CONFIG_COMPILER_APP)
@echo "CONFIG_COMPILER_KERNEL is :" $(CONFIG_COMPILER_KERNEL)
@echo "KERNELPATHS is :" $(KERNELPATHS)
@echo "TARGET is :" $(TARGET)
@echo "VPATH is :" $(VPATH)
@echo "BSP_ROOT is :" $(BSP_ROOT)
@ -100,6 +109,24 @@ menuconfig:
@$(CONFIG2H_EXE) .config
@cp $(KERNEL_ROOT)/.config $(BSP_ROOT)/.config
kernel_selector:
$(eval KCONFIG_CONFIG := .selector)
@if [ -f "$(BSP_ROOT)/.selector" ]; then \
cp $(BSP_ROOT)/.selector $(KERNEL_ROOT)/.selector; \
else if [ -f "$(BSP_ROOT)/.defselector" ]; then \
cp $(BSP_ROOT)/.defselector $(KERNEL_ROOT)/.selector ;\
fi ;fi
@$(GEN_KSELECTOR_EXE)
@cp "$(UBIQUITOUS_ROOT)/Kselector_features_meta" $(UBIQUITOUS_ROOT)/Kselector_features
@cp "$(UBIQUITOUS_ROOT)/Kselector_params_meta" $(UBIQUITOUS_ROOT)/Kselector_params
@kconfig-mconf $(BSP_ROOT)/Kselector
@$(FEATURE2YAML_EXE) .selector
@if [ -f "$(BSP_ROOT)/requirement.yaml" ]; then \
cp $(BSP_ROOT)/requirement.yaml $(KERNEL_ROOT)/requirement.yaml; \
fi
@$(KSELECTOR_EXE)
@cp $(KERNEL_ROOT)/.selector $(BSP_ROOT)/.selector
clean:
@echo Clean target and build_dir
@rm -rf build

40
Ubiquitous/XiUOS/board/aiit-arm32-board/Kselector Normal file
View File

@ -0,0 +1,40 @@
mainmenu "Ubiquitous Kernel Selector"
config BSP_DIR
string
option env="BSP_ROOT"
default "."
config KERNEL_DIR
string
option env="KERNEL_ROOT"
default "../.."
config UBIQUITOUS_DIR
string
option env="UBIQUITOUS_ROOT"
default "../../.."
source "$UBIQUITOUS_DIR/Kselector_features"
source "$UBIQUITOUS_DIR/Kselector_params"
config MANUALLY_SELECT
bool "Manually select a kernel"
default n
if MANUALLY_SELECT
menu "Required Kernel"
choice
prompt "Select OS Kernel"
default SELECT_XIUOS
config SELECT_RT_THREAD
bool "select RT_Thread"
config SELECT_NUTTX
bool "select Nuttx"
config SELECT_XIUOS
bool "select XiUOS"
endchoice
endmenu
endif

73
Ubiquitous/XiUOS/tool/hosttools/generate_kselector.py Executable file
View File

@ -0,0 +1,73 @@
#! /usr/bin/python3
import os
ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT')
bsp_dir = os.environ.get('BSP_ROOT')
kernel_names = []
def get_kernel_names():
for d in os.scandir(ubiquitous_dir):
if d.is_dir() and d.name!= "feature_yaml":
kernel_names.append(d.name)
def generate_features():
with open(f"{ubiquitous_dir}/Kselector_features","w") as f:
# f.write("")
pass
def generate_params():
with open(f"{ubiquitous_dir}/Kselector_params","w") as f:
# f.write("")
pass
def generate():
get_kernel_names()
generate_features()
generate_params()
template = r"""
mainmenu "Ubiquitous Kernel Selector"
config BSP_DIR
string
option env="BSP_ROOT"
default "."
config KERNEL_DIR
string
option env="KERNEL_ROOT"
default "../.."
config UBIQUITOUS_DIR
string
option env="UBIQUITOUS_ROOT"
default "../../.."
source "$UBIQUITOUS_DIR/Kselector_features"
source "$UBIQUITOUS_DIR/Kselector_params"
config MANUALLY_SELECT
bool "Manually select a kernel"
default n
if MANUALLY_SELECT
menu "Required Kernel"
choice
prompt "Select OS Kernel"
default SELECT_XIUOS
"""
for kernel_name in kernel_names:
template += f"\tconfig SELECT_{kernel_name.upper()}\n"
template += f'\t\tbool "select {kernel_name}"\n'
# the last `\n` is very important, otherwise, it cannot be recognized as valid Kconfig file
template += "endchoice\nendmenu\nendif\n"
with open(f"{bsp_dir}/Kselector", "w") as f:
f.write(template)
if __name__ == '__main__':
generate()

159
Ubiquitous/XiUOS/tool/hosttools/kernel_selector.py Executable file
View File

@ -0,0 +1,159 @@
#! /usr/bin/python3
# this script builds a decision tree to select suitable kernels from developers' preferences
from __future__ import annotations
from typing import TypeVar, Generic, Tuple, Optional
import os
import yaml
try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
ubiquitous_dir = os.environ.get('UBIQUITOUS_ROOT')
kernel_dir = os.environ.get('KERNEL_ROOT')
YAMLS_PATH = f"{ubiquitous_dir}/feature_yaml/"
REQUIREMENT_PATH = f"{kernel_dir}/requirement.yaml"
COMPONENT_KEY = "Feature"
PARAMETER_KEY = "Parameter"
MANUAL_KEY = "Kernel"
# tree node for kernel features
V = TypeVar("V")
class TreeNode(Generic[V]):
def __init__(self, depth: int, name: str, value: V):
self.depth = depth
self.name = name
self.value = value
self.children = dict()
def __repr__(self):
res = f"L{self.depth}: name {self.name} - value {self.value} - {len(self.children)} children\n"
for c in self.children:
# recursively build representation
res += self.children[c].__repr__()
return res
def node_number(self):
if len(self.children) == 0:
return 1
number = 1
for c in self.children:
number += self.children[c].node_number()
return number
# this class should be constructed from the yaml file
class Kernel:
def __init__(self, name, feature_yaml: dict):
self.name = name
self.features_root: Optional[TreeNode[bool]] = None
self.parameters_root: Optional[TreeNode[int]] = None
self.extract_components(feature_yaml)
self.extract_parameters(feature_yaml)
def __repr__(self):
return f"{self.name}\nFeature Tree:\n{self.features_root}Parameter Tree:\n{self.parameters_root}\n"
def extract_components(self, feature_yaml: dict):
self.features_root = TreeNode(0, COMPONENT_KEY, True)
for k,v in feature_yaml[COMPONENT_KEY].items():
node = TreeNode(1, k, v)
if k in feature_yaml:
self.build_sub_tree(2, node, feature_yaml)
self.features_root.children[k] = node
def build_sub_tree(self, depth, node: TreeNode[bool], feature_yaml: dict):
if feature_yaml[node.name] is None:
return
for k,v in feature_yaml[node.name].items():
child = TreeNode(depth, k, v)
if child.name in feature_yaml:
self.build_sub_tree(depth+1, child, feature_yaml)
node.children[k] = child
def extract_parameters(self, feature_yaml: dict):
self.parameters_root = TreeNode(0, PARAMETER_KEY, 0)
for k,v in feature_yaml[PARAMETER_KEY].items():
child = TreeNode(1, k, v)
self.parameters_root.children[k] = child
def check(self, requirements_root, root) -> Tuple[bool, int]:
distance = 0
q = list()
q.append((requirements_root, root))
while len(q) > 0:
(r_node, s_node) = q.pop()
for c in r_node.children:
if c in s_node.children and self.compare(c, s_node.children[c].value, r_node.children[c].value):
q.append((r_node.children[c], s_node.children[c]))
distance += (r_node.children[c].value - s_node.children[c].value)**2
else:
return False, distance
return True, distance
def compare(self, name, v1, v2):
if name.endswith("GREATER_THAN"):
return v1 >= v2
elif name.endswith("LESS_THAN"):
return v1 <= v2
else:
return v1 == v2
# load the requirements from the yaml file
def load_required_features()->dict:
with open(REQUIREMENT_PATH, 'r') as f:
content = f.read()
feature_yaml = yaml.load(content, Loader=Loader)
print(feature_yaml)
return feature_yaml
# load kernel yamls from ubiquitous directory and build Kernel class
def load_kernel_features()->list[Kernel]:
kernels = {}
for file_name in os.listdir(YAMLS_PATH):
kernel_name = file_name.rstrip(".yaml")
with open(os.path.join(YAMLS_PATH, file_name), 'r') as f:
content = f.read()
feature_yaml = yaml.load(content, Loader=Loader)
kernels[kernel_name] = Kernel(kernel_name, feature_yaml)
return kernels
# recommend a suitable kernel according to the requirement
def select_kernel(requirement: dict, kernels: dict[Kernel]) -> str:
selected = None
selected_node_number = 0
selected_distance = 0
requirement = Kernel("requirement", requirement)
for name, kernel in kernels.items():
# here should be a tree matching algorithm
# requirement trees (both features and parameters)
# should be subtrees of the recommended kernel
pass_features, _ = kernel.check(requirement.features_root, kernel.features_root)
pass_parameters, distance = kernel.check(requirement.parameters_root, kernel.parameters_root)
print(name, pass_features, pass_parameters, distance)
if pass_features and pass_parameters:
if selected is None:
selected = kernel
selected_node_number = kernel.features_root.node_number()
selected_node_number = distance
else:
node_number = kernel.features_root.node_number()
if selected_node_number > node_number or (selected_node_number == node_number and selected_distance > distance):
selected = kernel
selected_node_number = kernel.features_root.node_number()
selected_node_number = distance
if selected is None:
return "Cannot find suitable kernel"
else:
return selected.name
if __name__ == "__main__":
requirement = load_required_features()
if MANUAL_KEY in requirement:
selected = list(requirement[MANUAL_KEY].keys())[0][7:]
else:
kernels = load_kernel_features()
selected = select_kernel(requirement, kernels)
print(f"Selected kernel is: {selected}")

73
Ubiquitous/XiUOS/tool/hosttools/kernel_selector.sh Executable file
View File

@ -0,0 +1,73 @@
#!/bin/bash
# use this script to read the `.feature file` generated from `kconfig-mconf `, which further parse Kfeature to build the menu
# this script will genearte a more structured yaml file,
# the json file would further be sent to a decision tree for kernel selection.
VERSION=0.1.0
function generate_requirement_file()
{
local SELECTOR_NAME=${1}
# destination file using file descriptor 8
exec 8>${2}
echo -ne "version: ${VERSION}\n" >&8
EMPTY_LINE='true'
while read LN
do
LINE=`echo $LN | sed 's/[ \t\r\n]*$//g'`
if [ -z "$LINE" ]; then
continue
fi
if [ '#' = ${LINE:0:1} ]; then
if [ ${#LINE} -eq 1 ]; then
# empty line
if $EMPTY_LINE; then
continue
fi
echo >&8
EMPTY_LINE='true'
continue
fi
if echo -ne "$LINE" | egrep '^# Required' >/dev/null 2>/dev/null; then
# the key in yaml
echo -ne "${LINE:11}:\n" >&8
else
LINE=${LINE:1}
echo -ne "# ${LINE}\n" >&8
fi
EMPTY_LINE='false'
else
EMPTY_LINE='false'
OLD_IFS="$IFS"
IFS='='
REQUIREMENTS=($LINE)
IFS="$OLD_IFS"
if [ ${#REQUIREMENTS[@]} -ge 2 ]; then
if echo -ne "$REQUIREMENTS[0]" | egrep '^CONFIG_' >/dev/null 2>/dev/null; then
REQUIREMENTS[0]="${REQUIREMENTS[0]:7}"
fi
if [ "${REQUIREMENTS[1]}" = 'y' ]; then
echo -ne " ${REQUIREMENTS[0]}: true\n" >&8
else
echo -ne " ${REQUIREMENTS[0]}: ${LINE#*=}\n" >&8
fi
fi
fi
done < $SELECTOR_NAME
exec 8<&-
}
generate_requirement_file $1 $BSP_ROOT/requirement.yaml

21
Ubiquitous/feature_yaml/Nuttx.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 20
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 10
ROM_GREATER_THAN: 10000

21
Ubiquitous/feature_yaml/RT_Thread.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 5
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 5
ROM_GREATER_THAN: 10000

21
Ubiquitous/feature_yaml/XiUOS.yaml Executable file
View File

@ -0,0 +1,21 @@
version: 0.1.0
# Automatically generated file; DO NOT EDIT.
# Ubiquitous Kernel Selector
Feature:
# CONFIG_FS is not set
BUS: true
BUS:
USB: true
# CONFIG_SERIAL is not set
Parameter:
RAM_LESS_THAN: 3
RAM_GREATER_THAN: 10000
ROM_LESS_THAN: 10
ROM_GREATER_THAN: 10000