提交 989d57b4 编写于 作者: E Evan Lezar

Merge branch 'refactor-nvidia-container-runtime' into 'master'

Refactor NVIDIA Container runtime

See merge request nvidia/container-toolkit/container-runtime!50
dist
testing
.git
test/output
/nvidia-container-runtime
/runc
......@@ -2,4 +2,5 @@ dist
*.swp
*.swo
test/output
runc
/runc
/nvidia-container-runtime
......@@ -28,11 +28,11 @@ MODULE := .
docker-native:
include $(CURDIR)/docker/docker.mk
binary:
go build -ldflags "-s -w" -o "$(LIB_NAME)" $(MODULE)/cmd/nvidia-container-runtime/...
binaries:
go build -ldflags "-s -w" $(MODULE)/cmd/...
build:
@go build -ldflags "-s -w" $(MODULE)/...
build: binaries
go build -ldflags "-s -w" $(MODULE)/...
# Define the check targets for the Golang codebase
MODULE := .
......@@ -77,8 +77,8 @@ mock-hook:
[ ! -e /etc/nvidia-container-runtime/config.toml ] && echo "" > /etc/nvidia-container-runtime/config.toml || true
[ ! -e /usr/bin/nvidia-container-runtime-hook ] && echo "" > /usr/bin/nvidia-container-runtime-hook && chmod +x /usr/bin/nvidia-container-runtime-hook || true
test: binary mock-runc mock-hook
@go test -v $(MODULE)/...
test: build mock-runc mock-hook
@go test -v -coverprofile=coverage.out $(MODULE)/...
@${RM} $(MOCK_RUNC)
.PHONY: docker-test
......@@ -90,4 +90,3 @@ docker-test:
-w $(PWD) \
golang:$(GOLANG_VERSION) \
make test
......@@ -25,12 +25,14 @@ import (
"github.com/tsaikd/KDGoLib/logrusutil"
)
// Logger adds a way to manage output to a log file to a logrus.Logger
type Logger struct {
*logrus.Logger
previousOutput io.Writer
logFile *os.File
}
// NewLogger constructs a Logger with a preddefined formatter
func NewLogger() *Logger {
logrusLogger := logrus.New()
......@@ -47,6 +49,9 @@ func NewLogger() *Logger {
return logger
}
// LogToFile opens the specified file for appending and sets the logger to
// output to the opened file. A reference to the file pointer is stored to
// allow this to be closed.
func (l *Logger) LogToFile(filename string) error {
logFile, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
......@@ -60,6 +65,8 @@ func (l *Logger) LogToFile(filename string) error {
return nil
}
// CloseFile closes the log file (if any) and resets the logger output to what it
// was before LogToFile was called.
func (l *Logger) CloseFile() error {
if l.logFile == nil {
return nil
......
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"syscall"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/pelletier/go-toml"
)
......@@ -28,151 +21,6 @@ var (
var logger = NewLogger()
type args struct {
bundleDirPath string
cmd string
}
type config struct {
debugFilePath string
}
func getConfig() (*config, error) {
cfg := &config{}
if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
configDir = XDGConfigDir
}
configFilePath := path.Join(configDir, configFilePath)
tomlContent, err := ioutil.ReadFile(configFilePath)
if err != nil {
return nil, err
}
toml, err := toml.Load(string(tomlContent))
if err != nil {
return nil, err
}
cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
return cfg, nil
}
// getArgs checks the specified slice of strings (argv) for a 'bundle' flag and a 'create'
// command line argument as allowed by runc.
// The following are supported:
// --bundle{{SEP}}BUNDLE_PATH
// -bundle{{SEP}}BUNDLE_PATH
// -b{{SEP}}BUNDLE_PATH
// where {{SEP}} is either ' ' or '='
func getArgs(argv []string) (*args, error) {
args := &args{}
for i := 0; i < len(argv); i++ {
param := argv[i]
if param == "create" {
args.cmd = param
continue
}
if !strings.HasPrefix(param, "-") {
continue
}
trimmed := strings.TrimLeft(param, "-")
if len(trimmed) == 0 {
continue
}
parts := strings.SplitN(trimmed, "=", 2)
if parts[0] != "bundle" && parts[0] != "b" {
continue
}
if len(parts) == 2 {
args.bundleDirPath = parts[1]
continue
}
if len(argv)-i <= 1 {
return nil, fmt.Errorf("bundle option needs an argument")
}
args.bundleDirPath = argv[i+1]
i++
}
return args, nil
}
// execRunc discovers the runc binary and issues an exec syscall.
func execRunc() error {
runcCandidates := []string{
"docker-runc",
"runc",
}
var err error
var runcPath string
for _, candidate := range runcCandidates {
logger.Printf("Looking for \"%v\" binary", candidate)
runcPath, err = exec.LookPath(candidate)
if err == nil {
break
}
logger.Printf("\"%v\" binary not found: %v", candidate, err)
}
if err != nil {
return fmt.Errorf("error locating runc: %v", err)
}
logger.Printf("Runc path: %s\n", runcPath)
err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ())
if err != nil {
return fmt.Errorf("could not exec '%v': %v", runcPath, err)
}
// syscall.Exec is not expected to return. This is an error state regardless of whether
// err is nil or not.
return fmt.Errorf("unexpected return from exec '%v'", runcPath)
}
func addNVIDIAHook(spec *specs.Spec) error {
path, err := exec.LookPath("nvidia-container-runtime-hook")
if err != nil {
path = hookDefaultFilePath
_, err = os.Stat(path)
if err != nil {
return err
}
}
logger.Printf("prestart hook path: %s\n", path)
args := []string{path}
if spec.Hooks == nil {
spec.Hooks = &specs.Hooks{}
} else if len(spec.Hooks.Prestart) != 0 {
for _, hook := range spec.Hooks.Prestart {
if !strings.Contains(hook.Path, "nvidia-container-runtime-hook") {
continue
}
logger.Println("existing nvidia prestart hook in OCI spec file")
return nil
}
}
spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
Path: path,
Args: append(args, "prestart"),
})
return nil
}
func main() {
err := run(os.Args)
if err != nil {
......@@ -201,83 +49,41 @@ func run(argv []string) (err error) {
logger.CloseFile()
}()
logger.Printf("Running %s\n", argv[0])
args, err := getArgs(argv)
if err != nil {
return fmt.Errorf("error getting processing command line arguments: %v", err)
}
if args.cmd != "create" {
logger.Println("Command is not \"create\", executing runc doing nothing")
err = execRunc()
if err != nil {
return fmt.Errorf("error forwarding command to runc: %v", err)
}
}
configFilePath, err := args.getConfigFilePath()
r, err := newRuntime(argv)
if err != nil {
return fmt.Errorf("error getting config file path: %v", err)
return fmt.Errorf("error creating runtime: %v", err)
}
logger.Printf("Using OCI specification file path: %v", configFilePath)
jsonFile, err := os.OpenFile(configFilePath, os.O_RDWR, 0644)
if err != nil {
return fmt.Errorf("error opening OCI specification file: %v", err)
}
defer jsonFile.Close()
logger.Printf("Running %s\n", argv[0])
return r.Exec(argv)
}
jsonContent, err := ioutil.ReadAll(jsonFile)
if err != nil {
return fmt.Errorf("error reading OCI specificaiton: %v", err)
}
type config struct {
debugFilePath string
}
var spec specs.Spec
err = json.Unmarshal(jsonContent, &spec)
if err != nil {
return fmt.Errorf("error unmarshalling OCI specification: %v", err)
}
// getConfig sets up the config struct. Values are read from a toml file
// or set via the environment.
func getConfig() (*config, error) {
cfg := &config{}
err = addNVIDIAHook(&spec)
if err != nil {
return fmt.Errorf("error injecting NVIDIA Container Runtime hook: %v", err)
if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
configDir = XDGConfigDir
}
jsonOutput, err := json.Marshal(spec)
if err != nil {
return fmt.Errorf("error marshalling modified OCI specification: %v", err)
}
configFilePath := path.Join(configDir, configFilePath)
_, err = jsonFile.WriteAt(jsonOutput, 0)
tomlContent, err := os.ReadFile(configFilePath)
if err != nil {
return fmt.Errorf("error writing modifed OCI specification to file: %v", err)
return nil, err
}
logger.Print("Prestart hook added, executing runc")
err = execRunc()
toml, err := toml.Load(string(tomlContent))
if err != nil {
return fmt.Errorf("error forwarding 'create' command to runc: %v", err)
}
return nil
}
func (a args) getConfigFilePath() (string, error) {
configRoot := a.bundleDirPath
if configRoot == "" {
logger.Printf("Bundle directory path is empty, using working directory.")
workingDirectory, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("error getting working directory: %v", err)
}
configRoot = workingDirectory
return nil, err
}
logger.Printf("Using bundle directory: %v", configRoot)
configFilePath := filepath.Join(configRoot, "config.json")
cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
return configFilePath, nil
return cfg, nil
}
......@@ -5,10 +5,8 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"strings"
......@@ -40,7 +38,7 @@ func TestMain(m *testing.M) {
var err error
moduleRoot, err := getModuleRoot(filename)
if err != nil {
log.Fatalf("error in test setup: could not get module root: %v", err)
logger.Fatalf("error in test setup: could not get module root: %v", err)
}
paths := strings.Split(os.Getenv("PATH"), ":")
......@@ -50,7 +48,7 @@ func TestMain(m *testing.M) {
// Confirm path setup correctly
runcPath, err := exec.LookPath("runc")
if err != nil || !strings.HasPrefix(runcPath, moduleRoot) {
log.Fatal("error in test setup: mock runc path set incorrectly in TestMain()")
logger.Fatal("error in test setup: mock runc path set incorrectly in TestMain()")
}
cfg = &testConfig{
......@@ -91,8 +89,8 @@ func TestBadInput(t *testing.T) {
cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
err = cmdRun.Run()
require.Error(t, err, "runtime should return an error")
output, err := cmdRun.CombinedOutput()
require.Errorf(t, err, "runtime should return an error", "output=%v", string(output))
cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
......@@ -112,8 +110,8 @@ func TestGoodInput(t *testing.T) {
cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", cfg.bundlePath(), "testcontainer")
t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
err = cmdRun.Run()
require.NoError(t, err, "runtime should not return an error")
output, err := cmdRun.CombinedOutput()
require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
// Check config.json and confirm there are no hooks
spec, err := cfg.getRuntimeSpec()
......@@ -167,8 +165,8 @@ func TestDuplicateHook(t *testing.T) {
// Test how runtime handles already existing prestart hook in config.json
cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
err = cmdCreate.Run()
require.NoError(t, err, "runtime should not return an error")
output, err := cmdCreate.CombinedOutput()
require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
// Check config.json for NVIDIA prestart hook
spec, err = cfg.getRuntimeSpec()
......@@ -177,6 +175,13 @@ func TestDuplicateHook(t *testing.T) {
require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
}
// addNVIDIAHook is a basic wrapper for nvidiaContainerRunime.addNVIDIAHook that is used for
// testing.
func addNVIDIAHook(spec *specs.Spec) error {
r := nvidiaContainerRuntime{logger: logger.Logger}
return r.addNVIDIAHook(spec)
}
func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
filePath := c.specFilePath()
......@@ -252,8 +257,8 @@ func TestGetConfigWithCustomConfig(t *testing.T) {
// By default debug is disabled
contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
testDir := path.Join(wd, "test")
filename := path.Join(testDir, configFilePath)
testDir := filepath.Join(wd, "test")
filename := filepath.Join(testDir, configFilePath)
os.Setenv(configOverride, testDir)
......@@ -266,124 +271,3 @@ func TestGetConfigWithCustomConfig(t *testing.T) {
require.NoError(t, err)
require.Equal(t, cfg.debugFilePath, "/nvidia-container-toolkit.log")
}
func TestArgsGetConfigFilePath(t *testing.T) {
wd, err := os.Getwd()
require.NoError(t, err)
testCases := []struct {
args args
configPath string
}{
{
args: args{},
configPath: fmt.Sprintf("%v/config.json", wd),
},
{
args: args{bundleDirPath: "/foo/bar"},
configPath: "/foo/bar/config.json",
},
{
args: args{bundleDirPath: "/foo/bar/"},
configPath: "/foo/bar/config.json",
},
}
for i, tc := range testCases {
cp, err := tc.args.getConfigFilePath()
require.NoErrorf(t, err, "%d: %v", i, tc)
require.Equalf(t, tc.configPath, cp, "%d: %v", i, tc)
}
}
func TestGetArgs(t *testing.T) {
testCases := []struct {
argv []string
expected *args
isError bool
}{
{
argv: []string{},
expected: &args{},
},
{
argv: []string{"create"},
expected: &args{
cmd: "create",
},
},
{
argv: []string{"--bundle"},
expected: nil,
isError: true,
},
{
argv: []string{"-b"},
expected: nil,
isError: true,
},
{
argv: []string{"--bundle", "/foo/bar"},
expected: &args{bundleDirPath: "/foo/bar"},
},
{
argv: []string{"-bundle", "/foo/bar"},
expected: &args{bundleDirPath: "/foo/bar"},
},
{
argv: []string{"--bundle=/foo/bar"},
expected: &args{bundleDirPath: "/foo/bar"},
},
{
argv: []string{"-b=/foo/bar"},
expected: &args{bundleDirPath: "/foo/bar"},
},
{
argv: []string{"-b=/foo/=bar"},
expected: &args{bundleDirPath: "/foo/=bar"},
},
{
argv: []string{"-b", "/foo/bar"},
expected: &args{bundleDirPath: "/foo/bar"},
},
{
argv: []string{"create", "-b", "/foo/bar"},
expected: &args{
cmd: "create",
bundleDirPath: "/foo/bar",
},
},
{
argv: []string{"-b", "create", "create"},
expected: &args{
cmd: "create",
bundleDirPath: "create",
},
},
{
argv: []string{"-b=create", "create"},
expected: &args{
cmd: "create",
bundleDirPath: "create",
},
},
{
argv: []string{"-b", "create"},
expected: &args{
bundleDirPath: "create",
},
},
}
for i, tc := range testCases {
args, err := getArgs(tc.argv)
if tc.isError {
require.Errorf(t, err, "%d: %v", i, tc)
} else {
require.NoErrorf(t, err, "%d: %v", i, tc)
}
require.EqualValuesf(t, tc.expected, args, "%d: %v", i, tc)
}
}
/*
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/
package main
import (
"fmt"
"os"
"os/exec"
"strings"
"github.com/nvidia/nvidia-container-runtime/pkg/oci"
"github.com/opencontainers/runtime-spec/specs-go"
log "github.com/sirupsen/logrus"
)
// nvidiaContainerRuntime encapsulates the NVIDIA Container Runtime. It wraps the specified runtime, conditionally
// modifying the specified OCI specification before invoking the runtime.
type nvidiaContainerRuntime struct {
logger *log.Logger
runtime oci.Runtime
ociSpec oci.Spec
}
var _ oci.Runtime = (*nvidiaContainerRuntime)(nil)
// newNvidiaContainerRuntime is a constructor for a standard runtime shim.
func newNvidiaContainerRuntimeWithLogger(logger *log.Logger, runtime oci.Runtime, ociSpec oci.Spec) (oci.Runtime, error) {
r := nvidiaContainerRuntime{
logger: logger,
runtime: runtime,
ociSpec: ociSpec,
}
return &r, nil
}
// Exec defines the entrypoint for the NVIDIA Container Runtime. A check is performed to see whether modifications
// to the OCI spec are required -- and applicable modifcations applied. The supplied arguments are then
// forwarded to the underlying runtime's Exec method.
func (r nvidiaContainerRuntime) Exec(args []string) error {
if r.modificationRequired(args) {
err := r.modifyOCISpec()
if err != nil {
return fmt.Errorf("error modifying OCI spec: %v", err)
}
}
r.logger.Println("Forwarding command to runtime")
return r.runtime.Exec(args)
}
// modificationRequired checks the intput arguments to determine whether a modification