// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
// Package vm provides an abstract test machine (VM, physical machine, etc)
// interface for the rest of the system.
// For convenience test machines are subsequently collectively called VMs.
// Package wraps vmimpl package interface with some common functionality
// and higher-level interface.
package vm
import (
"bytes"
"fmt"
"os"
"time"
"github.com/google/syzkaller/pkg/mgrconfig"
"github.com/google/syzkaller/pkg/osutil"
"github.com/google/syzkaller/pkg/report"
"github.com/google/syzkaller/vm/vmimpl"
// Import all VM implementations, so that users only need to import vm.
_ "github.com/google/syzkaller/vm/adb"
_ "github.com/google/syzkaller/vm/gce"
_ "github.com/google/syzkaller/vm/gvisor"
_ "github.com/google/syzkaller/vm/isolated"
_ "github.com/google/syzkaller/vm/kvm"
_ "github.com/google/syzkaller/vm/odroid"
_ "github.com/google/syzkaller/vm/qemu"
)
type Pool struct {
impl vmimpl.Pool
workdir string
}
type Instance struct {
impl vmimpl.Instance
workdir string
index int
}
var (
Shutdown = vmimpl.Shutdown
ErrTimeout = vmimpl.ErrTimeout
)
type BootErrorer interface {
BootError() (string, []byte)
}
func Create(cfg *mgrconfig.Config, debug bool) (*Pool, error) {
env := &vmimpl.Env{
Name: cfg.Name,
OS: cfg.TargetOS,
Arch: cfg.TargetVMArch,
Workdir: cfg.Workdir,
Image: cfg.Image,
SSHKey: cfg.SSHKey,
SSHUser: cfg.SSHUser,
Debug: debug,
Config: cfg.VM,
}
impl, err := vmimpl.Create(cfg.Type, env)
if err != nil {
return nil, err
}
return &Pool{
impl: impl,
workdir: env.Workdir,
}, nil
}
func (pool *Pool) Count() int {
return pool.impl.Count()
}
func (pool *Pool) Create(index int) (*Instance, error) {
if index < 0 || index >= pool.Count() {
return nil, fmt.Errorf("invalid VM index %v (count %v)", index, pool.Count())
}
workdir, err := osutil.ProcessTempDir(pool.workdir)
if err != nil {
return nil, fmt.Errorf("failed to create instance temp dir: %v", err)
}
impl, err := pool.impl.Create(workdir, index)
if err != nil {
os.RemoveAll(workdir)
return nil, err
}
return &Instance{
impl: impl,
workdir: workdir,
index: index,
}, nil
}
func (inst *Instance) Copy(hostSrc string) (string, error) {
return inst.impl.Copy(hostSrc)
}
func (inst *Instance) Forward(port int) (string, error) {
return inst.impl.Forward(port)
}
func (inst *Instance) Run(timeout time.Duration, stop <-chan bool, command string) (
outc <-chan []byte, errc <-chan error, err error) {
return inst.impl.Run(timeout, stop, command)
}
func (inst *Instance) Diagnose() bool {
return inst.impl.Diagnose()
}
func (inst *Instance) Close() {
inst.impl.Close()
os.RemoveAll(inst.workdir)
}
// MonitorExecution monitors execution of a program running inside of a VM.
// It detects kernel oopses in output, lost connections, hangs, etc.
// outc/errc is what vm.Instance.Run returns, reporter parses kernel output for oopses.
// If canExit is false and the program exits, it is treated as an error.
// Returns a non-symbolized crash report, or nil if no error happens.
func (inst *Instance) MonitorExecution(outc <-chan []byte, errc <-chan error,
reporter report.Reporter, canExit bool) (rep *report.Report) {
mon := &monitor{
inst: inst,
outc: outc,
errc: errc,
reporter: reporter,
canExit: canExit,
}
lastExecuteTime := time.Now()
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case err := <-errc:
switch err {
case nil:
// The program has exited without errors,
// but wait for kernel output in case there is some delayed oops.
return mon.extractError("")
case ErrTimeout:
return nil
default:
// Note: connection lost can race with a kernel oops message.
// In such case we want to return the kernel oops.
return mon.extractError("lost connection to test machine")
}
case out := <-outc:
lastPos := len(mon.output)
mon.output = append(mon.output, out...)
if bytes.Contains(mon.output[lastPos:], executingProgram1) ||
bytes.Contains(mon.output[lastPos:], executingProgram2) {
lastExecuteTime = time.Now()
}
if reporter.ContainsCrash(mon.output[mon.matchPos:]) {
return mon.extractError("unknown error")
}
if len(mon.output) > 2*beforeContext {
copy(mon.output, mon.output[len(mon.output)-beforeContext:])
mon.output = mon.output[:beforeContext]
}
mon.matchPos = len(mon.output) - maxErrorLength
if mon.matchPos < 0 {
mon.matchPos = 0
}
case <-ticker.C:
// Detect both "not output whatsoever" and "kernel episodically prints
// something to console, but fuzzer is not actually executing programs".
// The timeout used to be 3 mins for a long time.
// But (1) we were seeing flakes on linux where net namespace
// destruction can be really slow, and (2) gVisor watchdog timeout
// is 3 mins + 1/4 of that for checking period = 3m45s.
// Current linux max timeout is CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=140
// and workqueue.watchdog_thresh=140 which both actually result
// in 140-280s detection delay.
// So the current timeout is 5 mins (300s).
// We don't want it to be too long too because it will waste time on real hangs.
if time.Since(lastExecuteTime) < 5*time.Minute {
break
}
if inst.Diagnose() {
mon.waitForOutput()
}
rep := &report.Report{
Title: "no output from test machine",
Output: mon.output,
Suppressed: report.IsSuppressed(mon.reporter, mon.output),
}
return rep
case <-Shutdown:
return nil
}
}
}
type monitor struct {
inst *Instance
outc <-chan []byte
errc <-chan error
reporter report.Reporter
canExit bool
output []byte
matchPos int
}
func (mon *monitor) extractError(defaultError string) *report.Report {
// Give it some time to finish writing the error message.
mon.inst.Diagnose()
mon.waitForOutput()
if bytes.Contains(mon.output, []byte("SYZ-FUZZER: PREEMPTED")) {
return nil
}
if !mon.reporter.ContainsCrash(mon.output[mon.matchPos:]) {
if defaultError == "" {
if mon.canExit {
return nil
}
defaultError = "lost connection to test machine"
}
rep := &report.Report{
Title: defaultError,
Output: mon.output,
Suppressed: report.IsSuppressed(mon.reporter, mon.output),
}
return rep
}
rep := mon.reporter.Parse(mon.output[mon.matchPos:])
if rep == nil {
panic(fmt.Sprintf("reporter.ContainsCrash/Parse disagree:\n%s", mon.output[mon.matchPos:]))
}
start := mon.matchPos + rep.StartPos - beforeContext
if start < 0 {
start = 0
}
end := mon.matchPos + rep.EndPos + afterContext
if end > len(mon.output) {
end = len(mon.output)
}
rep.Output = mon.output[start:end]
rep.StartPos += mon.matchPos - start
rep.EndPos += mon.matchPos - start
return rep
}
func (mon *monitor) waitForOutput() {
timer := time.NewTimer(10 * time.Second)
for {
select {
case out, ok := <-mon.outc:
if !ok {
timer.Stop()
return
}
mon.output = append(mon.output, out...)
case <-timer.C:
return
}
}
}
const (
beforeContext = 1024 << 10
afterContext = 128 << 10
maxErrorLength = 512
)
var (
executingProgram1 = []byte("executing program") // syz-fuzzer output
executingProgram2 = []byte("executed programs:") // syz-execprog output
)