// Copyright 2017 syzkaller project authors. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. package main import ( "fmt" "io/ioutil" "os" "path/filepath" "time" "github.com/google/syzkaller/dashboard/dashapi" "github.com/google/syzkaller/pkg/build" "github.com/google/syzkaller/pkg/instance" "github.com/google/syzkaller/pkg/log" "github.com/google/syzkaller/pkg/mgrconfig" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/vcs" ) type JobProcessor struct { name string managers []*Manager dash *dashapi.Dashboard syzkallerRepo string syzkallerBranch string } func newJobProcessor(cfg *Config, managers []*Manager) *JobProcessor { jp := &JobProcessor{ name: fmt.Sprintf("%v-job", cfg.Name), managers: managers, syzkallerRepo: cfg.SyzkallerRepo, syzkallerBranch: cfg.SyzkallerBranch, } if cfg.DashboardAddr != "" && cfg.DashboardClient != "" { jp.dash = dashapi.New(cfg.DashboardClient, cfg.DashboardAddr, cfg.DashboardKey) } return jp } func (jp *JobProcessor) loop(stop chan struct{}) { if jp.dash == nil { return } ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { select { case <-ticker.C: jp.poll() case <-stop: log.Logf(0, "job loop stopped") return } } } func (jp *JobProcessor) poll() { var names []string for _, mgr := range jp.managers { names = append(names, mgr.name) } req, err := jp.dash.JobPoll(names) if err != nil { jp.Errorf("failed to poll jobs: %v", err) return } if req.ID == "" { return } var mgr *Manager for _, m := range jp.managers { if m.name == req.Manager { mgr = m break } } if mgr == nil { jp.Errorf("got job for unknown manager: %v", req.Manager) return } job := &Job{ req: req, mgr: mgr, } log.Logf(0, "starting job %v for manager %v on %v/%v", req.ID, req.Manager, req.KernelRepo, req.KernelBranch) resp := jp.process(job) log.Logf(0, "done job %v: commit %v, crash %q, error: %s", resp.ID, resp.Build.KernelCommit, resp.CrashTitle, resp.Error) if err := jp.dash.JobDone(resp); err != nil { jp.Errorf("failed to mark job as done: %v", err) return } } type Job struct { req *dashapi.JobPollResp resp *dashapi.JobDoneReq mgr *Manager } func (jp *JobProcessor) process(job *Job) *dashapi.JobDoneReq { req, mgr := job.req, job.mgr build := dashapi.Build{ Manager: mgr.name, ID: req.ID, OS: mgr.managercfg.TargetOS, Arch: mgr.managercfg.TargetArch, VMArch: mgr.managercfg.TargetVMArch, CompilerID: mgr.compilerID, KernelRepo: req.KernelRepo, KernelBranch: req.KernelBranch, KernelCommit: "[unknown]", SyzkallerCommit: "[unknown]", } job.resp = &dashapi.JobDoneReq{ ID: req.ID, Build: build, } required := []struct { name string ok bool }{ {"kernel repository", req.KernelRepo != ""}, {"kernel branch", req.KernelBranch != ""}, {"kernel config", len(req.KernelConfig) != 0}, {"syzkaller commit", req.SyzkallerCommit != ""}, {"reproducer options", len(req.ReproOpts) != 0}, {"reproducer program", len(req.ReproSyz) != 0}, } for _, req := range required { if !req.ok { job.resp.Error = []byte(req.name + " is empty") jp.Errorf("%s", job.resp.Error) return job.resp } } // TODO(dvyukov): this will only work for qemu/gce, // because e.g. adb requires unique device IDs and we can't use what // manager already uses. For qemu/gce this is also bad, because we // override resource limits specified in config (e.g. can OOM), but works. switch typ := mgr.managercfg.Type; typ { case "gce", "qemu": default: job.resp.Error = []byte(fmt.Sprintf("testing is not yet supported for %v machine type.", typ)) jp.Errorf("%s", job.resp.Error) return job.resp } if err := jp.test(job); err != nil { job.resp.Error = []byte(err.Error()) } return job.resp } func (jp *JobProcessor) test(job *Job) error { kernelBuildSem <- struct{}{} defer func() { <-kernelBuildSem }() req, resp, mgr := job.req, job.resp, job.mgr dir := osutil.Abs(filepath.Join("jobs", mgr.managercfg.TargetOS)) kernelDir := filepath.Join(dir, "kernel") mgrcfg := new(mgrconfig.Config) *mgrcfg = *mgr.managercfg mgrcfg.Name += "-job" mgrcfg.Workdir = filepath.Join(dir, "workdir") mgrcfg.KernelSrc = kernelDir mgrcfg.Syzkaller = filepath.Join(dir, "gopath", "src", "github.com", "google", "syzkaller") os.RemoveAll(mgrcfg.Workdir) defer os.RemoveAll(mgrcfg.Workdir) env, err := instance.NewEnv(mgrcfg) if err != nil { return err } log.Logf(0, "job: building syzkaller on %v...", req.SyzkallerCommit) resp.Build.SyzkallerCommit = req.SyzkallerCommit if err := env.BuildSyzkaller(jp.syzkallerRepo, req.SyzkallerCommit); err != nil { return err } log.Logf(0, "job: fetching kernel...") repo, err := vcs.NewRepo(mgrcfg.TargetOS, mgrcfg.Type, kernelDir) if err != nil { return fmt.Errorf("failed to create kernel repo: %v", err) } var kernelCommit *vcs.Commit if vcs.CheckCommitHash(req.KernelBranch) { kernelCommit, err = repo.CheckoutCommit(req.KernelRepo, req.KernelBranch) if err != nil { return fmt.Errorf("failed to checkout kernel repo %v on commit %v: %v", req.KernelRepo, req.KernelBranch, err) } resp.Build.KernelBranch = "" } else { kernelCommit, err = repo.CheckoutBranch(req.KernelRepo, req.KernelBranch) if err != nil { return fmt.Errorf("failed to checkout kernel repo %v/%v: %v", req.KernelRepo, req.KernelBranch, err) } } resp.Build.KernelCommit = kernelCommit.Hash resp.Build.KernelCommitTitle = kernelCommit.Title resp.Build.KernelCommitDate = kernelCommit.Date if err := build.Clean(mgrcfg.TargetOS, mgrcfg.TargetVMArch, mgrcfg.Type, kernelDir); err != nil { return fmt.Errorf("kernel clean failed: %v", err) } if len(req.Patch) != 0 { if err := vcs.Patch(kernelDir, req.Patch); err != nil { return err } } log.Logf(0, "job: building kernel...") if err := env.BuildKernel(mgr.mgrcfg.Compiler, mgr.mgrcfg.Userspace, mgr.mgrcfg.KernelCmdline, mgr.mgrcfg.KernelSysctl, req.KernelConfig); err != nil { return err } resp.Build.KernelConfig, err = ioutil.ReadFile(filepath.Join(mgrcfg.KernelSrc, ".config")) if err != nil { return fmt.Errorf("failed to read config file: %v", err) } log.Logf(0, "job: testing...") results, err := env.Test(3, req.ReproSyz, req.ReproOpts, req.ReproC) if err != nil { return err } // We can have transient errors and other errors of different types. // We need to avoid reporting transient "failed to boot" or "failed to copy binary" errors. // If any of the instances crash during testing, we report this with the highest priority. // Then if any of the runs succeed, we report that (to avoid transient errors). // If all instances failed to boot, then we report one of these errors. anySuccess := false var anyErr, testErr error for _, res := range results { if res == nil { anySuccess = true continue } anyErr = res switch err := res.(type) { case *instance.TestError: // We should not put rep into resp.CrashTitle/CrashReport, // because that will be treated as patch not fixing the bug. if rep := err.Report; rep != nil { testErr = fmt.Errorf("%v\n\n%s\n\n%s", rep.Title, rep.Report, rep.Output) } else { testErr = fmt.Errorf("%v\n\n%s", err.Title, err.Output) } case *instance.CrashError: resp.CrashTitle = err.Report.Title resp.CrashReport = err.Report.Report resp.CrashLog = err.Report.Output return nil } } if anySuccess { return nil } if testErr != nil { return testErr } return anyErr } // Errorf logs non-fatal error and sends it to dashboard. func (jp *JobProcessor) Errorf(msg string, args ...interface{}) { log.Logf(0, "job: "+msg, args...) if jp.dash != nil { jp.dash.LogError(jp.name, msg, args...) } }