// Copyright 2016 syzkaller project authors. All rights reserved. // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. // Package gce provides wrappers around Google Compute Engine (GCE) APIs. // It is assumed that the program itself also runs on GCE as APIs operate on the current project/zone. // // See https://cloud.google.com/compute/docs for details. // In particular, API reference: // https://cloud.google.com/compute/docs/reference/latest // and Go API wrappers: // https://godoc.org/google.golang.org/api/compute/v0.beta package gce import ( "fmt" "io/ioutil" "math/rand" "net/http" "strings" "time" "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/compute/v0.beta" "google.golang.org/api/googleapi" ) type Context struct { ProjectID string ZoneID string Instance string InternalIP string ExternalIP string Network string Subnetwork string computeService *compute.Service // apiCallTicker ticks regularly, preventing us from accidentally making // GCE API calls too quickly. Our quota is 20 QPS, but we limit ourselves // to less than that because several independent programs can do API calls. apiRateGate <-chan time.Time } func NewContext() (*Context, error) { ctx := &Context{ apiRateGate: time.NewTicker(time.Second).C, } background := context.Background() tokenSource, err := google.DefaultTokenSource(background, compute.CloudPlatformScope) if err != nil { return nil, fmt.Errorf("failed to get a token source: %v", err) } httpClient := oauth2.NewClient(background, tokenSource) ctx.computeService, _ = compute.New(httpClient) // Obtain project name, zone and current instance IP address. ctx.ProjectID, err = ctx.getMeta("project/project-id") if err != nil { return nil, fmt.Errorf("failed to query gce project-id: %v", err) } ctx.ZoneID, err = ctx.getMeta("instance/zone") if err != nil { return nil, fmt.Errorf("failed to query gce zone: %v", err) } if i := strings.LastIndexByte(ctx.ZoneID, '/'); i != -1 { ctx.ZoneID = ctx.ZoneID[i+1:] // the query returns some nonsense prefix } ctx.Instance, err = ctx.getMeta("instance/name") if err != nil { return nil, fmt.Errorf("failed to query gce instance name: %v", err) } inst, err := ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, ctx.Instance).Do() if err != nil { return nil, fmt.Errorf("error getting instance info: %v", err) } for _, iface := range inst.NetworkInterfaces { if strings.HasPrefix(iface.NetworkIP, "10.") { ctx.InternalIP = iface.NetworkIP } for _, ac := range iface.AccessConfigs { if ac.NatIP != "" { ctx.ExternalIP = ac.NatIP } } ctx.Network = iface.Network ctx.Subnetwork = iface.Subnetwork } if ctx.InternalIP == "" { return nil, fmt.Errorf("failed to get current instance internal IP") } return ctx, nil } func (ctx *Context) CreateInstance(name, machineType, image, sshkey string) (string, error) { prefix := "https://www.googleapis.com/compute/v1/projects/" + ctx.ProjectID sshkeyAttr := "syzkaller:" + sshkey oneAttr := "1" falseAttr := false instance := &compute.Instance{ Name: name, Description: "syzkaller worker", MachineType: prefix + "/zones/" + ctx.ZoneID + "/machineTypes/" + machineType, Disks: []*compute.AttachedDisk{ { AutoDelete: true, Boot: true, Type: "PERSISTENT", InitializeParams: &compute.AttachedDiskInitializeParams{ DiskName: name, SourceImage: prefix + "/global/images/" + image, }, }, }, Metadata: &compute.Metadata{ Items: []*compute.MetadataItems{ { Key: "ssh-keys", Value: &sshkeyAttr, }, { Key: "serial-port-enable", Value: &oneAttr, }, }, }, NetworkInterfaces: []*compute.NetworkInterface{ { Network: ctx.Network, Subnetwork: ctx.Subnetwork, }, }, Scheduling: &compute.Scheduling{ AutomaticRestart: &falseAttr, Preemptible: true, OnHostMaintenance: "TERMINATE", }, } retry: var op *compute.Operation err := ctx.apiCall(func() (err error) { op, err = ctx.computeService.Instances.Insert(ctx.ProjectID, ctx.ZoneID, instance).Do() return }) if err != nil { return "", fmt.Errorf("failed to create instance: %v", err) } if err := ctx.waitForCompletion("zone", "create image", op.Name, false); err != nil { if _, ok := err.(resourcePoolExhaustedError); ok && instance.Scheduling.Preemptible { instance.Scheduling.Preemptible = false goto retry } return "", err } var inst *compute.Instance err = ctx.apiCall(func() (err error) { inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do() return }) if err != nil { return "", fmt.Errorf("error getting instance %s details after creation: %v", name, err) } // Finds its internal IP. ip := "" for _, iface := range inst.NetworkInterfaces { if strings.HasPrefix(iface.NetworkIP, "10.") { ip = iface.NetworkIP break } } if ip == "" { return "", fmt.Errorf("didn't find instance internal IP address") } return ip, nil } func (ctx *Context) DeleteInstance(name string, wait bool) error { var op *compute.Operation err := ctx.apiCall(func() (err error) { op, err = ctx.computeService.Instances.Delete(ctx.ProjectID, ctx.ZoneID, name).Do() return }) if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == 404 { return nil } if err != nil { return fmt.Errorf("failed to delete instance: %v", err) } if wait { if err := ctx.waitForCompletion("zone", "delete image", op.Name, true); err != nil { return err } } return nil } func (ctx *Context) IsInstanceRunning(name string) bool { var inst *compute.Instance err := ctx.apiCall(func() (err error) { inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do() return }) if err != nil { return false } return inst.Status == "RUNNING" } func (ctx *Context) CreateImage(imageName, gcsFile string) error { image := &compute.Image{ Name: imageName, RawDisk: &compute.ImageRawDisk{ Source: "https://storage.googleapis.com/" + gcsFile, }, Licenses: []string{ "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx", }, } var op *compute.Operation err := ctx.apiCall(func() (err error) { op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do() return }) if err != nil { // Try again without the vmx license in case it is not supported. image.Licenses = nil err := ctx.apiCall(func() (err error) { op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do() return }) if err != nil { return fmt.Errorf("failed to create image: %v", err) } } if err := ctx.waitForCompletion("global", "create image", op.Name, false); err != nil { return err } return nil } func (ctx *Context) DeleteImage(imageName string) error { var op *compute.Operation err := ctx.apiCall(func() (err error) { op, err = ctx.computeService.Images.Delete(ctx.ProjectID, imageName).Do() return }) if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == 404 { return nil } if err != nil { return fmt.Errorf("failed to delete image: %v", err) } if err := ctx.waitForCompletion("global", "delete image", op.Name, true); err != nil { return err } return nil } type resourcePoolExhaustedError string func (err resourcePoolExhaustedError) Error() string { return string(err) } func (ctx *Context) waitForCompletion(typ, desc, opName string, ignoreNotFound bool) error { for { time.Sleep(2 * time.Second) var op *compute.Operation err := ctx.apiCall(func() (err error) { switch typ { case "global": op, err = ctx.computeService.GlobalOperations.Get(ctx.ProjectID, opName).Do() case "zone": op, err = ctx.computeService.ZoneOperations.Get(ctx.ProjectID, ctx.ZoneID, opName).Do() default: panic("unknown operation type: " + typ) } return }) if err != nil { return fmt.Errorf("failed to get %v operation %v: %v", desc, opName, err) } switch op.Status { case "PENDING", "RUNNING": continue case "DONE": if op.Error != nil { reason := "" for _, operr := range op.Error.Errors { if operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED" { return resourcePoolExhaustedError(fmt.Sprintf("%+v", operr)) } if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" { return nil } reason += fmt.Sprintf("%+v.", operr) } return fmt.Errorf("%v operation failed: %v", desc, reason) } return nil default: return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op) } } } func (ctx *Context) getMeta(path string) (string, error) { req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil) if err != nil { return "", err } req.Header.Add("Metadata-Flavor", "Google") resp, err := http.DefaultClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return "", err } return string(body), nil } func (ctx *Context) apiCall(fn func() error) error { rateLimited := 0 for { <-ctx.apiRateGate err := fn() if err != nil { if strings.Contains(err.Error(), "Rate Limit Exceeded") || strings.Contains(err.Error(), "rateLimitExceeded") { rateLimited++ backoff := time.Duration(float64(rateLimited) * 1e9 * (rand.Float64() + 1)) time.Sleep(backoff) if rateLimited < 20 { continue } } } return err } }