// Copyright 2016 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
// Package gce provides wrappers around Google Compute Engine (GCE) APIs.
// It is assumed that the program itself also runs on GCE as APIs operate on the current project/zone.
//
// See https://cloud.google.com/compute/docs for details.
// In particular, API reference:
// https://cloud.google.com/compute/docs/reference/latest
// and Go API wrappers:
// https://godoc.org/google.golang.org/api/compute/v0.beta
package gce
import (
"fmt"
"io/ioutil"
"math/rand"
"net/http"
"strings"
"time"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/compute/v0.beta"
"google.golang.org/api/googleapi"
)
type Context struct {
ProjectID string
ZoneID string
Instance string
InternalIP string
ExternalIP string
Network string
Subnetwork string
computeService *compute.Service
// apiCallTicker ticks regularly, preventing us from accidentally making
// GCE API calls too quickly. Our quota is 20 QPS, but we limit ourselves
// to less than that because several independent programs can do API calls.
apiRateGate <-chan time.Time
}
func NewContext() (*Context, error) {
ctx := &Context{
apiRateGate: time.NewTicker(time.Second).C,
}
background := context.Background()
tokenSource, err := google.DefaultTokenSource(background, compute.CloudPlatformScope)
if err != nil {
return nil, fmt.Errorf("failed to get a token source: %v", err)
}
httpClient := oauth2.NewClient(background, tokenSource)
ctx.computeService, _ = compute.New(httpClient)
// Obtain project name, zone and current instance IP address.
ctx.ProjectID, err = ctx.getMeta("project/project-id")
if err != nil {
return nil, fmt.Errorf("failed to query gce project-id: %v", err)
}
ctx.ZoneID, err = ctx.getMeta("instance/zone")
if err != nil {
return nil, fmt.Errorf("failed to query gce zone: %v", err)
}
if i := strings.LastIndexByte(ctx.ZoneID, '/'); i != -1 {
ctx.ZoneID = ctx.ZoneID[i+1:] // the query returns some nonsense prefix
}
ctx.Instance, err = ctx.getMeta("instance/name")
if err != nil {
return nil, fmt.Errorf("failed to query gce instance name: %v", err)
}
inst, err := ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, ctx.Instance).Do()
if err != nil {
return nil, fmt.Errorf("error getting instance info: %v", err)
}
for _, iface := range inst.NetworkInterfaces {
if strings.HasPrefix(iface.NetworkIP, "10.") {
ctx.InternalIP = iface.NetworkIP
}
for _, ac := range iface.AccessConfigs {
if ac.NatIP != "" {
ctx.ExternalIP = ac.NatIP
}
}
ctx.Network = iface.Network
ctx.Subnetwork = iface.Subnetwork
}
if ctx.InternalIP == "" {
return nil, fmt.Errorf("failed to get current instance internal IP")
}
return ctx, nil
}
func (ctx *Context) CreateInstance(name, machineType, image, sshkey string) (string, error) {
prefix := "https://www.googleapis.com/compute/v1/projects/" + ctx.ProjectID
sshkeyAttr := "syzkaller:" + sshkey
oneAttr := "1"
falseAttr := false
instance := &compute.Instance{
Name: name,
Description: "syzkaller worker",
MachineType: prefix + "/zones/" + ctx.ZoneID + "/machineTypes/" + machineType,
Disks: []*compute.AttachedDisk{
{
AutoDelete: true,
Boot: true,
Type: "PERSISTENT",
InitializeParams: &compute.AttachedDiskInitializeParams{
DiskName: name,
SourceImage: prefix + "/global/images/" + image,
},
},
},
Metadata: &compute.Metadata{
Items: []*compute.MetadataItems{
{
Key: "ssh-keys",
Value: &sshkeyAttr,
},
{
Key: "serial-port-enable",
Value: &oneAttr,
},
},
},
NetworkInterfaces: []*compute.NetworkInterface{
{
Network: ctx.Network,
Subnetwork: ctx.Subnetwork,
},
},
Scheduling: &compute.Scheduling{
AutomaticRestart: &falseAttr,
Preemptible: true,
OnHostMaintenance: "TERMINATE",
},
}
retry:
var op *compute.Operation
err := ctx.apiCall(func() (err error) {
op, err = ctx.computeService.Instances.Insert(ctx.ProjectID, ctx.ZoneID, instance).Do()
return
})
if err != nil {
return "", fmt.Errorf("failed to create instance: %v", err)
}
if err := ctx.waitForCompletion("zone", "create image", op.Name, false); err != nil {
if _, ok := err.(resourcePoolExhaustedError); ok && instance.Scheduling.Preemptible {
instance.Scheduling.Preemptible = false
goto retry
}
return "", err
}
var inst *compute.Instance
err = ctx.apiCall(func() (err error) {
inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
return
})
if err != nil {
return "", fmt.Errorf("error getting instance %s details after creation: %v", name, err)
}
// Finds its internal IP.
ip := ""
for _, iface := range inst.NetworkInterfaces {
if strings.HasPrefix(iface.NetworkIP, "10.") {
ip = iface.NetworkIP
break
}
}
if ip == "" {
return "", fmt.Errorf("didn't find instance internal IP address")
}
return ip, nil
}
func (ctx *Context) DeleteInstance(name string, wait bool) error {
var op *compute.Operation
err := ctx.apiCall(func() (err error) {
op, err = ctx.computeService.Instances.Delete(ctx.ProjectID, ctx.ZoneID, name).Do()
return
})
if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == 404 {
return nil
}
if err != nil {
return fmt.Errorf("failed to delete instance: %v", err)
}
if wait {
if err := ctx.waitForCompletion("zone", "delete image", op.Name, true); err != nil {
return err
}
}
return nil
}
func (ctx *Context) IsInstanceRunning(name string) bool {
var inst *compute.Instance
err := ctx.apiCall(func() (err error) {
inst, err = ctx.computeService.Instances.Get(ctx.ProjectID, ctx.ZoneID, name).Do()
return
})
if err != nil {
return false
}
return inst.Status == "RUNNING"
}
func (ctx *Context) CreateImage(imageName, gcsFile string) error {
image := &compute.Image{
Name: imageName,
RawDisk: &compute.ImageRawDisk{
Source: "https://storage.googleapis.com/" + gcsFile,
},
Licenses: []string{
"https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx",
},
}
var op *compute.Operation
err := ctx.apiCall(func() (err error) {
op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
return
})
if err != nil {
// Try again without the vmx license in case it is not supported.
image.Licenses = nil
err := ctx.apiCall(func() (err error) {
op, err = ctx.computeService.Images.Insert(ctx.ProjectID, image).Do()
return
})
if err != nil {
return fmt.Errorf("failed to create image: %v", err)
}
}
if err := ctx.waitForCompletion("global", "create image", op.Name, false); err != nil {
return err
}
return nil
}
func (ctx *Context) DeleteImage(imageName string) error {
var op *compute.Operation
err := ctx.apiCall(func() (err error) {
op, err = ctx.computeService.Images.Delete(ctx.ProjectID, imageName).Do()
return
})
if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == 404 {
return nil
}
if err != nil {
return fmt.Errorf("failed to delete image: %v", err)
}
if err := ctx.waitForCompletion("global", "delete image", op.Name, true); err != nil {
return err
}
return nil
}
type resourcePoolExhaustedError string
func (err resourcePoolExhaustedError) Error() string {
return string(err)
}
func (ctx *Context) waitForCompletion(typ, desc, opName string, ignoreNotFound bool) error {
for {
time.Sleep(2 * time.Second)
var op *compute.Operation
err := ctx.apiCall(func() (err error) {
switch typ {
case "global":
op, err = ctx.computeService.GlobalOperations.Get(ctx.ProjectID, opName).Do()
case "zone":
op, err = ctx.computeService.ZoneOperations.Get(ctx.ProjectID, ctx.ZoneID, opName).Do()
default:
panic("unknown operation type: " + typ)
}
return
})
if err != nil {
return fmt.Errorf("failed to get %v operation %v: %v", desc, opName, err)
}
switch op.Status {
case "PENDING", "RUNNING":
continue
case "DONE":
if op.Error != nil {
reason := ""
for _, operr := range op.Error.Errors {
if operr.Code == "ZONE_RESOURCE_POOL_EXHAUSTED" {
return resourcePoolExhaustedError(fmt.Sprintf("%+v", operr))
}
if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" {
return nil
}
reason += fmt.Sprintf("%+v.", operr)
}
return fmt.Errorf("%v operation failed: %v", desc, reason)
}
return nil
default:
return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op)
}
}
}
func (ctx *Context) getMeta(path string) (string, error) {
req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil)
if err != nil {
return "", err
}
req.Header.Add("Metadata-Flavor", "Google")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(body), nil
}
func (ctx *Context) apiCall(fn func() error) error {
rateLimited := 0
for {
<-ctx.apiRateGate
err := fn()
if err != nil {
if strings.Contains(err.Error(), "Rate Limit Exceeded") ||
strings.Contains(err.Error(), "rateLimitExceeded") {
rateLimited++
backoff := time.Duration(float64(rateLimited) * 1e9 * (rand.Float64() + 1))
time.Sleep(backoff)
if rateLimited < 20 {
continue
}
}
}
return err
}
}