dashboard: create buildlet client package, move coordinator code into it

Operation Packification, step 2 of tons.

Eventually the buildlet client binary will use this stuff now.

Change-Id: I4cf5f3e6beb9e56bdc795ed513ce6daaf61425e3
Reviewed-on: https://go-review.googlesource.com/2921
Reviewed-by: Brad Fitzpatrick <[email protected]>

dashboard/builders.go

@@ -36,6 +36,17 @@ type BuildConfig struct {
 	tool    string   // the tool this configuration is for
 }
 
+func (c *BuildConfig) GOOS() string { return c.Name[:strings.Index(c.Name, "-")] }
+
+func (c *BuildConfig) GOARCH() string {
+	arch := c.Name[strings.Index(c.Name, "-")+1:]
+	i := strings.Index(arch, "-")
+	if i == -1 {
+		return arch
+	}
+	return arch[:i]
+}
+
 func (c *BuildConfig) UsesDocker() bool { return c.VMImage == "" }
 func (c *BuildConfig) UsesVM() bool     { return c.VMImage != "" }
 

dashboard/builders_test.go

@@ -0,0 +1,22 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package dashboard
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestOSARCHAccessors(t *testing.T) {
+	valid := func(s string) bool { return s != "" && !strings.Contains(s, "-") }
+	for _, conf := range Builders {
+		os := conf.GOOS()
+		arch := conf.GOARCH()
+		osArch := os + "-" + arch
+		if !valid(os) || !valid(arch) || !(conf.Name == osArch || strings.HasPrefix(conf.Name, osArch+"-")) {
+			t.Errorf("OS+ARCH(%q) = %q, %q; invalid", conf.Name, os, arch)
+		}
+	}
+}

dashboard/buildlet/buildletclient.go

@@ -0,0 +1,72 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build extdep
+
+// Package buildlet contains client tools for working with a buildlet
+// server.
+package buildlet // import "golang.org/x/tools/dashboard/buildlet"
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// KeyPair is the TLS public certificate PEM file and its associated
+// private key PEM file that a builder will use for its HTTPS
+// server. The zero value means no HTTPs, which is used by the
+// coordinator for machines running within a firewall.
+type KeyPair struct {
+	CertPEM string
+	KeyPEM  string
+}
+
+// NoKeyPair is used by the coordinator to speak http directly to buildlets,
+// inside their firewall, without TLS.
+var NoKeyPair = KeyPair{}
+
+// NewClient returns a *Client that will manipulate ipPort,
+// authenticated using the provided keypair.
+//
+// This constructor returns immediately without testing the host or auth.
+func NewClient(ipPort string, tls KeyPair) *Client {
+	return &Client{
+		ipPort: ipPort,
+		tls:    tls,
+	}
+}
+
+// A Client interacts with a single buildlet.
+type Client struct {
+	ipPort string
+	tls    KeyPair
+}
+
+// URL returns the buildlet's URL prefix, without a trailing slash.
+func (c *Client) URL() string {
+	if c.tls != NoKeyPair {
+		return "http://" + strings.TrimSuffix(c.ipPort, ":80")
+	}
+	return "https://" + strings.TrimSuffix(c.ipPort, ":443")
+}
+
+func (c *Client) PutTarball(r io.Reader) error {
+	req, err := http.NewRequest("PUT", c.URL()+"/writetgz", r)
+	if err != nil {
+		return err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode/100 != 2 {
+		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
+		return fmt.Errorf("%v; body: %s", res.Status, slurp)
+	}
+	return nil
+}

dashboard/buildlet/gce.go

@@ -0,0 +1,245 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build extdep
+
+package buildlet
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/tools/dashboard"
+	"google.golang.org/api/compute/v1"
+)
+
+type VMOpts struct {
+	// Zone is the GCE zone to create the VM in. Required.
+	Zone string
+
+	// ProjectID is the GCE project ID. Required.
+	ProjectID string
+
+	// TLS optionally specifies the TLS keypair to use.
+	// If zero, http without auth is used.
+	TLS KeyPair
+
+	// Optional description of the VM.
+	Description string
+
+	// Optional metadata to put on the instance.
+	Meta map[string]string
+
+	// DeleteIn optionally specifies a duration at which
+	// to delete the VM.
+	DeleteIn time.Duration
+
+	// OnInstanceRequested optionally specifies a hook to run synchronously
+	// after the computeService.Instances.Insert call, but before
+	// waiting for its operation to proceed.
+	OnInstanceRequested func()
+
+	// OnInstanceCreated optionally specifies a hook to run synchronously
+	// after the instance operation succeeds.
+	OnInstanceCreated func()
+
+	// OnInstanceCreated optionally specifies a hook to run synchronously
+	// after the computeService.Instances.Get call.
+	OnGotInstanceInfo func()
+}
+
+// StartNewVM boots a new VM on GCE and returns a buildlet client
+// configured to speak to it.
+func StartNewVM(ts oauth2.TokenSource, instName, builderType string, opts VMOpts) (*Client, error) {
+	computeService, _ := compute.New(oauth2.NewClient(oauth2.NoContext, ts))
+
+	conf, ok := dashboard.Builders[builderType]
+	if !ok {
+		return nil, fmt.Errorf("invalid builder type %q", builderType)
+	}
+
+	zone := opts.Zone
+	if zone == "" {
+		// TODO: automatic? maybe that's not useful.
+		// For now just return an error.
+		return nil, errors.New("buildlet: missing required Zone option")
+	}
+	projectID := opts.ProjectID
+	if projectID == "" {
+		return nil, errors.New("buildlet: missing required ProjectID option")
+	}
+
+	prefix := "https://www.googleapis.com/compute/v1/projects/" + projectID
+	machType := prefix + "/zones/" + zone + "/machineTypes/" + conf.MachineType()
+
+	instance := &compute.Instance{
+		Name:        instName,
+		Description: opts.Description,
+		MachineType: machType,
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				Type:       "PERSISTENT",
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					DiskName:    instName,
+					SourceImage: "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + conf.VMImage,
+					DiskType:    "https://www.googleapis.com/compute/v1/projects/" + projectID + "/zones/" + zone + "/diskTypes/pd-ssd",
+				},
+			},
+		},
+		Tags: &compute.Tags{
+			// Warning: do NOT list "http-server" or "allow-ssh" (our
+			// project's custom tag to allow ssh access) here; the
+			// buildlet provides full remote code execution.
+			// The https-server is authenticated, though.
+			Items: []string{"https-server"},
+		},
+		Metadata: &compute.Metadata{
+			Items: []*compute.MetadataItems{
+				// The buildlet-binary-url is the URL of the buildlet binary
+				// which the VMs are configured to download at boot and run.
+				// This lets us/ update the buildlet more easily than
+				// rebuilding the whole VM image.
+				{
+					Key:   "buildlet-binary-url",
+					Value: "http://storage.googleapis.com/go-builder-data/buildlet." + conf.GOOS() + "-" + conf.GOARCH(),
+				},
+			},
+		},
+		NetworkInterfaces: []*compute.NetworkInterface{
+			&compute.NetworkInterface{
+				AccessConfigs: []*compute.AccessConfig{
+					&compute.AccessConfig{
+						Type: "ONE_TO_ONE_NAT",
+						Name: "External NAT",
+					},
+				},
+				Network: prefix + "/global/networks/default",
+			},
+		},
+	}
+
+	if opts.DeleteIn != 0 {
+		// In case the VM gets away from us (generally: if the
+		// coordinator dies while a build is running), then we
+		// set this attribute of when it should be killed so
+		// we can kill it later when the coordinator is
+		// restarted. The cleanUpOldVMs goroutine loop handles
+		// that killing.
+		instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
+			Key:   "delete-at",
+			Value: fmt.Sprint(time.Now().Add(opts.DeleteIn).Unix()),
+		})
+	}
+	for k, v := range opts.Meta {
+		instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
+			Key:   k,
+			Value: v,
+		})
+	}
+
+	op, err := computeService.Instances.Insert(projectID, zone, instance).Do()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create instance: %v", err)
+	}
+	if fn := opts.OnInstanceRequested; fn != nil {
+		fn()
+	}
+	createOp := op.Name
+
+	// Wait for instance create operation to succeed.
+OpLoop:
+	for {
+		time.Sleep(2 * time.Second)
+		op, err := computeService.ZoneOperations.Get(projectID, zone, createOp).Do()
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get op %s: %v", createOp, err)
+		}
+		switch op.Status {
+		case "PENDING", "RUNNING":
+			continue
+		case "DONE":
+			if op.Error != nil {
+				for _, operr := range op.Error.Errors {
+					return nil, fmt.Errorf("Error creating instance: %+v", operr)
+				}
+				return nil, errors.New("Failed to start.")
+			}
+			break OpLoop
+		default:
+			return nil, fmt.Errorf("Unknown create status %q: %+v", op.Status, op)
+		}
+	}
+	if fn := opts.OnInstanceCreated; fn != nil {
+		fn()
+	}
+
+	inst, err := computeService.Instances.Get(projectID, zone, instName).Do()
+	if err != nil {
+		return nil, fmt.Errorf("Error getting instance %s details after creation: %v", instName, err)
+	}
+
+	// Find its internal IP.
+	var ip string
+	for _, iface := range inst.NetworkInterfaces {
+		if strings.HasPrefix(iface.NetworkIP, "10.") {
+			ip = iface.NetworkIP
+		}
+	}
+	if ip == "" {
+		return nil, errors.New("didn't find its internal IP address")
+	}
+
+	// Wait for it to boot and its buildlet to come up.
+	var buildletURL string
+	var ipPort string
+	if opts.TLS != NoKeyPair {
+		buildletURL = "https://" + ip
+		ipPort = ip + ":443"
+	} else {
+		buildletURL = "http://" + ip
+		ipPort = ip + ":80"
+	}
+	if fn := opts.OnGotInstanceInfo; fn != nil {
+		fn()
+	}
+
+	const timeout = 90 * time.Second
+	var alive bool
+	impatientClient := &http.Client{
+		Timeout: 5 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	deadline := time.Now().Add(timeout)
+	try := 0
+	for time.Now().Before(deadline) {
+		try++
+		res, err := impatientClient.Get(buildletURL)
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		res.Body.Close()
+		if res.StatusCode != 200 {
+			return nil, fmt.Errorf("buildlet returned HTTP status code %d on try number %d", res.StatusCode, try)
+		}
+		alive = true
+		break
+	}
+	if !alive {
+		return nil, fmt.Errorf("buildlet didn't come up in %v", timeout)
+	}
+
+	return NewClient(ipPort, opts.TLS), nil
+}