Skip to content

Commit

Permalink
Merge pull request #16 from erizocosmico/feature/manage-drivers
Browse files Browse the repository at this point in the history 
 *: implement driver management
  • Loading branch information
@erizocosmico
erizocosmico committed Aug 27, 2018
2 parents b30ba48 + 1f2cb97 commit 85ce721
Show file tree
Hide file tree
Showing 7 changed files with 614 additions and 232 deletions.
262 changes: 224 additions & 38 deletions api/api.pb.go
View file

Large diffs are not rendered by default.

26 changes: 25 additions & 1 deletion api/api.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ service Engine {
// List all drivers.
rpc ListDrivers(ListDriversRequest) returns (ListDriversResponse) {}

// Install drivers.
rpc InstallDriver(VersionedDriver) returns (InstallDriverResponse) {}

// Update drivers.
rpc UpdateDriver(VersionedDriver) returns (UpdateDriverResponse) {}

// Remove drivers.
rpc RemoveDriver(RemoveDriverRequest) returns (RemoveDriverResponse) {}

// SQL stuff.
rpc SQL(SQLRequest) returns (SQLResponse) {}

Expand Down Expand Up @@ -89,4 +98,19 @@ message StopComponentRequest {
string name = 1;
}

message StopComponentResponse {}
message StopComponentResponse {}

message VersionedDriver {
string language = 1;
string version = 2;
}

message InstallDriverResponse {}

message UpdateDriverResponse {}

message RemoveDriverRequest {
string language = 1;
}

message RemoveDriverResponse {}
87 changes: 81 additions & 6 deletions cmd/srcd-server/engine/drivers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ package engine

import (
"context"
"encoding/json"
"fmt"
"strings"

drivers "github.com/bblfsh/bblfshd/daemon/protocol"
"github.com/pkg/errors"
Expand All @@ -12,7 +12,9 @@ import (
"google.golang.org/grpc"
)

func (s *Server) ListDrivers(ctx context.Context, req *api.ListDriversRequest) (*api.ListDriversResponse, error) {
var ErrDriverAlreadyInstalled = errors.New("driver already installed")

func (s *Server) bblfshDriverClient() (drivers.ProtocolServiceClient, error) {
if err := s.startComponent(bblfshd.Name); err != nil {
return nil, err
}
Expand All @@ -23,21 +25,94 @@ func (s *Server) ListDrivers(ctx context.Context, req *api.ListDriversRequest) (
if err != nil {
return nil, errors.Wrap(err, "could not connect to bblfsh drivers")
}
client := drivers.NewProtocolServiceClient(conn)

return drivers.NewProtocolServiceClient(conn), nil
}

func (s *Server) ListDrivers(ctx context.Context, req *api.ListDriversRequest) (*api.ListDriversResponse, error) {
client, err := s.bblfshDriverClient()
if err != nil {
return nil, err
}

res, err := client.DriverStates(ctx, &drivers.DriverStatesRequest{})
if err != nil {
return nil, errors.Wrap(err, "could not list drivers from bblfsh")
}

j, _ := json.MarshalIndent(res, "", " ")
logrus.Infof("response: %s", j)

var list api.ListDriversResponse
for _, state := range res.State {
list.Drivers = append(list.Drivers, &api.ListDriversResponse_DriverInfo{
Lang: state.Language,
Version: state.Version,
})
}

return &list, nil
}

func (s *Server) InstallDriver(
ctx context.Context,
r *api.VersionedDriver,
) (*api.InstallDriverResponse, error) {
err := s.installDriver(ctx, r.Language, r.Version, false)
return new(api.InstallDriverResponse), err
}

func (s *Server) UpdateDriver(
ctx context.Context,
r *api.VersionedDriver,
) (*api.UpdateDriverResponse, error) {
err := s.installDriver(ctx, r.Language, r.Version, true)
return new(api.UpdateDriverResponse), err
}

func (s *Server) RemoveDriver(
ctx context.Context,
r *api.RemoveDriverRequest,
) (*api.RemoveDriverResponse, error) {
client, err := s.bblfshDriverClient()
if err != nil {
return nil, err
}

_, err = client.RemoveDriver(ctx, &drivers.RemoveDriverRequest{Language: r.Language})
return new(api.RemoveDriverResponse), err
}

func (s *Server) installDriver(ctx context.Context, lang, version string, update bool) error {
client, err := s.bblfshDriverClient()
if err != nil {
return err
}

if update {
_, err := client.RemoveDriver(ctx, &drivers.RemoveDriverRequest{Language: lang})
if err != nil {
return err
}
}

resp, err := client.InstallDriver(ctx, &drivers.InstallDriverRequest{
Language: lang,
ImageReference: driverImage(lang, version),
Update: update,
})
if err != nil {
return err
}

if len(resp.Errors) > 0 {
// TODO(campoy): file an issue regarding this error, it should be in err above.
if !strings.HasPrefix(resp.Errors[0], "driver already installed") {
return ErrDriverAlreadyInstalled
}
return fmt.Errorf("can't install %s driver: %s", lang, strings.Join(resp.Errors, "; "))
}

return nil
}

func driverImage(lang, version string) string {
return fmt.Sprintf("docker://bblfsh/%s-driver:%s", lang, version)
}
156 changes: 0 additions & 156 deletions cmd/srcd-server/engine/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,8 @@ package engine

import (
"context"
"fmt"
"strings"
"time"

"github.com/src-d/engine-cli/components"

"github.com/docker/docker/api/types/container"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
bblfsh "gopkg.in/bblfsh/client-go.v2"
"gopkg.in/bblfsh/client-go.v2/tools"
"gopkg.in/bblfsh/sdk.v1/uast"
enry "gopkg.in/src-d/enry.v1"

drivers "github.com/bblfsh/bblfshd/daemon/protocol"
api "github.com/src-d/engine-cli/api"
"github.com/src-d/engine-cli/docker"
)

var _ api.EngineServer = new(Server)
Expand All @@ -39,143 +23,3 @@ func NewServer(version, workdir string) *Server {
func (s *Server) Version(ctx context.Context, req *api.VersionRequest) (*api.VersionResponse, error) {
return &api.VersionResponse{Version: s.version}, nil
}

const (
bblfshParsePort = 9432
bblfshControlPort = 9433
)

var bblfshd = components.Bblfshd

type logf func(format string, args ...interface{})

func (s *Server) ParseWithLogs(req *api.ParseRequest, stream api.Engine_ParseWithLogsServer) error {
log := func(format string, args ...interface{}) {
logrus.Infof(format, args...)
err := stream.Send(&api.ParseResponse{
Kind: api.ParseResponse_LOG,
Log: fmt.Sprintf(format, args...),
})
if err != nil {
logrus.Errorf("could not stream log: %v", err)
}
}

res, err := s.parse(stream.Context(), req, log)
if err != nil {
return err
}
return stream.Send(res)
}

func (s *Server) Parse(ctx context.Context, req *api.ParseRequest) (*api.ParseResponse, error) {
return s.parse(ctx, req, logrus.Infof)
}

func (s *Server) parse(ctx context.Context, req *api.ParseRequest, log logf) (*api.ParseResponse, error) {
log("got parse request")
lang := req.Lang
if lang == "" {
lang = enry.GetLanguage(req.Name, req.Content)
}
lang = strings.ToLower(lang)
if req.Kind == api.ParseRequest_LANG {
return &api.ParseResponse{Lang: lang}, nil
}

// TODO(campoy): this should be a bit more flexible, might need to a table somewhere.

if err := s.startComponent(bblfshd.Name); err != nil {
return nil, err
}

// TODO(campoy): add security
{
addr := fmt.Sprintf("%s:%d", bblfshd.Name, bblfshControlPort)
log("connecting to bblfsh management on %s", addr)
conn, err := grpc.Dial(addr, grpc.WithInsecure())
if err != nil {
return nil, errors.Wrap(err, "could not connect to bblfsh drivers")
}
log("installing driver for %s", lang)
driverClient := drivers.NewProtocolServiceClient(conn)
res, err := driverClient.InstallDriver(ctx, &drivers.InstallDriverRequest{
Language: lang,
// TODO(campoy): latest might not always be what we want, is it?
ImageReference: fmt.Sprintf("docker://bblfsh/%s-driver:latest", lang),
})
if err != nil {
return nil, errors.Wrap(err, "could not install driver")
}

// TODO(campoy): file an issue regarding this error, it should be in err above.
if len(res.Errors) == 1 {
if !strings.HasPrefix(res.Errors[0], "driver already installed") {
return nil, fmt.Errorf(res.Errors[0])
}
log("driver was already installed")
} else if len(res.Errors) > 1 {
return nil, fmt.Errorf("multiple errors: %s", strings.Join(res.Errors, "; "))
}
}

addr := fmt.Sprintf("%s:%d", bblfshd.Name, bblfshParsePort)
log("connecting to bblfsh parsing on %s", addr)
client, err := bblfsh.NewClient(addr)
if err != nil {
return nil, errors.Wrap(err, "could not connect to bblfsh")
}

res, err := client.NewParseRequest().
Language(lang).
Content(string(req.Content)).
Filename(req.Name).
DoWithContext(ctx)
if err != nil {
return nil, errors.Wrap(err, "could not parse")
}

var nodes = []*uast.Node{res.UAST}
if req.Query != "" {
filtered, err := tools.Filter(res.UAST, req.Query)
if err != nil {
return nil, errors.Wrapf(err, "could not apply query %s", req.Query)
}
nodes = filtered
}

resp := &api.ParseResponse{Kind: api.ParseResponse_FINAL, Lang: lang}
for _, node := range nodes {
uast, err := node.Marshal()
if err != nil {
return nil, errors.Wrap(err, "could not marshal uast")
}
resp.Uast = append(resp.Uast, uast)
}
return resp, nil
}

func (s *Server) withWorkdirMounted(at string) docker.ConfigOption {
return docker.WithVolume(s.workdir, at)
}

func createBbblfshd() error {
if err := docker.EnsureInstalled(bblfshd.Image, ""); err != nil {
return err
}

logrus.Infof("starting bblfshd daemon")

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

config := &container.Config{
Image: bblfshd.Image,
Cmd: []string{"-ctl-address=0.0.0.0:9433", "-ctl-network=tcp"},
}

// TODO: add volume to store drivers
host := &container.HostConfig{Privileged: true}

return docker.Start(ctx, config, host, bblfshd.Name)
}
Loading

0 comments on commit 85ce721

Please sign in to comment.