Upgraded to a version of etcd compatible with newer grpc (#2376)

Trillian has been stuck on grpc < 1.30 - #2195. This uses the v3 version of etcd which isn't using the experimental (and now deleted) grpc APIs

I've deleted the prometheus `etcdiscover` tool in this commit. This can be re-added if it is used, and we can find a way to test it. The upgrade path for this looks non-trivial (the etcd watcher API has changed significantly).

.gitignore

@@ -12,7 +12,6 @@
 /ct_hammer
 /ct_server
 /dump_tree
-/etcdiscover
 /licenses
 /loglb
 /maphammer

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## HEAD
 
+### Dependency updates
+ * Upgraded to etcd v3 in order to allow grpc to be upgraded (#2195)
+   * etcd was `v0.5.0-alpha.5`, now `v3.5.0-alpha.0`
+ * grpc upgraded from `v1.29.1` to `v1.36.0`
+
 ### Cleanup
  * Removed the deprecated crypto.NewSHA256Signer function.
  * Finish removing the `LogMetadata.GetUnsequencedCounts()` method.

cmd/internal/serverutil/main.go

@@ -17,6 +17,7 @@ package serverutil
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"time"
@@ -30,14 +31,13 @@ import (
 	"github.com/google/trillian/util"
 	"github.com/google/trillian/util/clock"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/client/v3/naming/endpoints"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/naming"
 	"google.golang.org/grpc/reflection"
 
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	etcdnaming "go.etcd.io/etcd/clientv3/naming"
+	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
 const (
@@ -215,25 +215,26 @@ func AnnounceSelf(ctx context.Context, client *clientv3.Client, etcdService, end
 		return func() {}
 	}
 
-	res := etcdnaming.GRPCResolver{Client: client}
-
 	// Get a lease so our entry self-destructs.
 	leaseRsp, err := client.Grant(ctx, 30)
 	if err != nil {
 		glog.Exitf("Failed to get lease from etcd: %v", err)
 	}
 	client.KeepAlive(ctx, leaseRsp.ID)
 
-	update := naming.Update{Op: naming.Add, Addr: endpoint} // nolint: megacheck
-	res.Update(ctx, etcdService, update, clientv3.WithLease(leaseRsp.ID))
-	glog.Infof("Announcing our presence in %v with %+v", etcdService, update)
+	em, err := endpoints.NewManager(client, etcdService)
+	if err != nil {
+		glog.Exitf("Failed to create etcd manager: %v", err)
+	}
+	fullEndpoint := fmt.Sprintf("%s/%s", etcdService, endpoint)
+	em.AddEndpoint(ctx, fullEndpoint, endpoints.Endpoint{Addr: endpoint})
+	glog.Infof("Announcing our presence in %v", etcdService)
 
-	bye := naming.Update{Op: naming.Delete, Addr: endpoint} // nolint: megacheck
 	return func() {
 		// Use a background context because the original context may have been cancelled.
-		glog.Infof("Removing our presence in %v with %+v", etcdService, bye)
+		glog.Infof("Removing our presence in %v", etcdService)
 		ctx := context.Background()
-		res.Update(ctx, etcdService, bye)
+		em.DeleteEndpoint(ctx, fullEndpoint)
 		client.Revoke(ctx, leaseRsp.ID)
 	}
 }

cmd/trillian_log_server/main.go

@@ -44,7 +44,7 @@ import (
 	"github.com/google/trillian/server"
 	"github.com/google/trillian/storage"
 	"github.com/google/trillian/util/clock"
-	"go.etcd.io/etcd/clientv3"
+	clientv3 "go.etcd.io/etcd/client/v3"
 	"google.golang.org/grpc"
 
 	// Register key ProtoHandlers

cmd/trillian_log_signer/main.go

@@ -41,7 +41,7 @@ import (
 	"github.com/google/trillian/util/election"
 	"github.com/google/trillian/util/election2"
 	etcdelect "github.com/google/trillian/util/election2/etcd"
-	"go.etcd.io/etcd/clientv3"
+	clientv3 "go.etcd.io/etcd/client/v3"
 	"google.golang.org/grpc"
 
 	// Register key ProtoHandlers

go.mod

@@ -10,56 +10,39 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/apache/beam v2.27.0+incompatible
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
-	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/fullstorydev/grpcurl v1.6.0
+	github.com/fullstorydev/grpcurl v1.8.0
 	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/go-sql-driver/mysql v1.5.0
-	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/mock v1.5.0
 	github.com/golang/protobuf v1.4.3
 	github.com/google/btree v1.0.0
 	github.com/google/certificate-transparency-go v1.0.21
 	github.com/google/go-cmp v0.5.4
-	github.com/google/uuid v1.1.1 // indirect
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
 	github.com/huandu/xstrings v1.2.0 // indirect
-	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/letsencrypt/pkcs11key/v4 v4.0.0
 	github.com/lib/pq v1.9.0
-	github.com/mattn/go-colorable v0.1.4 // indirect
-	github.com/mattn/go-isatty v0.0.10 // indirect
-	github.com/mattn/go-runewidth v0.0.6 // indirect
 	github.com/miekg/pkcs11 v1.0.3 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/mwitkow/go-proto-validators v0.2.0 // indirect
-	github.com/olekukonko/tablewriter v0.0.2 // indirect
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect
 	github.com/prometheus/client_golang v1.9.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/pseudomuto/protoc-gen-doc v1.3.2
-	github.com/spf13/cobra v0.0.5 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
-	go.etcd.io/bbolt v1.3.4 // indirect
-	go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5
+	go.etcd.io/etcd/client/v3 v3.5.0-alpha.0
+	go.etcd.io/etcd/etcdctl/v3 v3.5.0-alpha.0
+	go.etcd.io/etcd/server/v3 v3.5.0-alpha.0
+	go.etcd.io/etcd/v3 v3.5.0-alpha.0
 	go.opencensus.io v0.22.4
-	go.uber.org/multierr v1.4.0 // indirect
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e
-	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
-	golang.org/x/tools v0.0.0-20200706234117-b22de6825cf7
+	golang.org/x/tools v0.0.0-20201014170642-d1624618ad65
 	google.golang.org/api v0.29.0
 	google.golang.org/genproto v0.0.0-20200707001353-8e8330bf89df
-	google.golang.org/grpc v1.29.1
+	google.golang.org/grpc v1.36.0
 	google.golang.org/protobuf v1.25.0
-	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 )