diff --git a/glide.lock b/glide.lock index 8ad847931d6e..2931fee1b6af 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: ce2c9c5c1323a030dfa6afeaba2801780a27b7c1b38b1ee1de2b00baf254377d -updated: 2018-06-13T11:06:49.527989847-04:00 +hash: 1511c1c08d724d5b2f6c586490bfa154f305c5b0038fb2a4bf3c0404aabc72a6 +updated: 2018-06-13T17:31:37.381725-04:00 imports: - name: bitbucket.org/ww/goautoneg version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675 @@ -185,7 +185,8 @@ imports: - name: github.com/coreos/bbolt version: 32c383e75ce054674c53b5a07e55de85332aee14 - name: github.com/coreos/etcd - version: 121edf0467052d55876a817b89875fb39a99bf78 + version: 135cf9b40738d17886f499b40bc176fc892ba5e9 + repo: https://github.com/openshift/coreos-etcd.git subpackages: - alarm - auth @@ -501,7 +502,7 @@ imports: - name: github.com/google/btree version: e89373fe6b4a7413d7acd6da1725b83ef713e6e4 - name: github.com/google/cadvisor - version: 5287e255428f16a3c0c8083107dfa0c980d7cdac + version: 7de6d3a20159d0d4c404436989dcdbbcfb3b20ae repo: https://github.com/openshift/google-cadvisor.git subpackages: - accelerators diff --git a/glide.yaml b/glide.yaml index d9c8b5c7bcdd..7c0c32a3834f 100644 --- a/glide.yaml +++ b/glide.yaml @@ -99,7 +99,8 @@ import: version: 01a732e01d00cb9a81bb0ca050d3e6d2b947927b # master: etcd - package: github.com/coreos/etcd - version: v3.2.16 + repo: https://github.com/openshift/coreos-etcd.git + version: origin-3.10-etcd-v3.2.16 - package: google.golang.org/grpc version: v1.7.5 - package: github.com/grpc-ecosystem/grpc-gateway diff --git a/pkg/cmd/server/origin/webconsole_proxy.go b/pkg/cmd/server/origin/webconsole_proxy.go index 657beb1336f0..32257cea4a89 100644 --- a/pkg/cmd/server/origin/webconsole_proxy.go +++ b/pkg/cmd/server/origin/webconsole_proxy.go @@ -43,7 +43,7 @@ func withAssetServerRedirect(handler http.Handler, accessor *webConsolePublicURL func (c *MasterConfig) withConsoleRedirection(handler, assetServerHandler http.Handler, accessor *webConsolePublicURLAccessor) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { // blacklist well known paths so we do not risk recursion deadlocks - for _, prefix := range []string{"/apis", "/api", "/oapi", "/healtz", "/version"} { + for _, prefix := range []string{"/apis", "/api", "/oapi", "/healthz", "/version"} { if req.URL.Path == prefix || strings.HasPrefix(req.URL.Path, prefix+"/") { // Dispatch to the next handler handler.ServeHTTP(w, req) diff --git a/vendor/github.com/coreos/etcd/clientv3/client.go b/vendor/github.com/coreos/etcd/clientv3/client.go index 2bdd928771f1..5dc93af200a2 100644 --- a/vendor/github.com/coreos/etcd/clientv3/client.go +++ b/vendor/github.com/coreos/etcd/clientv3/client.go @@ -529,6 +529,20 @@ func isHaltErr(ctx context.Context, err error) bool { return ev.Code() != codes.Unavailable && ev.Code() != codes.Internal } +// isUnavailableErr returns true if the given error is an unavailable error +func isUnavailableErr(ctx context.Context, err error) bool { + if ctx != nil && ctx.Err() != nil { + return false + } + if err == nil { + return false + } + ev, _ := status.FromError(err) + // Unavailable codes mean the system will be right back. + // (e.g., can't connect, lost leader) + return ev.Code() == codes.Unavailable +} + func toErr(ctx context.Context, err error) error { if err == nil { return nil diff --git a/vendor/github.com/coreos/etcd/clientv3/watch.go b/vendor/github.com/coreos/etcd/clientv3/watch.go index 16a91fdff404..f39ae0656b24 100644 --- a/vendor/github.com/coreos/etcd/clientv3/watch.go +++ b/vendor/github.com/coreos/etcd/clientv3/watch.go @@ -769,10 +769,13 @@ func (w *watchGrpcStream) joinSubstreams() { } } +var maxBackoff = 100 * time.Millisecond + // openWatchClient retries opening a watch client until success or halt. // manually retry in case "ws==nil && err==nil" // TODO: remove FailFast=false func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) { + backoff := time.Millisecond for { select { case <-w.ctx.Done(): @@ -788,6 +791,17 @@ func (w *watchGrpcStream) openWatchClient() (ws pb.Watch_WatchClient, err error) if isHaltErr(w.ctx, err) { return nil, v3rpc.Error(err) } + if isUnavailableErr(w.ctx, err) { + // retry, but backoff + if backoff < maxBackoff { + // 25% backoff factor + backoff = backoff + backoff/4 + if backoff > maxBackoff { + backoff = maxBackoff + } + } + time.Sleep(backoff) + } } return ws, nil } diff --git a/vendor/github.com/google/cadvisor/container/crio/handler.go b/vendor/github.com/google/cadvisor/container/crio/handler.go index 024341da8ab2..0eecb9804428 100644 --- a/vendor/github.com/google/cadvisor/container/crio/handler.go +++ b/vendor/github.com/google/cadvisor/container/crio/handler.go @@ -81,9 +81,6 @@ type crioContainerHandler struct { ipAddress string ignoreMetrics container.MetricSet - - // container restart count - restartCount int } var _ container.ContainerHandler = &crioContainerHandler{} @@ -175,7 +172,10 @@ func newCrioContainerHandler( // ignore err and get zero as default, this happens with sandboxes, not sure why... // kube isn't sending restart count in labels for sandboxes. restartCount, _ := strconv.Atoi(cInfo.Annotations["io.kubernetes.container.restartCount"]) - handler.restartCount = restartCount + // Only adds restartcount label if it's greater than 0 + if restartCount > 0 { + handler.labels["restartcount"] = strconv.Itoa(restartCount) + } handler.ipAddress = cInfo.IP @@ -225,10 +225,6 @@ func (self *crioContainerHandler) GetSpec() (info.ContainerSpec, error) { spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem) spec.Labels = self.labels - // Only adds restartcount label if it's greater than 0 - if self.restartCount > 0 { - spec.Labels["restartcount"] = strconv.Itoa(self.restartCount) - } spec.Envs = self.envs spec.Image = self.image diff --git a/vendor/github.com/google/cadvisor/container/docker/handler.go b/vendor/github.com/google/cadvisor/container/docker/handler.go index 8f63d02eb040..a037b092e6ff 100644 --- a/vendor/github.com/google/cadvisor/container/docker/handler.go +++ b/vendor/github.com/google/cadvisor/container/docker/handler.go @@ -115,9 +115,6 @@ type dockerContainerHandler struct { // zfs watcher zfsWatcher *zfs.ZfsWatcher - - // container restart count - restartCount int } var _ container.ContainerHandler = &dockerContainerHandler{} @@ -250,7 +247,10 @@ func newDockerContainerHandler( handler.image = ctnr.Config.Image handler.networkMode = ctnr.HostConfig.NetworkMode handler.deviceID = ctnr.GraphDriver.Data["DeviceId"] - handler.restartCount = ctnr.RestartCount + // Only adds restartcount label if it's greater than 0 + if ctnr.RestartCount > 0 { + handler.labels["restartcount"] = strconv.Itoa(ctnr.RestartCount) + } // Obtain the IP address for the contianer. // If the NetworkMode starts with 'container:' then we need to use the IP address of the container specified. @@ -386,10 +386,6 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) { spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem) spec.Labels = self.labels - // Only adds restartcount label if it's greater than 0 - if self.restartCount > 0 { - spec.Labels["restartcount"] = strconv.Itoa(self.restartCount) - } spec.Envs = self.envs spec.Image = self.image spec.CreationTime = self.creationTime diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 506e3eef8007..fefe83854897 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -51,8 +51,8 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/resourceconfig:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", - "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library", diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 5b8f1b9bb1ec..8f78ac0f16e3 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -32,8 +32,8 @@ import ( "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/healthz" serverstorage "k8s.io/apiserver/pkg/server/storage" - "k8s.io/apiserver/pkg/storage/etcd3/preflight" "k8s.io/apiserver/pkg/storage/storagebackend" + storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory" ) type EtcdOptions struct { @@ -166,29 +166,30 @@ func (s *EtcdOptions) ApplyTo(c *server.Config) error { if s == nil { return nil } - - s.addEtcdHealthEndpoint(c) + if err := s.addEtcdHealthEndpoint(c); err != nil { + return err + } c.RESTOptionsGetter = &SimpleRestOptionsFactory{Options: *s} return nil } func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error { - s.addEtcdHealthEndpoint(c) + if err := s.addEtcdHealthEndpoint(c); err != nil { + return err + } c.RESTOptionsGetter = &storageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory} return nil } -func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) { +func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error { + healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig) + if err != nil { + return err + } c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error { - done, err := preflight.EtcdConnection{ServerList: s.StorageConfig.ServerList}.CheckEtcdServers() - if !done { - return fmt.Errorf("etcd failed") - } - if err != nil { - return err - } - return nil + return healthCheck() })) + return nil } type SimpleRestOptionsFactory struct { diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go index 84b0381049b9..d2d232abd825 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go @@ -17,6 +17,8 @@ limitations under the License. package factory import ( + "context" + "fmt" "net" "net/http" "time" @@ -30,6 +32,29 @@ import ( "k8s.io/apiserver/pkg/storage/storagebackend" ) +func newETCD2HealthCheck(c storagebackend.Config) (func() error, error) { + tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile) + if err != nil { + return nil, err + } + + client, err := newETCD2Client(tr, c.ServerList) + if err != nil { + return nil, err + } + + members := etcd2client.NewMembersAPI(client) + + return func() error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, err := members.List(ctx); err != nil { + return fmt.Errorf("error listing etcd members: %v", err) + } + return nil + }, nil +} + func newETCD2Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) { tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile) if err != nil { diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go index 90a0a28a1214..159f775df9d9 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go @@ -17,12 +17,15 @@ limitations under the License. package factory import ( + "fmt" + "sync/atomic" "time" "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/pkg/transport" "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage" "k8s.io/apiserver/pkg/storage/etcd3" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -38,7 +41,41 @@ var ( dialTimeout = 10 * time.Second ) -func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) { +func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) { + // constructing the etcd v3 client blocks and times out if etcd is not available. + // retry in a loop in the background until we successfully create the client, storing the client or error encountered + + clientValue := &atomic.Value{} + + clientErrMsg := &atomic.Value{} + clientErrMsg.Store("etcd client connection not yet established") + + go wait.PollUntil(time.Second, func() (bool, error) { + client, err := newETCD3Client(c) + if err != nil { + clientErrMsg.Store(err.Error()) + return false, nil + } + clientValue.Store(client) + clientErrMsg.Store("") + return true, nil + }, wait.NeverStop) + + return func() error { + if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 { + return fmt.Errorf(errMsg) + } + client := clientValue.Load().(*clientv3.Client) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, err := client.Cluster.MemberList(ctx); err != nil { + return fmt.Errorf("error listing etcd members: %v", err) + } + return nil + }, nil +} + +func newETCD3Client(c storagebackend.Config) (*clientv3.Client, error) { tlsInfo := transport.TLSInfo{ CertFile: c.CertFile, KeyFile: c.KeyFile, @@ -46,7 +83,7 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e } tlsConfig, err := tlsInfo.ClientConfig() if err != nil { - return nil, nil, err + return nil, err } // NOTE: Client relies on nil tlsConfig // for non-secure connections, update the implicit variable @@ -61,6 +98,11 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e TLS: tlsConfig, } client, err := clientv3.New(cfg) + return client, err +} + +func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) { + client, err := newETCD3Client(c) if err != nil { return nil, nil, err } diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go index 101207b9fa41..bba1fa209a0e 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go @@ -41,3 +41,15 @@ func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) { return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type) } } + +// CreateHealthCheck creates a healthcheck function based on given config. +func CreateHealthCheck(c storagebackend.Config) (func() error, error) { + switch c.Type { + case storagebackend.StorageTypeETCD2: + return newETCD2HealthCheck(c) + case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3: + return newETCD3HealthCheck(c) + default: + return nil, fmt.Errorf("unknown storage type: %s", c.Type) + } +}