diff --git a/contrib/completions/bash/openshift b/contrib/completions/bash/openshift index fd5f662d966f..63cd27a1b73f 100644 --- a/contrib/completions/bash/openshift +++ b/contrib/completions/bash/openshift @@ -20094,6 +20094,8 @@ _openshift_infra_router() local_nonpersistent_flags+=("--allowed-domains=") flags+=("--as=") local_nonpersistent_flags+=("--as=") + flags+=("--bind-ports-after-sync") + local_nonpersistent_flags+=("--bind-ports-after-sync") flags+=("--certificate-authority=") flags_with_completion+=("--certificate-authority") flags_completion+=("_filedir") diff --git a/contrib/completions/zsh/openshift b/contrib/completions/zsh/openshift index 60aba9615129..51053d9ad049 100644 --- a/contrib/completions/zsh/openshift +++ b/contrib/completions/zsh/openshift @@ -20255,6 +20255,8 @@ _openshift_infra_router() local_nonpersistent_flags+=("--allowed-domains=") flags+=("--as=") local_nonpersistent_flags+=("--as=") + flags+=("--bind-ports-after-sync") + local_nonpersistent_flags+=("--bind-ports-after-sync") flags+=("--certificate-authority=") flags_with_completion+=("--certificate-authority") flags_completion+=("_filedir") diff --git a/docs/man/man1/openshift-infra-router.1 b/docs/man/man1/openshift-infra-router.1 index 6d651cb34b27..5005ac9d7208 100644 --- a/docs/man/man1/openshift-infra-router.1 +++ b/docs/man/man1/openshift-infra-router.1 @@ -45,6 +45,10 @@ You may restrict the set of routes exposed to a single project (with \-\-namespa \fB\-\-as\fP="" Username to impersonate for the operation +.PP +\fB\-\-bind\-ports\-after\-sync\fP=false + Bind ports only after route state has been synchronized + .PP \fB\-\-certificate\-authority\fP="" Path to a cert. file for the certificate authority diff --git a/images/router/haproxy/conf/haproxy-config.template b/images/router/haproxy/conf/haproxy-config.template index 67986d4659ca..0a2465296302 100644 --- a/images/router/haproxy/conf/haproxy-config.template +++ b/images/router/haproxy/conf/haproxy-config.template @@ -94,6 +94,7 @@ listen stats :1936 stats auth {{.StatsUser}}:{{.StatsPassword}} {{ end }} +{{ if .BindPorts }} frontend public bind :{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} mode http @@ -523,6 +524,9 @@ backend be_secure_{{$cfgIdx}} {{ end }}{{/* end range over serviceUnitNames */}} {{ end }}{{/* end tls==reencrypt */}} {{ end }}{{/* end loop over routes */}} +{{ else }} +# Avoiding binding ports until routing configuration has been synchronized. +{{ end }}{{/* end bind ports after sync */}} {{ end }}{{/* end haproxy config template */}} {{/*--------------------------------- END OF HAPROXY CONFIG, BELOW ARE MAPPING FILES ------------------------*/}} diff --git a/pkg/client/cache/eventqueue.go b/pkg/client/cache/eventqueue.go index 1638b0e07a7a..c98dc93e5933 100644 --- a/pkg/client/cache/eventqueue.go +++ b/pkg/client/cache/eventqueue.go @@ -49,6 +49,10 @@ type EventQueue struct { // item it refers to is explicitly deleted from the store or the // event is read via Pop(). lastReplaceKey string + // Tracks whether the Replace() method has been called at least once. + replaceCalled bool + // Tracks the number of items queued by the last Replace() call. + replaceCount int } // EventQueue implements kcache.Store @@ -322,6 +326,9 @@ func (eq *EventQueue) Replace(objects []interface{}, resourceVersion string) err eq.lock.Lock() defer eq.lock.Unlock() + eq.replaceCalled = true + eq.replaceCount = len(objects) + eq.events = map[string]watch.EventType{} eq.queue = eq.queue[:0] @@ -346,6 +353,23 @@ func (eq *EventQueue) Replace(objects []interface{}, resourceVersion string) err return nil } +// ListSuccessfulAtLeastOnce indicates whether a List operation was +// successfully completed regardless of whether any items were queued. +func (eq *EventQueue) ListSuccessfulAtLeastOnce() bool { + eq.lock.Lock() + defer eq.lock.Unlock() + + return eq.replaceCalled +} + +// ListCount returns how many objects were queued by the most recent List operation. +func (eq *EventQueue) ListCount() int { + eq.lock.Lock() + defer eq.lock.Unlock() + + return eq.replaceCount +} + // ListConsumed indicates whether the items queued by a List/Relist // operation have been consumed. func (eq *EventQueue) ListConsumed() bool { diff --git a/pkg/cmd/infra/router/template.go b/pkg/cmd/infra/router/template.go index da9055dfe65f..b42ba62b2d1b 100644 --- a/pkg/cmd/infra/router/template.go +++ b/pkg/cmd/infra/router/template.go @@ -62,6 +62,7 @@ type TemplateRouter struct { DefaultCertificateDir string ExtendedValidation bool RouterService *ktypes.NamespacedName + BindPortsAfterSync bool } // reloadInterval returns how often to run the router reloads. The interval @@ -86,6 +87,7 @@ func (o *TemplateRouter) Bind(flag *pflag.FlagSet) { flag.StringVar(&o.ReloadScript, "reload", util.Env("RELOAD_SCRIPT", ""), "The path to the reload script to use") flag.DurationVar(&o.ReloadInterval, "interval", reloadInterval(), "Controls how often router reloads are invoked. Mutiple router reload requests are coalesced for the duration of this interval since the last reload time.") flag.BoolVar(&o.ExtendedValidation, "extended-validation", util.Env("EXTENDED_VALIDATION", "true") == "true", "If set, then an additional extended validation step is performed on all routes admitted in by this router. Defaults to true and enables the extended validation checks.") + flag.BoolVar(&o.BindPortsAfterSync, "bind-ports-after-sync", util.Env("ROUTER_BIND_PORTS_AFTER_SYNC", "") == "true", "Bind ports only after route state has been synchronized") } type RouterStats struct { @@ -188,6 +190,7 @@ func (o *TemplateRouterOptions) Run() error { StatsUsername: o.StatsUsername, StatsPassword: o.StatsPassword, PeerService: o.RouterService, + BindPortsAfterSync: o.BindPortsAfterSync, IncludeUDP: o.RouterSelection.IncludeUDP, AllowWildcardRoutes: o.RouterSelection.AllowWildcardRoutes, } diff --git a/pkg/router/controller/controller.go b/pkg/router/controller/controller.go index 2a125280c946..629fdf51c345 100644 --- a/pkg/router/controller/controller.go +++ b/pkg/router/controller/controller.go @@ -37,6 +37,11 @@ type RouterController struct { endpointsListConsumed bool filteredByNamespace bool + RoutesListSuccessfulAtLeastOnce func() bool + EndpointsListSuccessfulAtLeastOnce func() bool + RoutesListCount func() int + EndpointsListCount func() int + WatchNodes bool Namespaces NamespaceLister @@ -57,6 +62,51 @@ func (c *RouterController) Run() { if c.WatchNodes { go utilwait.Forever(c.HandleNode, 0) } + go c.watchForFirstSync() +} + +// handleFirstSync signals the router when it sees that the various +// watchers have successfully listed data from the api. +func (c *RouterController) handleFirstSync() bool { + c.lock.Lock() + defer c.lock.Unlock() + + synced := c.RoutesListSuccessfulAtLeastOnce() && + c.EndpointsListSuccessfulAtLeastOnce() && + (c.Namespaces == nil || c.filteredByNamespace) + if !synced { + return false + } + + // If either of the event queues were empty after the initial + // List, the tracking listConsumed variable's default value of + // 'false' may prevent the router from reloading to indicate the + // readiness status. Set the value to 'true' to ensure that a + // reload will be performed if necessary. + if c.RoutesListCount() == 0 { + c.routesListConsumed = true + } + if c.EndpointsListCount() == 0 { + c.endpointsListConsumed = true + } + c.updateLastSyncProcessed() + + err := c.Plugin.SetSyncedAtLeastOnce() + if err == nil { + return true + } + utilruntime.HandleError(err) + return false +} + +// watchForFirstSync loops until the first sync has been handled. +func (c *RouterController) watchForFirstSync() { + for { + if c.handleFirstSync() { + return + } + time.Sleep(50 * time.Millisecond) + } } func (c *RouterController) HandleNamespaces() { diff --git a/pkg/router/controller/controller_test.go b/pkg/router/controller/controller_test.go index b3c183d6e16e..8db76ef4afdf 100644 --- a/pkg/router/controller/controller_test.go +++ b/pkg/router/controller/controller_test.go @@ -12,6 +12,7 @@ import ( type fakeRouterPlugin struct { lastSyncProcessed bool + syncedAtLeastOnce bool } func (p *fakeRouterPlugin) HandleRoute(t watch.EventType, route *routeapi.Route) error { @@ -26,11 +27,17 @@ func (p *fakeRouterPlugin) HandleEndpoints(watch.EventType, *kapi.Endpoints) err func (p *fakeRouterPlugin) HandleNamespaces(namespaces sets.String) error { return nil } + func (p *fakeRouterPlugin) SetLastSyncProcessed(processed bool) error { p.lastSyncProcessed = processed return nil } +func (p *fakeRouterPlugin) SetSyncedAtLeastOnce() error { + p.syncedAtLeastOnce = true + return nil +} + type fakeNamespaceLister struct { } diff --git a/pkg/router/controller/extended_validator.go b/pkg/router/controller/extended_validator.go index e87b1fab6573..aeb8e9e39057 100644 --- a/pkg/router/controller/extended_validator.go +++ b/pkg/router/controller/extended_validator.go @@ -82,3 +82,7 @@ func (p *ExtendedValidator) HandleNamespaces(namespaces sets.String) error { func (p *ExtendedValidator) SetLastSyncProcessed(processed bool) error { return p.plugin.SetLastSyncProcessed(processed) } + +func (p *ExtendedValidator) SetSyncedAtLeastOnce() error { + return p.plugin.SetSyncedAtLeastOnce() +} diff --git a/pkg/router/controller/factory/factory.go b/pkg/router/controller/factory/factory.go index 40ed873c0c30..e04c48d02347 100644 --- a/pkg/router/controller/factory/factory.go +++ b/pkg/router/controller/factory/factory.go @@ -99,6 +99,18 @@ func (factory *RouterControllerFactory) Create(plugin router.Plugin, watchNodes } return eventType, obj.(*kapi.Node), nil }, + EndpointsListCount: func() int { + return endpointsEventQueue.ListCount() + }, + RoutesListCount: func() int { + return routeEventQueue.ListCount() + }, + EndpointsListSuccessfulAtLeastOnce: func() bool { + return endpointsEventQueue.ListSuccessfulAtLeastOnce() + }, + RoutesListSuccessfulAtLeastOnce: func() bool { + return routeEventQueue.ListSuccessfulAtLeastOnce() + }, EndpointsListConsumed: func() bool { return endpointsEventQueue.ListConsumed() }, diff --git a/pkg/router/controller/host_admitter.go b/pkg/router/controller/host_admitter.go index 0e7d4424fe3f..00dc6f7bce3a 100644 --- a/pkg/router/controller/host_admitter.go +++ b/pkg/router/controller/host_admitter.go @@ -152,6 +152,10 @@ func (p *HostAdmitter) SetLastSyncProcessed(processed bool) error { return p.plugin.SetLastSyncProcessed(processed) } +func (p *HostAdmitter) SetSyncedAtLeastOnce() error { + return p.plugin.SetSyncedAtLeastOnce() +} + // addRoute admits routes based on subdomain ownership - returns errors if the route is not admitted. func (p *HostAdmitter) addRoute(route *routeapi.Route) error { // Find displaced routes (or error if an existing route displaces us) diff --git a/pkg/router/controller/status.go b/pkg/router/controller/status.go index 173f4ed6cc4c..aef18762ae8e 100644 --- a/pkg/router/controller/status.go +++ b/pkg/router/controller/status.go @@ -315,3 +315,7 @@ func (a *StatusAdmitter) HandleNamespaces(namespaces sets.String) error { func (a *StatusAdmitter) SetLastSyncProcessed(processed bool) error { return a.plugin.SetLastSyncProcessed(processed) } + +func (a *StatusAdmitter) SetSyncedAtLeastOnce() error { + return a.plugin.SetSyncedAtLeastOnce() +} diff --git a/pkg/router/controller/status_test.go b/pkg/router/controller/status_test.go index cc748f2ad404..b4fb8f96b3ac 100644 --- a/pkg/router/controller/status_test.go +++ b/pkg/router/controller/status_test.go @@ -43,6 +43,10 @@ func (p *fakePlugin) SetLastSyncProcessed(processed bool) error { return fmt.Errorf("not expected") } +func (p *fakePlugin) SetSyncedAtLeastOnce() error { + return fmt.Errorf("not expected") +} + func TestStatusNoOp(t *testing.T) { now := nowFn() touched := unversioned.Time{Time: now.Add(-time.Minute)} diff --git a/pkg/router/controller/unique_host.go b/pkg/router/controller/unique_host.go index 8accbcc810c7..1e21b4674db6 100644 --- a/pkg/router/controller/unique_host.go +++ b/pkg/router/controller/unique_host.go @@ -259,6 +259,10 @@ func (p *UniqueHost) SetLastSyncProcessed(processed bool) error { return p.plugin.SetLastSyncProcessed(processed) } +func (p *UniqueHost) SetSyncedAtLeastOnce() error { + return p.plugin.SetSyncedAtLeastOnce() +} + // routeKeys returns the internal router key to use for the given Route. func routeKeys(route *routeapi.Route) []string { keys := make([]string, 1+len(route.Spec.AlternateBackends)) diff --git a/pkg/router/f5/plugin.go b/pkg/router/f5/plugin.go index 31aaebeb62d2..acb12b1ff666 100644 --- a/pkg/router/f5/plugin.go +++ b/pkg/router/f5/plugin.go @@ -618,3 +618,8 @@ func (p *F5Plugin) HandleRoute(eventType watch.EventType, func (p *F5Plugin) SetLastSyncProcessed(processed bool) error { return nil } + +// No-op since f5 has its own concept of what 'ready' means +func (p *F5Plugin) SetSyncedAtLeastOnce() error { + return nil +} diff --git a/pkg/router/interfaces.go b/pkg/router/interfaces.go index c4c74748a2ed..9308d0bb6af6 100644 --- a/pkg/router/interfaces.go +++ b/pkg/router/interfaces.go @@ -17,4 +17,5 @@ type Plugin interface { HandleNamespaces(namespaces sets.String) error HandleNode(watch.EventType, *kapi.Node) error SetLastSyncProcessed(processed bool) error + SetSyncedAtLeastOnce() error } diff --git a/pkg/router/template/plugin.go b/pkg/router/template/plugin.go index a7cd97e56095..fd4a72e445ce 100644 --- a/pkg/router/template/plugin.go +++ b/pkg/router/template/plugin.go @@ -50,6 +50,7 @@ type TemplatePluginConfig struct { IncludeUDP bool AllowWildcardRoutes bool PeerService *ktypes.NamespacedName + BindPortsAfterSync bool } // routerInterface controls the interaction of the plugin with the underlying router implementation @@ -89,6 +90,9 @@ type routerInterface interface { // SetSkipCommit indicates to the router whether commits should be skipped SetSkipCommit(skipCommit bool) + + // SetSyncedAtLeastOnce indicates to the router that state has been read from the api at least once + SetSyncedAtLeastOnce() } func env(name, defaultValue string) string { @@ -143,6 +147,7 @@ func NewTemplatePlugin(cfg TemplatePluginConfig, lookupSvc ServiceLookup) (*Temp statsPort: cfg.StatsPort, allowWildcardRoutes: cfg.AllowWildcardRoutes, peerEndpointsKey: peerKey, + bindPortsAfterSync: cfg.BindPortsAfterSync, } router, err := newTemplateRouter(templateRouterCfg) return newDefaultTemplatePlugin(router, cfg.IncludeUDP, lookupSvc), err @@ -239,6 +244,12 @@ func (p *TemplatePlugin) SetLastSyncProcessed(processed bool) error { return nil } +func (p *TemplatePlugin) SetSyncedAtLeastOnce() error { + p.Router.SetSyncedAtLeastOnce() + p.Router.Commit() + return nil +} + // routeKeys returns the internal router keys to use for the given Route. // A route can have several services that it can point to, now func routeKeys(route *routeapi.Route) ([]string, []int32) { diff --git a/pkg/router/template/plugin_test.go b/pkg/router/template/plugin_test.go index a429305d560f..907af531b226 100644 --- a/pkg/router/template/plugin_test.go +++ b/pkg/router/template/plugin_test.go @@ -243,6 +243,9 @@ func (r *TestRouter) Commit() { func (r *TestRouter) SetSkipCommit(skipCommit bool) { } +func (r *TestRouter) SetSyncedAtLeastOnce() { +} + func (r *TestRouter) HasServiceUnit(key string) bool { return false } diff --git a/pkg/router/template/router.go b/pkg/router/template/router.go index dd055f5813d1..90dacfc28a32 100644 --- a/pkg/router/template/router.go +++ b/pkg/router/template/router.go @@ -86,6 +86,10 @@ type templateRouter struct { lock sync.Mutex // the router should only reload when the value is false skipCommit bool + // If true, haproxy should only bind ports when it has route and endpoint state + bindPortsAfterSync bool + // whether the router state has been read from the api at least once + syncedAtLeastOnce bool } // templateRouterCfg holds all configuration items required to initialize the template router @@ -103,6 +107,7 @@ type templateRouterCfg struct { allowWildcardRoutes bool peerEndpointsKey string includeUDP bool + bindPortsAfterSync bool } // templateConfig is a subset of the templateRouter information that should be passed to the template for generating @@ -124,6 +129,8 @@ type templateData struct { StatsPassword string //port to expose stats with (if the template supports it) StatsPort int + // whether the router should bind the default ports + BindPorts bool } func newTemplateRouter(cfg templateRouterCfg) (*templateRouter, error) { @@ -162,6 +169,7 @@ func newTemplateRouter(cfg templateRouterCfg) (*templateRouter, error) { allowWildcardRoutes: cfg.allowWildcardRoutes, peerEndpointsKey: cfg.peerEndpointsKey, peerEndpoints: []Endpoint{}, + bindPortsAfterSync: cfg.bindPortsAfterSync, rateLimitedCommitFunction: nil, rateLimitedCommitStopChannel: make(chan struct{}), @@ -394,6 +402,7 @@ func (r *templateRouter) writeConfig() error { StatsUser: r.statsUser, StatsPassword: r.statsPassword, StatsPort: r.statsPort, + BindPorts: !r.bindPortsAfterSync || r.syncedAtLeastOnce, } if err := template.Execute(file, data); err != nil { file.Close() @@ -729,6 +738,13 @@ func (r *templateRouter) SetSkipCommit(skipCommit bool) { } } +// SetSyncedAtLeastOnce indicates to the router that state has been +// read from the api. +func (r *templateRouter) SetSyncedAtLeastOnce() { + r.syncedAtLeastOnce = true + glog.V(4).Infof("Router state synchronized for the first time") +} + // HasServiceUnit attempts to retrieve a service unit for the given // key, returning a boolean indication of whether the key is known. func (r *templateRouter) HasServiceUnit(key string) bool { diff --git a/test/integration/router_stress_test.go b/test/integration/router_stress_test.go index d7ed723da279..f17705325d6c 100644 --- a/test/integration/router_stress_test.go +++ b/test/integration/router_stress_test.go @@ -257,6 +257,10 @@ func (p *DelayPlugin) SetLastSyncProcessed(processed bool) error { return p.plugin.SetLastSyncProcessed(processed) } +func (p *DelayPlugin) SetSyncedAtLeastOnce() error { + return p.plugin.SetSyncedAtLeastOnce() +} + // launchRouter launches a template router that communicates with the // api via the provided clients. func launchRouter(oc osclient.Interface, kc kclient.Interface, maxDelay int32, name string, reloadInterval int, reloadCounts map[string]int) (templatePlugin *templateplugin.TemplatePlugin) { diff --git a/test/integration/router_test.go b/test/integration/router_test.go index 1e984869084d..e22a42eaba4d 100644 --- a/test/integration/router_test.go +++ b/test/integration/router_test.go @@ -1256,6 +1256,10 @@ u3YLAbyW/lHhOCiZu2iAI8AbmXem9lW6Tr7p/97s0w== // createAndStartRouterContainer is responsible for deploying the router image in docker. It assumes that all router images // will use a command line flag that can take --master which points to the master url func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int, reloadInterval int) (containerId string, err error) { + return createAndStartRouterContainerBindAfterSync(dockerCli, masterIp, routerStatsPort, reloadInterval, false) +} + +func createAndStartRouterContainerBindAfterSync(dockerCli *dockerClient.Client, masterIp string, routerStatsPort int, reloadInterval int, bindPortsAfterSync bool) (containerId string, err error) { ports := []string{"80", "443"} if routerStatsPort > 0 { ports = append(ports, fmt.Sprintf("%d", routerStatsPort)) @@ -1291,6 +1295,7 @@ func createAndStartRouterContainer(dockerCli *dockerClient.Client, masterIp stri fmt.Sprintf("STATS_USERNAME=%s", statsUser), fmt.Sprintf("STATS_PASSWORD=%s", statsPassword), fmt.Sprintf("DEFAULT_CERTIFICATE=%s", defaultCert), + fmt.Sprintf("ROUTER_BIND_PORTS_AFTER_SYNC=%s", strconv.FormatBool(bindPortsAfterSync)), } reloadIntVar := fmt.Sprintf("RELOAD_INTERVAL=%ds", reloadInterval) @@ -1582,8 +1587,80 @@ func TestRouterReloadCoalesce(t *testing.T) { // waitForRouterToBecomeAvailable checks for the router start up and waits // till it becomes available. -func waitForRouterToBecomeAvailable(host string, port int) { +func waitForRouterToBecomeAvailable(host string, port int) error { hostAndPort := fmt.Sprintf("%s:%d", host, port) uri := fmt.Sprintf("%s/healthz", hostAndPort) - waitForRoute(uri, hostAndPort, "http", nil, "") + return waitForRoute(uri, hostAndPort, "http", nil, "") +} + +// Ensure that when configured with ROUTER_BIND_PORTS_AFTER_SYNC=true, +// haproxy binds ports only when an initial sync has been performed. +func TestRouterBindsPortsAfterSync(t *testing.T) { + // Create a new master but do not start it yet to simulate a router without api access. + fakeMasterAndPod := tr.NewTestHttpService() + + dockerCli, err := testutil.NewDockerClient() + if err != nil { + t.Fatalf("Unable to get docker client: %v", err) + } + + bindPortsAfterSync := true + reloadInterval := 1 + routerId, err := createAndStartRouterContainerBindAfterSync(dockerCli, fakeMasterAndPod.MasterHttpAddr, statsPort, reloadInterval, bindPortsAfterSync) + if err != nil { + t.Fatalf("Error starting container %s : %v", getRouterImage(), err) + } + defer cleanUp(t, dockerCli, routerId) + + routerIP := "127.0.0.1" + + if err = waitForRouterToBecomeAvailable(routerIP, statsPort); err != nil { + t.Fatalf("Unexpected error while waiting for the router to become available: %v", err) + } + + routeAddress := getRouteAddress() + + // Validate that the default ports are not yet bound + schemes := []string{"http", "https"} + for _, scheme := range schemes { + _, err = getRoute(routeAddress, routeAddress, scheme, nil, "") + if err == nil { + t.Fatalf("Router is unexpectedly accepting connections via %v", scheme) + } else if !strings.HasSuffix(fmt.Sprintf("%v", err), "connection refused") { + t.Fatalf("Unexpected error when dispatching %v request: %v", scheme, err) + } + } + + // Start the master + defer fakeMasterAndPod.Stop() + err = fakeMasterAndPod.Start() + validateServer(fakeMasterAndPod, t) + if err != nil { + t.Fatalf("Unable to start http server: %v", err) + } + + // Validate that the default ports are now bound + var lastErr error + for _, scheme := range schemes { + err := wait.Poll(time.Millisecond*100, time.Duration(reloadInterval)*2*time.Second, func() (bool, error) { + _, err := getRoute(routeAddress, routeAddress, scheme, nil, "") + lastErr = nil + switch err { + case ErrUnavailable: + return true, nil + case nil: + return false, fmt.Errorf("Router is unexpectedly accepting connections via %v", scheme) + default: + lastErr = fmt.Errorf("Unexpected error when dispatching %v request: %v", scheme, err) + return false, nil + } + + }) + if err == wait.ErrWaitTimeout && lastErr != nil { + err = lastErr + } + if err != nil { + t.Fatalf(err.Error()) + } + } }