"Fossies" - the Fresh Open Source Software Archive

Member "minikube-1.26.1/test/integration/addons_test.go" (2 Aug 2022, 29266 Bytes) of package /linux/misc/minikube-1.26.1.tar.gz:

As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "addons_test.go": 1.25.2_vs_1.26.0.

A hint: This file contains one or more very long lines, so maybe it is better readable using the pure text view mode that shows the contents as wrapped lines within the browser window.

    1 //go:build integration
    2 
    3 /*
    4 Copyright 2016 The Kubernetes Authors All rights reserved.
    5 
    6 Licensed under the Apache License, Version 2.0 (the "License");
    7 you may not use this file except in compliance with the License.
    8 You may obtain a copy of the License at
    9 
   10     http://www.apache.org/licenses/LICENSE-2.0
   11 
   12 Unless required by applicable law or agreed to in writing, software
   13 distributed under the License is distributed on an "AS IS" BASIS,
   14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   15 See the License for the specific language governing permissions and
   16 limitations under the License.
   17 */
   18 
   19 package integration
   20 
   21 import (
   22     "bytes"
   23     "context"
   24     "encoding/json"
   25     "fmt"
   26     "net/http"
   27     "net/url"
   28     "os"
   29     "os/exec"
   30     "path/filepath"
   31     "reflect"
   32     "regexp"
   33     "strings"
   34     "testing"
   35     "time"
   36 
   37     "github.com/blang/semver/v4"
   38     retryablehttp "github.com/hashicorp/go-retryablehttp"
   39     "k8s.io/minikube/pkg/kapi"
   40     "k8s.io/minikube/pkg/minikube/detect"
   41     "k8s.io/minikube/pkg/util/retry"
   42 )
   43 
   44 // TestAddons tests addons that require no special environment in parallel
   45 func TestAddons(t *testing.T) {
   46     profile := UniqueProfileName("addons")
   47     ctx, cancel := context.WithTimeout(context.Background(), Minutes(40))
   48     defer Cleanup(t, profile, cancel)
   49 
   50     setupSucceeded := t.Run("Setup", func(t *testing.T) {
   51         // Set an env var to point to our dummy credentials file
   52         // don't use t.Setenv because we sometimes manually unset the env var later manually
   53         err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", filepath.Join(*testdataDir, "gcp-creds.json"))
   54         t.Cleanup(func() {
   55             os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS")
   56         })
   57         if err != nil {
   58             t.Fatalf("Failed setting GOOGLE_APPLICATION_CREDENTIALS env var: %v", err)
   59         }
   60 
   61         err = os.Setenv("GOOGLE_CLOUD_PROJECT", "this_is_fake")
   62         t.Cleanup(func() {
   63             os.Unsetenv("GOOGLE_CLOUD_PROJECT")
   64         })
   65         if err != nil {
   66             t.Fatalf("Failed setting GOOGLE_CLOUD_PROJECT env var: %v", err)
   67         }
   68 
   69         args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth"}, StartArgs()...)
   70         if !NoneDriver() { // none driver does not support ingress
   71             args = append(args, "--addons=ingress", "--addons=ingress-dns")
   72         }
   73         if !arm64Platform() {
   74             args = append(args, "--addons=helm-tiller")
   75         }
   76         rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
   77         if err != nil {
   78             t.Fatalf("%s failed: %v", rr.Command(), err)
   79         }
   80 
   81     })
   82 
   83     if !setupSucceeded {
   84         t.Fatalf("Failed setup for addon tests")
   85     }
   86 
   87     // Parallelized tests
   88     t.Run("parallel", func(t *testing.T) {
   89         tests := []struct {
   90             name      string
   91             validator validateFunc
   92         }{
   93             {"Registry", validateRegistryAddon},
   94             {"Ingress", validateIngressAddon},
   95             {"MetricsServer", validateMetricsServerAddon},
   96             {"HelmTiller", validateHelmTillerAddon},
   97             {"Olm", validateOlmAddon},
   98             {"CSI", validateCSIDriverAndSnapshots},
   99             {"Headlamp", validateHeadlampAddon},
  100         }
  101         for _, tc := range tests {
  102             tc := tc
  103             if ctx.Err() == context.DeadlineExceeded {
  104                 t.Fatalf("Unable to run more tests (deadline exceeded)")
  105             }
  106             t.Run(tc.name, func(t *testing.T) {
  107                 MaybeParallel(t)
  108                 tc.validator(ctx, t, profile)
  109             })
  110         }
  111     })
  112 
  113     // Run other tests after to avoid collision
  114     t.Run("serial", func(t *testing.T) {
  115         tests := []struct {
  116             name      string
  117             validator validateFunc
  118         }{
  119             {"GCPAuth", validateGCPAuthAddon},
  120         }
  121         for _, tc := range tests {
  122             tc := tc
  123             if ctx.Err() == context.DeadlineExceeded {
  124                 t.Fatalf("Unable to run more tests (deadline exceeded)")
  125             }
  126             t.Run(tc.name, func(t *testing.T) {
  127                 tc.validator(ctx, t, profile)
  128             })
  129         }
  130     })
  131 
  132     t.Run("StoppedEnableDisable", func(t *testing.T) {
  133         // Assert that disable/enable works offline
  134         rr, err := Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile))
  135         if err != nil {
  136             t.Errorf("failed to stop minikube. args %q : %v", rr.Command(), err)
  137         }
  138         rr, err = Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "dashboard", "-p", profile))
  139         if err != nil {
  140             t.Errorf("failed to enable dashboard addon: args %q : %v", rr.Command(), err)
  141         }
  142         rr, err = Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "dashboard", "-p", profile))
  143         if err != nil {
  144             t.Errorf("failed to disable dashboard addon: args %q : %v", rr.Command(), err)
  145         }
  146     })
  147 }
  148 
  149 // validateIngressAddon tests the ingress addon by deploying a default nginx pod
  150 func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
  151     defer PostMortemLogs(t, profile)
  152     if NoneDriver() {
  153         t.Skipf("skipping: ingress not supported")
  154     }
  155 
  156     client, err := kapi.Client(profile)
  157     if err != nil {
  158         t.Fatalf("failed to get Kubernetes client: %v", client)
  159     }
  160 
  161     // avoid timeouts like:
  162     // Error from server (InternalError): Internal error occurred: failed calling webhook "validate.nginx.ingress.kubernetes.io": Post "https://ingress-nginx-controller-admission.ingress-nginx.svc:443/networking/v1/ingresses?timeout=10s": dial tcp 10.107.218.58:443: i/o timeout
  163     // Error from server (InternalError): Internal error occurred: failed calling webhook "validate.nginx.ingress.kubernetes.io": Post "https://ingress-nginx-controller-admission.ingress-nginx.svc:443/networking/v1/ingresses?timeout=10s": context deadline exceeded
  164     if _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "wait", "--for=condition=ready", "--namespace=ingress-nginx", "pod", "--selector=app.kubernetes.io/component=controller", "--timeout=90s")); err != nil {
  165         t.Fatalf("failed waiting for ingress-nginx-controller : %v", err)
  166     }
  167 
  168     // use nginx ingress yaml that corresponds to k8s version
  169     // default: k8s >= v1.19, ingress api v1
  170     ingressYaml := "nginx-ingress-v1.yaml"
  171     ingressDNSYaml := "ingress-dns-example-v1.yaml"
  172     v, err := client.ServerVersion()
  173     if err != nil {
  174         t.Log("failed to get k8s version, assuming v1.19+ => ingress api v1")
  175     } else if semver.MustParseRange("<1.19.0")(semver.MustParse(fmt.Sprintf("%s.%s.0", v.Major, v.Minor))) {
  176         // legacy: k8s < v1.19 & ingress api v1beta1
  177         ingressYaml = "nginx-ingress-v1beta1.yaml"
  178         ingressDNSYaml = "ingress-dns-example-v1beta1.yaml"
  179     }
  180 
  181     // create networking.k8s.io/v1 ingress
  182     createv1Ingress := func() error {
  183         // apply networking.k8s.io/v1 ingress
  184         rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressYaml)))
  185         if err != nil {
  186             return err
  187         }
  188         if rr.Stderr.String() != "" {
  189             t.Logf("%v: unexpected stderr: %s (may be temporary)", rr.Command(), rr.Stderr)
  190         }
  191         return nil
  192     }
  193     if err := retry.Expo(createv1Ingress, 1*time.Second, Seconds(90)); err != nil {
  194         t.Errorf("failed to create ingress: %v", err)
  195     }
  196 
  197     rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml")))
  198     if err != nil {
  199         t.Errorf("failed to kubectl replace nginx-pod-svc. args %q. %v", rr.Command(), err)
  200     }
  201 
  202     if _, err := PodWait(ctx, t, profile, "default", "run=nginx", Minutes(8)); err != nil {
  203         t.Fatalf("failed waiting for ngnix pod: %v", err)
  204     }
  205     if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, Minutes(10)); err != nil {
  206         t.Errorf("failed waiting for nginx service to be up: %v", err)
  207     }
  208 
  209     want := "Welcome to nginx!"
  210     addr := "http://127.0.0.1/"
  211 
  212     // check if the ingress can route nginx app with networking.k8s.io/v1 ingress
  213     checkv1Ingress := func() error {
  214         rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl -s %s -H 'Host: nginx.example.com'", addr)))
  215         if err != nil {
  216             return err
  217         }
  218 
  219         stderr := rr.Stderr.String()
  220         if rr.Stderr.String() != "" {
  221             t.Logf("debug: unexpected stderr for %v:\n%s", rr.Command(), stderr)
  222         }
  223         stdout := rr.Stdout.String()
  224         if !strings.Contains(stdout, want) {
  225             return fmt.Errorf("%v stdout = %q, want %q", rr.Command(), stdout, want)
  226         }
  227         return nil
  228     }
  229     if err := retry.Expo(checkv1Ingress, 500*time.Millisecond, Seconds(90)); err != nil {
  230         t.Errorf("failed to get expected response from %s within minikube: %v", addr, err)
  231     }
  232 
  233     if NeedsPortForward() {
  234         t.Skip("skipping ingress DNS test for any combination that needs port forwarding")
  235     }
  236 
  237     // check the ingress-dns addon here as well
  238     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressDNSYaml)))
  239     if err != nil {
  240         t.Errorf("failed to kubectl replace ingress-dns-example. args %q. %v", rr.Command(), err)
  241     }
  242 
  243     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ip"))
  244     if err != nil {
  245         t.Errorf("failed to retrieve minikube ip. args %q : %v", rr.Command(), err)
  246     }
  247     ip := strings.TrimSuffix(rr.Stdout.String(), "\n")
  248 
  249     rr, err = Run(t, exec.CommandContext(ctx, "nslookup", "hello-john.test", ip))
  250     if err != nil {
  251         t.Errorf("failed to nslookup hello-john.test host. args %q : %v", rr.Command(), err)
  252     }
  253     // nslookup should include info about the hello-john.test host, including minikube's ip
  254     if !strings.Contains(rr.Stdout.String(), ip) {
  255         t.Errorf("unexpected output from nslookup. stdout: %v\nstderr: %v", rr.Stdout.String(), rr.Stderr.String())
  256     }
  257 
  258     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "ingress-dns", "--alsologtostderr", "-v=1"))
  259     if err != nil {
  260         t.Errorf("failed to disable ingress-dns addon. args %q : %v", rr.Command(), err)
  261     }
  262 
  263     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "ingress", "--alsologtostderr", "-v=1"))
  264     if err != nil {
  265         t.Errorf("failed to disable ingress addon. args %q : %v", rr.Command(), err)
  266     }
  267 }
  268 
  269 // validateRegistryAddon tests the registry addon
  270 func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) {
  271     defer PostMortemLogs(t, profile)
  272 
  273     client, err := kapi.Client(profile)
  274     if err != nil {
  275         t.Fatalf("failed to get Kubernetes client for %s : %v", profile, err)
  276     }
  277 
  278     start := time.Now()
  279     if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", Minutes(6)); err != nil {
  280         t.Errorf("failed waiting for registry replicacontroller to stabilize: %v", err)
  281     }
  282     t.Logf("registry stabilized in %s", time.Since(start))
  283 
  284     if _, err := PodWait(ctx, t, profile, "kube-system", "actual-registry=true", Minutes(6)); err != nil {
  285         t.Fatalf("failed waiting for pod actual-registry: %v", err)
  286     }
  287     if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", Minutes(10)); err != nil {
  288         t.Fatalf("failed waiting for pod registry-proxy: %v", err)
  289     }
  290 
  291     // Test from inside the cluster (no curl available on busybox)
  292     rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "po", "-l", "run=registry-test", "--now"))
  293     if err != nil {
  294         t.Logf("pre-cleanup %s failed: %v (not a problem)", rr.Command(), err)
  295     }
  296 
  297     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "run", "--rm", "registry-test", "--restart=Never", "--image=gcr.io/k8s-minikube/busybox", "-it", "--", "sh", "-c", "wget --spider -S http://registry.kube-system.svc.cluster.local"))
  298     if err != nil {
  299         t.Errorf("failed to hit registry.kube-system.svc.cluster.local. args %q failed: %v", rr.Command(), err)
  300     }
  301     want := "HTTP/1.1 200"
  302     if !strings.Contains(rr.Stdout.String(), want) {
  303         t.Errorf("expected curl response be %q, but got *%s*", want, rr.Stdout.String())
  304     }
  305 
  306     if NeedsPortForward() {
  307         t.Skipf("Unable to complete rest of the test due to connectivity assumptions")
  308     }
  309 
  310     // Test from outside the cluster
  311     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ip"))
  312     if err != nil {
  313         t.Fatalf("failed run minikube ip. args %q : %v", rr.Command(), err)
  314     }
  315     if rr.Stderr.String() != "" {
  316         t.Errorf("expected stderr to be -empty- but got: *%q* .  args %q", rr.Stderr, rr.Command())
  317     }
  318 
  319     endpoint := fmt.Sprintf("http://%s:%d", strings.TrimSpace(rr.Stdout.String()), 5000)
  320     u, err := url.Parse(endpoint)
  321     if err != nil {
  322         t.Fatalf("failed to parse %q: %v", endpoint, err)
  323     }
  324 
  325     checkExternalAccess := func() error {
  326         resp, err := retryablehttp.Get(u.String())
  327         if err != nil {
  328             return err
  329         }
  330         if resp.StatusCode != http.StatusOK {
  331             return fmt.Errorf("%s = status code %d, want %d", u, resp.StatusCode, http.StatusOK)
  332         }
  333         return nil
  334     }
  335 
  336     if err := retry.Expo(checkExternalAccess, 500*time.Millisecond, Seconds(150)); err != nil {
  337         t.Errorf("failed to check external access to %s: %v", u.String(), err.Error())
  338     }
  339 
  340     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "registry", "--alsologtostderr", "-v=1"))
  341     if err != nil {
  342         t.Errorf("failed to disable registry addon. args %q: %v", rr.Command(), err)
  343     }
  344 }
  345 
  346 // validateMetricsServerAddon tests the metrics server addon by making sure "kubectl top pods" returns a sensible result
  347 func validateMetricsServerAddon(ctx context.Context, t *testing.T, profile string) {
  348     defer PostMortemLogs(t, profile)
  349 
  350     client, err := kapi.Client(profile)
  351     if err != nil {
  352         t.Fatalf("failed to get Kubernetes client for %s: %v", profile, err)
  353     }
  354 
  355     start := time.Now()
  356     if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "metrics-server", Minutes(6)); err != nil {
  357         t.Errorf("failed waiting for metrics-server deployment to stabilize: %v", err)
  358     }
  359     t.Logf("metrics-server stabilized in %s", time.Since(start))
  360 
  361     if _, err := PodWait(ctx, t, profile, "kube-system", "k8s-app=metrics-server", Minutes(6)); err != nil {
  362         t.Fatalf("failed waiting for k8s-app=metrics-server pod: %v", err)
  363     }
  364 
  365     want := "CPU(cores)"
  366     checkMetricsServer := func() error {
  367         rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "top", "pods", "-n", "kube-system"))
  368         if err != nil {
  369             return err
  370         }
  371         if rr.Stderr.String() != "" {
  372             t.Logf("%v: unexpected stderr: %s", rr.Command(), rr.Stderr)
  373         }
  374         if !strings.Contains(rr.Stdout.String(), want) {
  375             return fmt.Errorf("%v stdout = %q, want %q", rr.Command(), rr.Stdout, want)
  376         }
  377         return nil
  378     }
  379 
  380     if err := retry.Expo(checkMetricsServer, time.Second*3, Minutes(6)); err != nil {
  381         t.Errorf("failed checking metric server: %v", err.Error())
  382     }
  383 
  384     rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "metrics-server", "--alsologtostderr", "-v=1"))
  385     if err != nil {
  386         t.Errorf("failed to disable metrics-server addon: args %q: %v", rr.Command(), err)
  387     }
  388 }
  389 
  390 // validateHelmTillerAddon tests the helm tiller addon by running "helm version" inside the cluster
  391 func validateHelmTillerAddon(ctx context.Context, t *testing.T, profile string) {
  392 
  393     defer PostMortemLogs(t, profile)
  394 
  395     if arm64Platform() {
  396         t.Skip("skip Helm test on arm64")
  397     }
  398 
  399     client, err := kapi.Client(profile)
  400     if err != nil {
  401         t.Fatalf("failed to get Kubernetes client for %s: %v", profile, err)
  402     }
  403 
  404     start := time.Now()
  405     if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "tiller-deploy", Minutes(6)); err != nil {
  406         t.Errorf("failed waiting for tiller-deploy deployment to stabilize: %v", err)
  407     }
  408     t.Logf("tiller-deploy stabilized in %s", time.Since(start))
  409 
  410     if _, err := PodWait(ctx, t, profile, "kube-system", "app=helm", Minutes(6)); err != nil {
  411         t.Fatalf("failed waiting for helm pod: %v", err)
  412     }
  413 
  414     if NoneDriver() {
  415         _, err := exec.LookPath("socat")
  416         if err != nil {
  417             t.Skipf("socat is required by kubectl to complete this test")
  418         }
  419     }
  420 
  421     want := "Server: &version.Version"
  422     // Test from inside the cluster (`helm version` use pod.list permission.)
  423     checkHelmTiller := func() error {
  424 
  425         rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "run", "--rm", "helm-test", "--restart=Never", "--image=alpine/helm:2.16.3", "-it", "--namespace=kube-system", "--", "version"))
  426         if err != nil {
  427             return err
  428         }
  429         if rr.Stderr.String() != "" {
  430             t.Logf("%v: unexpected stderr: %s", rr.Command(), rr.Stderr)
  431         }
  432         if !strings.Contains(rr.Stdout.String(), want) {
  433             return fmt.Errorf("%v stdout = %q, want %q", rr.Command(), rr.Stdout, want)
  434         }
  435         return nil
  436     }
  437 
  438     if err := retry.Expo(checkHelmTiller, 500*time.Millisecond, Minutes(2)); err != nil {
  439         t.Errorf("failed checking helm tiller: %v", err.Error())
  440     }
  441 
  442     rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "helm-tiller", "--alsologtostderr", "-v=1"))
  443     if err != nil {
  444         t.Errorf("failed disabling helm-tiller addon. arg %q.s %v", rr.Command(), err)
  445     }
  446 }
  447 
  448 // validateOlmAddon tests the OLM addon
  449 func validateOlmAddon(ctx context.Context, t *testing.T, profile string) {
  450     t.Skip("Skipping OLM addon test until https://github.com/operator-framework/operator-lifecycle-manager/issues/2534 is resolved")
  451     defer PostMortemLogs(t, profile)
  452     start := time.Now()
  453 
  454     if _, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "wait", "--for=condition=ready", "--namespace=olm", "pod", "--selector=app=catalog-operator", "--timeout=90s")); err != nil {
  455         t.Fatalf("failed waiting for pod catalog-operator: %v", err)
  456     }
  457     t.Logf("catalog-operator stabilized in %s", time.Since(start))
  458 
  459     if _, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "wait", "--for=condition=ready", "--namespace=olm", "pod", "--selector=app=olm-operator", "--timeout=90s")); err != nil {
  460         t.Fatalf("failed waiting for pod olm-operator: %v", err)
  461     }
  462     t.Logf("olm-operator stabilized in %s", time.Since(start))
  463 
  464     if _, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "wait", "--for=condition=ready", "--namespace=olm", "pod", "--selector=app=packageserver", "--timeout=90s")); err != nil {
  465         t.Fatalf("failed waiting for pod olm-operator: %v", err)
  466     }
  467     t.Logf("packageserver stabilized in %s", time.Since(start))
  468 
  469     if _, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "wait", "--for=condition=ready", "--namespace=olm", "pod", "--selector=olm.catalogSource=operatorhubio-catalog", "--timeout=90s")); err != nil {
  470         t.Fatalf("failed waiting for pod operatorhubio-catalog: %v", err)
  471     }
  472     t.Logf("operatorhubio-catalog stabilized in %s", time.Since(start))
  473 
  474     // Install one sample Operator such as etcd
  475     rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "etcd.yaml")))
  476     if err != nil {
  477         t.Logf("etcd operator installation with %s failed: %v", rr.Command(), err)
  478     }
  479 
  480     want := "Succeeded"
  481     checkOperatorInstalled := func() error {
  482         rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "csv", "-n", "my-etcd"))
  483         if err != nil {
  484             return err
  485         }
  486         if rr.Stderr.String() != "" {
  487             t.Logf("%v: unexpected stderr: %s", rr.Command(), rr.Stderr)
  488         }
  489         if !strings.Contains(rr.Stdout.String(), want) {
  490             return fmt.Errorf("%v stdout = %q, want %q", rr.Command(), rr.Stdout, want)
  491         }
  492         return nil
  493     }
  494     // Operator installation takes a while
  495     if err := retry.Expo(checkOperatorInstalled, time.Second*3, Minutes(10)); err != nil {
  496         t.Errorf("failed checking operator installed: %v", err.Error())
  497     }
  498 }
  499 
  500 // validateCSIDriverAndSnapshots tests the csi hostpath driver by creating a persistent volume, snapshotting it and restoring it.
  501 func validateCSIDriverAndSnapshots(ctx context.Context, t *testing.T, profile string) {
  502     defer PostMortemLogs(t, profile)
  503 
  504     client, err := kapi.Client(profile)
  505     if err != nil {
  506         t.Fatalf("failed to get Kubernetes client for %s: %v", profile, err)
  507     }
  508 
  509     start := time.Now()
  510     if err := kapi.WaitForPods(client, "kube-system", "kubernetes.io/minikube-addons=csi-hostpath-driver", Minutes(6)); err != nil {
  511         t.Errorf("failed waiting for csi-hostpath-driver pods to stabilize: %v", err)
  512     }
  513     t.Logf("csi-hostpath-driver pods stabilized in %s", time.Since(start))
  514 
  515     // create sample PVC
  516     rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "pvc.yaml")))
  517     if err != nil {
  518         t.Logf("creating sample PVC with %s failed: %v", rr.Command(), err)
  519     }
  520 
  521     if err := PVCWait(ctx, t, profile, "default", "hpvc", Minutes(6)); err != nil {
  522         t.Fatalf("failed waiting for PVC hpvc: %v", err)
  523     }
  524 
  525     // create sample pod with the PVC
  526     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "pv-pod.yaml")))
  527     if err != nil {
  528         t.Logf("creating pod with %s failed: %v", rr.Command(), err)
  529     }
  530 
  531     if _, err := PodWait(ctx, t, profile, "default", "app=task-pv-pod", Minutes(6)); err != nil {
  532         t.Fatalf("failed waiting for pod task-pv-pod: %v", err)
  533     }
  534 
  535     // create volume snapshot
  536     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "snapshot.yaml")))
  537     if err != nil {
  538         t.Logf("creating pod with %s failed: %v", rr.Command(), err)
  539     }
  540 
  541     if err := VolumeSnapshotWait(ctx, t, profile, "default", "new-snapshot-demo", Minutes(6)); err != nil {
  542         t.Fatalf("failed waiting for volume snapshot new-snapshot-demo: %v", err)
  543     }
  544 
  545     // delete pod
  546     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "pod", "task-pv-pod"))
  547     if err != nil {
  548         t.Logf("deleting pod with %s failed: %v", rr.Command(), err)
  549     }
  550 
  551     // delete pvc
  552     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "pvc", "hpvc"))
  553     if err != nil {
  554         t.Logf("deleting pod with %s failed: %v", rr.Command(), err)
  555     }
  556 
  557     // restore pv from snapshot
  558     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "pvc-restore.yaml")))
  559     if err != nil {
  560         t.Logf("creating pvc with %s failed: %v", rr.Command(), err)
  561     }
  562 
  563     if err = PVCWait(ctx, t, profile, "default", "hpvc-restore", Minutes(6)); err != nil {
  564         t.Fatalf("failed waiting for PVC hpvc-restore: %v", err)
  565     }
  566 
  567     // create pod from restored snapshot
  568     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "csi-hostpath-driver", "pv-pod-restore.yaml")))
  569     if err != nil {
  570         t.Logf("creating pod with %s failed: %v", rr.Command(), err)
  571     }
  572 
  573     if _, err := PodWait(ctx, t, profile, "default", "app=task-pv-pod-restore", Minutes(6)); err != nil {
  574         t.Fatalf("failed waiting for pod task-pv-pod-restore: %v", err)
  575     }
  576 
  577     // CLEANUP
  578     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "pod", "task-pv-pod-restore"))
  579     if err != nil {
  580         t.Logf("cleanup with %s failed: %v", rr.Command(), err)
  581     }
  582     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "pvc", "hpvc-restore"))
  583     if err != nil {
  584         t.Logf("cleanup with %s failed: %v", rr.Command(), err)
  585     }
  586     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "volumesnapshot", "new-snapshot-demo"))
  587     if err != nil {
  588         t.Logf("cleanup with %s failed: %v", rr.Command(), err)
  589     }
  590     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "csi-hostpath-driver", "--alsologtostderr", "-v=1"))
  591     if err != nil {
  592         t.Errorf("failed to disable csi-hostpath-driver addon: args %q: %v", rr.Command(), err)
  593     }
  594     rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "volumesnapshots", "--alsologtostderr", "-v=1"))
  595     if err != nil {
  596         t.Errorf("failed to disable volumesnapshots addon: args %q: %v", rr.Command(), err)
  597     }
  598 }
  599 
  600 // validateGCPAuthAddon tests the GCP Auth addon with either phony or real credentials and makes sure the files are mounted into pods correctly
  601 func validateGCPAuthAddon(ctx context.Context, t *testing.T, profile string) {
  602     defer PostMortemLogs(t, profile)
  603 
  604     // schedule a pod to check environment variables
  605     rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "-f", filepath.Join(*testdataDir, "busybox.yaml")))
  606     if err != nil {
  607         t.Fatalf("%s failed: %v", rr.Command(), err)
  608     }
  609 
  610     serviceAccountName := "gcp-auth-test"
  611     // create a dummy service account so we know the pull secret got added
  612     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "create", "sa", serviceAccountName))
  613     if err != nil {
  614         t.Fatalf("%s failed: %v", rr.Command(), err)
  615     }
  616 
  617     // 8 minutes, because 4 is not enough for images to pull in all cases.
  618     names, err := PodWait(ctx, t, profile, "default", "integration-test=busybox", Minutes(8))
  619     if err != nil {
  620         t.Fatalf("wait: %v", err)
  621     }
  622 
  623     // Use this pod to confirm that the env vars are set correctly
  624     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "printenv GOOGLE_APPLICATION_CREDENTIALS"))
  625     if err != nil {
  626         t.Fatalf("printenv creds: %v", err)
  627     }
  628 
  629     got := strings.TrimSpace(rr.Stdout.String())
  630     expected := "/google-app-creds.json"
  631     if got != expected {
  632         t.Errorf("'printenv GOOGLE_APPLICATION_CREDENTIALS' returned %s, expected %s", got, expected)
  633     }
  634 
  635     // Now check the service account and make sure the "gcp-auth" image pull secret is present
  636     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "sa", serviceAccountName))
  637     if err != nil {
  638         t.Fatalf("%s failed: %v", rr.Command(), err)
  639     }
  640 
  641     expectedPullSecret := "gcp-auth"
  642     re := regexp.MustCompile(`.*Image pull secrets:.*`)
  643     secrets := re.FindString(rr.Stdout.String())
  644     if !strings.Contains(secrets, expectedPullSecret) {
  645         t.Errorf("Unexpected image pull secrets. expected %s, got %s", expectedPullSecret, secrets)
  646     }
  647 
  648     if !detect.IsOnGCE() || detect.IsCloudShell() {
  649         // Make sure the file contents are correct
  650         rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "cat /google-app-creds.json"))
  651         if err != nil {
  652             t.Fatalf("cat creds: %v", err)
  653         }
  654 
  655         var gotJSON map[string]string
  656         err = json.Unmarshal(bytes.TrimSpace(rr.Stdout.Bytes()), &gotJSON)
  657         if err != nil {
  658             t.Fatalf("unmarshal json: %v", err)
  659         }
  660         expectedJSON := map[string]string{
  661             "client_id":        "haha",
  662             "client_secret":    "nice_try",
  663             "quota_project_id": "this_is_fake",
  664             "refresh_token":    "maybe_next_time",
  665             "type":             "authorized_user",
  666         }
  667 
  668         if !reflect.DeepEqual(gotJSON, expectedJSON) {
  669             t.Fatalf("unexpected creds file: got %v, expected %v", gotJSON, expectedJSON)
  670         }
  671     }
  672 
  673     // Check the GOOGLE_CLOUD_PROJECT env var as well
  674     rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "/bin/sh", "-c", "printenv GOOGLE_CLOUD_PROJECT"))
  675     if err != nil {
  676         t.Fatalf("print env project: %v", err)
  677     }
  678 
  679     got = strings.TrimSpace(rr.Stdout.String())
  680     expected = "this_is_fake"
  681 
  682     if got != expected {
  683         t.Errorf("'printenv GOOGLE_CLOUD_PROJECT' returned %s, expected %s", got, expected)
  684     }
  685 
  686     disableGCPAuth := func() error {
  687         _, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "gcp-auth", "--alsologtostderr", "-v=1"))
  688         if err != nil {
  689             return err
  690         }
  691         return nil
  692     }
  693 
  694     if err := retry.Expo(disableGCPAuth, Minutes(2), Minutes(10), 5); err != nil {
  695         t.Errorf("failed to disable GCP auth addon: %v", err)
  696     }
  697 
  698     // If we're on GCE, we have proper credentials and can test the registry secrets with an artifact registry image
  699     if detect.IsOnGCE() && !detect.IsCloudShell() {
  700         os.Unsetenv("GOOGLE_APPLICATION_CREDENTIALS")
  701         os.Unsetenv("GOOGLE_CLOUD_PROJECT")
  702         args := []string{"-p", profile, "addons", "enable", "gcp-auth"}
  703         rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
  704         if err != nil {
  705             t.Errorf("%s failed: %v", rr.Command(), err)
  706         } else if !strings.Contains(rr.Output(), "It seems that you are running in GCE") {
  707             t.Errorf("Unexpected error message: %v", rr.Output())
  708         }
  709         _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "private-image.yaml")))
  710         if err != nil {
  711             t.Fatalf("print env project: %v", err)
  712         }
  713 
  714         // Make sure the pod is up and running, which means we successfully pulled the private image down
  715         // 8 minutes, because 4 is not enough for images to pull in all cases.
  716         _, err = PodWait(ctx, t, profile, "default", "integration-test=private-image", Minutes(8))
  717         if err != nil {
  718             t.Fatalf("wait for private image: %v", err)
  719         }
  720 
  721         // Try it with a European mirror as well
  722         _, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "private-image-eu.yaml")))
  723         if err != nil {
  724             t.Fatalf("print env project: %v", err)
  725         }
  726 
  727         _, err = PodWait(ctx, t, profile, "default", "integration-test=private-image-eu", Minutes(8))
  728         if err != nil {
  729             t.Fatalf("wait for private image: %v", err)
  730         }
  731     }
  732 }
  733 
  734 func validateHeadlampAddon(ctx context.Context, t *testing.T, profile string) {
  735     defer PostMortemLogs(t, profile)
  736 
  737     rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "enable", "headlamp", "-p", profile, "--alsologtostderr", "-v=1"))
  738     if err != nil {
  739         t.Fatalf("failed to enable headlamp addon: args: %q: %v", rr.Command(), err)
  740     }
  741 
  742     if _, err := PodWait(ctx, t, profile, "headlamp", "app.kubernetes.io/name=headlamp", Minutes(8)); err != nil {
  743         t.Fatalf("failed waiting for headlamp pod: %v", err)
  744     }
  745 }