From 762fa4f7475637702ac0427198228c406591348d Mon Sep 17 00:00:00 2001 From: Joakim Olsson Date: Fri, 30 Sep 2022 22:07:56 +0200 Subject: [PATCH] chore: remove some duplication and add a first few tests --- cmd/handler/handler.go | 2 +- gitlab/client.go | 63 ++++---- gitlab/client_test.go | 118 ++++++++++++++ go.mod | 3 + go.sum | 2 + kube/collector.go | 30 ++++ kube/fetcher.go | 35 +--- kube/fetcher_test.go | 359 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 542 insertions(+), 70 deletions(-) create mode 100644 gitlab/client_test.go create mode 100644 kube/collector.go create mode 100644 kube/fetcher_test.go diff --git a/cmd/handler/handler.go b/cmd/handler/handler.go index 9eca9ba..a2913eb 100644 --- a/cmd/handler/handler.go +++ b/cmd/handler/handler.go @@ -28,7 +28,7 @@ func main() { if kubecfg, exists := os.LookupEnv("KUBECONFIG"); exists { kubeClient = kube.New(kube.WithKubeConfigProvider(kubecfg)) } else { - kubeClient = kube.New(kube.WithInClusterProvider()) + kubeClient = kube.New() } gitlabClient := gitlab.New(cli.GitlabToken) if err := handle(cli, logger, kubeClient, gitlabClient); err != nil { diff --git a/gitlab/client.go b/gitlab/client.go index 599305d..3c9c52d 100644 --- a/gitlab/client.go +++ b/gitlab/client.go @@ -12,20 +12,16 @@ import ( ) func New(token string) *RestClient { - return &RestClient{token: token, client: http.DefaultClient} + return &RestClient{token: token, client: http.DefaultClient, baseUrl: "https://gitlab.com"} } type RestClient struct { - client *http.Client - token string + client *http.Client + token string + baseUrl string } func (r *RestClient) UpdateCleanupPolicy(project string, versions []string) error { - encoded := url.QueryEscape(project) - reqUrl, err := url.Parse(fmt.Sprintf("https://gitlab.com/api/v4/projects/%s", encoded)) - if err != nil { - return err - } options := ProjectConfig{ ContainerExpirationPolicyAttributes: ContainerExpirationPolicyAttributes{ Cadence: "1d", @@ -38,7 +34,22 @@ func (r *RestClient) UpdateCleanupPolicy(project string, versions []string) erro } buff := &bytes.Buffer{} encoder := json.NewEncoder(buff) - err = encoder.Encode(&options) + err := encoder.Encode(&options) + if err != nil { + return err + } + return r.projectApiCall("PUT", project, "", io.NopCloser(buff), nil) +} + +func (r *RestClient) GetTags(project string) ([]Tag, error) { + var tags []Tag + err := r.projectApiCall("GET", project, "/repository/tags", nil, &tags) + return tags, err +} + +func (r *RestClient) projectApiCall(method, project string, api string, body io.ReadCloser, response interface{}) error { + encoded := url.QueryEscape(project) + reqUrl, err := url.Parse(fmt.Sprintf("%s/api/v4/projects/%s%s", r.baseUrl, encoded, api)) if err != nil { return err } @@ -46,38 +57,20 @@ func (r *RestClient) UpdateCleanupPolicy(project string, versions []string) erro header.Add("Content-Type", "application/json;charset=UTF-8") header.Add("PRIVATE-TOKEN", r.token) req := &http.Request{ - Method: "PUT", - URL: reqUrl, - Header: header, - Body: io.NopCloser(buff), - } - _, err = r.client.Do(req) - return err -} - -func (r *RestClient) GetTags(project string) ([]Tag, error) { - encoded := url.QueryEscape(project) - reqUrl, err := url.Parse(fmt.Sprintf("https://gitlab.com/api/v4/projects/%s/repository/tags", encoded)) - if err != nil { - return nil, err - } - header := http.Header{} - header.Add("Content-Type", "application/json;charset=UTF-8") - header.Add("PRIVATE-TOKEN", r.token) - req := &http.Request{ - Method: "GET", + Method: method, URL: reqUrl, Header: header, + Body: body, } resp, err := r.client.Do(req) if err != nil { - return nil, err + return err } - - var tags []Tag - decoder := json.NewDecoder(resp.Body) - err = decoder.Decode(&tags) - return tags, err + if resp.StatusCode == http.StatusOK && response != nil { + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(response) + } + return err } type ProjectConfig struct { diff --git a/gitlab/client_test.go b/gitlab/client_test.go new file mode 100644 index 0000000..c5029d7 --- /dev/null +++ b/gitlab/client_test.go @@ -0,0 +1,118 @@ +package gitlab + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRestClient_UpdateCleanupPolicy(t *testing.T) { + type args struct { + project string + versions []string + } + tests := []struct { + name string + args args + handler func(t *testing.T) http.HandlerFunc + wantErr assert.ErrorAssertionFunc + }{ + { + name: "success", + args: args{ + project: "unboundsoftware/dummy", + versions: []string{"1.0", "1.1"}, + }, + handler: func(t *testing.T) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + buff, err := io.ReadAll(request.Body) + assert.NoError(t, err) + assert.Equal(t, "{\"container_expiration_policy_attributes\":{\"cadence\":\"1d\",\"enabled\":true,\"keep_n\":10,\"older_than\":\"14d\",\"name_regex\":\".*\",\"name_regex_keep\":\"(main|master|1.0|1.1)\"}}\n", string(buff)) + writer.WriteHeader(http.StatusOK) + } + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(tt.handler(t)) + defer server.Close() + r := &RestClient{ + client: http.DefaultClient, + token: "some-gitlab-token", + baseUrl: server.URL, + } + tt.wantErr(t, r.UpdateCleanupPolicy(tt.args.project, tt.args.versions), fmt.Sprintf("UpdateCleanupPolicy(%v, %v)", tt.args.project, tt.args.versions)) + }) + } +} + +func TestRestClient_GetTags(t *testing.T) { + type args struct { + project string + } + tests := []struct { + name string + args args + handler func(t *testing.T) http.HandlerFunc + want []Tag + wantErr assert.ErrorAssertionFunc + }{ + { + name: "error", + args: args{ + project: "unboundsoftware/dummy", + }, + handler: func(t *testing.T) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Length", "23") + writer.WriteHeader(http.StatusOK) + _, _ = writer.Write([]byte("abc")) + } + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.EqualError(t, err, "invalid character 'a' looking for beginning of value") + }, + }, + { + name: "success", + args: args{ + project: "unboundsoftware/dummy", + }, + handler: func(t *testing.T) http.HandlerFunc { + return func(writer http.ResponseWriter, request *http.Request) { + //writer.Header().Set("Content-Length", "23") + writer.WriteHeader(http.StatusOK) + _, _ = writer.Write([]byte(`[{"name":"1.0"},{"name": "1.1"}]`)) + } + }, + want: []Tag{ + {Name: "1.0"}, + {Name: "1.1"}, + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(tt.handler(t)) + defer server.Close() + r := &RestClient{ + client: http.DefaultClient, + token: "some-gitlab-token", + baseUrl: server.URL, + } + got, err := r.GetTags(tt.args.project) + if !tt.wantErr(t, err, fmt.Sprintf("GetTags(%v)", tt.args.project)) { + return + } + assert.Equalf(t, tt.want, got, "GetTags(%v)", tt.args.project) + }) + } +} diff --git a/go.mod b/go.mod index bceb496..00817b7 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.19 require ( github.com/alecthomas/kong v0.6.1 github.com/apex/log v1.9.0 + github.com/stretchr/testify v1.7.2 + gitlab.com/unboundsoftware/apex-mocks v0.2.0 k8s.io/apimachinery v0.25.2 k8s.io/client-go v0.25.2 ) @@ -30,6 +32,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect diff --git a/go.sum b/go.sum index 4dd294b..522b77a 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +gitlab.com/unboundsoftware/apex-mocks v0.2.0 h1:IFt+uyIoOkSl4qdUBLUSIvOhaRdQRGB6TnpZqfRuXqY= +gitlab.com/unboundsoftware/apex-mocks v0.2.0/go.mod h1:FGsQjCu/nS6b+QaBpAFvms6p0Chr0aobGcUPeeZNSNo= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/kube/collector.go b/kube/collector.go new file mode 100644 index 0000000..9bc3621 --- /dev/null +++ b/kube/collector.go @@ -0,0 +1,30 @@ +package kube + +import "strings" + +type ImageCollector map[string]map[string]struct{} + +func NewImageCollector() ImageCollector { + return make(map[string]map[string]struct{}) +} + +func (c *ImageCollector) Add(image string) { + parts := strings.Split(image[20:], ":") + if x, exists := (*c)[parts[0]]; exists { + x[parts[1]] = struct{}{} + } else { + (*c)[parts[0]] = map[string]struct{}{ + parts[1]: {}, + } + } +} + +func (c *ImageCollector) Images() map[string][]string { + images := make(map[string][]string) + for i, x := range *c { + for v := range x { + images[i] = append(images[i], v) + } + } + return images +} diff --git a/kube/fetcher.go b/kube/fetcher.go index 1d66f93..8c0a5db 100644 --- a/kube/fetcher.go +++ b/kube/fetcher.go @@ -19,12 +19,6 @@ type Client struct { type Options func(*Client) -func WithInClusterProvider() func(c *Client) { - return func(c *Client) { - c.provider = &DefaultProvider{provider: &InClusterProvider{}} - } -} - func WithKubeConfigProvider(kubeconfig string) func(c *Client) { return func(c *Client) { c.provider = &DefaultProvider{provider: &PathConfigProvider{kubecfg: kubeconfig}} @@ -32,7 +26,7 @@ func WithKubeConfigProvider(kubeconfig string) func(c *Client) { } func New(opts ...Options) *Client { - c := &Client{} + c := &Client{provider: &DefaultProvider{provider: &InClusterProvider{}}} for _, opt := range opts { opt(c) } @@ -119,30 +113,3 @@ func (k PathConfigProvider) Provide() (*rest.Config, error) { } var _ ConfigProvider = &PathConfigProvider{} - -type ImageCollector map[string]map[string]struct{} - -func NewImageCollector() ImageCollector { - return make(map[string]map[string]struct{}) -} - -func (c *ImageCollector) Add(image string) { - parts := strings.Split(image[20:], ":") - if x, exists := (*c)[parts[0]]; exists { - x[parts[1]] = struct{}{} - } else { - (*c)[parts[0]] = map[string]struct{}{ - parts[1]: {}, - } - } -} - -func (c *ImageCollector) Images() map[string][]string { - images := make(map[string][]string) - for i, x := range *c { - for v := range x { - images[i] = append(images[i], v) - } - } - return images -} diff --git a/kube/fetcher_test.go b/kube/fetcher_test.go new file mode 100644 index 0000000..31b656c --- /dev/null +++ b/kube/fetcher_test.go @@ -0,0 +1,359 @@ +package kube + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "gitlab.com/unboundsoftware/apex-mocks" + v1 "k8s.io/api/apps/v1" + batchapiv1 "k8s.io/api/batch/v1" + v12 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" +) + +func TestClient_GetImages(t *testing.T) { + type fields struct { + provider ClientProvider + } + type args struct { + namespaces []string + } + tests := []struct { + name string + fields fields + args args + want map[string][]string + wantLogged []string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "error getting client", + fields: fields{ + provider: MockClientProvider(func() (APIClient, error) { + return nil, fmt.Errorf("error") + }), + }, + args: args{}, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.EqualError(t, err, "error") + }, + }, + { + name: "error fetching deployments", + fields: fields{ + provider: MockClientProvider(func() (APIClient, error) { + return &MockAPIClient{ + Apps: func() appsv1.AppsV1Interface { + return &MockApps{ + DeploymentsFn: func(namespace string) appsv1.DeploymentInterface { + assert.Equal(t, "default", namespace) + return &MockDeployments{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) { + return nil, fmt.Errorf("error") + }, + } + }, + } + }, + }, nil + }), + }, + args: args{ + namespaces: []string{"default"}, + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.EqualError(t, err, "error") + }, + }, + { + name: "error fetching cron jobs", + fields: fields{ + provider: MockClientProvider(func() (APIClient, error) { + return &MockAPIClient{ + Apps: func() appsv1.AppsV1Interface { + return &MockApps{ + DeploymentsFn: func(namespace string) appsv1.DeploymentInterface { + return &MockDeployments{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) { + return &v1.DeploymentList{}, nil + }, + } + }, + } + }, + Batch: func() batchv1.BatchV1Interface { + return &MockBatch{ + CronJobsFn: func(namespace string) batchv1.CronJobInterface { + assert.Equal(t, "default", namespace) + return &MockCronJobs{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*batchapiv1.CronJobList, error) { + return nil, fmt.Errorf("error") + }, + } + }, + } + }, + }, nil + }), + }, + args: args{ + namespaces: []string{"default"}, + }, + want: nil, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.EqualError(t, err, "error") + }, + }, + { + name: "no deployments or cronjobs", + fields: fields{ + provider: MockClientProvider(func() (APIClient, error) { + return &MockAPIClient{ + Apps: func() appsv1.AppsV1Interface { + return &MockApps{ + DeploymentsFn: func(namespace string) appsv1.DeploymentInterface { + return &MockDeployments{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) { + return &v1.DeploymentList{}, nil + }, + } + }, + } + }, + Batch: func() batchv1.BatchV1Interface { + return &MockBatch{ + CronJobsFn: func(namespace string) batchv1.CronJobInterface { + return &MockCronJobs{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*batchapiv1.CronJobList, error) { + return &batchapiv1.CronJobList{}, nil + }, + } + }, + } + }, + }, nil + }), + }, + args: args{ + namespaces: []string{"default"}, + }, + want: map[string][]string{}, + wantErr: assert.NoError, + }, + { + name: "deployments and cronjobs in multiple namespaces", + fields: fields{ + provider: MockClientProvider(func() (APIClient, error) { + return &MockAPIClient{ + Apps: func() appsv1.AppsV1Interface { + return &MockApps{ + DeploymentsFn: func(namespace string) appsv1.DeploymentInterface { + return &MockDeployments{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) { + if namespace == "default" { + return &v1.DeploymentList{ + Items: []v1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "some-deployment", + }, + Spec: v1.DeploymentSpec{ + Template: v12.PodTemplateSpec{ + Spec: v12.PodSpec{ + Containers: []v12.Container{ + { + Image: "registry.gitlab.com/unboundsoftware/dummy:abc123", + }, + }, + }, + }, + }, + }, + }, + }, nil + } + return &v1.DeploymentList{ + Items: []v1.Deployment{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "other-deployment", + }, + Spec: v1.DeploymentSpec{ + Template: v12.PodTemplateSpec{ + Spec: v12.PodSpec{ + Containers: []v12.Container{ + { + Image: "registry.gitlab.com/unboundsoftware/dummy:def456", + }, + }, + }, + }, + }, + }, + }, + }, nil + }, + } + }, + } + }, + Batch: func() batchv1.BatchV1Interface { + return &MockBatch{ + CronJobsFn: func(namespace string) batchv1.CronJobInterface { + return &MockCronJobs{ + ListFn: func(ctx context.Context, opts metav1.ListOptions) (*batchapiv1.CronJobList, error) { + if namespace == "other" { + return &batchapiv1.CronJobList{ + Items: []batchapiv1.CronJob{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "some-cronjob", + }, + Spec: batchapiv1.CronJobSpec{ + JobTemplate: batchapiv1.JobTemplateSpec{ + Spec: batchapiv1.JobSpec{ + Template: v12.PodTemplateSpec{ + Spec: v12.PodSpec{ + Containers: []v12.Container{ + { + Image: "registry.gitlab.com/unboundsoftware/other:xxx111", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, nil + } + return &batchapiv1.CronJobList{}, nil + }, + } + }, + } + }, + }, nil + }), + }, + args: args{ + namespaces: []string{"default", "other"}, + }, + want: map[string][]string{ + "unboundsoftware/dummy": {"abc123", "def456"}, + "unboundsoftware/other": {"xxx111"}, + }, + wantLogged: []string{ + "info: Found image 'unboundsoftware/dummy:abc123' in deployment default.some-deployment", + "info: Found image 'unboundsoftware/dummy:def456' in deployment other.other-deployment", + "info: Found image 'unboundsoftware/other:xxx111' in cronjob other.some-cronjob", + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Client{ + provider: tt.fields.provider, + } + logger := apex.New() + ctx := context.Background() + got, err := c.GetImages(ctx, logger, tt.args.namespaces) + if !tt.wantErr(t, err, fmt.Sprintf("GetImages(%v, %v, %v)", ctx, logger, tt.args.namespaces)) { + return + } + assert.Equalf(t, tt.want, got, "GetImages(%v, %v, %v)", ctx, logger, tt.args.namespaces) + logger.Check(t, tt.wantLogged) + }) + } +} + +type MockClientProvider func() (APIClient, error) + +func (m MockClientProvider) Provide() (APIClient, error) { + return m() +} + +type MockAPIClient struct { + Apps func() appsv1.AppsV1Interface + Batch func() batchv1.BatchV1Interface +} + +func (m *MockAPIClient) AppsV1() appsv1.AppsV1Interface { + if m.Apps == nil { + return nil + } + return m.Apps() +} + +func (m *MockAPIClient) BatchV1() batchv1.BatchV1Interface { + if m.Batch == nil { + return nil + } + return m.Batch() +} + +var _ APIClient = &MockAPIClient{} + +type MockApps struct { + appsv1.AppsV1Interface + DeploymentsFn func(namespace string) appsv1.DeploymentInterface +} + +func (a *MockApps) Deployments(namespace string) appsv1.DeploymentInterface { + if a.DeploymentsFn == nil { + return nil + } + return a.DeploymentsFn(namespace) +} + +var _ appsv1.AppsV1Interface = &MockApps{} + +type MockDeployments struct { + appsv1.DeploymentInterface + ListFn func(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) +} + +func (d *MockDeployments) List(ctx context.Context, opts metav1.ListOptions) (*v1.DeploymentList, error) { + if d.ListFn == nil { + return nil, nil + } + return d.ListFn(ctx, opts) +} + +var _ appsv1.DeploymentInterface = &MockDeployments{} + +type MockBatch struct { + batchv1.BatchV1Interface + CronJobsFn func(namespace string) batchv1.CronJobInterface +} + +func (b *MockBatch) CronJobs(namespace string) batchv1.CronJobInterface { + if b.CronJobsFn == nil { + return nil + } + return b.CronJobsFn(namespace) +} + +var _ batchv1.BatchV1Interface = &MockBatch{} + +type MockCronJobs struct { + batchv1.CronJobInterface + ListFn func(ctx context.Context, opts metav1.ListOptions) (*batchapiv1.CronJobList, error) +} + +func (m *MockCronJobs) List(ctx context.Context, opts metav1.ListOptions) (*batchapiv1.CronJobList, error) { + if m.ListFn == nil { + return nil, nil + } + return m.ListFn(ctx, opts) +} + +var _ batchv1.CronJobInterface = &MockCronJobs{}