commit 4c6483d971a556c4c1a3e5bc406d68ce1cbc6e7a Author: Joakim Olsson Date: Thu Nov 25 18:27:08 2021 +0100 feat: initial version diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..504853f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.gitignore +.git +Dockerfile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4976690 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea +exported diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..37ad9bf --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,37 @@ +include: +- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' + +stages: +- build +- deploy-prod + +variables: + DOCKER_HOST: tcp://docker:2375/ + +image: buildtool/build-tools:${BUILDTOOLS_VERSION} + +build: + stage: build + services: + - docker:dind + script: + - build + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov -t ${CODECOV_TOKEN} + - push + artifacts: + paths: + - release/ + - k8s + +deploy-to-prod: + stage: deploy-prod + script: + - echo Deploy to prod. + - deploy prod + environment: + name: prod + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual diff --git a/.gitlab/dependabot.yml b/.gitlab/dependabot.yml new file mode 100644 index 0000000..f0b56e8 --- /dev/null +++ b/.gitlab/dependabot.yml @@ -0,0 +1,17 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: +- package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + open-pull-requests-limit: 20 +- package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + open-pull-requests-limit: 20 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d864de7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM golang:1.17.3 as build +WORKDIR /build +ENV CGO_ENABLED=0 +ADD . /build +RUN if [ $(go mod tidy -v 2>&1 | grep -c unused) != 0 ]; then echo "Unused modules, please run 'go mod tidy'"; exit 1; fi +RUN go fmt ./... +RUN go vet ./... +RUN CGO_ENABLED=1 go test -mod=readonly -race -coverprofile=coverage.txt.tmp -covermode=atomic -coverpkg=$(go list ./... | tr '\n' , | sed 's/,$//') ./... +RUN ["/bin/bash", "-c", "cat coverage.txt.tmp | grep -v -f <(find . -type f | xargs grep -l 'Code generated') > coverage.txt"] +RUN go tool cover -html=coverage.txt -o coverage.html +RUN go tool cover -func=coverage.txt +RUN rm coverage.txt.tmp + +RUN GOOS=linux GOARCH=amd64 go build \ + -tags prod \ + -a -installsuffix cgo \ + -mod=readonly \ + -o /release/service \ + -ldflags '-w -s' \ + ./cmd/service/service.go + +FROM scratch as export +COPY --from=build /build/coverage.txt / + +FROM scratch +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=build /release/service / +CMD ["/service"] diff --git a/cmd/service/service.go b/cmd/service/service.go new file mode 100644 index 0000000..03f1f9a --- /dev/null +++ b/cmd/service/service.go @@ -0,0 +1,96 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/alecthomas/kong" + "github.com/apex/log" + "github.com/apex/log/handlers/json" + + "gitlab.com/unboundsoftware/s3uploader/server" + "gitlab.com/unboundsoftware/s3uploader/storage" +) + +var CLI struct { + Port int `name:"port" env:"PORT" help:"Port which the service listens to" default:"80"` + Bucket string `name:"bucket" env:"BUCKET" help:"The AWS S3 bucket where the uploaded objects should be stored" required:"true"` + ReturnURL string `name:"return-url" env:"RETURN_URL" help:"Base-url to be prepended to all returned locations" required:"true"` +} + +func main() { + _ = kong.Parse(&CLI) + log.SetHandler(json.New(os.Stdout)) + logger := log.WithField("service", "s3uploader") + + if err := start(logger); err != nil { + logger.WithError(err).Error("process error") + } +} + +func start(logger log.Interface) error { + rootCtx, rootCancel := context.WithCancel(context.Background()) + defer rootCancel() + + s3, err := storage.New(CLI.Bucket) + if err != nil { + return fmt.Errorf("storage failed: %w", err) + } + srv := server.New(s3, CLI.ReturnURL, logger) + httpSrvAddr := fmt.Sprintf(":%d", CLI.Port) + httpSrv := &http.Server{Addr: httpSrvAddr, Handler: srv} + + wg := sync.WaitGroup{} + + sigint := make(chan os.Signal, 1) + signal.Notify(sigint, os.Interrupt, syscall.SIGTERM) + + wg.Add(1) + go func() { + defer wg.Done() + sig := <-sigint + if sig != nil { + // In case our shutdown logic is broken/incomplete we reset signal + // handlers so next signal goes to go itself. Go is more aggressive when + // shutting down goroutines + signal.Reset(os.Interrupt, syscall.SIGTERM) + logger.Info("Got shutdown signal..") + rootCancel() + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + <-rootCtx.Done() + + close(sigint) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + err := httpSrv.Shutdown(ctx) + logger.WithError(err).Info("Shutdown of HTTP server complete") + }() + + wg.Add(1) + go func() { + defer wg.Done() + logger.Info(fmt.Sprintf("Serving HTTP API on %s", httpSrvAddr)) + err := httpSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.WithError(err).Error("HTTP server failed") + rootCancel() + } + }() + + wg.Wait() + + return nil +} diff --git a/cmd/service/service_test.go b/cmd/service/service_test.go new file mode 100644 index 0000000..9261a4f --- /dev/null +++ b/cmd/service/service_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + "syscall" + "testing" + "time" + + mocks "gitlab.com/unboundsoftware/apex-mocks" +) + +func TestMainFunc_Success(t *testing.T) { + os.Args = []string{"s3uploader", "--port", "7777", "--bucket", "test-bucket-somewhere", "--return-url", "https://example.org"} + go func() { + time.Sleep(time.Second) + _ = syscall.Kill(syscall.Getpid(), syscall.SIGTERM) + }() + main() +} + +func TestMainFunc_Invalid_AWS_Config(t *testing.T) { + os.Args = []string{"s3uploader", "--port", "7777", "--bucket", "test-bucket-somewhere", "--return-url", "https://example.org"} + _ = os.Setenv("AWS_STS_REGIONAL_ENDPOINTS", "unknown_value") + defer func() { + _ = os.Unsetenv("AWS_STS_REGIONAL_ENDPOINTS") + }() + main() +} + +func Test_start(t *testing.T) { + type args struct { + port int + bucket string + url string + } + tests := []struct { + name string + args args + wantErr bool + wantLogged []string + }{ + { + name: "invalid port", + args: args{ + port: 77777, + bucket: "some-bucket", + url: "https://example.org", + }, + wantErr: false, + wantLogged: []string{ + "info: Serving HTTP API on :77777", + "error: HTTP server failed", + "info: Shutdown of HTTP server complete", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logger := mocks.New() + CLI.Port = tt.args.port + CLI.Bucket = tt.args.bucket + CLI.ReturnURL = tt.args.url + if err := start(logger.Logger); (err != nil) != tt.wantErr { + t.Errorf("start() error = %v, wantErr %v", err, tt.wantErr) + } + logger.Check(t, tt.wantLogged) + }) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b0d4734 --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module gitlab.com/unboundsoftware/s3uploader + +go 1.17 + +require ( + github.com/alecthomas/kong v0.2.18 + github.com/apex/log v1.9.0 + github.com/aws/aws-sdk-go v1.42.9 + github.com/stretchr/testify v1.7.0 + gitlab.com/unboundsoftware/apex-mocks v0.0.4 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sanity-io/litter v1.3.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..33b0aff --- /dev/null +++ b/go.sum @@ -0,0 +1,102 @@ +github.com/alecthomas/kong v0.2.18 h1:H05f55eRO5f9gusObxgjpqKtozJNvniqMTuOPnf+2SQ= +github.com/alecthomas/kong v0.2.18/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ= +github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= +github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.42.9 h1:8ptAGgA+uC2TUbdvUeOVSfBocIZvGE2NKiLxkAcn1GA= +github.com/aws/aws-sdk-go v1.42.9/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sanity-io/litter v1.3.0 h1:5ZO+weUsqdSWMUng5JnpkW/Oz8iTXiIdeumhQr1sSjs= +github.com/sanity-io/litter v1.3.0/go.mod h1:5Z71SvaYy5kcGtyglXOC9rrUi3c1E8CamFWjQsazTh0= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +gitlab.com/unboundsoftware/apex-mocks v0.0.4 h1:W8cUMstZTAK5JqWZIxuUnpOaKpneIy9wKxA0C6bEAJs= +gitlab.com/unboundsoftware/apex-mocks v0.0.4/go.mod h1:wGPNmTNDecBLrpTeJbCtAXR6jvRVqzaMxB2JSr8gtjE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/k8s/deploy.yaml b/k8s/deploy.yaml new file mode 100644 index 0000000..97dfcd9 --- /dev/null +++ b/k8s/deploy.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: s3uploader + namespace: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: s3uploader + namespace: default + labels: + app.kubernetes.io/component: s3uploader + annotations: + kubernetes.io/change-cause: "${TIMESTAMP} Deployed commit id: ${COMMIT}" +spec: + selector: + matchLabels: + app.kubernetes.io/component: s3uploader + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: '10%' + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/component: s3uploader + spec: + serviceAccountName: s3uploader + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/component" + operator: In + values: + - s3uploader + topologyKey: kubernetes.io/hostname + containers: + - name: s3uploader + image: registry.gitlab.com/unboundsoftware/s3uploader:${COMMIT} + imagePullPolicy: IfNotPresent + env: + - name: BUCKET + value: upload.unbound.se + - name: RETURN_URL + value: https://uploads.unbound.se + - name: AWS_DEFAULT_REGION + value: "eu-west-1" + - name: AWS_REGION + value: "eu-west-1" + - name: AWS_ROLE_ARN + value: "arn:aws:iam::724902258495:role/s3uploader.default.sa.k8s.unbound.se" + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: "regional" + ports: + - containerPort: 80 + name: http + resources: + requests: + memory: 10Mi + cpu: 10m + limits: + memory: 100Mi + cpu: 100m + readinessProbe: + httpGet: + path: /health + port: 80 + failureThreshold: 1 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: 80 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + volumeMounts: + - mountPath: "/var/run/secrets/amazonaws.com/serviceaccount/" + name: aws-token + restartPolicy: Always + volumes: + - name: aws-token + projected: + sources: + - serviceAccountToken: + audience: "amazonaws.com" + expirationSeconds: 86400 + path: token + securityContext: + fsGroup: 65534 +--- +apiVersion: v1 +kind: Service +metadata: + name: s3uploader + labels: + app.kubernetes.io/component: s3uploader +spec: + selector: + app.kubernetes.io/component: s3uploader + ports: + - port: 80 + name: http + targetPort: 80 + type: NodePort +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: s3uploader-ingress + annotations: + kubernetes.io/ingress.class: "alb" + alb.ingress.kubernetes.io/group.name: unbound + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: instance + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80},{"HTTPS": 443}]' + alb.ingress.kubernetes.io/ssl-redirect: "443" + alb.ingress.kubernetes.io/healthcheck-path: /health +spec: + rules: + - host: "upload.unbound.se" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: s3uploader + port: + number: 80 diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..422654f --- /dev/null +++ b/server/server.go @@ -0,0 +1,122 @@ +package server + +import ( + "crypto/rand" + "crypto/sha1" + "fmt" + "math/big" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/apex/log" + + "gitlab.com/unboundsoftware/s3uploader/storage" +) + +type Server struct { + *http.ServeMux + store storage.Storage + returnUrl string + logger log.Interface + now func() time.Time +} + +func (s *Server) HandlePut(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodOptions: + s.cors(w) + case http.MethodPut: + path := strings.TrimPrefix(req.URL.Path, "/put") + s.logger.Infof("uploading to %s", path) + s.upload(path, w, req) + default: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("This endpoint requires PUT")) + } +} + +func (s *Server) HandleUpload(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodOptions: + s.cors(w) + case http.MethodPut: + prefix, err := randomString(64) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("An error occurred")) + return + } + now := s.now() + day := now.Format("20060102") + prefixHash := hash(prefix) + dayHash := hash(now.Format(time.RFC3339Nano)) + s.logger.Infof("uploading to %s/%s%s", day, prefixHash, dayHash) + s.upload(fmt.Sprintf("%s/%s%s", day, prefixHash, dayHash), w, req) + default: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("This endpoint requires PUT")) + } +} + +func (s *Server) HandleHealth(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("OK")) +} + +func (s *Server) upload(path string, w http.ResponseWriter, req *http.Request) { + err := s.store.Store(path, req.Body) + if err != nil { + s.logger.WithError(err).Error("error storing object in bucket") + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("error storing object in bucket")) + return + } + w.Header().Add("Access-Control-Expose-Headers", "X-File-URL") + fileUrl := filepath.Join(s.returnUrl, path) + w.Header().Add("X-File-URL", fileUrl) + s.logger.Infof("success - file is available at %s", fileUrl) + _, _ = w.Write([]byte("success")) +} + +func (s *Server) cors(w http.ResponseWriter) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "PUT,OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With") + w.Header().Set("Access-Control-Max-Age", "1728000") + w.WriteHeader(http.StatusNoContent) +} + +func New(store storage.Storage, url string, logger log.Interface) http.Handler { + mux := &Server{ + ServeMux: http.NewServeMux(), + store: store, + returnUrl: url, + logger: logger, + now: time.Now, + } + mux.HandleFunc("/put/", mux.HandlePut) + mux.HandleFunc("/upload", mux.HandleUpload) + mux.HandleFunc("/health", mux.HandleHealth) + return mux +} + +func hash(s string) string { + hasher := sha1.New() + hasher.Write([]byte(s)) + return fmt.Sprintf("%x", hasher.Sum(nil)) +} + +func randomString(n int) (string, error) { + const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + ret := make([]byte, n) + for i := 0; i < n; i++ { + num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) + if err != nil { + return "", err + } + ret[i] = letters[num.Int64()] + } + + return string(ret), nil +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..aff2075 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,254 @@ +package server + +import ( + "crypto/rand" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + mocks "gitlab.com/unboundsoftware/apex-mocks" +) + +func TestServer(t *testing.T) { + type args struct { + store func(t *testing.T) StoreFunc + url string + random io.Reader + } + type req struct { + method string + path string + body string + } + tests := []struct { + name string + args args + req req + wantStatus int + wantResp string + wantHeaders map[string]string + }{ + { + name: "unhandled path", + args: args{}, + req: req{ + method: http.MethodPut, + path: "/missing", + body: "xyz", + }, + wantStatus: 404, + wantResp: "404 page not found\n", + }, + { + name: "health", + args: args{}, + req: req{ + method: http.MethodPut, + path: "/health", + body: "", + }, + wantStatus: 200, + wantResp: "OK", + }, + { + name: "GET on /put", + args: args{}, + req: req{ + method: http.MethodGet, + path: "/put/some/file", + body: "abc", + }, + wantStatus: 400, + wantResp: "This endpoint requires PUT", + }, + { + name: "OPTIONS on /put", + args: args{}, + req: req{ + method: http.MethodOptions, + path: "/put/some/file", + body: "abc", + }, + wantStatus: 204, + wantResp: "", + wantHeaders: map[string]string{ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "PUT,OPTIONS", + "Access-Control-Allow-Headers": "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With", + "Access-Control-Max-Age": "1728000", + }, + }, + { + name: "PUT on /put - error", + args: args{ + store: func(t *testing.T) StoreFunc { + return func(path string, content io.Reader) error { + assert.Equal(t, "/some/file", path) + temp, err := ioutil.ReadAll(content) + assert.NoError(t, err) + assert.Equal(t, "abc", string(temp)) + return fmt.Errorf("error") + } + }, + }, + req: req{ + method: http.MethodPut, + path: "/put/some/file", + body: "abc", + }, + wantStatus: 500, + wantResp: "error storing object in bucket", + }, + { + name: "PUT on /put - success", + args: args{ + store: func(t *testing.T) StoreFunc { + return func(path string, content io.Reader) error { + return nil + } + }, + url: "https://example.org", + }, + req: req{ + method: http.MethodPut, + path: "/put/some/file", + body: "abc", + }, + wantStatus: 200, + wantResp: "success", + wantHeaders: map[string]string{ + "Access-Control-Expose-Headers": "X-File-URL", + "X-File-URL": "https:/example.org/some/file", + }, + }, + { + name: "GET on /upload", + args: args{}, + req: req{ + method: http.MethodGet, + path: "/upload", + body: "abc", + }, + wantStatus: 400, + wantResp: "This endpoint requires PUT", + }, + { + name: "OPTIONS on /upload", + args: args{}, + req: req{ + method: http.MethodOptions, + path: "/upload", + body: "abc", + }, + wantStatus: 204, + wantResp: "", + wantHeaders: map[string]string{ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "PUT,OPTIONS", + "Access-Control-Allow-Headers": "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With", + "Access-Control-Max-Age": "1728000", + }, + }, + { + name: "PUT on /upload - error with random", + args: args{ + random: strings.NewReader("too short"), + }, + req: req{ + method: http.MethodPut, + path: "/upload", + body: "abc", + }, + wantStatus: 500, + wantResp: "An error occurred", + }, + { + name: "PUT on /upload - error", + args: args{ + store: func(t *testing.T) StoreFunc { + return func(path string, content io.Reader) error { + assert.Equal(t, fmt.Sprintf("%s/588b41ebf261820104615b83201c729bd16016d6e43649b28b0ef77d54ca5aaf8da0ce74ae3f20a4", time.Now().Format("20060102")), path) + temp, err := ioutil.ReadAll(content) + assert.NoError(t, err) + assert.Equal(t, "abc", string(temp)) + return fmt.Errorf("error") + } + }, + }, + req: req{ + method: http.MethodPut, + path: "/upload", + body: "abc", + }, + wantStatus: 500, + wantResp: "error storing object in bucket", + }, + { + name: "PUT on /upload - success", + args: args{ + store: func(t *testing.T) StoreFunc { + return func(path string, content io.Reader) error { + return nil + } + }, + url: "https://example.org", + }, + req: req{ + method: http.MethodPut, + path: "/upload", + body: "abc", + }, + wantStatus: 200, + wantResp: "success", + wantHeaders: map[string]string{ + "Access-Control-Expose-Headers": "X-File-URL", + "X-File-URL": "https:/example.org/20211125/588b41ebf261820104615b83201c729bd16016d6e43649b28b0ef77d54ca5aaf8da0ce74ae3f20a4", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.args.random != nil { + rand.Reader = tt.args.random + } else { + rand.Reader = strings.NewReader("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + } + logger := mocks.New() + var store StoreFunc + if tt.args.store != nil { + store = tt.args.store(t) + } + server := New(store, tt.args.url, logger.Logger) + server.(*Server).now = func() time.Time { + return time.Date(2021, 11, 25, 7, 43, 12, 0, time.UTC) + } + recorder := httptest.NewRecorder() + u, err := url.Parse(fmt.Sprintf("https://example.org%s", tt.req.path)) + require.NoError(t, err) + server.ServeHTTP(recorder, &http.Request{ + Method: tt.req.method, + URL: u, + Body: io.NopCloser(strings.NewReader(tt.req.body)), + }) + assert.Equal(t, tt.wantStatus, recorder.Code, "StatusCode") + assert.Equal(t, tt.wantResp, recorder.Body.String(), "Body") + for k, v := range tt.wantHeaders { + assert.Equal(t, v, recorder.Header().Get(k), "Header") + } + }) + } +} + +type StoreFunc func(path string, content io.Reader) error + +func (f StoreFunc) Store(path string, content io.Reader) error { + return f(path, content) +} diff --git a/storage/s3.go b/storage/s3.go new file mode 100644 index 0000000..6f4a3c7 --- /dev/null +++ b/storage/s3.go @@ -0,0 +1,39 @@ +package storage + +import ( + "io" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface" +) + +type Storage interface { + Store(path string, content io.Reader) error +} + +type S3 struct { + bucket string + svc s3manageriface.UploaderAPI +} + +func (s *S3) Store(path string, content io.Reader) error { + _, err := s.svc.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + Body: content, + }) + return err +} + +func New(bucket string) (Storage, error) { + sess, err := session.NewSession(aws.NewConfig()) + if err != nil { + return nil, err + } + uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) { + u.PartSize = 5 * 1024 * 1024 + }) + return &S3{bucket: bucket, svc: uploader}, nil +} diff --git a/storage/s3_test.go b/storage/s3_test.go new file mode 100644 index 0000000..c7f677a --- /dev/null +++ b/storage/s3_test.go @@ -0,0 +1,131 @@ +package storage + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface" + "github.com/stretchr/testify/assert" +) + +func TestNew(t *testing.T) { + type args struct { + bucket string + } + tests := []struct { + name string + args args + setup func() func() + wantErr bool + }{ + { + name: "invalid AWS config", + args: args{}, + setup: func() func() { + _ = os.Setenv("AWS_STS_REGIONAL_ENDPOINTS", "unknown_value") + return func() { + _ = os.Unsetenv("AWS_STS_REGIONAL_ENDPOINTS") + } + }, + wantErr: true, + }, + { + name: "success", + args: args{}, + setup: func() func() { + return func() { + } + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer tt.setup()() + _, err := New(tt.args.bucket) + if (err != nil) != tt.wantErr { + t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestS3_Store(t *testing.T) { + type fields struct { + bucket string + svc func(t *testing.T) s3manageriface.UploaderAPI + } + type args struct { + path string + content io.Reader + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "upload error", + fields: fields{ + bucket: "some-bucket", + svc: func(t *testing.T) s3manageriface.UploaderAPI { + return &mock{ + upload: func(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) { + assert.Equal(t, aws.String("some-bucket"), input.Bucket) + assert.Equal(t, aws.String("/some/path"), input.Key) + buff, err := ioutil.ReadAll(input.Body) + assert.NoError(t, err) + assert.Equal(t, "some content", string(buff)) + return nil, fmt.Errorf("error") + }, + } + }, + }, + args: args{ + path: "/some/path", + content: strings.NewReader("some content"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var svc s3manageriface.UploaderAPI + if tt.fields.svc != nil { + svc = tt.fields.svc(t) + } + s := &S3{ + bucket: tt.fields.bucket, + svc: svc, + } + if err := s.Store(tt.args.path, tt.args.content); (err != nil) != tt.wantErr { + t.Errorf("Store() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +type mock struct { + upload func(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) +} + +func (m *mock) Upload(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) { + if m.upload != nil { + return m.upload(input, f...) + } + return nil, nil +} + +func (m *mock) UploadWithContext(aws.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) { + panic("implement me") +} + +var _ s3manageriface.UploaderAPI = &mock{}