feat: initial version
This commit is contained in:
@@ -0,0 +1,3 @@
|
|||||||
|
.gitignore
|
||||||
|
.git
|
||||||
|
Dockerfile
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
.idea
|
||||||
|
exported
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
include:
|
||||||
|
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
|
||||||
|
|
||||||
|
stages:
|
||||||
|
- build
|
||||||
|
- deploy-prod
|
||||||
|
|
||||||
|
variables:
|
||||||
|
DOCKER_HOST: tcp://docker:2375/
|
||||||
|
|
||||||
|
image: buildtool/build-tools:${BUILDTOOLS_VERSION}
|
||||||
|
|
||||||
|
build:
|
||||||
|
stage: build
|
||||||
|
services:
|
||||||
|
- docker:dind
|
||||||
|
script:
|
||||||
|
- build
|
||||||
|
- curl -Os https://uploader.codecov.io/latest/linux/codecov
|
||||||
|
- chmod +x codecov
|
||||||
|
- ./codecov -t ${CODECOV_TOKEN}
|
||||||
|
- push
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- release/
|
||||||
|
- k8s
|
||||||
|
|
||||||
|
deploy-to-prod:
|
||||||
|
stage: deploy-prod
|
||||||
|
script:
|
||||||
|
- echo Deploy to prod.
|
||||||
|
- deploy prod
|
||||||
|
environment:
|
||||||
|
name: prod
|
||||||
|
rules:
|
||||||
|
- if: $CI_COMMIT_BRANCH == "main"
|
||||||
|
when: manual
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# To get started with Dependabot version updates, you'll need to specify which
|
||||||
|
# package ecosystems to update and where the package manifests are located.
|
||||||
|
# Please see the documentation for all configuration options:
|
||||||
|
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||||
|
|
||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: "gomod"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 20
|
||||||
|
- package-ecosystem: "docker"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 20
|
||||||
+29
@@ -0,0 +1,29 @@
|
|||||||
|
FROM golang:1.17.3 as build
|
||||||
|
WORKDIR /build
|
||||||
|
ENV CGO_ENABLED=0
|
||||||
|
ADD . /build
|
||||||
|
RUN if [ $(go mod tidy -v 2>&1 | grep -c unused) != 0 ]; then echo "Unused modules, please run 'go mod tidy'"; exit 1; fi
|
||||||
|
RUN go fmt ./...
|
||||||
|
RUN go vet ./...
|
||||||
|
RUN CGO_ENABLED=1 go test -mod=readonly -race -coverprofile=coverage.txt.tmp -covermode=atomic -coverpkg=$(go list ./... | tr '\n' , | sed 's/,$//') ./...
|
||||||
|
RUN ["/bin/bash", "-c", "cat coverage.txt.tmp | grep -v -f <(find . -type f | xargs grep -l 'Code generated') > coverage.txt"]
|
||||||
|
RUN go tool cover -html=coverage.txt -o coverage.html
|
||||||
|
RUN go tool cover -func=coverage.txt
|
||||||
|
RUN rm coverage.txt.tmp
|
||||||
|
|
||||||
|
RUN GOOS=linux GOARCH=amd64 go build \
|
||||||
|
-tags prod \
|
||||||
|
-a -installsuffix cgo \
|
||||||
|
-mod=readonly \
|
||||||
|
-o /release/service \
|
||||||
|
-ldflags '-w -s' \
|
||||||
|
./cmd/service/service.go
|
||||||
|
|
||||||
|
FROM scratch as export
|
||||||
|
COPY --from=build /build/coverage.txt /
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo
|
||||||
|
COPY --from=build /release/service /
|
||||||
|
CMD ["/service"]
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/alecthomas/kong"
|
||||||
|
"github.com/apex/log"
|
||||||
|
"github.com/apex/log/handlers/json"
|
||||||
|
|
||||||
|
"gitlab.com/unboundsoftware/s3uploader/server"
|
||||||
|
"gitlab.com/unboundsoftware/s3uploader/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
var CLI struct {
|
||||||
|
Port int `name:"port" env:"PORT" help:"Port which the service listens to" default:"80"`
|
||||||
|
Bucket string `name:"bucket" env:"BUCKET" help:"The AWS S3 bucket where the uploaded objects should be stored" required:"true"`
|
||||||
|
ReturnURL string `name:"return-url" env:"RETURN_URL" help:"Base-url to be prepended to all returned locations" required:"true"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
_ = kong.Parse(&CLI)
|
||||||
|
log.SetHandler(json.New(os.Stdout))
|
||||||
|
logger := log.WithField("service", "s3uploader")
|
||||||
|
|
||||||
|
if err := start(logger); err != nil {
|
||||||
|
logger.WithError(err).Error("process error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func start(logger log.Interface) error {
|
||||||
|
rootCtx, rootCancel := context.WithCancel(context.Background())
|
||||||
|
defer rootCancel()
|
||||||
|
|
||||||
|
s3, err := storage.New(CLI.Bucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("storage failed: %w", err)
|
||||||
|
}
|
||||||
|
srv := server.New(s3, CLI.ReturnURL, logger)
|
||||||
|
httpSrvAddr := fmt.Sprintf(":%d", CLI.Port)
|
||||||
|
httpSrv := &http.Server{Addr: httpSrvAddr, Handler: srv}
|
||||||
|
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
|
||||||
|
sigint := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigint, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
sig := <-sigint
|
||||||
|
if sig != nil {
|
||||||
|
// In case our shutdown logic is broken/incomplete we reset signal
|
||||||
|
// handlers so next signal goes to go itself. Go is more aggressive when
|
||||||
|
// shutting down goroutines
|
||||||
|
signal.Reset(os.Interrupt, syscall.SIGTERM)
|
||||||
|
logger.Info("Got shutdown signal..")
|
||||||
|
rootCancel()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
<-rootCtx.Done()
|
||||||
|
|
||||||
|
close(sigint)
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
err := httpSrv.Shutdown(ctx)
|
||||||
|
logger.WithError(err).Info("Shutdown of HTTP server complete")
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
logger.Info(fmt.Sprintf("Serving HTTP API on %s", httpSrvAddr))
|
||||||
|
err := httpSrv.ListenAndServe()
|
||||||
|
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
logger.WithError(err).Error("HTTP server failed")
|
||||||
|
rootCancel()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
mocks "gitlab.com/unboundsoftware/apex-mocks"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMainFunc_Success(t *testing.T) {
|
||||||
|
os.Args = []string{"s3uploader", "--port", "7777", "--bucket", "test-bucket-somewhere", "--return-url", "https://example.org"}
|
||||||
|
go func() {
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
_ = syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
|
||||||
|
}()
|
||||||
|
main()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMainFunc_Invalid_AWS_Config(t *testing.T) {
|
||||||
|
os.Args = []string{"s3uploader", "--port", "7777", "--bucket", "test-bucket-somewhere", "--return-url", "https://example.org"}
|
||||||
|
_ = os.Setenv("AWS_STS_REGIONAL_ENDPOINTS", "unknown_value")
|
||||||
|
defer func() {
|
||||||
|
_ = os.Unsetenv("AWS_STS_REGIONAL_ENDPOINTS")
|
||||||
|
}()
|
||||||
|
main()
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_start(t *testing.T) {
|
||||||
|
type args struct {
|
||||||
|
port int
|
||||||
|
bucket string
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args args
|
||||||
|
wantErr bool
|
||||||
|
wantLogged []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid port",
|
||||||
|
args: args{
|
||||||
|
port: 77777,
|
||||||
|
bucket: "some-bucket",
|
||||||
|
url: "https://example.org",
|
||||||
|
},
|
||||||
|
wantErr: false,
|
||||||
|
wantLogged: []string{
|
||||||
|
"info: Serving HTTP API on :77777",
|
||||||
|
"error: HTTP server failed",
|
||||||
|
"info: Shutdown of HTTP server complete",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
logger := mocks.New()
|
||||||
|
CLI.Port = tt.args.port
|
||||||
|
CLI.Bucket = tt.args.bucket
|
||||||
|
CLI.ReturnURL = tt.args.url
|
||||||
|
if err := start(logger.Logger); (err != nil) != tt.wantErr {
|
||||||
|
t.Errorf("start() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
logger.Check(t, tt.wantLogged)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
module gitlab.com/unboundsoftware/s3uploader
|
||||||
|
|
||||||
|
go 1.17
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/alecthomas/kong v0.2.18
|
||||||
|
github.com/apex/log v1.9.0
|
||||||
|
github.com/aws/aws-sdk-go v1.42.9
|
||||||
|
github.com/stretchr/testify v1.7.0
|
||||||
|
gitlab.com/unboundsoftware/apex-mocks v0.0.4
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||||
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
github.com/sanity-io/litter v1.3.0 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
github.com/alecthomas/kong v0.2.18 h1:H05f55eRO5f9gusObxgjpqKtozJNvniqMTuOPnf+2SQ=
|
||||||
|
github.com/alecthomas/kong v0.2.18/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ=
|
||||||
|
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
|
||||||
|
github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
|
||||||
|
github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
|
||||||
|
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
|
||||||
|
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
|
||||||
|
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||||
|
github.com/aws/aws-sdk-go v1.42.9 h1:8ptAGgA+uC2TUbdvUeOVSfBocIZvGE2NKiLxkAcn1GA=
|
||||||
|
github.com/aws/aws-sdk-go v1.42.9/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
|
||||||
|
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||||
|
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||||
|
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||||
|
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||||
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
|
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||||
|
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||||
|
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||||
|
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||||
|
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
|
||||||
|
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||||
|
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||||
|
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||||
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||||
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||||
|
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||||
|
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||||
|
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||||
|
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||||
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||||
|
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||||
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||||
|
github.com/sanity-io/litter v1.3.0 h1:5ZO+weUsqdSWMUng5JnpkW/Oz8iTXiIdeumhQr1sSjs=
|
||||||
|
github.com/sanity-io/litter v1.3.0/go.mod h1:5Z71SvaYy5kcGtyglXOC9rrUi3c1E8CamFWjQsazTh0=
|
||||||
|
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||||
|
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||||
|
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
||||||
|
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||||
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
|
||||||
|
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
||||||
|
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
|
||||||
|
github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
|
||||||
|
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||||
|
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||||
|
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||||
|
gitlab.com/unboundsoftware/apex-mocks v0.0.4 h1:W8cUMstZTAK5JqWZIxuUnpOaKpneIy9wKxA0C6bEAJs=
|
||||||
|
gitlab.com/unboundsoftware/apex-mocks v0.0.4/go.mod h1:wGPNmTNDecBLrpTeJbCtAXR6jvRVqzaMxB2JSr8gtjE=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
|
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
|
||||||
|
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
|
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||||
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||||
|
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||||
|
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||||
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||||
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
+143
@@ -0,0 +1,143 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: s3uploader
|
||||||
|
namespace: default
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: s3uploader
|
||||||
|
namespace: default
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: s3uploader
|
||||||
|
annotations:
|
||||||
|
kubernetes.io/change-cause: "${TIMESTAMP} Deployed commit id: ${COMMIT}"
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/component: s3uploader
|
||||||
|
strategy:
|
||||||
|
rollingUpdate:
|
||||||
|
maxSurge: 1
|
||||||
|
maxUnavailable: '10%'
|
||||||
|
type: RollingUpdate
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: s3uploader
|
||||||
|
spec:
|
||||||
|
serviceAccountName: s3uploader
|
||||||
|
affinity:
|
||||||
|
podAntiAffinity:
|
||||||
|
preferredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
- weight: 100
|
||||||
|
podAffinityTerm:
|
||||||
|
labelSelector:
|
||||||
|
matchExpressions:
|
||||||
|
- key: "app.kubernetes.io/component"
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- s3uploader
|
||||||
|
topologyKey: kubernetes.io/hostname
|
||||||
|
containers:
|
||||||
|
- name: s3uploader
|
||||||
|
image: registry.gitlab.com/unboundsoftware/s3uploader:${COMMIT}
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: BUCKET
|
||||||
|
value: upload.unbound.se
|
||||||
|
- name: RETURN_URL
|
||||||
|
value: https://uploads.unbound.se
|
||||||
|
- name: AWS_DEFAULT_REGION
|
||||||
|
value: "eu-west-1"
|
||||||
|
- name: AWS_REGION
|
||||||
|
value: "eu-west-1"
|
||||||
|
- name: AWS_ROLE_ARN
|
||||||
|
value: "arn:aws:iam::724902258495:role/s3uploader.default.sa.k8s.unbound.se"
|
||||||
|
- name: AWS_WEB_IDENTITY_TOKEN_FILE
|
||||||
|
value: "/var/run/secrets/amazonaws.com/serviceaccount/token"
|
||||||
|
- name: AWS_STS_REGIONAL_ENDPOINTS
|
||||||
|
value: "regional"
|
||||||
|
ports:
|
||||||
|
- containerPort: 80
|
||||||
|
name: http
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 10Mi
|
||||||
|
cpu: 10m
|
||||||
|
limits:
|
||||||
|
memory: 100Mi
|
||||||
|
cpu: 100m
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 80
|
||||||
|
failureThreshold: 1
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
successThreshold: 1
|
||||||
|
timeoutSeconds: 5
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 80
|
||||||
|
failureThreshold: 3
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
successThreshold: 1
|
||||||
|
timeoutSeconds: 5
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: "/var/run/secrets/amazonaws.com/serviceaccount/"
|
||||||
|
name: aws-token
|
||||||
|
restartPolicy: Always
|
||||||
|
volumes:
|
||||||
|
- name: aws-token
|
||||||
|
projected:
|
||||||
|
sources:
|
||||||
|
- serviceAccountToken:
|
||||||
|
audience: "amazonaws.com"
|
||||||
|
expirationSeconds: 86400
|
||||||
|
path: token
|
||||||
|
securityContext:
|
||||||
|
fsGroup: 65534
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: s3uploader
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/component: s3uploader
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/component: s3uploader
|
||||||
|
ports:
|
||||||
|
- port: 80
|
||||||
|
name: http
|
||||||
|
targetPort: 80
|
||||||
|
type: NodePort
|
||||||
|
---
|
||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
name: s3uploader-ingress
|
||||||
|
annotations:
|
||||||
|
kubernetes.io/ingress.class: "alb"
|
||||||
|
alb.ingress.kubernetes.io/group.name: unbound
|
||||||
|
alb.ingress.kubernetes.io/scheme: internet-facing
|
||||||
|
alb.ingress.kubernetes.io/target-type: instance
|
||||||
|
alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80},{"HTTPS": 443}]'
|
||||||
|
alb.ingress.kubernetes.io/ssl-redirect: "443"
|
||||||
|
alb.ingress.kubernetes.io/healthcheck-path: /health
|
||||||
|
spec:
|
||||||
|
rules:
|
||||||
|
- host: "upload.unbound.se"
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- path: /
|
||||||
|
pathType: Prefix
|
||||||
|
backend:
|
||||||
|
service:
|
||||||
|
name: s3uploader
|
||||||
|
port:
|
||||||
|
number: 80
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha1"
|
||||||
|
"fmt"
|
||||||
|
"math/big"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/apex/log"
|
||||||
|
|
||||||
|
"gitlab.com/unboundsoftware/s3uploader/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Server struct {
|
||||||
|
*http.ServeMux
|
||||||
|
store storage.Storage
|
||||||
|
returnUrl string
|
||||||
|
logger log.Interface
|
||||||
|
now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) HandlePut(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.Method {
|
||||||
|
case http.MethodOptions:
|
||||||
|
s.cors(w)
|
||||||
|
case http.MethodPut:
|
||||||
|
path := strings.TrimPrefix(req.URL.Path, "/put")
|
||||||
|
s.logger.Infof("uploading to %s", path)
|
||||||
|
s.upload(path, w, req)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
_, _ = w.Write([]byte("This endpoint requires PUT"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) HandleUpload(w http.ResponseWriter, req *http.Request) {
|
||||||
|
switch req.Method {
|
||||||
|
case http.MethodOptions:
|
||||||
|
s.cors(w)
|
||||||
|
case http.MethodPut:
|
||||||
|
prefix, err := randomString(64)
|
||||||
|
if err != nil {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = w.Write([]byte("An error occurred"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := s.now()
|
||||||
|
day := now.Format("20060102")
|
||||||
|
prefixHash := hash(prefix)
|
||||||
|
dayHash := hash(now.Format(time.RFC3339Nano))
|
||||||
|
s.logger.Infof("uploading to %s/%s%s", day, prefixHash, dayHash)
|
||||||
|
s.upload(fmt.Sprintf("%s/%s%s", day, prefixHash, dayHash), w, req)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
_, _ = w.Write([]byte("This endpoint requires PUT"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) HandleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = w.Write([]byte("OK"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) upload(path string, w http.ResponseWriter, req *http.Request) {
|
||||||
|
err := s.store.Store(path, req.Body)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.WithError(err).Error("error storing object in bucket")
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
_, _ = w.Write([]byte("error storing object in bucket"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Add("Access-Control-Expose-Headers", "X-File-URL")
|
||||||
|
fileUrl := filepath.Join(s.returnUrl, path)
|
||||||
|
w.Header().Add("X-File-URL", fileUrl)
|
||||||
|
s.logger.Infof("success - file is available at %s", fileUrl)
|
||||||
|
_, _ = w.Write([]byte("success"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) cors(w http.ResponseWriter) {
|
||||||
|
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||||
|
w.Header().Set("Access-Control-Allow-Methods", "PUT,OPTIONS")
|
||||||
|
w.Header().Set("Access-Control-Allow-Headers", "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With")
|
||||||
|
w.Header().Set("Access-Control-Max-Age", "1728000")
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(store storage.Storage, url string, logger log.Interface) http.Handler {
|
||||||
|
mux := &Server{
|
||||||
|
ServeMux: http.NewServeMux(),
|
||||||
|
store: store,
|
||||||
|
returnUrl: url,
|
||||||
|
logger: logger,
|
||||||
|
now: time.Now,
|
||||||
|
}
|
||||||
|
mux.HandleFunc("/put/", mux.HandlePut)
|
||||||
|
mux.HandleFunc("/upload", mux.HandleUpload)
|
||||||
|
mux.HandleFunc("/health", mux.HandleHealth)
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
func hash(s string) string {
|
||||||
|
hasher := sha1.New()
|
||||||
|
hasher.Write([]byte(s))
|
||||||
|
return fmt.Sprintf("%x", hasher.Sum(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func randomString(n int) (string, error) {
|
||||||
|
const letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||||
|
ret := make([]byte, n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letters))))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
ret[i] = letters[num.Int64()]
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(ret), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
mocks "gitlab.com/unboundsoftware/apex-mocks"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestServer(t *testing.T) {
|
||||||
|
type args struct {
|
||||||
|
store func(t *testing.T) StoreFunc
|
||||||
|
url string
|
||||||
|
random io.Reader
|
||||||
|
}
|
||||||
|
type req struct {
|
||||||
|
method string
|
||||||
|
path string
|
||||||
|
body string
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args args
|
||||||
|
req req
|
||||||
|
wantStatus int
|
||||||
|
wantResp string
|
||||||
|
wantHeaders map[string]string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "unhandled path",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/missing",
|
||||||
|
body: "xyz",
|
||||||
|
},
|
||||||
|
wantStatus: 404,
|
||||||
|
wantResp: "404 page not found\n",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "health",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/health",
|
||||||
|
body: "",
|
||||||
|
},
|
||||||
|
wantStatus: 200,
|
||||||
|
wantResp: "OK",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GET on /put",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodGet,
|
||||||
|
path: "/put/some/file",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 400,
|
||||||
|
wantResp: "This endpoint requires PUT",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "OPTIONS on /put",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodOptions,
|
||||||
|
path: "/put/some/file",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 204,
|
||||||
|
wantResp: "",
|
||||||
|
wantHeaders: map[string]string{
|
||||||
|
"Access-Control-Allow-Origin": "*",
|
||||||
|
"Access-Control-Allow-Methods": "PUT,OPTIONS",
|
||||||
|
"Access-Control-Allow-Headers": "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With",
|
||||||
|
"Access-Control-Max-Age": "1728000",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PUT on /put - error",
|
||||||
|
args: args{
|
||||||
|
store: func(t *testing.T) StoreFunc {
|
||||||
|
return func(path string, content io.Reader) error {
|
||||||
|
assert.Equal(t, "/some/file", path)
|
||||||
|
temp, err := ioutil.ReadAll(content)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "abc", string(temp))
|
||||||
|
return fmt.Errorf("error")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/put/some/file",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 500,
|
||||||
|
wantResp: "error storing object in bucket",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PUT on /put - success",
|
||||||
|
args: args{
|
||||||
|
store: func(t *testing.T) StoreFunc {
|
||||||
|
return func(path string, content io.Reader) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
},
|
||||||
|
url: "https://example.org",
|
||||||
|
},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/put/some/file",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 200,
|
||||||
|
wantResp: "success",
|
||||||
|
wantHeaders: map[string]string{
|
||||||
|
"Access-Control-Expose-Headers": "X-File-URL",
|
||||||
|
"X-File-URL": "https:/example.org/some/file",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GET on /upload",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodGet,
|
||||||
|
path: "/upload",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 400,
|
||||||
|
wantResp: "This endpoint requires PUT",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "OPTIONS on /upload",
|
||||||
|
args: args{},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodOptions,
|
||||||
|
path: "/upload",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 204,
|
||||||
|
wantResp: "",
|
||||||
|
wantHeaders: map[string]string{
|
||||||
|
"Access-Control-Allow-Origin": "*",
|
||||||
|
"Access-Control-Allow-Methods": "PUT,OPTIONS",
|
||||||
|
"Access-Control-Allow-Headers": "Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With",
|
||||||
|
"Access-Control-Max-Age": "1728000",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PUT on /upload - error with random",
|
||||||
|
args: args{
|
||||||
|
random: strings.NewReader("too short"),
|
||||||
|
},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/upload",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 500,
|
||||||
|
wantResp: "An error occurred",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PUT on /upload - error",
|
||||||
|
args: args{
|
||||||
|
store: func(t *testing.T) StoreFunc {
|
||||||
|
return func(path string, content io.Reader) error {
|
||||||
|
assert.Equal(t, fmt.Sprintf("%s/588b41ebf261820104615b83201c729bd16016d6e43649b28b0ef77d54ca5aaf8da0ce74ae3f20a4", time.Now().Format("20060102")), path)
|
||||||
|
temp, err := ioutil.ReadAll(content)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "abc", string(temp))
|
||||||
|
return fmt.Errorf("error")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/upload",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 500,
|
||||||
|
wantResp: "error storing object in bucket",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PUT on /upload - success",
|
||||||
|
args: args{
|
||||||
|
store: func(t *testing.T) StoreFunc {
|
||||||
|
return func(path string, content io.Reader) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
},
|
||||||
|
url: "https://example.org",
|
||||||
|
},
|
||||||
|
req: req{
|
||||||
|
method: http.MethodPut,
|
||||||
|
path: "/upload",
|
||||||
|
body: "abc",
|
||||||
|
},
|
||||||
|
wantStatus: 200,
|
||||||
|
wantResp: "success",
|
||||||
|
wantHeaders: map[string]string{
|
||||||
|
"Access-Control-Expose-Headers": "X-File-URL",
|
||||||
|
"X-File-URL": "https:/example.org/20211125/588b41ebf261820104615b83201c729bd16016d6e43649b28b0ef77d54ca5aaf8da0ce74ae3f20a4",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if tt.args.random != nil {
|
||||||
|
rand.Reader = tt.args.random
|
||||||
|
} else {
|
||||||
|
rand.Reader = strings.NewReader("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
|
||||||
|
}
|
||||||
|
logger := mocks.New()
|
||||||
|
var store StoreFunc
|
||||||
|
if tt.args.store != nil {
|
||||||
|
store = tt.args.store(t)
|
||||||
|
}
|
||||||
|
server := New(store, tt.args.url, logger.Logger)
|
||||||
|
server.(*Server).now = func() time.Time {
|
||||||
|
return time.Date(2021, 11, 25, 7, 43, 12, 0, time.UTC)
|
||||||
|
}
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
u, err := url.Parse(fmt.Sprintf("https://example.org%s", tt.req.path))
|
||||||
|
require.NoError(t, err)
|
||||||
|
server.ServeHTTP(recorder, &http.Request{
|
||||||
|
Method: tt.req.method,
|
||||||
|
URL: u,
|
||||||
|
Body: io.NopCloser(strings.NewReader(tt.req.body)),
|
||||||
|
})
|
||||||
|
assert.Equal(t, tt.wantStatus, recorder.Code, "StatusCode")
|
||||||
|
assert.Equal(t, tt.wantResp, recorder.Body.String(), "Body")
|
||||||
|
for k, v := range tt.wantHeaders {
|
||||||
|
assert.Equal(t, v, recorder.Header().Get(k), "Header")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type StoreFunc func(path string, content io.Reader) error
|
||||||
|
|
||||||
|
func (f StoreFunc) Store(path string, content io.Reader) error {
|
||||||
|
return f(path, content)
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/aws/session"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Storage interface {
|
||||||
|
Store(path string, content io.Reader) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type S3 struct {
|
||||||
|
bucket string
|
||||||
|
svc s3manageriface.UploaderAPI
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3) Store(path string, content io.Reader) error {
|
||||||
|
_, err := s.svc.Upload(&s3manager.UploadInput{
|
||||||
|
Bucket: aws.String(s.bucket),
|
||||||
|
Key: aws.String(path),
|
||||||
|
Body: content,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(bucket string) (Storage, error) {
|
||||||
|
sess, err := session.NewSession(aws.NewConfig())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
|
||||||
|
u.PartSize = 5 * 1024 * 1024
|
||||||
|
})
|
||||||
|
return &S3{bucket: bucket, svc: uploader}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||||
|
"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNew(t *testing.T) {
|
||||||
|
type args struct {
|
||||||
|
bucket string
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args args
|
||||||
|
setup func() func()
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "invalid AWS config",
|
||||||
|
args: args{},
|
||||||
|
setup: func() func() {
|
||||||
|
_ = os.Setenv("AWS_STS_REGIONAL_ENDPOINTS", "unknown_value")
|
||||||
|
return func() {
|
||||||
|
_ = os.Unsetenv("AWS_STS_REGIONAL_ENDPOINTS")
|
||||||
|
}
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "success",
|
||||||
|
args: args{},
|
||||||
|
setup: func() func() {
|
||||||
|
return func() {
|
||||||
|
}
|
||||||
|
},
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
defer tt.setup()()
|
||||||
|
_, err := New(tt.args.bucket)
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Errorf("New() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestS3_Store(t *testing.T) {
|
||||||
|
type fields struct {
|
||||||
|
bucket string
|
||||||
|
svc func(t *testing.T) s3manageriface.UploaderAPI
|
||||||
|
}
|
||||||
|
type args struct {
|
||||||
|
path string
|
||||||
|
content io.Reader
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fields fields
|
||||||
|
args args
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "upload error",
|
||||||
|
fields: fields{
|
||||||
|
bucket: "some-bucket",
|
||||||
|
svc: func(t *testing.T) s3manageriface.UploaderAPI {
|
||||||
|
return &mock{
|
||||||
|
upload: func(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
|
||||||
|
assert.Equal(t, aws.String("some-bucket"), input.Bucket)
|
||||||
|
assert.Equal(t, aws.String("/some/path"), input.Key)
|
||||||
|
buff, err := ioutil.ReadAll(input.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, "some content", string(buff))
|
||||||
|
return nil, fmt.Errorf("error")
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
args: args{
|
||||||
|
path: "/some/path",
|
||||||
|
content: strings.NewReader("some content"),
|
||||||
|
},
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
var svc s3manageriface.UploaderAPI
|
||||||
|
if tt.fields.svc != nil {
|
||||||
|
svc = tt.fields.svc(t)
|
||||||
|
}
|
||||||
|
s := &S3{
|
||||||
|
bucket: tt.fields.bucket,
|
||||||
|
svc: svc,
|
||||||
|
}
|
||||||
|
if err := s.Store(tt.args.path, tt.args.content); (err != nil) != tt.wantErr {
|
||||||
|
t.Errorf("Store() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type mock struct {
|
||||||
|
upload func(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mock) Upload(input *s3manager.UploadInput, f ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
|
||||||
|
if m.upload != nil {
|
||||||
|
return m.upload(input, f...)
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mock) UploadWithContext(aws.Context, *s3manager.UploadInput, ...func(*s3manager.Uploader)) (*s3manager.UploadOutput, error) {
|
||||||
|
panic("implement me")
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ s3manageriface.UploaderAPI = &mock{}
|
||||||
Reference in New Issue
Block a user