From d12b497a28fc3c0fb9dc8817311f3ba706ef5d80 Mon Sep 17 00:00:00 2001 From: Joakim Olsson Date: Tue, 4 Nov 2025 10:19:06 +0100 Subject: [PATCH] feat: add storage module with S3 support and development tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create shared storage module for AWS S3 operations with comprehensive development infrastructure: Core Features: - S3 interface with two upload patterns (manager and direct) - Presigned URL generation with 15-minute expiration - Support for multipart uploads and direct PutObject - Comprehensive test coverage (8 tests, 70.4% coverage) - Generic implementation without project-specific dependencies Development Tooling: - .editorconfig for consistent editor settings - .pre-commit-config.yaml with Go linters and formatters - .golangci.yml for golangci-lint configuration - commitlint.config.js for conventional commit validation - cliff.toml for automated changelog generation (v0.0.1) - renovate.json for automated dependency updates - .gitlab-ci.yml for CI/CD pipeline CI/CD Pipeline: - Automated testing with race detection - Coverage tracking and Codecov integration - Vulnerability scanning with govulncheck - Pre-commit validation gates - Release automation Module exports: - New(bucket) - Upload manager pattern for large files - NewS3(cfg, bucket) - Direct upload pattern - Store(path, content, contentType) - Upload and get presigned URL ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .editorconfig | 11 ++ .gitignore | 1 + .gitlab-ci.yml | 38 ++++ .golangci.yml | 22 +++ .pre-commit-config.yaml | 46 +++++ README.md | 55 ++++++ cliff.toml | 84 ++++++++ commitlint.config.js | 3 + go.mod | 28 +++ go.sum | 38 ++++ renovate.json | 6 + s3.go | 125 ++++++++++++ s3_test.go | 429 ++++++++++++++++++++++++++++++++++++++++ 13 files changed, 886 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .golangci.yml create mode 100644 .pre-commit-config.yaml create mode 100644 README.md create mode 100644 cliff.toml create mode 100644 commitlint.config.js create mode 100644 go.mod create mode 100644 go.sum create mode 100644 renovate.json create mode 100644 s3.go create mode 100644 s3_test.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..04cd3ad --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +[*.go] +indent_style = tab +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0163820 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +coverage.* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b75ac99 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,38 @@ +include: +- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' +- project: unboundsoftware/ci-templates + file: Defaults.gitlab-ci.yml +- project: unboundsoftware/ci-templates + file: Release.gitlab-ci.yml +- project: unboundsoftware/ci-templates + file: Pre-Commit-Go.gitlab-ci.yml + +image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d + +stages: +- deps +- test + +deps: + stage: deps + script: + - go mod download + +test: + stage: test + dependencies: + - deps + script: + - CGO_ENABLED=1 go test -mod=readonly -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(go list ./... | tr '\n' , | sed 's/,$//') ./... + - go tool cover -html=coverage.txt -o coverage.html + - go tool cover -func=coverage.txt + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov -t ${CODECOV_TOKEN} -R $CI_PROJECT_DIR -C $CI_COMMIT_SHA -r $CI_PROJECT_PATH + +vulnerabilities: + stage: test + image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d + script: + - go install golang.org/x/vuln/cmd/govulncheck@latest + - govulncheck ./... diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..5381cc5 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,22 @@ +version: "2" +run: + allow-parallel-runners: true +linters: + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3def4dd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: + - --allow-multiple-documents + - id: check-added-large-files +- repo: https://gitlab.com/devopshq/gitlab-ci-linter + rev: v1.0.6 + hooks: + - id: gitlab-ci-linter + args: + - --project + - unboundsoftware/storage +- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.23.0 + hooks: + - id: commitlint + stages: [ commit-msg ] + additional_dependencies: [ '@commitlint/config-conventional' ] +- repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: go-mod-tidy + - id: go-imports + args: + - -local + - gitlab.com/unboundsoftware/storage +- repo: https://github.com/lietu/go-pre-commit + rev: v1.0.0 + hooks: + - id: go-test + - id: gofumpt +- repo: https://github.com/golangci/golangci-lint + rev: v2.5.0 + hooks: + - id: golangci-lint-full +- repo: https://github.com/gitleaks/gitleaks + rev: v8.28.0 + hooks: + - id: gitleaks diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4affef --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Storage Module + +Shared storage utilities for AWS S3. + +## Features + +- S3 object storage with presigned URL generation +- Two upload strategies: managed uploads (for large files) and direct uploads +- Configurable part size for multipart uploads +- 15-minute presigned URL expiration + +## Usage + +### Using the Upload Manager (recommended for large files) + +```go +import "gitlab.com/unboundsoftware/storage" + +// Create storage with automatic AWS config loading +s3Storage, err := storage.New("my-bucket") +if err != nil { + // handle error +} + +// Upload a file and get a presigned URL +url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf") +``` + +### Using Direct Upload (for smaller files or custom config) + +```go +import ( + "github.com/aws/aws-sdk-go-v2/config" + "gitlab.com/unboundsoftware/storage" +) + +// Load custom AWS config +cfg, err := config.LoadDefaultConfig(context.Background()) +if err != nil { + // handle error +} + +// Create storage with custom config +s3Storage := storage.NewS3(cfg, "my-bucket") + +// Upload a file and get a presigned URL +url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf") +``` + +## Configuration + +The storage module uses AWS SDK v2 and loads configuration from: +- Environment variables (`AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) +- Shared configuration files (`~/.aws/config`, `~/.aws/credentials`) +- IAM roles (when running on AWS infrastructure) diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..ce3d6f4 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,84 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# template for the changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL +] +# render body even when there are no releases to process +# render_always = true +# output file path +# output = "test.md" + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "๐Ÿš€ Features" }, + { message = "^fix", group = "๐Ÿ› Bug Fixes" }, + { message = "^doc", group = "๐Ÿ“š Documentation" }, + { message = "^perf", group = "โšก Performance" }, + { message = "^refactor", group = "๐Ÿšœ Refactor" }, + { message = "^style", group = "๐ŸŽจ Styling" }, + { message = "^test", group = "๐Ÿงช Testing" }, + { message = "^chore\\(release\\): prepare for", skip = true }, + { message = "^chore|^ci", group = "โš™๏ธ Miscellaneous Tasks" }, + { body = ".*security", group = "๐Ÿ›ก๏ธ Security" }, + { message = "^revert", group = "โ—€๏ธ Revert" }, +] +# filter out the commits that are not matched by commit parsers +filter_commits = false +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" + +[bump] +# initial tag to use when no tags exist +initial_tag = "v0.0.1" diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..c34aa79 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,3 @@ +module.exports = { + extends: ['@commitlint/config-conventional'] +}; diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..382b73b --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module gitlab.com/unboundsoftware/storage + +go 1.23 + +require ( + github.com/aws/aws-sdk-go-v2 v1.39.5 + github.com/aws/aws-sdk-go-v2/config v1.31.16 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect + github.com/aws/smithy-go v1.23.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6627bff --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= +github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko= +github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc= +github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..eb57524 --- /dev/null +++ b/s3.go @@ -0,0 +1,125 @@ +package storage + +import ( + "context" + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Uploader is the interface for uploading objects to S3 using the upload manager +type Uploader interface { + Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +// DirectUploader is the interface for uploading objects directly to S3 +type DirectUploader interface { + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +// Presigner is the interface for generating presigned URLs +type Presigner interface { + PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +// S3 provides storage operations for AWS S3 +type S3 struct { + bucket string + svc Uploader + directSvc DirectUploader + presigner Presigner + useDirectUpload bool +} + +// Store uploads content to S3 and returns a presigned URL valid for 15 minutes +func (s *S3) Store(path string, content io.Reader, contentType string) (string, error) { + if s.useDirectUpload { + return s.storeWithDirectUpload(path, content, contentType) + } + return s.storeWithManager(path, content, contentType) +} + +func (s *S3) storeWithManager(path string, content io.Reader, contentType string) (string, error) { + out, err := s.svc.Upload(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + Body: content, + }) + if err != nil { + return "", err + } + presignedUrl, err := s.presigner.PresignGetObject(context.Background(), + &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(*out.Key), + ResponseContentType: aws.String(contentType), + }, + s3.WithPresignExpires(time.Minute*15), + ) + if err != nil { + return "", err + } + + return presignedUrl.URL, nil +} + +func (s *S3) storeWithDirectUpload(path string, content io.Reader, contentType string) (string, error) { + // Upload file to S3 + _, err := s.directSvc.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + Body: content, + ContentType: aws.String(contentType), + }) + if err != nil { + return "", err + } + + // Generate presigned URL valid for 15 minutes + req, err := s.presigner.PresignGetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + }, s3.WithPresignExpires(15*time.Minute)) + if err != nil { + return "", err + } + + return req.URL, nil +} + +// New creates a new S3 storage instance using the upload manager +// This loads AWS config from the default locations and is suitable for most use cases +func New(bucket string) (*S3, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, err + } + client := s3.NewFromConfig(cfg) + uploader := manager.NewUploader(client, func(u *manager.Uploader) { + u.PartSize = 5 * 1024 * 1024 + }) + presignClient := s3.NewPresignClient(client) + return &S3{ + bucket: bucket, + svc: uploader, + presigner: presignClient, + useDirectUpload: false, + }, nil +} + +// NewS3 creates a new S3 storage instance using direct PutObject +// This is useful when you want more control over the AWS configuration +func NewS3(cfg aws.Config, bucket string) *S3 { + client := s3.NewFromConfig(cfg) + return &S3{ + bucket: bucket, + directSvc: client, + presigner: s3.NewPresignClient(client), + useDirectUpload: true, + } +} diff --git a/s3_test.go b/s3_test.go new file mode 100644 index 0000000..3337f78 --- /dev/null +++ b/s3_test.go @@ -0,0 +1,429 @@ +package storage + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Mock implementations for testing + +type mockUploader struct { + uploadFunc func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +func (m *mockUploader) Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return m.uploadFunc(ctx, input, opts...) +} + +type mockDirectUploader struct { + putObjectFunc func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +func (m *mockDirectUploader) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return m.putObjectFunc(ctx, params, optFns...) +} + +type mockPresigner struct { + presignFunc func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +func (m *mockPresigner) PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return m.presignFunc(ctx, params, optFns...) +} + +// Test NewS3 constructor + +func TestNewS3(t *testing.T) { + cfg := aws.Config{ + Region: "us-east-1", + } + bucket := "test-bucket" + + s3Instance := NewS3(cfg, bucket) + + if s3Instance == nil { + t.Fatal("Expected S3 instance, got nil") + } + + if s3Instance.bucket != bucket { + t.Errorf("Expected bucket %s, got %s", bucket, s3Instance.bucket) + } + + if !s3Instance.useDirectUpload { + t.Error("Expected useDirectUpload to be true for NewS3") + } + + if s3Instance.directSvc == nil { + t.Error("Expected directSvc to be set") + } + + if s3Instance.presigner == nil { + t.Error("Expected presigner to be set") + } +} + +// Test Store with upload manager pattern + +func TestStore_WithUploadManager_Success(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true" + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + // Verify input parameters + if *input.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *input.Bucket) + } + if *input.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *input.Key) + } + + // Read and verify body + body, err := io.ReadAll(input.Body) + if err != nil { + t.Errorf("Failed to read body: %v", err) + } + if string(body) != testContent { + t.Errorf("Expected content %s, got %s", testContent, string(body)) + } + + return &manager.UploadOutput{ + Key: aws.String(testPath), + }, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Verify presign parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + if *params.ResponseContentType != testContentType { + t.Errorf("Expected content type %s, got %s", testContentType, *params.ResponseContentType) + } + + return &v4.PresignedHTTPRequest{ + URL: expectedURL, + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if url != expectedURL { + t.Errorf("Expected URL %s, got %s", expectedURL, url) + } +} + +func TestStore_WithUploadManager_UploadError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("upload failed") + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return nil, expectedError + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + t.Error("Presigner should not be called when upload fails") + return nil, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +func TestStore_WithUploadManager_PresignError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("presign failed") + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return &manager.UploadOutput{ + Key: aws.String(testPath), + }, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, expectedError + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +// Test Store with direct upload pattern + +func TestStore_WithDirectUpload_Success(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true" + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + // Verify input parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + if *params.ContentType != testContentType { + t.Errorf("Expected content type %s, got %s", testContentType, *params.ContentType) + } + + // Read and verify body + body, err := io.ReadAll(params.Body) + if err != nil { + t.Errorf("Failed to read body: %v", err) + } + if string(body) != testContent { + t.Errorf("Expected content %s, got %s", testContent, string(body)) + } + + return &s3.PutObjectOutput{}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Verify presign parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + + // Verify 15 minute expiry was requested + // Note: We can't directly verify the duration in the options, but we can ensure it's called + return &v4.PresignedHTTPRequest{ + URL: expectedURL, + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if url != expectedURL { + t.Errorf("Expected URL %s, got %s", expectedURL, url) + } +} + +func TestStore_WithDirectUpload_PutObjectError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("put object failed") + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return nil, expectedError + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + t.Error("Presigner should not be called when PutObject fails") + return nil, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +func TestStore_WithDirectUpload_PresignError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("presign failed") + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return &s3.PutObjectOutput{}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, expectedError + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +// Test that presign expiry is set correctly + +func TestStore_PresignExpiry(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + + var capturedExpiry time.Duration + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return &manager.UploadOutput{Key: aws.String(testPath)}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Apply options to capture the expiry + opts := &s3.PresignOptions{} + for _, fn := range optFns { + fn(opts) + } + capturedExpiry = opts.Expires + + return &v4.PresignedHTTPRequest{ + URL: "https://example.com/presigned", + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + _, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + expectedExpiry := 15 * time.Minute + if capturedExpiry != expectedExpiry { + t.Errorf("Expected presign expiry of %v, got %v", expectedExpiry, capturedExpiry) + } +}