commit d12b497a28fc3c0fb9dc8817311f3ba706ef5d80 Author: Joakim Olsson Date: Tue Nov 4 10:19:06 2025 +0100 feat: add storage module with S3 support and development tooling Create shared storage module for AWS S3 operations with comprehensive development infrastructure: Core Features: - S3 interface with two upload patterns (manager and direct) - Presigned URL generation with 15-minute expiration - Support for multipart uploads and direct PutObject - Comprehensive test coverage (8 tests, 70.4% coverage) - Generic implementation without project-specific dependencies Development Tooling: - .editorconfig for consistent editor settings - .pre-commit-config.yaml with Go linters and formatters - .golangci.yml for golangci-lint configuration - commitlint.config.js for conventional commit validation - cliff.toml for automated changelog generation (v0.0.1) - renovate.json for automated dependency updates - .gitlab-ci.yml for CI/CD pipeline CI/CD Pipeline: - Automated testing with race detection - Coverage tracking and Codecov integration - Vulnerability scanning with govulncheck - Pre-commit validation gates - Release automation Module exports: - New(bucket) - Upload manager pattern for large files - NewS3(cfg, bucket) - Direct upload pattern - Store(path, content, contentType) - Upload and get presigned URL ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..04cd3ad --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +[*.go] +indent_style = tab +indent_size = 2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0163820 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +coverage.* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b75ac99 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,38 @@ +include: +- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' +- project: unboundsoftware/ci-templates + file: Defaults.gitlab-ci.yml +- project: unboundsoftware/ci-templates + file: Release.gitlab-ci.yml +- project: unboundsoftware/ci-templates + file: Pre-Commit-Go.gitlab-ci.yml + +image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d + +stages: +- deps +- test + +deps: + stage: deps + script: + - go mod download + +test: + stage: test + dependencies: + - deps + script: + - CGO_ENABLED=1 go test -mod=readonly -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(go list ./... | tr '\n' , | sed 's/,$//') ./... + - go tool cover -html=coverage.txt -o coverage.html + - go tool cover -func=coverage.txt + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov -t ${CODECOV_TOKEN} -R $CI_PROJECT_DIR -C $CI_COMMIT_SHA -r $CI_PROJECT_PATH + +vulnerabilities: + stage: test + image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d + script: + - go install golang.org/x/vuln/cmd/govulncheck@latest + - govulncheck ./... diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..5381cc5 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,22 @@ +version: "2" +run: + allow-parallel-runners: true +linters: + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3def4dd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: + - --allow-multiple-documents + - id: check-added-large-files +- repo: https://gitlab.com/devopshq/gitlab-ci-linter + rev: v1.0.6 + hooks: + - id: gitlab-ci-linter + args: + - --project + - unboundsoftware/storage +- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.23.0 + hooks: + - id: commitlint + stages: [ commit-msg ] + additional_dependencies: [ '@commitlint/config-conventional' ] +- repo: https://github.com/dnephin/pre-commit-golang + rev: v0.5.1 + hooks: + - id: go-mod-tidy + - id: go-imports + args: + - -local + - gitlab.com/unboundsoftware/storage +- repo: https://github.com/lietu/go-pre-commit + rev: v1.0.0 + hooks: + - id: go-test + - id: gofumpt +- repo: https://github.com/golangci/golangci-lint + rev: v2.5.0 + hooks: + - id: golangci-lint-full +- repo: https://github.com/gitleaks/gitleaks + rev: v8.28.0 + hooks: + - id: gitleaks diff --git a/README.md b/README.md new file mode 100644 index 0000000..e4affef --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# Storage Module + +Shared storage utilities for AWS S3. + +## Features + +- S3 object storage with presigned URL generation +- Two upload strategies: managed uploads (for large files) and direct uploads +- Configurable part size for multipart uploads +- 15-minute presigned URL expiration + +## Usage + +### Using the Upload Manager (recommended for large files) + +```go +import "gitlab.com/unboundsoftware/storage" + +// Create storage with automatic AWS config loading +s3Storage, err := storage.New("my-bucket") +if err != nil { + // handle error +} + +// Upload a file and get a presigned URL +url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf") +``` + +### Using Direct Upload (for smaller files or custom config) + +```go +import ( + "github.com/aws/aws-sdk-go-v2/config" + "gitlab.com/unboundsoftware/storage" +) + +// Load custom AWS config +cfg, err := config.LoadDefaultConfig(context.Background()) +if err != nil { + // handle error +} + +// Create storage with custom config +s3Storage := storage.NewS3(cfg, "my-bucket") + +// Upload a file and get a presigned URL +url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf") +``` + +## Configuration + +The storage module uses AWS SDK v2 and loads configuration from: +- Environment variables (`AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) +- Shared configuration files (`~/.aws/config`, `~/.aws/credentials`) +- IAM roles (when running on AWS infrastructure) diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..ce3d6f4 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,84 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# template for the changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL +] +# render body even when there are no releases to process +# render_always = true +# output file path +# output = "test.md" + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "๐Ÿš€ Features" }, + { message = "^fix", group = "๐Ÿ› Bug Fixes" }, + { message = "^doc", group = "๐Ÿ“š Documentation" }, + { message = "^perf", group = "โšก Performance" }, + { message = "^refactor", group = "๐Ÿšœ Refactor" }, + { message = "^style", group = "๐ŸŽจ Styling" }, + { message = "^test", group = "๐Ÿงช Testing" }, + { message = "^chore\\(release\\): prepare for", skip = true }, + { message = "^chore|^ci", group = "โš™๏ธ Miscellaneous Tasks" }, + { body = ".*security", group = "๐Ÿ›ก๏ธ Security" }, + { message = "^revert", group = "โ—€๏ธ Revert" }, +] +# filter out the commits that are not matched by commit parsers +filter_commits = false +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" + +[bump] +# initial tag to use when no tags exist +initial_tag = "v0.0.1" diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..c34aa79 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,3 @@ +module.exports = { + extends: ['@commitlint/config-conventional'] +}; diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..382b73b --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module gitlab.com/unboundsoftware/storage + +go 1.23 + +require ( + github.com/aws/aws-sdk-go-v2 v1.39.5 + github.com/aws/aws-sdk-go-v2/config v1.31.16 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect + github.com/aws/smithy-go v1.23.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..6627bff --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= +github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko= +github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc= +github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..eb57524 --- /dev/null +++ b/s3.go @@ -0,0 +1,125 @@ +package storage + +import ( + "context" + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Uploader is the interface for uploading objects to S3 using the upload manager +type Uploader interface { + Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +// DirectUploader is the interface for uploading objects directly to S3 +type DirectUploader interface { + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +// Presigner is the interface for generating presigned URLs +type Presigner interface { + PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +// S3 provides storage operations for AWS S3 +type S3 struct { + bucket string + svc Uploader + directSvc DirectUploader + presigner Presigner + useDirectUpload bool +} + +// Store uploads content to S3 and returns a presigned URL valid for 15 minutes +func (s *S3) Store(path string, content io.Reader, contentType string) (string, error) { + if s.useDirectUpload { + return s.storeWithDirectUpload(path, content, contentType) + } + return s.storeWithManager(path, content, contentType) +} + +func (s *S3) storeWithManager(path string, content io.Reader, contentType string) (string, error) { + out, err := s.svc.Upload(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + Body: content, + }) + if err != nil { + return "", err + } + presignedUrl, err := s.presigner.PresignGetObject(context.Background(), + &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(*out.Key), + ResponseContentType: aws.String(contentType), + }, + s3.WithPresignExpires(time.Minute*15), + ) + if err != nil { + return "", err + } + + return presignedUrl.URL, nil +} + +func (s *S3) storeWithDirectUpload(path string, content io.Reader, contentType string) (string, error) { + // Upload file to S3 + _, err := s.directSvc.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + Body: content, + ContentType: aws.String(contentType), + }) + if err != nil { + return "", err + } + + // Generate presigned URL valid for 15 minutes + req, err := s.presigner.PresignGetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + }, s3.WithPresignExpires(15*time.Minute)) + if err != nil { + return "", err + } + + return req.URL, nil +} + +// New creates a new S3 storage instance using the upload manager +// This loads AWS config from the default locations and is suitable for most use cases +func New(bucket string) (*S3, error) { + cfg, err := config.LoadDefaultConfig(context.Background()) + if err != nil { + return nil, err + } + client := s3.NewFromConfig(cfg) + uploader := manager.NewUploader(client, func(u *manager.Uploader) { + u.PartSize = 5 * 1024 * 1024 + }) + presignClient := s3.NewPresignClient(client) + return &S3{ + bucket: bucket, + svc: uploader, + presigner: presignClient, + useDirectUpload: false, + }, nil +} + +// NewS3 creates a new S3 storage instance using direct PutObject +// This is useful when you want more control over the AWS configuration +func NewS3(cfg aws.Config, bucket string) *S3 { + client := s3.NewFromConfig(cfg) + return &S3{ + bucket: bucket, + directSvc: client, + presigner: s3.NewPresignClient(client), + useDirectUpload: true, + } +} diff --git a/s3_test.go b/s3_test.go new file mode 100644 index 0000000..3337f78 --- /dev/null +++ b/s3_test.go @@ -0,0 +1,429 @@ +package storage + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Mock implementations for testing + +type mockUploader struct { + uploadFunc func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +func (m *mockUploader) Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return m.uploadFunc(ctx, input, opts...) +} + +type mockDirectUploader struct { + putObjectFunc func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +func (m *mockDirectUploader) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return m.putObjectFunc(ctx, params, optFns...) +} + +type mockPresigner struct { + presignFunc func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) +} + +func (m *mockPresigner) PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return m.presignFunc(ctx, params, optFns...) +} + +// Test NewS3 constructor + +func TestNewS3(t *testing.T) { + cfg := aws.Config{ + Region: "us-east-1", + } + bucket := "test-bucket" + + s3Instance := NewS3(cfg, bucket) + + if s3Instance == nil { + t.Fatal("Expected S3 instance, got nil") + } + + if s3Instance.bucket != bucket { + t.Errorf("Expected bucket %s, got %s", bucket, s3Instance.bucket) + } + + if !s3Instance.useDirectUpload { + t.Error("Expected useDirectUpload to be true for NewS3") + } + + if s3Instance.directSvc == nil { + t.Error("Expected directSvc to be set") + } + + if s3Instance.presigner == nil { + t.Error("Expected presigner to be set") + } +} + +// Test Store with upload manager pattern + +func TestStore_WithUploadManager_Success(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true" + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + // Verify input parameters + if *input.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *input.Bucket) + } + if *input.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *input.Key) + } + + // Read and verify body + body, err := io.ReadAll(input.Body) + if err != nil { + t.Errorf("Failed to read body: %v", err) + } + if string(body) != testContent { + t.Errorf("Expected content %s, got %s", testContent, string(body)) + } + + return &manager.UploadOutput{ + Key: aws.String(testPath), + }, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Verify presign parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + if *params.ResponseContentType != testContentType { + t.Errorf("Expected content type %s, got %s", testContentType, *params.ResponseContentType) + } + + return &v4.PresignedHTTPRequest{ + URL: expectedURL, + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if url != expectedURL { + t.Errorf("Expected URL %s, got %s", expectedURL, url) + } +} + +func TestStore_WithUploadManager_UploadError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("upload failed") + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return nil, expectedError + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + t.Error("Presigner should not be called when upload fails") + return nil, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +func TestStore_WithUploadManager_PresignError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("presign failed") + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return &manager.UploadOutput{ + Key: aws.String(testPath), + }, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, expectedError + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +// Test Store with direct upload pattern + +func TestStore_WithDirectUpload_Success(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true" + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + // Verify input parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + if *params.ContentType != testContentType { + t.Errorf("Expected content type %s, got %s", testContentType, *params.ContentType) + } + + // Read and verify body + body, err := io.ReadAll(params.Body) + if err != nil { + t.Errorf("Failed to read body: %v", err) + } + if string(body) != testContent { + t.Errorf("Expected content %s, got %s", testContent, string(body)) + } + + return &s3.PutObjectOutput{}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Verify presign parameters + if *params.Bucket != testBucket { + t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket) + } + if *params.Key != testPath { + t.Errorf("Expected key %s, got %s", testPath, *params.Key) + } + + // Verify 15 minute expiry was requested + // Note: We can't directly verify the duration in the options, but we can ensure it's called + return &v4.PresignedHTTPRequest{ + URL: expectedURL, + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + if url != expectedURL { + t.Errorf("Expected URL %s, got %s", expectedURL, url) + } +} + +func TestStore_WithDirectUpload_PutObjectError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("put object failed") + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return nil, expectedError + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + t.Error("Presigner should not be called when PutObject fails") + return nil, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +func TestStore_WithDirectUpload_PresignError(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + expectedError := errors.New("presign failed") + + mockDirectUploader := &mockDirectUploader{ + putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return &s3.PutObjectOutput{}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + return nil, expectedError + }, + } + + s3Instance := &S3{ + bucket: testBucket, + directSvc: mockDirectUploader, + presigner: mockPresigner, + useDirectUpload: true, + } + + url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + + if err == nil { + t.Fatal("Expected error, got nil") + } + + if err != expectedError { + t.Errorf("Expected error %v, got %v", expectedError, err) + } + + if url != "" { + t.Errorf("Expected empty URL, got %s", url) + } +} + +// Test that presign expiry is set correctly + +func TestStore_PresignExpiry(t *testing.T) { + testBucket := "test-bucket" + testPath := "path/to/file.pdf" + testContent := "test content" + testContentType := "application/pdf" + + var capturedExpiry time.Duration + + mockUploader := &mockUploader{ + uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) { + return &manager.UploadOutput{Key: aws.String(testPath)}, nil + }, + } + + mockPresigner := &mockPresigner{ + presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) { + // Apply options to capture the expiry + opts := &s3.PresignOptions{} + for _, fn := range optFns { + fn(opts) + } + capturedExpiry = opts.Expires + + return &v4.PresignedHTTPRequest{ + URL: "https://example.com/presigned", + }, nil + }, + } + + s3Instance := &S3{ + bucket: testBucket, + svc: mockUploader, + presigner: mockPresigner, + useDirectUpload: false, + } + + _, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + expectedExpiry := 15 * time.Minute + if capturedExpiry != expectedExpiry { + t.Errorf("Expected presign expiry of %v, got %v", expectedExpiry, capturedExpiry) + } +}