feat: add storage module with S3 support and development tooling
Create shared storage module for AWS S3 operations with comprehensive development infrastructure: Core Features: - S3 interface with two upload patterns (manager and direct) - Presigned URL generation with 15-minute expiration - Support for multipart uploads and direct PutObject - Comprehensive test coverage (8 tests, 70.4% coverage) - Generic implementation without project-specific dependencies Development Tooling: - .editorconfig for consistent editor settings - .pre-commit-config.yaml with Go linters and formatters - .golangci.yml for golangci-lint configuration - commitlint.config.js for conventional commit validation - cliff.toml for automated changelog generation (v0.0.1) - renovate.json for automated dependency updates - .gitlab-ci.yml for CI/CD pipeline CI/CD Pipeline: - Automated testing with race detection - Coverage tracking and Codecov integration - Vulnerability scanning with govulncheck - Pre-commit validation gates - Release automation Module exports: - New(bucket) - Upload manager pattern for large files - NewS3(cfg, bucket) - Direct upload pattern - Store(path, content, contentType) - Upload and get presigned URL 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
indent_size = 2
|
||||
@@ -0,0 +1 @@
|
||||
coverage.*
|
||||
@@ -0,0 +1,38 @@
|
||||
include:
|
||||
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
|
||||
- project: unboundsoftware/ci-templates
|
||||
file: Defaults.gitlab-ci.yml
|
||||
- project: unboundsoftware/ci-templates
|
||||
file: Release.gitlab-ci.yml
|
||||
- project: unboundsoftware/ci-templates
|
||||
file: Pre-Commit-Go.gitlab-ci.yml
|
||||
|
||||
image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d
|
||||
|
||||
stages:
|
||||
- deps
|
||||
- test
|
||||
|
||||
deps:
|
||||
stage: deps
|
||||
script:
|
||||
- go mod download
|
||||
|
||||
test:
|
||||
stage: test
|
||||
dependencies:
|
||||
- deps
|
||||
script:
|
||||
- CGO_ENABLED=1 go test -mod=readonly -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(go list ./... | tr '\n' , | sed 's/,$//') ./...
|
||||
- go tool cover -html=coverage.txt -o coverage.html
|
||||
- go tool cover -func=coverage.txt
|
||||
- curl -Os https://uploader.codecov.io/latest/linux/codecov
|
||||
- chmod +x codecov
|
||||
- ./codecov -t ${CODECOV_TOKEN} -R $CI_PROJECT_DIR -C $CI_COMMIT_SHA -r $CI_PROJECT_PATH
|
||||
|
||||
vulnerabilities:
|
||||
stage: test
|
||||
image: amd64/golang:1.25.3@sha256:69d10098be2e990bb1d987daec0e36d18ad287e139450dc7d98a0ded3498888d
|
||||
script:
|
||||
- go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
- govulncheck ./...
|
||||
@@ -0,0 +1,22 @@
|
||||
version: "2"
|
||||
run:
|
||||
allow-parallel-runners: true
|
||||
linters:
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
formatters:
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
@@ -0,0 +1,46 @@
|
||||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
args:
|
||||
- --allow-multiple-documents
|
||||
- id: check-added-large-files
|
||||
- repo: https://gitlab.com/devopshq/gitlab-ci-linter
|
||||
rev: v1.0.6
|
||||
hooks:
|
||||
- id: gitlab-ci-linter
|
||||
args:
|
||||
- --project
|
||||
- unboundsoftware/storage
|
||||
- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
|
||||
rev: v9.23.0
|
||||
hooks:
|
||||
- id: commitlint
|
||||
stages: [ commit-msg ]
|
||||
additional_dependencies: [ '@commitlint/config-conventional' ]
|
||||
- repo: https://github.com/dnephin/pre-commit-golang
|
||||
rev: v0.5.1
|
||||
hooks:
|
||||
- id: go-mod-tidy
|
||||
- id: go-imports
|
||||
args:
|
||||
- -local
|
||||
- gitlab.com/unboundsoftware/storage
|
||||
- repo: https://github.com/lietu/go-pre-commit
|
||||
rev: v1.0.0
|
||||
hooks:
|
||||
- id: go-test
|
||||
- id: gofumpt
|
||||
- repo: https://github.com/golangci/golangci-lint
|
||||
rev: v2.5.0
|
||||
hooks:
|
||||
- id: golangci-lint-full
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.28.0
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
@@ -0,0 +1,55 @@
|
||||
# Storage Module
|
||||
|
||||
Shared storage utilities for AWS S3.
|
||||
|
||||
## Features
|
||||
|
||||
- S3 object storage with presigned URL generation
|
||||
- Two upload strategies: managed uploads (for large files) and direct uploads
|
||||
- Configurable part size for multipart uploads
|
||||
- 15-minute presigned URL expiration
|
||||
|
||||
## Usage
|
||||
|
||||
### Using the Upload Manager (recommended for large files)
|
||||
|
||||
```go
|
||||
import "gitlab.com/unboundsoftware/storage"
|
||||
|
||||
// Create storage with automatic AWS config loading
|
||||
s3Storage, err := storage.New("my-bucket")
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
// Upload a file and get a presigned URL
|
||||
url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf")
|
||||
```
|
||||
|
||||
### Using Direct Upload (for smaller files or custom config)
|
||||
|
||||
```go
|
||||
import (
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"gitlab.com/unboundsoftware/storage"
|
||||
)
|
||||
|
||||
// Load custom AWS config
|
||||
cfg, err := config.LoadDefaultConfig(context.Background())
|
||||
if err != nil {
|
||||
// handle error
|
||||
}
|
||||
|
||||
// Create storage with custom config
|
||||
s3Storage := storage.NewS3(cfg, "my-bucket")
|
||||
|
||||
// Upload a file and get a presigned URL
|
||||
url, err := s3Storage.Store("path/to/file.pdf", fileReader, "application/pdf")
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The storage module uses AWS SDK v2 and loads configuration from:
|
||||
- Environment variables (`AWS_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
|
||||
- Shared configuration files (`~/.aws/config`, `~/.aws/credentials`)
|
||||
- IAM roles (when running on AWS infrastructure)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
# git-cliff ~ default configuration file
|
||||
# https://git-cliff.org/docs/configuration
|
||||
#
|
||||
# Lines starting with "#" are comments.
|
||||
# Configuration options are organized into tables and keys.
|
||||
# See documentation for more information on available options.
|
||||
|
||||
[changelog]
|
||||
# template for the changelog header
|
||||
header = """
|
||||
# Changelog\n
|
||||
All notable changes to this project will be documented in this file.\n
|
||||
"""
|
||||
# template for the changelog body
|
||||
# https://keats.github.io/tera/docs/#introduction
|
||||
body = """
|
||||
{% if version %}\
|
||||
## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
|
||||
{% else %}\
|
||||
## [unreleased]
|
||||
{% endif %}\
|
||||
{% for group, commits in commits | group_by(attribute="group") %}
|
||||
### {{ group | striptags | trim | upper_first }}
|
||||
{% for commit in commits %}
|
||||
- {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
|
||||
{% if commit.breaking %}[**breaking**] {% endif %}\
|
||||
{{ commit.message | upper_first }}\
|
||||
{% endfor %}
|
||||
{% endfor %}\n
|
||||
"""
|
||||
# template for the changelog footer
|
||||
footer = """
|
||||
<!-- generated by git-cliff -->
|
||||
"""
|
||||
# remove the leading and trailing s
|
||||
trim = true
|
||||
# postprocessors
|
||||
postprocessors = [
|
||||
# { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
|
||||
]
|
||||
# render body even when there are no releases to process
|
||||
# render_always = true
|
||||
# output file path
|
||||
# output = "test.md"
|
||||
|
||||
[git]
|
||||
# parse the commits based on https://www.conventionalcommits.org
|
||||
conventional_commits = true
|
||||
# filter out the commits that are not conventional
|
||||
filter_unconventional = true
|
||||
# process each line of a commit as an individual commit
|
||||
split_commits = false
|
||||
# regex for preprocessing the commit messages
|
||||
commit_preprocessors = [
|
||||
# Replace issue numbers
|
||||
#{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
|
||||
# Check spelling of the commit with https://github.com/crate-ci/typos
|
||||
# If the spelling is incorrect, it will be automatically fixed.
|
||||
#{ pattern = '.*', replace_command = 'typos --write-changes -' },
|
||||
]
|
||||
# regex for parsing and grouping commits
|
||||
commit_parsers = [
|
||||
{ message = "^feat", group = "<!-- 0 -->🚀 Features" },
|
||||
{ message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
|
||||
{ message = "^doc", group = "<!-- 3 -->📚 Documentation" },
|
||||
{ message = "^perf", group = "<!-- 4 -->⚡ Performance" },
|
||||
{ message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
|
||||
{ message = "^style", group = "<!-- 5 -->🎨 Styling" },
|
||||
{ message = "^test", group = "<!-- 6 -->🧪 Testing" },
|
||||
{ message = "^chore\\(release\\): prepare for", skip = true },
|
||||
{ message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
|
||||
{ body = ".*security", group = "<!-- 8 -->🛡️ Security" },
|
||||
{ message = "^revert", group = "<!-- 9 -->◀️ Revert" },
|
||||
]
|
||||
# filter out the commits that are not matched by commit parsers
|
||||
filter_commits = false
|
||||
# sort the tags topologically
|
||||
topo_order = false
|
||||
# sort the commits inside sections by oldest/newest order
|
||||
sort_commits = "oldest"
|
||||
|
||||
[bump]
|
||||
# initial tag to use when no tags exist
|
||||
initial_tag = "v0.0.1"
|
||||
@@ -0,0 +1,3 @@
|
||||
module.exports = {
|
||||
extends: ['@commitlint/config-conventional']
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
module gitlab.com/unboundsoftware/storage
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.39.5
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.23.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,38 @@
|
||||
github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w=
|
||||
github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M=
|
||||
github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:recommended"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// Uploader is the interface for uploading objects to S3 using the upload manager
|
||||
type Uploader interface {
|
||||
Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error)
|
||||
}
|
||||
|
||||
// DirectUploader is the interface for uploading objects directly to S3
|
||||
type DirectUploader interface {
|
||||
PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
}
|
||||
|
||||
// Presigner is the interface for generating presigned URLs
|
||||
type Presigner interface {
|
||||
PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error)
|
||||
}
|
||||
|
||||
// S3 provides storage operations for AWS S3
|
||||
type S3 struct {
|
||||
bucket string
|
||||
svc Uploader
|
||||
directSvc DirectUploader
|
||||
presigner Presigner
|
||||
useDirectUpload bool
|
||||
}
|
||||
|
||||
// Store uploads content to S3 and returns a presigned URL valid for 15 minutes
|
||||
func (s *S3) Store(path string, content io.Reader, contentType string) (string, error) {
|
||||
if s.useDirectUpload {
|
||||
return s.storeWithDirectUpload(path, content, contentType)
|
||||
}
|
||||
return s.storeWithManager(path, content, contentType)
|
||||
}
|
||||
|
||||
func (s *S3) storeWithManager(path string, content io.Reader, contentType string) (string, error) {
|
||||
out, err := s.svc.Upload(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(path),
|
||||
Body: content,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
presignedUrl, err := s.presigner.PresignGetObject(context.Background(),
|
||||
&s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(*out.Key),
|
||||
ResponseContentType: aws.String(contentType),
|
||||
},
|
||||
s3.WithPresignExpires(time.Minute*15),
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return presignedUrl.URL, nil
|
||||
}
|
||||
|
||||
func (s *S3) storeWithDirectUpload(path string, content io.Reader, contentType string) (string, error) {
|
||||
// Upload file to S3
|
||||
_, err := s.directSvc.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(path),
|
||||
Body: content,
|
||||
ContentType: aws.String(contentType),
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Generate presigned URL valid for 15 minutes
|
||||
req, err := s.presigner.PresignGetObject(context.Background(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(s.bucket),
|
||||
Key: aws.String(path),
|
||||
}, s3.WithPresignExpires(15*time.Minute))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return req.URL, nil
|
||||
}
|
||||
|
||||
// New creates a new S3 storage instance using the upload manager
|
||||
// This loads AWS config from the default locations and is suitable for most use cases
|
||||
func New(bucket string) (*S3, error) {
|
||||
cfg, err := config.LoadDefaultConfig(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client := s3.NewFromConfig(cfg)
|
||||
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
|
||||
u.PartSize = 5 * 1024 * 1024
|
||||
})
|
||||
presignClient := s3.NewPresignClient(client)
|
||||
return &S3{
|
||||
bucket: bucket,
|
||||
svc: uploader,
|
||||
presigner: presignClient,
|
||||
useDirectUpload: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewS3 creates a new S3 storage instance using direct PutObject
|
||||
// This is useful when you want more control over the AWS configuration
|
||||
func NewS3(cfg aws.Config, bucket string) *S3 {
|
||||
client := s3.NewFromConfig(cfg)
|
||||
return &S3{
|
||||
bucket: bucket,
|
||||
directSvc: client,
|
||||
presigner: s3.NewPresignClient(client),
|
||||
useDirectUpload: true,
|
||||
}
|
||||
}
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
// Mock implementations for testing
|
||||
|
||||
type mockUploader struct {
|
||||
uploadFunc func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error)
|
||||
}
|
||||
|
||||
func (m *mockUploader) Upload(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
|
||||
return m.uploadFunc(ctx, input, opts...)
|
||||
}
|
||||
|
||||
type mockDirectUploader struct {
|
||||
putObjectFunc func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
|
||||
}
|
||||
|
||||
func (m *mockDirectUploader) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
return m.putObjectFunc(ctx, params, optFns...)
|
||||
}
|
||||
|
||||
type mockPresigner struct {
|
||||
presignFunc func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error)
|
||||
}
|
||||
|
||||
func (m *mockPresigner) PresignGetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
return m.presignFunc(ctx, params, optFns...)
|
||||
}
|
||||
|
||||
// Test NewS3 constructor
|
||||
|
||||
func TestNewS3(t *testing.T) {
|
||||
cfg := aws.Config{
|
||||
Region: "us-east-1",
|
||||
}
|
||||
bucket := "test-bucket"
|
||||
|
||||
s3Instance := NewS3(cfg, bucket)
|
||||
|
||||
if s3Instance == nil {
|
||||
t.Fatal("Expected S3 instance, got nil")
|
||||
}
|
||||
|
||||
if s3Instance.bucket != bucket {
|
||||
t.Errorf("Expected bucket %s, got %s", bucket, s3Instance.bucket)
|
||||
}
|
||||
|
||||
if !s3Instance.useDirectUpload {
|
||||
t.Error("Expected useDirectUpload to be true for NewS3")
|
||||
}
|
||||
|
||||
if s3Instance.directSvc == nil {
|
||||
t.Error("Expected directSvc to be set")
|
||||
}
|
||||
|
||||
if s3Instance.presigner == nil {
|
||||
t.Error("Expected presigner to be set")
|
||||
}
|
||||
}
|
||||
|
||||
// Test Store with upload manager pattern
|
||||
|
||||
func TestStore_WithUploadManager_Success(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true"
|
||||
|
||||
mockUploader := &mockUploader{
|
||||
uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
|
||||
// Verify input parameters
|
||||
if *input.Bucket != testBucket {
|
||||
t.Errorf("Expected bucket %s, got %s", testBucket, *input.Bucket)
|
||||
}
|
||||
if *input.Key != testPath {
|
||||
t.Errorf("Expected key %s, got %s", testPath, *input.Key)
|
||||
}
|
||||
|
||||
// Read and verify body
|
||||
body, err := io.ReadAll(input.Body)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to read body: %v", err)
|
||||
}
|
||||
if string(body) != testContent {
|
||||
t.Errorf("Expected content %s, got %s", testContent, string(body))
|
||||
}
|
||||
|
||||
return &manager.UploadOutput{
|
||||
Key: aws.String(testPath),
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
// Verify presign parameters
|
||||
if *params.Bucket != testBucket {
|
||||
t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket)
|
||||
}
|
||||
if *params.Key != testPath {
|
||||
t.Errorf("Expected key %s, got %s", testPath, *params.Key)
|
||||
}
|
||||
if *params.ResponseContentType != testContentType {
|
||||
t.Errorf("Expected content type %s, got %s", testContentType, *params.ResponseContentType)
|
||||
}
|
||||
|
||||
return &v4.PresignedHTTPRequest{
|
||||
URL: expectedURL,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
svc: mockUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: false,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if url != expectedURL {
|
||||
t.Errorf("Expected URL %s, got %s", expectedURL, url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_WithUploadManager_UploadError(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedError := errors.New("upload failed")
|
||||
|
||||
mockUploader := &mockUploader{
|
||||
uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
|
||||
return nil, expectedError
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
t.Error("Presigner should not be called when upload fails")
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
svc: mockUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: false,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Expected error, got nil")
|
||||
}
|
||||
|
||||
if err != expectedError {
|
||||
t.Errorf("Expected error %v, got %v", expectedError, err)
|
||||
}
|
||||
|
||||
if url != "" {
|
||||
t.Errorf("Expected empty URL, got %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_WithUploadManager_PresignError(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedError := errors.New("presign failed")
|
||||
|
||||
mockUploader := &mockUploader{
|
||||
uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
|
||||
return &manager.UploadOutput{
|
||||
Key: aws.String(testPath),
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
return nil, expectedError
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
svc: mockUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: false,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Expected error, got nil")
|
||||
}
|
||||
|
||||
if err != expectedError {
|
||||
t.Errorf("Expected error %v, got %v", expectedError, err)
|
||||
}
|
||||
|
||||
if url != "" {
|
||||
t.Errorf("Expected empty URL, got %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
// Test Store with direct upload pattern
|
||||
|
||||
func TestStore_WithDirectUpload_Success(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedURL := "https://s3.amazonaws.com/test-bucket/path/to/file.pdf?presigned=true"
|
||||
|
||||
mockDirectUploader := &mockDirectUploader{
|
||||
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
// Verify input parameters
|
||||
if *params.Bucket != testBucket {
|
||||
t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket)
|
||||
}
|
||||
if *params.Key != testPath {
|
||||
t.Errorf("Expected key %s, got %s", testPath, *params.Key)
|
||||
}
|
||||
if *params.ContentType != testContentType {
|
||||
t.Errorf("Expected content type %s, got %s", testContentType, *params.ContentType)
|
||||
}
|
||||
|
||||
// Read and verify body
|
||||
body, err := io.ReadAll(params.Body)
|
||||
if err != nil {
|
||||
t.Errorf("Failed to read body: %v", err)
|
||||
}
|
||||
if string(body) != testContent {
|
||||
t.Errorf("Expected content %s, got %s", testContent, string(body))
|
||||
}
|
||||
|
||||
return &s3.PutObjectOutput{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
// Verify presign parameters
|
||||
if *params.Bucket != testBucket {
|
||||
t.Errorf("Expected bucket %s, got %s", testBucket, *params.Bucket)
|
||||
}
|
||||
if *params.Key != testPath {
|
||||
t.Errorf("Expected key %s, got %s", testPath, *params.Key)
|
||||
}
|
||||
|
||||
// Verify 15 minute expiry was requested
|
||||
// Note: We can't directly verify the duration in the options, but we can ensure it's called
|
||||
return &v4.PresignedHTTPRequest{
|
||||
URL: expectedURL,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
directSvc: mockDirectUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: true,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if url != expectedURL {
|
||||
t.Errorf("Expected URL %s, got %s", expectedURL, url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_WithDirectUpload_PutObjectError(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedError := errors.New("put object failed")
|
||||
|
||||
mockDirectUploader := &mockDirectUploader{
|
||||
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
return nil, expectedError
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
t.Error("Presigner should not be called when PutObject fails")
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
directSvc: mockDirectUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: true,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Expected error, got nil")
|
||||
}
|
||||
|
||||
if err != expectedError {
|
||||
t.Errorf("Expected error %v, got %v", expectedError, err)
|
||||
}
|
||||
|
||||
if url != "" {
|
||||
t.Errorf("Expected empty URL, got %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore_WithDirectUpload_PresignError(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
expectedError := errors.New("presign failed")
|
||||
|
||||
mockDirectUploader := &mockDirectUploader{
|
||||
putObjectFunc: func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
|
||||
return &s3.PutObjectOutput{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
return nil, expectedError
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
directSvc: mockDirectUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: true,
|
||||
}
|
||||
|
||||
url, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("Expected error, got nil")
|
||||
}
|
||||
|
||||
if err != expectedError {
|
||||
t.Errorf("Expected error %v, got %v", expectedError, err)
|
||||
}
|
||||
|
||||
if url != "" {
|
||||
t.Errorf("Expected empty URL, got %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that presign expiry is set correctly
|
||||
|
||||
func TestStore_PresignExpiry(t *testing.T) {
|
||||
testBucket := "test-bucket"
|
||||
testPath := "path/to/file.pdf"
|
||||
testContent := "test content"
|
||||
testContentType := "application/pdf"
|
||||
|
||||
var capturedExpiry time.Duration
|
||||
|
||||
mockUploader := &mockUploader{
|
||||
uploadFunc: func(ctx context.Context, input *s3.PutObjectInput, opts ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
|
||||
return &manager.UploadOutput{Key: aws.String(testPath)}, nil
|
||||
},
|
||||
}
|
||||
|
||||
mockPresigner := &mockPresigner{
|
||||
presignFunc: func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
|
||||
// Apply options to capture the expiry
|
||||
opts := &s3.PresignOptions{}
|
||||
for _, fn := range optFns {
|
||||
fn(opts)
|
||||
}
|
||||
capturedExpiry = opts.Expires
|
||||
|
||||
return &v4.PresignedHTTPRequest{
|
||||
URL: "https://example.com/presigned",
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
s3Instance := &S3{
|
||||
bucket: testBucket,
|
||||
svc: mockUploader,
|
||||
presigner: mockPresigner,
|
||||
useDirectUpload: false,
|
||||
}
|
||||
|
||||
_, err := s3Instance.Store(testPath, strings.NewReader(testContent), testContentType)
|
||||
if err != nil {
|
||||
t.Fatalf("Expected no error, got %v", err)
|
||||
}
|
||||
|
||||
expectedExpiry := 15 * time.Minute
|
||||
if capturedExpiry != expectedExpiry {
|
||||
t.Errorf("Expected presign expiry of %v, got %v", expectedExpiry, capturedExpiry)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user