feat: add put-endpoint

chore: update to latest build-tools

See merge request unboundsoftware/nginx-s3-upload!1

.gitlab-ci.yml

@@ -5,12 +5,12 @@ stages:
 variables:
   DOCKER_HOST: tcp://docker:2375/
 
-image: registry.gitlab.com/sparetimecoders/build-tools:master
+image: buildtool/build-tools:0.0.23
 
 build:
   stage: build
   services:
-    - docker:18.06-dind
+    - docker:19.03-dind
   script:
   - build
   - push

Dockerfile

@@ -20,6 +20,9 @@ RUN wget https://github.com/simpl/ngx_devel_kit/archive/v0.3.0.tar.gz -O ngx_dev
     wget https://github.com/openresty/lua-resty-core/archive/v0.1.17.tar.gz -O lua-resty-core.tar.gz && \
     mkdir lua-resty-core && \
     tar xf lua-resty-core.tar.gz -C lua-resty-core --strip-components=1 && \
+    wget https://github.com/ledgetech/lua-resty-http/archive/v0.14.tar.gz -O lua-resty-http.tar.gz && \
+    mkdir lua-resty-http && \
+    tar xf lua-resty-http.tar.gz -C lua-resty-http --strip-components=1 && \
     wget https://github.com/openresty/lua-nginx-module/archive/v0.10.15.tar.gz -O lua-nginx-module.tar.gz && \
     mkdir lua-nginx-module && \
     tar xf lua-nginx-module.tar.gz -C lua-nginx-module --strip-components=1 && \
@@ -56,7 +59,7 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log \
 
 # Apply Nginx config
 ADD nginx.conf /etc/nginx/nginx.conf
-ADD sign.lua hmac.lua sha2.lua /tmp/lua/
+ADD fetcher.lua hmac.lua JSON.lua sha2.lua sign.lua /tmp/lua/
 ADD start.sh /start.sh
 
 # Set default command

README.MD

@@ -8,12 +8,14 @@ Environment variables are used to provide configuration.
   | Variable name  | Comment |
   |----------------|---------|
   | S3_BUCKET_NAME | Mandatory |
+  | AWS_REGION | Mandatory |
+  | RETURN_URL | Mandatory |
   | AWS_ACCESS_KEY_ID | Optional - will be fetched from IAM policy on AWS |
   | AWS_SECRET_ACCESS_KEY | Optional - will be fetched from IAM policy on AWS |
 
 
 ## Testing
-    docker run --rm --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --env S3_BUCKET_NAME=upload.paidit.se -p 8000:80 875131241629.dkr.ecr.eu-west-1.amazonaws.com/nginx-s3-upload
+    docker run --rm --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --env S3_BUCKET_NAME=upload.example.com --env AWS_REGION=eu-west-1 --env RETURN_URL=images.example.com -p 8000:80 gitlab.com/unboundsoftware/nginx-s3-upload:1.1.0
 
 Try it out:
 
@@ -21,7 +23,7 @@ Try it out:
     ...
     > PUT /upload HTTP/1.1
      ...
-    < X-File-URL: https://uploads.paidit.se/59389abb021973a41d05e0d7d79949b17b83b142058a3704c98f274b6e563cfc403e64db6550f233
+    < X-File-URL: https://images.example.com/59389abb021973a41d05e0d7d79949b17b83b142058a3704c98f274b6e563cfc403e64db6550f233
     ...
 
 The header `X-File-URL` will contain the URL to the uploaded file.

fetcher.lua

@@ -0,0 +1,34 @@
+local fetcher = {}
+
+function fetcher.fetch()
+  local key = os.getenv("AWS_ACCESS_KEY_ID")
+  local secret = os.getenv("AWS_SECRET_ACCESS_KEY")
+  if (key and secret) then
+    ngx.log(ngx.STDERR, "Key: " .. key .. ", Secret: " .. secret)
+    return key, secret, nil
+  end
+
+  local http = require "resty.http"
+  local JSON = require "JSON"
+  local httpc = http.new()
+  local res, err = httpc:request_uri("http://169.254.169.254/latest/meta-data/iam/security-credentials/", { method = "GET" })
+  if not res then
+    ngx.log(ngx.STDERR, "failed to get IAM role: ", err)
+    return
+  end
+
+  local role = res.body
+  res, err = httpc:request_uri("http://169.254.169.254/latest/meta-data/iam/security-credentials/" .. role, { method = "GET" })
+  if not res then
+    ngx.log(ngx.STDERR, "failed to get role info: ", err)
+    return
+  end
+  local json = res.body
+  local table = JSON:decode(json)
+  key = table["AccessKeyId"]
+  secret = table["SecretAccessKey"]
+  local token = table["Token"]
+  return key, secret, token
+end
+
+return fetcher

k8s/deploy.yaml

@@ -3,7 +3,7 @@ kind: Deployment
 metadata:
   labels:
     app: nginx-s3-upload
-  name: nginx-s3-upload-deployment
+  name: nginx-s3-upload
   annotations:
     kubernetes.io/change-cause: "${TIMESTAMP} Deployed commit id: ${COMMIT}"
 spec:
@@ -76,13 +76,14 @@ spec:
 
 ---
 
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: nginx-s3-upload
   annotations:
     kubernetes.io/ingress.class: "nginx"
-    nginx.ingress.kubernetes.io/proxy-body-size: 1g
+    nginx.ingress.kubernetes.io/enable-cors: "true"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
 spec:
   rules:
   - host: 'upload.unbound.se'

nginx.conf

@@ -12,22 +12,15 @@ events {
 
 http {
   lua_load_resty_core off;
-  lua_package_path "/tmp/lua-resty-core/lib/?.lua;/tmp/lua/?.lua;;";
+  lua_package_path "/tmp/lua-resty-core/lib/?.lua;/tmp/lua-resty-http/lib/?.lua;/tmp/lua/?.lua;;";
   lua_package_cpath '/usr/lib/x86_64-linux-gnu/lua/5.1/?.so;;';
 
-  proxy_max_temp_file_size 0;
-  proxy_buffering off;
-  server_names_hash_bucket_size 256;
-  client_body_buffer_size 128k;
-  proxy_buffer_size 32k;
-  proxy_buffers 4 32k;
-
-  lua_need_request_body on;
   lua_socket_buffer_size 128k;
+  client_max_body_size 10m;
+  client_body_buffer_size 10m;
 
   server {
     listen 80;
-    client_max_body_size 0;
 
     location /healthcheck {
       add_header Content-Type text/plain;
@@ -35,7 +28,7 @@ http {
       access_log off;
     }
 
-    location ~* ^/upload {
+    location ~ ^/upload {
       if ($request_method = 'OPTIONS') {
         # Tell client that this pre-flight info is valid for 20 days
         add_header 'Access-Control-Allow-Origin' "*" ;
@@ -67,24 +60,28 @@ http {
       set $url https://$phost$ppath;
       set $returnurl https://$baseurl$ppath;
       set $acl public-read;
-      set $contentSha256 "";
+      set $contentSha256 "UNSIGNED-PAYLOAD";
       set $authorization "";
+      set $token "";
 
       access_by_lua_block {
         local sha2 = require("sha2")
-        ngx.req.read_body()
-        local body = ngx.req.get_body_data()
-        local contentSha256 = sha2.hash256(body)
-        ngx.var.contentSha256 = contentSha256
+        local fetcher = require("fetcher")
         local sign = require("sign")
+        local key, secret, token = fetcher.fetch()
         local headers = {["x-amz-acl"] = ngx.var.acl, ["x-amz-date"] = ngx.var.timestamp, ["x-amz-content-sha256"] = ngx.var.contentSha256, ["date"] = ngx.var.date, ["host"] = ngx.var.phost }
-        ngx.var.authorization = sign.sign(os.getenv("AWS_ACCESS_KEY_ID"), os.getenv("AWS_SECRET_ACCESS_KEY"), os.time(), ngx.var.ppath, headers, ngx.var.region)
+        if token then
+          ngx.var.token = token
+          headers["x-amz-security-token"] = token
+        end
+        ngx.var.authorization = sign.sign(key, secret, os.time(), ngx.var.ppath, headers, ngx.var.region)
       }
 
       proxy_set_header date $date;
       proxy_set_header host $phost;
       proxy_set_header x-amz-acl $acl;
       proxy_set_header x-amz-date $timestamp;
+      proxy_set_header x-amz-security-token $token;
       proxy_set_header x-amz-content-sha256 $contentSha256;
       proxy_set_header Authorization $authorization;
       proxy_hide_header x-amz-id-2;
@@ -95,7 +92,67 @@ http {
       resolver_timeout 10s;
       add_header 'Access-Control-Expose-Headers' 'X-File-Url';
 
-      add_header X-debug-message $authorization always;
+      proxy_pass $url;
+    }
+
+    location ~ ^/put/(.+)$ {
+      if ($request_method = 'OPTIONS') {
+        # Tell client that this pre-flight info is valid for 20 days
+        add_header 'Access-Control-Allow-Origin' "*" ;
+        add_header 'Access-Control-Allow-Credentials' 'true' ;
+        add_header 'Access-Control-Allow-Methods' 'GET, PUT, OPTIONS' ;
+        add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With';
+        add_header 'Access-Control-Max-Age' 1728000;
+        add_header 'Content-Type' 'text/plain charset=UTF-8';
+        add_header 'Content-Length' 0;
+        return 204;
+      }
+      if ($request_method != PUT) {
+        return 404;
+      }
+
+      set_by_lua $time "os.time()";
+      set_by_lua $timestamp "return os.date('%Y%m%dT%H%M%SZ', tonumber(ngx.var.time))";
+      set_by_lua $date "return os.date('%a, %d %b %Y %H:%M:%S GMT', tonumber(ngx.var.time))";
+      set_by_lua $bucket "return os.getenv('S3_BUCKET_NAME')";
+      set_by_lua $baseurl "return os.getenv('RETURN_URL')";
+      set_by_lua $region "return os.getenv('AWS_REGION')";
+      set $phost $bucket.s3-$region.amazonaws.com;
+      set $ppath /$1;
+      set $url https://$phost$ppath;
+      set $returnurl https://$baseurl$ppath;
+      set $acl public-read;
+      set $contentSha256 "UNSIGNED-PAYLOAD";
+      set $authorization "";
+      set $token "";
+
+      access_by_lua_block {
+        local sha2 = require("sha2")
+        local fetcher = require("fetcher")
+        local sign = require("sign")
+        local key, secret, token = fetcher.fetch()
+        local headers = {["x-amz-acl"] = ngx.var.acl, ["x-amz-date"] = ngx.var.timestamp, ["x-amz-content-sha256"] = ngx.var.contentSha256, ["date"] = ngx.var.date, ["host"] = ngx.var.phost }
+        if token then
+          ngx.var.token = token
+          headers["x-amz-security-token"] = token
+        end
+        ngx.var.authorization = sign.sign(key, secret, os.time(), ngx.var.ppath, headers, ngx.var.region)
+      }
+
+      proxy_set_header date $date;
+      proxy_set_header host $phost;
+      proxy_set_header x-amz-acl $acl;
+      proxy_set_header x-amz-date $timestamp;
+      proxy_set_header x-amz-security-token $token;
+      proxy_set_header x-amz-content-sha256 $contentSha256;
+      proxy_set_header Authorization $authorization;
+      proxy_hide_header x-amz-id-2;
+      proxy_hide_header x-amz-request-id;
+      add_header X-File-URL $returnurl;
+
+      resolver 8.8.8.8 valid=300s;
+      resolver_timeout 10s;
+      add_header 'Access-Control-Expose-Headers' 'X-File-Url';
 
       proxy_pass $url;
     }

start.sh

@@ -2,19 +2,9 @@
 
 set +u
 
-if [[ -z ${AWS_SECRET_ACCESS_KEY} ]]
-then
-  IAM_ROLE=$(curl http://169.254.169.254/latest/meta-data/iam/security-credentials/)
-  JSON=$(curl http://169.254.169.254/latest/meta-data/iam/security-credentials/${IAM_ROLE})
-  export AWS_ACCESS_KEY_ID=$(echo ${JSON} | jq -r '.AccessKeyId')
-  export AWS_SECRET_ACCESS_KEY=$(echo ${JSON} | jq -r '.SecretAccessKey')
-fi
-
 set -euo pipefail
 : ${S3_BUCKET_NAME:?"S3_BUCKET_NAME must be set"}
 : ${AWS_REGION:?"AWS_REGION must be set"}
 : ${RETURN_URL:?"RETURN_URL must be set"}
-: ${AWS_ACCESS_KEY_ID:?"AWS_ACCESS_KEY_ID must be set or be possible to fetch from meta-data service on AWS"}
-: ${AWS_SECRET_ACCESS_KEY:?"AWS_ACCESS_KEY_ID must be set or be possible to fetch from meta-data service on AWS"}
 
 exec nginx -g 'daemon off;'