RPJosh
/
docker-registry-proxy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'debug'

pull/7/head
Ricardo Pardini 2018-11-04 16:55:05 +01:00
parent 80bb415d84 7724f3ba15
commit 895627a073
No known key found for this signature in database
GPG Key ID: 3D38CA12A66C5D02
4 changed files with 127 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Dockerfile
View File

@ -1,10 +1,18 @@
# We start from my nginx fork which includes the proxy-connect module from tEngine
# Source is available at https://github.com/rpardini/nginx-proxy-connect-stable-alpine
# Its equivalent to nginx:stable-alpine 1.14.0, with alpine 3.7
FROM rpardini/nginx-proxy-connect-stable-alpine:latest
FROM rpardini/nginx-proxy-connect-stable-alpine:nginx-1.14.0-alpine-3.8
# Add openssl, bash and ca-certificates, then clean apk cache -- yeah complain all you want.
RUN apk add --update openssl bash ca-certificates && rm -rf /var/cache/apk/*
# Also added deps for mitmproxy.
RUN apk add --update openssl bash ca-certificates su-exec git g++ libffi libffi-dev libstdc++ openssl openssl-dev python3 python3-dev
RUN LDFLAGS=-L/lib pip3 install mitmproxy
RUN apk del --purge git g++ libffi-dev openssl-dev python3-dev && rm -rf /var/cache/apk/* && rm -rf ~/.cache/pip
# Required for mitmproxy
ENV LANG=en_US.UTF-8
# Check the installed mitmproxy version
RUN mitmproxy --version
# Create the cache directory and CA directory
RUN mkdir -p /docker_mirror_cache /ca
@ -27,6 +35,9 @@ RUN chmod +x /create_ca_cert.sh /entrypoint.sh
# Clients should only use 3128, not anything else.
EXPOSE 3128
# In debug mode, 8081 exposes the mitmweb interface.
EXPOSE 8081
## Default envs.
# A space delimited list of registries we should proxy and cache; this is in addition to the central DockerHub.
ENV REGISTRIES="k8s.gcr.io gcr.io quay.io"
@ -34,6 +45,10 @@ ENV REGISTRIES="k8s.gcr.io gcr.io quay.io"
ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.registry:user:password"
# Should we verify upstream's certificates? Default to true.
ENV VERIFY_SSL="true"
# Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
ENV DEBUG="true"
# Enable nginx debugging mode; this uses nginx-debug binary and enabled debug logging, which is VERY verbose so separate setting
ENV DEBUG_NGINX="false"
# Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
# Did you want a shell? Sorry, the entrypoint never returns, because it runs nginx itself. Use 'docker exec' if you need to mess around internally.
ENTRYPOINT ["/entrypoint.sh"]

7
create_ca_cert.sh
View File

@ -34,7 +34,7 @@ if [ -f "$CA_CRT_FILE" ] ; then
else
logInfo "No CA was found. Generating one."
logInfo "*** Please *** make sure to mount /ca as a volume -- if not, everytime this container starts, it will regenerate the CA and nothing will work."
openssl genrsa -des3 -passout pass:foobar -out ${CA_KEY_FILE} 4096
logInfo "generate CA cert with key and self sign it: ${CAID}"
@ -52,7 +52,7 @@ EOF
[[ ${DEBUG} -gt 0 ]] && logInfo "show the CA cert details"
[[ ${DEBUG} -gt 0 ]] && openssl x509 -noout -text -in ${CA_CRT_FILE}
echo 01 > ${CA_SRL_FILE}
fi
@ -116,3 +116,6 @@ openssl x509 -req -days 365 -in web.csr -CA ia.crt -CAkey ia.key -out web.crt -p
logInfo "Concatenating fullchain.pem..."
cat web.crt ia.crt ${CA_CRT_FILE} > fullchain.pem
logInfo "Concatenating fullchain_with_key.pem"
cat fullchain.pem web.key > fullchain_with_key.pem

35
entrypoint.sh
View File

@ -36,7 +36,7 @@ for ONEREGISTRYIN in ${AUTH_REGISTRIES}; do
done
echo "" > /etc/nginx/docker.verify.ssl.conf
if [ "a$VERIFY_SSL" == "atrue" ]; then
if [[ "a${VERIFY_SSL}" == "atrue" ]]; then
cat << EOD > /etc/nginx/docker.verify.ssl.conf
# We actually wanna be secure and avoid mitm attacks.
# Fitting, since this whole thing is a mitm...
@ -46,11 +46,38 @@ if [ "a$VERIFY_SSL" == "atrue" ]; then
proxy_ssl_verify_depth 2;
EOD
echo "Upstream SSL certificate verification enabled."
fi
fi
# create default config for the caching layer to listen on 443.
echo " listen 443 ssl default_server;" > /etc/nginx/caching.layer.listen
echo "error_log /var/log/nginx/error.log warn;" > /etc/nginx/error.log.debug.warn
# normally use non-debug version of nginx
NGINX_BIN="nginx"
if [[ "a${DEBUG}" == "atrue" ]]; then
# in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
echo " listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
echo "Starting in DEBUG MODE (mitmproxy)."
echo "Run mitmproxy with reverse pointing to the same certs..."
mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
--set keep_host_header=true --set ssl_insecure=true \
--mode reverse:https://127.0.0.1:444 --listen-host 0.0.0.0 \
--listen-port 443 --certs /certs/fullchain_with_key.pem \
-w /ca/outfile &
echo "Access mitmweb via http://127.0.0.1:8081/ "
fi
if [[ "a${DEBUG_NGINX}" == "atrue" ]]; then
echo "Starting in DEBUG MODE (nginx)."
echo "error_log /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
# use debug binary
NGINX_BIN="nginx-debug"
fi
echo "Testing nginx config..."
nginx -t
${NGINX_BIN} -t
echo "Starting nginx! Have a nice day."
nginx -g "daemon off;"
${NGINX_BIN} -g "daemon off;"

136
nginx.conf
View File

@ -1,7 +1,9 @@
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log warn;
# error log config comes from external file created by entrypoint, to toggle debug on/off.
include /etc/nginx/error.log.debug.warn;
pid /var/run/nginx.pid;
events {
@ -13,7 +15,7 @@ http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
# Use a debug-oriented logging format.
# Use a debug-oriented logging format.
log_format debugging '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent '
'"HOST: $host" "UPSTREAM: $upstream_addr" '
@ -22,19 +24,28 @@ http {
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
'"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
'"AUTH: $http_authorization" ' ;
log_format debug_proxy 'CONNECTPROXY: $remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent '
'"HOST: $host" "UPSTREAM: $upstream_addr" '
'"UPSTREAM-STATUS: $upstream_status" '
'"SSL-PROTO: $ssl_protocol" '
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
'"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
'"AUTH: $http_authorization" ' ;
log_format tweaked '$upstream_cache_status [$time_local] "$uri" '
'$status $body_bytes_sent '
'"HOST:$host" '
'"PROXY-HOST:$proxy_host" "UPSTREAM:$upstream_addr" ';
keepalive_timeout 300;
gzip off;
# The cache directory. This can get huge. Better to use a Docker volume pointing here!
# Set to 32gb which should be enough
proxy_cache_path /docker_mirror_cache levels=1:2 max_size=32g inactive=60d keys_zone=cache:10m use_temp_path=off;
# Just in case you want to rewrite some hosts. Default maps directly.
map $host $targetHost {
hostnames;
@ -48,22 +59,27 @@ http {
include /etc/nginx/docker.auth.map;
default "";
}
# @TODO: actually for auth.docker.io, if we want to support multiple authentications, we'll need to decide
# @TODO: based not only on the hostname, but also URI (/token) and query string (?scope)
# @TODO: I wonder if this would help gcr.io and quay.io with authentication also....
map $dockerAuth $finalAuth {
"" "$http_authorization"; # if empty, keep the original passed-in from the docker client.
default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
}
# Map to decide which hosts get directed to the caching portion.
# This is automatically generated from the list of cached registries, plus a few fixed hosts
# By default, we don't intercept, allowing free flow of non-registry traffic
map $connect_host $interceptedHost {
hostnames;
include /etc/nginx/docker.intercept.map;
default "$connect_host:443";
default "$connect_addr"; # $connect_addr is 'IP address and port of the remote host, e.g. "192.168.1.5:12345". IP address is resolved from host name of CONNECT request line.'
}
map $dockerAuth $finalAuth {
"" "$http_authorization"; # if empty, keep the original passed-in from the client
default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
}
# These maps parse the original Host and URI from a /forcecache redirect.
map $request_uri $realHost {
~/forcecacheinsecure/([^:/]+)/originalwas(/.+) $1;
@ -76,43 +92,46 @@ http {
~/forcecachesecure/([^:/]+)/originalwas(/.+) $2;
default "DID_NOT_MATCH_PATH";
}
# The proxy director layer, listens on 3128
server {
listen 3128;
server_name _;
# dont log the CONNECT proxy.
#access_log /var/log/nginx/access.log debug_proxy;
access_log off;
proxy_connect;
proxy_connect_address $interceptedHost;
proxy_max_temp_file_size 0;
# We need to resolve the real names of our proxied servers.
resolver 8.8.8.8 4.2.2.2 ipv6=off; # Avoid ipv6 addresses for now
# forward proxy for non-CONNECT request
location / {
return 403 "The docker caching proxy is working!";
add_header "Content-type" "text/plain" always;
return 200 "docker-registry-proxy: The docker caching proxy is working!";
}
location /ca.crt {
alias /ca/ca.crt;
alias /ca/ca.crt;
}
# @TODO: add a dynamic root path that generates instructions for usage on docker clients
}
# The caching layer
server {
# Listen on both 80 and 443, for all hostnames.
# actually could be 443 or 444, depending on debug. this is now generated by the entrypoint.
listen 80 default_server;
listen 443 ssl default_server;
include /etc/nginx/caching.layer.listen;
server_name _;
# Do some tweaked logging.
access_log /var/log/nginx/access.log tweaked;
@ -136,17 +155,17 @@ http {
if ($request_method = DELETE) {
return 405 "DELETE method is not allowed";
}
proxy_read_timeout 900;
# Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
# Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
# will wait for the first to finish instead of doing multiple upstream requests.
proxy_cache_lock on;
proxy_cache_lock_timeout 120s;
proxy_cache_lock_timeout 880s;
# Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
proxy_cache_valid 200 301 302 307 60d;
proxy_cache_valid 200 301 302 307 60d;
# Some extra settings to maximize cache hits and efficiency
proxy_force_ranges on;
proxy_ignore_client_abort on;
@ -155,50 +174,25 @@ http {
# Hide/ignore headers from caching. S3 especially likes to send Expires headers in the past in some situations.
proxy_hide_header Set-Cookie;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control Set-Cookie;
# Add the authentication info, if the map matched the target domain.
proxy_set_header Authorization $finalAuth;
# This comes from a include file generated by the entrypoint.
include /etc/nginx/docker.verify.ssl.conf;
# Some debugging info
# add_header X-Docker-Caching-Proxy-Real-Host $realHost;
# add_header X-Docker-Caching-Proxy-Real-Path $realPath;
# add_header X-Docker-Caching-Proxy-Auth $finalAuth;
# Block API v1. We dont know how to handle these.
# Docker-client should start with v2 and fallback to v1 if something fails, for example, if authentication failed to a protected v2 resource.
location /v1 {
return 405 "API v1 is invalid. Either the image does not exist upstream, or you need auth to get a v2 endpoint working against $host";
return 405 "docker-registry-proxy: docker is trying to use v1 API. Either the image does not exist upstream, or you need to configure docker-registry-proxy to authenticate against $host";
}
# don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
proxy_pass https://$targetHost;
}
# don't cache mutable entity /v2/<name>/tags/list
location ~ ^/v2/[^\/]+/tags/list {
proxy_pass https://$targetHost;
}
# don't cache mutable entity /v2/_catalog
location ~ ^/v2/_catalog$ {
proxy_pass https://$targetHost;
}
# force cache of the first hit which is always /v2/ - even for 401 unauthorized.
location = /v2/ {
proxy_pass https://$targetHost;
proxy_cache cache;
proxy_cache_valid 200 301 302 307 401 60d;
}
# cache everything else
location / {
# for the /v2/..../blobs/.... URIs, do cache, and treat redirects.
location ~ ^/v2/(.*)/blobs/ {
proxy_pass https://$targetHost;
proxy_cache cache;
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:blobs";
# Handling of redirects.
# Many registries (eg, quay.io, or k8s.gcr.io) emit a Location redirect
@ -210,14 +204,18 @@ http {
proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
}
# handling for the redirect case explained above, with https.
# The $realHost and $realPath variables come from a map defined at the top of this file.
location /forcecachesecure {
proxy_pass https://$realHost$realPath;
proxy_cache cache;
# Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
proxy_cache_key $proxy_host$uri;
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecachesecure";
}
# handling for the redirect case explained above, with http.
@ -225,9 +223,19 @@ http {
location /forcecacheinsecure {
proxy_pass http://$realHost$realPath;
proxy_cache cache;
# Change the cache key, so that we can cache signed S3 requests and such. Only host and path are considered.
proxy_cache_key $proxy_host$uri;
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
}
# by default, dont cache anything.
location / {
proxy_pass https://$targetHost;
proxy_cache off;
add_header X-Docker-Caching-Proxy-Debug-Cache "no:default";
}
}
}