completely reworked caching, now cache by exception (/blobs/ only essentially)
- now only /v2/.../blobs/... URIs are actually cached (together with their redirect catchers) - /manifests/, /token, and /v2/ are not cached anymore, which should solve a lot of problems - better messages for /v1 attempts - fix usage of $connect_host:443 (which is hostname:port and causes errors to be logged) to $connect_addr (which returns an IP:port) in the proxy layerpull/7/head
Ricardo Pardini
parent
8ff06e3dec
commit
7724f3ba15
|
|
@ -47,6 +47,8 @@ ENV AUTH_REGISTRIES="some.authenticated.registry:oneuser:onepassword another.reg
|
|||
ENV VERIFY_SSL="true"
|
||||
# Enable debugging mode; this inserts mitmproxy/mitmweb between the CONNECT proxy and the caching layer
|
||||
ENV DEBUG="true"
|
||||
# Enable nginx debugging mode; this uses nginx-debug binary and enabled debug logging, which is VERY verbose so separate setting
|
||||
ENV DEBUG_NGINX="false"
|
||||
|
||||
# Did you want a shell? Sorry. This only does one job; use exec /bin/bash if you wanna inspect stuff
|
||||
# Did you want a shell? Sorry, the entrypoint never returns, because it runs nginx itself. Use 'docker exec' if you need to mess around internally.
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
|
@ -58,12 +58,8 @@ NGINX_BIN="nginx"
|
|||
if [[ "a${DEBUG}" == "atrue" ]]; then
|
||||
# in debug mode, change caching layer to listen on 444, so that mitmproxy can sit in the middle.
|
||||
echo " listen 444 ssl default_server;" > /etc/nginx/caching.layer.listen
|
||||
echo "error_log /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
|
||||
|
||||
# use debug binary
|
||||
NGINX_BIN="nginx-debug"
|
||||
|
||||
echo "Starting in DEBUG MODE."
|
||||
echo "Starting in DEBUG MODE (mitmproxy)."
|
||||
echo "Run mitmproxy with reverse pointing to the same certs..."
|
||||
mitmweb --no-web-open-browser --web-iface 0.0.0.0 --web-port 8081 \
|
||||
--set keep_host_header=true --set ssl_insecure=true \
|
||||
|
|
@ -73,6 +69,12 @@ if [[ "a${DEBUG}" == "atrue" ]]; then
|
|||
echo "Access mitmweb via http://127.0.0.1:8081/ "
|
||||
fi
|
||||
|
||||
if [[ "a${DEBUG_NGINX}" == "atrue" ]]; then
|
||||
echo "Starting in DEBUG MODE (nginx)."
|
||||
echo "error_log /var/log/nginx/error.log debug;" > /etc/nginx/error.log.debug.warn
|
||||
# use debug binary
|
||||
NGINX_BIN="nginx-debug"
|
||||
fi
|
||||
|
||||
echo "Testing nginx config..."
|
||||
${NGINX_BIN} -t
|
||||
|
|
|
|||
88
nginx.conf
88
nginx.conf
|
|
@ -21,6 +21,15 @@ http {
|
|||
'"HOST: $host" "UPSTREAM: $upstream_addr" '
|
||||
'"UPSTREAM-STATUS: $upstream_status" '
|
||||
'"SSL-PROTO: $ssl_protocol" '
|
||||
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" '
|
||||
'"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
|
||||
'"AUTH: $http_authorization" ' ;
|
||||
|
||||
log_format debug_proxy 'CONNECTPROXY: $remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent '
|
||||
'"HOST: $host" "UPSTREAM: $upstream_addr" '
|
||||
'"UPSTREAM-STATUS: $upstream_status" '
|
||||
'"SSL-PROTO: $ssl_protocol" '
|
||||
'"CONNECT-HOST: $connect_host" "CONNECT-PORT: $connect_port" "CONNECT-ADDR: $connect_addr" "INTERCEPTED: $interceptedHost" '
|
||||
'"PROXY-HOST: $proxy_host" "UPSTREAM-REDIRECT: $upstream_http_location" "CACHE-STATUS: $upstream_cache_status" '
|
||||
'"AUTH: $http_authorization" ' ;
|
||||
|
|
@ -51,18 +60,23 @@ http {
|
|||
default "";
|
||||
}
|
||||
|
||||
# @TODO: actually for auth.docker.io, if we want to support multiple authentications, we'll need to decide
|
||||
# @TODO: based not only on the hostname, but also URI (/token) and query string (?scope)
|
||||
# @TODO: I wonder if this would help gcr.io and quay.io with authentication also....
|
||||
|
||||
map $dockerAuth $finalAuth {
|
||||
"" "$http_authorization"; # if empty, keep the original passed-in from the docker client.
|
||||
default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
|
||||
}
|
||||
|
||||
|
||||
# Map to decide which hosts get directed to the caching portion.
|
||||
# This is automatically generated from the list of cached registries, plus a few fixed hosts
|
||||
# By default, we don't intercept, allowing free flow of non-registry traffic
|
||||
map $connect_host $interceptedHost {
|
||||
hostnames;
|
||||
include /etc/nginx/docker.intercept.map;
|
||||
default "$connect_host:443";
|
||||
}
|
||||
|
||||
map $dockerAuth $finalAuth {
|
||||
"" "$http_authorization"; # if empty, keep the original passed-in from the client
|
||||
default "Basic $dockerAuth"; # if not empty, add the Basic preamble to the auth
|
||||
default "$connect_addr"; # $connect_addr is 'IP address and port of the remote host, e.g. "192.168.1.5:12345". IP address is resolved from host name of CONNECT request line.'
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -86,6 +100,7 @@ http {
|
|||
server_name _;
|
||||
|
||||
# dont log the CONNECT proxy.
|
||||
#access_log /var/log/nginx/access.log debug_proxy;
|
||||
access_log off;
|
||||
|
||||
proxy_connect;
|
||||
|
|
@ -97,7 +112,8 @@ http {
|
|||
|
||||
# forward proxy for non-CONNECT request
|
||||
location / {
|
||||
return 403 "The docker caching proxy is working!";
|
||||
add_header "Content-type" "text/plain" always;
|
||||
return 200 "docker-registry-proxy: The docker caching proxy is working!";
|
||||
}
|
||||
|
||||
location /ca.crt {
|
||||
|
|
@ -145,7 +161,7 @@ http {
|
|||
# Use cache locking, with a huge timeout, so that multiple Docker clients asking for the same blob at the same time
|
||||
# will wait for the first to finish instead of doing multiple upstream requests.
|
||||
proxy_cache_lock on;
|
||||
proxy_cache_lock_timeout 120s;
|
||||
proxy_cache_lock_timeout 880s;
|
||||
|
||||
# Cache all 200, 301, 302, and 307 (emitted by private registries) for 60 days.
|
||||
proxy_cache_valid 200 301 302 307 60d;
|
||||
|
|
@ -165,55 +181,18 @@ http {
|
|||
# This comes from a include file generated by the entrypoint.
|
||||
include /etc/nginx/docker.verify.ssl.conf;
|
||||
|
||||
# Some debugging info
|
||||
# add_header X-Docker-Caching-Proxy-Real-Host $realHost;
|
||||
# add_header X-Docker-Caching-Proxy-Real-Path $realPath;
|
||||
# add_header X-Docker-Caching-Proxy-Auth $finalAuth;
|
||||
|
||||
# Block API v1. We dont know how to handle these.
|
||||
# Docker-client should start with v2 and fallback to v1 if something fails, for example, if authentication failed to a protected v2 resource.
|
||||
location /v1 {
|
||||
return 405 "API v1 is invalid. Either the image does not exist upstream, or you need auth to get a v2 endpoint working against $host";
|
||||
return 405 "docker-registry-proxy: docker is trying to use v1 API. Either the image does not exist upstream, or you need to configure docker-registry-proxy to authenticate against $host";
|
||||
}
|
||||
|
||||
# don't cache mutable entity /v2/<name>/manifests/<reference> (unless the reference is a digest)
|
||||
location ~ ^/v2/[^\/]+/manifests/(?![A-Fa-f0-9_+.-]+:) {
|
||||
proxy_pass https://$targetHost;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:manifests";
|
||||
}
|
||||
|
||||
# don't cache mutable entity /v2/<name>/tags/list
|
||||
location ~ ^/v2/[^\/]+/tags/list {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache off;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:tagslist";
|
||||
}
|
||||
|
||||
# don't cache mutable entity /v2/_catalog
|
||||
location ~ ^/v2/_catalog$ {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache off;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:catalog";
|
||||
}
|
||||
|
||||
# dont cache the first hit which is always /v2/
|
||||
location = /v2/ {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache off;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:rootv2";
|
||||
}
|
||||
|
||||
# dont cache /token (done against auth servers)
|
||||
location = /token {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache off;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:token";
|
||||
}
|
||||
|
||||
# cache everything else
|
||||
location / {
|
||||
# for the /v2/..../blobs/.... URIs, do cache, and treat redirects.
|
||||
location ~ ^/v2/(.*)/blobs/ {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache cache;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:blobs";
|
||||
|
||||
# Handling of redirects.
|
||||
# Many registries (eg, quay.io, or k8s.gcr.io) emit a Location redirect
|
||||
|
|
@ -223,10 +202,9 @@ http {
|
|||
# We to it twice, one for http and another for https.
|
||||
proxy_redirect ~^https://([^:/]+)(/.+)$ https://docker.caching.proxy.internal/forcecachesecure/$1/originalwas$2;
|
||||
proxy_redirect ~^http://([^:/]+)(/.+)$ http://docker.caching.proxy.internal/forcecacheinsecure/$1/originalwas$2;
|
||||
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:everythingelse";
|
||||
}
|
||||
|
||||
|
||||
# handling for the redirect case explained above, with https.
|
||||
# The $realHost and $realPath variables come from a map defined at the top of this file.
|
||||
location /forcecachesecure {
|
||||
|
|
@ -251,5 +229,13 @@ http {
|
|||
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "yes:forcecacheinsecure";
|
||||
}
|
||||
|
||||
# by default, dont cache anything.
|
||||
location / {
|
||||
proxy_pass https://$targetHost;
|
||||
proxy_cache off;
|
||||
add_header X-Docker-Caching-Proxy-Debug-Cache "no:default";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue