Adding documentation on how to configure Kops to use registry-proxy (#64)
Marcelo Grebois
GitHub
parent
6dbce2f5cd
commit
18c9ca4983
|
|
@ -0,0 +1,159 @@
|
||||||
|
# How to use docker-registry-proxy with kops
|
||||||
|
|
||||||
|
## Install docker-registry-proxy
|
||||||
|
|
||||||
|
For running docker-registry-proxy with kops you will need to run it outside the cluster you want to configure, you can either use and EC2 instance and run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm --name docker_registry_proxy -it \
|
||||||
|
-p 0.0.0.0:3128:3128 -e ENABLE_MANIFEST_CACHE=true \
|
||||||
|
-v $(pwd)/docker_mirror_cache:/docker_mirror_cache \
|
||||||
|
-v $(pwd)/docker_mirror_certs:/ca \
|
||||||
|
rpardini/docker-registry-proxy:0.6.0
|
||||||
|
```
|
||||||
|
|
||||||
|
or you can run it from another cluster, maybe a management/observability one with provided yaml, in this case, you will need to change the following lines:
|
||||||
|
|
||||||
|
```
|
||||||
|
annotations:
|
||||||
|
external-dns.alpha.kubernetes.io/hostname: docker-registry-proxy.<your_domain>
|
||||||
|
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
|
||||||
|
```
|
||||||
|
|
||||||
|
with the correct domain name, so then you can reference the proxy as `http://docker-registry-proxy.<your_domain>:3128`
|
||||||
|
|
||||||
|
## Test the connection to the proxy
|
||||||
|
|
||||||
|
A simple curl should return:
|
||||||
|
|
||||||
|
```
|
||||||
|
❯ curl docker-registry-proxy.<your_domain>:3128
|
||||||
|
docker-registry-proxy: The docker caching proxy is working!%
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configure kops to use the proxy
|
||||||
|
|
||||||
|
Kops has the option to configure a cluster wide proxy, as explained [here](https://github.com/kubernetes/kops/blob/master/docs/http_proxy.md) but this wont work, as nodeup will fail to download the images, what you need is to use `additionalUserData`, which is part of the instance groups configuration.
|
||||||
|
|
||||||
|
So consider a node configuration like this one:
|
||||||
|
|
||||||
|
```
|
||||||
|
apiVersion: kops.k8s.io/v1alpha2
|
||||||
|
kind: InstanceGroup
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
kops.k8s.io/cluster: spot.k8s.local
|
||||||
|
name: spotgroup
|
||||||
|
spec:
|
||||||
|
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20200528
|
||||||
|
machineType: c3.xlarge
|
||||||
|
maxSize: 15
|
||||||
|
minSize: 2
|
||||||
|
mixedInstancesPolicy:
|
||||||
|
instances:
|
||||||
|
- c3.xlarge
|
||||||
|
- c4.xlarge
|
||||||
|
- c5.xlarge
|
||||||
|
- c5a.xlarge
|
||||||
|
onDemandAboveBase: 0
|
||||||
|
onDemandBase: 0
|
||||||
|
spotAllocationStrategy: capacity-optimized
|
||||||
|
nodeLabels:
|
||||||
|
kops.k8s.io/instancegroup: spotgroup
|
||||||
|
role: Node
|
||||||
|
subnets:
|
||||||
|
- us-east-1a
|
||||||
|
- us-east-1b
|
||||||
|
- us-east-1c
|
||||||
|
```
|
||||||
|
|
||||||
|
you will need to add the following:
|
||||||
|
|
||||||
|
```
|
||||||
|
additionalUserData:
|
||||||
|
- name: docker-registry-proxy.sh
|
||||||
|
type: text/x-shellscript
|
||||||
|
content: |
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Add environment vars pointing Docker to use the proxy
|
||||||
|
# https://docs.docker.com/config/daemon/systemd/#httphttps-proxy
|
||||||
|
|
||||||
|
mkdir -p /etc/systemd/system/docker.service.d
|
||||||
|
cat << EOD > /etc/systemd/system/docker.service.d/http-proxy.conf
|
||||||
|
[Service]
|
||||||
|
Environment="HTTP_PROXY=http://docker-registry-proxy.<your_domain>:3128/"
|
||||||
|
Environment="HTTPS_PROXY=http://docker-registry-proxy.<your_domain>:3128/"
|
||||||
|
EOD
|
||||||
|
|
||||||
|
# Get the CA certificate from the proxy and make it a trusted root.
|
||||||
|
curl http://docker-registry-proxy.<your_domain>:3128/ca.crt > /usr/share/ca-certificates/docker_registry_proxy.crt
|
||||||
|
echo "docker_registry_proxy.crt" >> /etc/ca-certificates.conf
|
||||||
|
update-ca-certificates --fresh
|
||||||
|
|
||||||
|
# Reload systemd
|
||||||
|
systemctl daemon-reload
|
||||||
|
|
||||||
|
# Restart dockerd
|
||||||
|
systemctl restart docker.service
|
||||||
|
```
|
||||||
|
|
||||||
|
so the final InstanceGroup will look like this:
|
||||||
|
|
||||||
|
```
|
||||||
|
apiVersion: kops.k8s.io/v1alpha2
|
||||||
|
kind: InstanceGroup
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
kops.k8s.io/cluster: spot.k8s.local
|
||||||
|
name: spotgroup
|
||||||
|
spec:
|
||||||
|
additionalUserData:
|
||||||
|
- name: docker-registry-proxy.sh
|
||||||
|
type: text/x-shellscript
|
||||||
|
content: |
|
||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Add environment vars pointing Docker to use the proxy
|
||||||
|
# https://docs.docker.com/config/daemon/systemd/#httphttps-proxy
|
||||||
|
|
||||||
|
mkdir -p /etc/systemd/system/docker.service.d
|
||||||
|
cat << EOD > /etc/systemd/system/docker.service.d/http-proxy.conf
|
||||||
|
[Service]
|
||||||
|
Environment="HTTP_PROXY=http://docker-registry-proxy.<your_domain>:3128/"
|
||||||
|
Environment="HTTPS_PROXY=http://docker-registry-proxy.<your_domain>:3128/"
|
||||||
|
EOD
|
||||||
|
|
||||||
|
# Get the CA certificate from the proxy and make it a trusted root.
|
||||||
|
curl http://docker-registry-proxy.<your_domain>:3128/ca.crt > /usr/share/ca-certificates/docker_registry_proxy.crt
|
||||||
|
echo "docker_registry_proxy.crt" >> /etc/ca-certificates.conf
|
||||||
|
update-ca-certificates --fresh
|
||||||
|
|
||||||
|
# Reload systemd
|
||||||
|
systemctl daemon-reload
|
||||||
|
|
||||||
|
# Restart dockerd
|
||||||
|
systemctl restart docker.service
|
||||||
|
image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20200528
|
||||||
|
machineType: c3.xlarge
|
||||||
|
maxSize: 15
|
||||||
|
minSize: 2
|
||||||
|
mixedInstancesPolicy:
|
||||||
|
instances:
|
||||||
|
- c3.xlarge
|
||||||
|
- c4.xlarge
|
||||||
|
- c5.xlarge
|
||||||
|
- c5a.xlarge
|
||||||
|
onDemandAboveBase: 0
|
||||||
|
onDemandBase: 0
|
||||||
|
spotAllocationStrategy: capacity-optimized
|
||||||
|
nodeLabels:
|
||||||
|
kops.k8s.io/instancegroup: spotgroup
|
||||||
|
role: Node
|
||||||
|
subnets:
|
||||||
|
- us-east-1a
|
||||||
|
- us-east-1b
|
||||||
|
- us-east-1c
|
||||||
|
```
|
||||||
|
|
||||||
|
Now all you need is to upgrade your cluster and do a rolling-update of the nodes, all images will be cached from now on.
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: docker-registry-proxy
|
||||||
|
namespace: registry-mirrors
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: docker-registry-proxy
|
||||||
|
spec:
|
||||||
|
serviceName: docker-registry
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: docker-registry-proxy
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: docker-registry-proxy
|
||||||
|
spec:
|
||||||
|
serviceAccountName: default
|
||||||
|
containers:
|
||||||
|
- name: docker-registry-proxy
|
||||||
|
image: ghcr.io/rpardini/docker-registry-proxy:0.6.0
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: ENABLE_MANIFEST_CACHE
|
||||||
|
value: "true"
|
||||||
|
- name: REGISTRIES
|
||||||
|
value: "k8s.gcr.io gcr.io quay.io us.gcr.io"
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 3128
|
||||||
|
protocol: TCP
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
volumeMounts:
|
||||||
|
- name: ca
|
||||||
|
mountPath: /ca
|
||||||
|
- name: docker-registry-cache
|
||||||
|
mountPath: /docker_mirror_cache
|
||||||
|
resources: {}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: ca
|
||||||
|
spec:
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
- metadata:
|
||||||
|
name: docker-registry-cache
|
||||||
|
spec:
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 100Gi
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: docker-registry-proxy
|
||||||
|
namespace: registry-mirrors
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: docker-registry-proxy
|
||||||
|
annotations:
|
||||||
|
external-dns.alpha.kubernetes.io/hostname: docker-registry-proxy.<your_domain>
|
||||||
|
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
|
||||||
|
spec:
|
||||||
|
type: LoadBalancer
|
||||||
|
ports:
|
||||||
|
- port: 3128
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
selector:
|
||||||
|
app.kubernetes.io/name: docker-registry-proxy
|
||||||
Loading…
Reference in New Issue