Allan's Blog

Friday, February 12, 2021

Create a local Kubernetes cluster and deploy your code end to end

These are the commands accompanying the video tutorial I've made on how to create a local Kubernetes cluster and deploy a simple stateless app from code, end to end. You can find the video tutorial here: https://youtu.be/MZr9Ls38uPw .

#
# Install k3s
#

curl -sfL https://get.k3s.io | K3S_CLUSTER_INIT=1 INSTALL_K3S_EXEC="--disable=servicelb" sh -

cat /var/lib/rancher/k3s/server/node-token

# Run on other nodes to join the cluster
    curl -sfL https://get.k3s.io | \
      INSTALL_K3S_EXEC=server \
      K3S_URL=https://192.168.50.203:6443 \
      K3S_TOKEN=... \
      sh -


kubectl get nodes --watch

journalctl --unit=k3s

kubectl get all --all-namespaces


#
# Install dashboard
#

GITHUB_URL=https://github.com/kubernetes/dashboard/releases
VERSION_KUBE_DASHBOARD=$(curl -w '%{url_effective}' -I -L -s -S ${GITHUB_URL}/latest -o /dev/null | sed -e 's|.*/||')
echo $VERSION_KUBE_DASHBOARD
kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/${VERSION_KUBE_DASHBOARD}/aio/deploy/recommended.yaml

cat>dashboard.admin-user.yml<<"EOF"
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
EOF

cat>dashboard.admin-user-role.yml<<"EOF"
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF

kubectl create -f dashboard.admin-user.yml -f dashboard.admin-user-role.yml


kubectl get all --all-namespaces

# Connect to node again, but with your port 8001 forwarded.
ssh -L 8001:127.0.0.1:8001 user1@ubuntutest1
kubectl proxy
# Open in browser:  http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/
kubectl --namespace kubernetes-dashboard describe secret admin-user-token | grep ^token


#
# Install MetalLB
#

GITHUB_URL=https://github.com/metallb/metallb/releases
VERSION=$(curl -w '%{url_effective}' -I -L -s -S ${GITHUB_URL}/latest -o /dev/null | sed -e 's|.*/||')
echo $VERSION

kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/$VERSION/manifests/namespace.yaml
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/$VERSION/manifests/metallb.yaml
kubectl create secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)"

cat>metallb-configmap.yml<<"EOF"
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 192.168.50.20-192.168.50.40
EOF
kubectl apply -f metallb-configmap.yml

# Check logs for errors
kubectl logs -lapp=metallb --namespace metallb-system --all-containers=true --prefix -f



#
# Install docker registry
#

apt-get install docker-compose
docker run -d -p 5000:5000 --restart=always --name registry registry
docker ps -a

# On each node:
mkdir -p /etc/rancher/k3s/
cat>/etc/rancher/k3s/registries.yaml<<"EOF"
mirrors:
  "ubuntutest1:5000":
    endpoint:
      - "http://ubuntutest1:5000"
EOF
systemctl restart k3s



#
# Create containerized app
#

mkdir -p /home/user1/dev/myapp1
cd /home/user1/dev/myapp1
cat>main.go<<"EOF"
package main

import (
  "github.com/gorilla/mux"
  "html/template"
  "log"
  "net/http"
  "os"
  "time"
)

var helloTemplate, _ = template.New("").Parse(`<!DOCTYPE html>
<html>
<head><title>testapp</title></head>
<body>
<h1>Test 1</h1>
<p>Now: {{.now.Format "2006-01-02 15:04:05" }}</p>
<p>Served from node: {{ .node }}</p>
<p>Served from pod: {{ .pod }}</p>
</body>
</html>
`)

type Msg struct {
  Ts  time.Time `json:"ts"`
}

func helloGet(w http.ResponseWriter, r *http.Request) {
  v := map[string]interface{}{
    "now":  time.Now(),
    "node": os.Getenv("NODE_NAME"),
    "pod": os.Getenv("POD_NAME"),
  }
  helloTemplate.Execute(w, v)
}

func main() {
  log.Println("Starting")

  router := mux.NewRouter()
  router.HandleFunc("/", helloGet).Methods("GET")
  handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    log.Println(r.Method + " " + r.URL.String())
    router.ServeHTTP(w, r)
  })

  log.Fatal(http.ListenAndServe(":8080", handler))
}
EOF


cat>go.mod<<"EOF"
module myapp1
EOF

touch go.sum

cat>Dockerfile<<"EOF"
FROM golang as builder
WORKDIR /app
COPY . ./
RUN go mod download
RUN CGO_ENABLED=0 GOOS=linux go build -v -o server

FROM alpine
RUN apk add --no-cache ca-certificates
COPY --from=builder /app/server /server
CMD ["/server"]
EOF


docker build -t "myapp1" .
docker run -it --rm -p 8080:8080 myapp1
docker tag myapp1 ubuntutest1:5000/myapp1:v1
docker push ubuntutest1:5000/myapp1:v1

curl -X GET ubuntutest1:5000/v2/_catalog
curl -X GET ubuntutest1:5000/v2/myapp1/tags/list


#
# Create and deploy our app in k8s
#

mkdir -p /home/user1/dev/myapp1/k8s
cd /home/user1/dev/myapp1

cat>k8s/deployment.yml<<"EOF"
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp1-deployment
spec:
  selector:
    matchLabels:
      app: myapp1
  replicas: 3
  template:
    metadata:
      labels:
        app: myapp1
    spec:
      containers:
      - name: myapp1
        image: ubuntutest1:5000/myapp1:v1
        ports:
          - containerPort: 8080
        env:
          - name: NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: "kubernetes.io/hostname"
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: myapp1
EOF
kubectl apply -f k8s/deployment.yml


cat>k8s/service.yml<<"EOF"
kind: Service
apiVersion: v1
metadata:
  name: myapp1-service
spec:
  type: ClusterIP
  selector:
    app: myapp1
  ports:
  - name: http-myapp1
    protocol: TCP
    port: 8080
EOF
kubectl apply -f k8s/service.yml


# Expose app via HTTP proxy (aka. "ingress")
cat>k8s/ingress.yml<<"EOF"
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp1-ingress
spec:
  rules:
  - http:
      paths:
        - path: /
          pathType: Prefix
          backend:
            service:
              name: myapp1-service
              port:
                number: 8080
EOF
kubectl apply -f k8s/ingress.yml

kubectl logs -lapp=myapp1 --all-containers=true --prefix -f


#
# Upgrading to a new image. Either use this command or update the same field in deployment.yml.
#


cd /home/user1/dev/myapp1
docker build -t "myapp1" .
docker tag myapp1 ubuntutest1:5000/myapp1:v2
docker push ubuntutest1:5000/myapp1:v2

kubectl set image deployments/myapp1-deployment myapp1=ubuntutest1:5000/myapp1:v2
kubectl rollout status deployments/myapp1-deployment

# Rolling back (calling repeatedly switches between two newest versions).
kubectl rollout undo deployments/myapp1-deployment



#
# Testing HA
#

kubectl get nodes ; kubectl get pod -o=custom-columns=NAME:.metadata.name,STATUS:.status.phase,NODE:.spec.nodeName --all-namespaces

kubectl drain ubuntutest1 --ignore-daemonsets --delete-emptydir-data
kubectl uncordon ubuntutest1

Sunday, February 07, 2021

k3s and MetalLB "destination unreachable" issue

I was trying out MetalLB with a bare-metal Kubernetes cluster (using the k3s distro), and was hitting issues with requests to the cluster IP seemingly randomly giving me "destination unreachable". I noticed with Wireshark that I was getting a ton of "gratuitous" ARP packets. Every few seconds.

I inspected MetalLB's logs with the following command:

kubectl logs -lapp=metallb --namespace metallb-system --all-containers=true --prefix -f

It was logging the message "IP allocated by controller not allowed by config" every few seconds.

Turned out to be k3s's builtin internal load balancer that was interferring with MetalLB. Disabling it with the "--disable=servicelb" flag during k3s's installation fixed the issue.

curl -sfL https://get.k3s.io | K3S_CLUSTER_INIT=1 INSTALL_K3S_EXEC="--disable=servicelb" sh -

Monday, July 13, 2020

A "super ping" command to run while diagnosing network issues

Here's a bash function for your bash profile that continuously prints some basic checks I usually do while diagnosing network issues:

Show my LAN IPs
Show the default gateway IP
Ping the default gateway IP
Ping 8.8.8.8 (Google's public DNS servers)
DNS resolve amazon.com
Ping amazon.com
Get my WAN IP by curl'ing ifconfig.me

Tested on Ubuntu 20.04 and MacOS Catalina

function pingg() {
    while true; do
        printf "%s  " $(TZ=UTC date "+%Y-%m-%dT%H:%M:%S")

        if [[ "$OSTYPE" == "darwin"* ]]; then
            lanIps=$(ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | paste -d, -s -)
            gw=$(netstat -rn -f inet | grep default | awk '{print $2}')
        else
            lanIps=$(ip addr | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | paste -d, -s -)
            gw=$(ip route | grep default | awk '{print $3}')
        fi
        printf "lanIp=%-15s " $lanIps
        printf "gw=%-15s " $gw

        pingGw="-------"
        if [ "$gw" != "" ]; then
            pingGw=$(ping -W1 -c1 $gw 2>/dev/null | tail -1 | grep -v "0 packets received" | cut -d '=' -f 2 | cut -d '/' -f 2)
            if [ "$pingGw" == "" ]; then
                pingGw="-------"
            fi
        fi
        printf "pGw=%-7s " $pingGw

        ping8888=$(ping -W1 -c1 8.8.8.8 2>/dev/null | tail -1 | grep -v "0 packets received" | cut -d '=' -f 2 | cut -d '/' -f 2)
        if [ "$ping8888" == "" ]; then
            ping8888="-------"
        fi
        printf "p8=%-7s " $ping8888

        dnsAmzn=$(host -4 -W 1 -t a amazon.com | head -n1 | grep "has address" | awk '{print $4}')
        if [ "$dnsAmzn" == "" ]; then
            dnsAmzn="-------"
        fi
        printf "dnsAmz=%-15s " $dnsAmzn

        pingAmzn="-------"
        if [ "$dnsAmzn" != "-------" ]; then
            pingAmzn=$(ping -W1 -c1 $dnsAmzn 2>/dev/null | tail -1 | grep -v "0 packets received" | cut -d '=' -f 2 | cut -d '/' -f 2)
            if [ "$pingAmzn" == "" ]; then
                pingAmzn="-------"
            fi
        fi
        printf "pAmz=%-7s " $pingAmzn

        wanIp="-------"
        dnsIfconfigMe=$(host -4 -W 1 -t a ifconfig.me | head -n1 | grep "has address" | awk '{print $4}')
        if [ "$dnsIfconfigMe" != "" ]; then
            wanIp=$(curl -s --max-time 2 --connect-timeout 2 -H "Host: ifconfig.me" $dnsIfconfigMe | head -n 1 | awk '{print $1'})
            if [ "$wanIp" == "" ]; then
                wanIp="-------"
            fi
        fi
        printf "wanIp=%-15s " $wanIp

        printf "\n"

        sleep 0.7
    done
}

Example output:

2020-07-13T23:43:59  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.264   p8=53.338  dnsAmz=176.32.98.166   pAmz=127.532 wanIp=1.2.3.4
2020-07-13T23:44:01  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.190   p8=47.107  dnsAmz=176.32.103.205  pAmz=111.309 wanIp=1.2.3.4
2020-07-13T23:44:02  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.290   p8=------- dnsAmz=176.32.103.205  pAmz=118.834 wanIp=1.2.3.4
2020-07-13T23:44:04  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.328   p8=143.387 dnsAmz=176.32.103.205  pAmz=135.333 wanIp=1.2.3.4
2020-07-13T23:44:06  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.313   p8=17.944  dnsAmz=176.32.103.205  pAmz=105.977 wanIp=1.2.3.4
2020-07-13T23:44:07  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.223   p8=22.505  dnsAmz=176.32.103.205  pAmz=97.623  wanIp=1.2.3.4
2020-07-13T23:44:08  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.250   p8=23.917  dnsAmz=176.32.103.205  pAmz=98.403  wanIp=1.2.3.4
2020-07-13T23:44:09  lanIp=192.168.1.6/24 gw=192.168.1.1    pGw=0.220   p8=41.262  dnsAmz=176.32.103.205  pAmz=100.828 wanIp=1.2.3.4

Tuesday, June 23, 2020

Upgrading Alpine Linux diskless setup on a Raspberry Pi

Here's a procedure I've come up with to fully upgrade a diskless setup of Alpine Linux on a running Raspberry Pi. It's not pretty, but seems gets the job done, based on the tests I did.

I tested this procedure on each version from 3.3 to to 3.12. The system I tested on had a working setup of Apache2 with PHP, as well as Samba. Here's the result of the testing.

Started with v3.3.
Confirmed 3.3 to 3.4. Package "php-apache2" package had disappeared and had to be manually replaced by "php5-apache2".
Confirmed 3.4 to 3.5.
Confirmed 3.5 to 3.6.
Confirmed 3.6 to 3.7.
Confirmed 3.7 to 3.8.
Could not boot after upgrading from 3.8 to 3.9. But a fresh installation of 3.9 was also unbootable, so assuming 3.9 is broken. Seems to be a known issue.
Confirmed 3.8 to 3.10. Package "php5-apache2" package had disappeared and had to be manually replaced by "php7-apache2".

The process I came up with can be summarized to:

Remount the SD card in read-write mode.
Move all files in the SD card FAT32 root to a subdirectory such as old/2020-06-23T04_22_54.
Download a fresh alpine-rpi-x.y.z-armhf.tar.gz file and untar it to the SD card root. This will give you Alpine Linux's newest Raspberry compatible Linux kernel image.
Remount the SD card back to read-only mode.
Update the /etc/apk/repositories (in the in-memory file system) to point to the target version.
Do an "apk upgrade --available" to download the updated versions of the packages you used, so they will be ready on the device after rebooting.
Reboot into the new setup.

If anything goes wrong, you can put the SD card in another PC, delete the broken setup, and copy your old working setup back out from the "old" directory.

Here are the commands. As I don't know if this process will keep working in future versions, I recommend running them one line at a time and ensuring the line does what you expect.

targetVersion=v3.12

# Move everything to a safe place as it is now.
mount -oremount,rw /media/mmcblk0p1
cd /media/mmcblk0p1
olddir=/media/mmcblk0p1/old/$(date --utc "+%Y-%m-%dT%H_%M_%S")
mkdir -p $olddir
mv *  $olddir   # This will give a warning that it cannot move old into old. That's ok.
mv .* $olddir   # This will give a warning that it cannot move . and .. . That's ok.

# Download fresh Alpine Linux image and replace existing.
newImgFile=$(curl http://dl-cdn.alpinelinux.org/alpine/$targetVersion/releases/armhf/latest-releases.yaml --silent | grep "file: alpine-rpi-" | sed "s/ *file: //g")
echo $newImgFile
mount -oremount,rw /media/mmcblk0p1
wget http://dl-cdn.alpinelinux.org/alpine/$targetVersion/releases/armhf/$newImgFile
wget http://dl-cdn.alpinelinux.org/alpine/$targetVersion/releases/armhf/${newImgFile}.sha512
sha512sum -c ${newImgFile}.sha512    # Ensure it says OK
tar -xzf $newImgFile
rm ${newImgFile}*
cp $olddir/usercfg.txt .
mount -oremount,ro /media/mmcblk0p1

# Update packages to the target distro.
cat > /etc/apk/repositories <<EOF
/media/mmcblk0p1/apks
http://dl-cdn.alpinelinux.org/alpine/$targetVersion/main
http://dl-cdn.alpinelinux.org/alpine/$targetVersion/community
EOF
setup-apkcache /media/mmcblk0p1/cache
apk update
apk upgrade --available
apk add
apk cache download
apk -v cache clean
lbu commit -d

reboot

# After reboot, ensure there aren't packages that no longer exist in this version.
apk add

Sunday, June 21, 2020

Add shortcut keys to Google Drawing

Here's a quick and dirty way to add shortcut keys to Google Drawing. It probably only works in Chrome.

It's a JavaScript snippet that can be run in the context of the Google Drawing page in order to automate clicking on the toolbar buttons.

The keys this code adds are:

Alt-A: Arrow
Alt-L: Line
Alt-B: Rectangle (box)
Alt-C: Cylinder
Alt-X: Text box

Here's a bookmarklet with the script: GDraw shortcuts. Drag this hyperlink to your bookmarks bar and click it when you are inside Google Drawing in order to enable the shortcut keys. (Thanks to Peter Coles for his bookmarklet creation tool)

And here's the code, in case you want to extend it with additional shortcut keys.

function newMouseEvent(eventType, x, y) {
    return new MouseEvent(eventType, {
        "view": window,
        "bubbles": true,
        "cancelable": true,
        "screenX": x,
        "screenY": y
    });
}

function myclick(x, y) {
    var el = document.elementFromPoint(x, y);

    el.dispatchEvent(newMouseEvent("mouseover", x, y));
    el.dispatchEvent(newMouseEvent("mousedown", x, y));
    el.dispatchEvent(newMouseEvent("mouseup", x, y));
    el.dispatchEvent(newMouseEvent("click", x, y));
}

function clickLineButton() {
  var r = document.getElementById("lineMenuButton").getBoundingClientRect();
  myclick(r.x+3, r.y+3);

  setTimeout(function() {
    myclick(r.x+3, r.y+50);
  }, 200);
}

function clickArrowButton() {
  var r = document.getElementById("lineMenuButton").getBoundingClientRect();
  myclick(r.x+3, r.y+3);

  setTimeout(function() {
    myclick(r.x+3, r.y+80);
  }, 200);
}

function clickBoxButton() {
  var r = document.getElementById("shapeButton").getBoundingClientRect();
  myclick(r.x+3, r.y+3);

  setTimeout(function() {
    myclick(r.x+3, r.y+40);

    setTimeout(function() {
      myclick(r.x+172, r.y+56);
    }, 200);

  }, 200);
}

function clickCylinderButton() {
  var r = document.getElementById("shapeButton").getBoundingClientRect();
  myclick(r.x+3, r.y+3);

  setTimeout(function() {
    myclick(r.x+3, r.y+40);

    setTimeout(function() {
      myclick(r.x+197, r.y+223);
    }, 200);

  }, 200);
}

function myKeydownHandler(e) {
  /* A key */
  if (e.keyCode == 65 && e.altKey == true) { 
    clickArrowButton();
    e.preventDefault();
  }

  /* L key */
  if (e.keyCode == 76 && e.altKey == true) { 
    clickLineButton();
    e.preventDefault();
  }

  /* B key */
  if (e.keyCode == 66 && e.altKey == true) {
    clickBoxButton();
    e.preventDefault();
  }

  /* C key */
  if (e.keyCode == 67 && e.altKey == true) { 
    clickCylinderButton();
    e.preventDefault();
  }
}

window.addEventListener("keydown", myKeydownHandler, true);

var elements = document.getElementsByTagName("iframe");
for (var i = 0; i < elements.length; i++) {
  if (elements[i].src == "" || elements[i].src == "about:blank") {
    elements[i].contentWindow.addEventListener("keydown", myKeydownHandler, true);
  }
}

Thursday, June 18, 2020

Reach servers behind NAT with WireGuard and a VPS, while preserving source IP

This post describes an approach to let your Linux machine behind a NAT be reachable from the public internet, using a cheap VPS and Wireguard.

It's similar to this post in Ralph's Blog, but with with one distinction: it preserves the source IP of the incoming connection. This may be important depending on your use. For me, I was running a PHP web app for which I needed the $_SERVER["REMOTE_ADDR"] field to be the true and correct IP of the client, and not just the IP of the gateway that forwarded the traffic.

Values used in this example:

192.168.51.0/24 is the private VPN network between the cloud VPS host and the server you want to expose.
192.168.51.1 is the VPN network IP of the cloud VPS.
192.168.51.2 is the VPN network IP of the server you want to expose.
50.40.30.20 is the public IP of the cloud VPS.
59014 the UDP port number I randomly chose for WireGuard on the VPS.
80 and 443 are the TCP port numbers I want to forward from my VPS to my local server.

On the cloud VPS:

mkdir -p /etc/wireguard
cd /etc/wireguard
wg genkey | tee privatekey | wg pubkey > publickey
chown 600 privatekey
cat privatekey
cat publickey

# Replace the PrivateKey in the [Interface] section with the actual privatekey value generated above.
# Replace the PublicKey in the [Peer] section with the publickey value you generate ON YOUR LOCAL SERVER in the later steps.
cat > /etc/wireguard/wg0.conf <<EOF
[Interface]
Address = 192.168.51.1
PrivateKey = KF+X0SO2lkBFnxTCK4R5PCv/FC6+wgH6rt6wWHkVVHE=
ListenPort = 59014

[Peer]
PublicKey = y700qaTgjDjMrLc9tGF7+PKFrX7uHyIJJ/s2zvadKVY=
AllowedIPs = 192.168.51.2/32
EOF

chmod 600 wg0.conf

# Start the VPN server
wg-quick up wg0

echo 0 > /proc/sys/net/ipv4/ip_forward
iptables -P FORWARD DROP
iptables -A FORWARD -i eth0 -o wg0  -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A FORWARD -i wg0  -o eth0 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
iptables -A FORWARD -i wg0  -o eth0 -m conntrack --ctstate NEW -j ACCEPT
iptables -A FORWARD -i eth0 -o wg0  -p tcp --syn --dport 80  -m conntrack --ctstate NEW -j ACCEPT
iptables -A FORWARD -i eth0 -o wg0  -p tcp --syn --dport 443 -m conntrack --ctstate NEW -j ACCEPT
iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80  -j DNAT --to-destination 192.168.51.2
iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j DNAT --to-destination 192.168.51.2
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
echo 1 > /proc/sys/net/ipv4/ip_forward

On your local server, instead of using wg-quick, we will set up the interface a bit more manually. This is in order to have finer control over routing. We want connections originating on this server to exit via this server's normal internet connection, and not via the WireGuard VPN connection. However, packets part of TCP connections that got forwarded to us on the VPN from the VPS should be routed back via the VPS.

On your local server:

mkdir -p /etc/wireguard
cd /etc/wireguard
wg genkey | tee privatekey | wg pubkey > publickey
chown 600 privatekey
cat privatekey
cat publickey

# Replace the PrivateKey in the [Interface] section with the actual privatekey value generated above.
# Replace the PublicKey in the [Peer] section with the publickey value you generated ON YOUR CLOUD VPS in the earlier steps.
cat > /etc/wireguard/wg0.conf <<EOF
[Interface]
PrivateKey = YIOy80YJdcMqI1WCMROoyUfsfkc6GhP0KqRS3BfcoEs=

[Peer]
PublicKey = i1VQnnODEBbf+P9cyd9XmB9G58qEpM53TbMXn1UAJx8=
Endpoint = 50.40.30.20:59014
AllowedIPs = 0.0.0.0/0
PersistentKeepalive = 25
EOF

chmod 600 wg0.conf

# Set up VPN connection
ip link add wg0 type wireguard
wg setconf wg0 /etc/wireguard/wg0.conf
ip address add 192.168.51.2 dev wg0
ip link set mtu 1420 up dev wg0
ip rule add from 192.168.51.2 table 51820
ip rule add iif wg0 table 51820
ip route add 192.168.51.0/24 dev wg0
ip route add 0.0.0.0/0 via 192.168.51.1 dev wg0 table 51820

# Shut down VPN connection (don't run these steps now, but keep them handy)
ip rule delete from 192.168.51.2 table 51820
ip rule delete iif wg0 table 51820
ip link delete dev wg0

You should now be able to receive TCP traffic on port 80 and 443 on your local server, and the src IP address of the actual real client will be preserved all the way to your local server.

WireGuard kernel module for Alpine Linux on diskless Raspberry Pi

As of writing (Alpine Linux v3.12), the WireGuard kernel module is not part of the Alpine Linux Raspberry Pi kernel by default. There's a community module called wireguard-rpi2, but it doesn't work in a system set up in diskless mode. The reason it doesn't work, is because in diskless mode on a Raspberry Pi, Alpine Linux uses a read-only loopback file system to store its kernel modules. This filesystem is mounted on /.modloop .

The workaround I came up with, is to use an overlay file system to make the kernel modules directory writable.

Here's the workaround:

# Remount the kernel module directory /.modloop as an overlay, to allow writing
modprobe overlay
mkdir -p /.modloop.lower /.modloop.upper /.modloop.workdir
mount /dev/loop0 /.modloop.lower
umount /.modloop/
mount -t overlay -o lowerdir=/.modloop.lower,upperdir=/.modloop.upper,workdir=/.modloop.workdir none /.modloop
lbu include /.modloop.upper
lbu commit -d

# Manually get the Wireguard kernel module to avoid installing the wireguard-rpi2 which does not work with diskless systems
cd /tmp
pkgname=$(apk list | grep wireguard-rpi2 | cut -d " " -f 1)
wget http://dl-cdn.alpinelinux.org/alpine/v3.12/community/armhf/$pkgname.apk
mkdir /tmp/$pkgname
tar -xzf $pkgname.apk -C /tmp/$pkgname
mkdir -p /lib/modules/$(uname -r)/extra/
cp /tmp/$pkgname/lib/modules/$(uname -r)/extra/wireguard.ko /lib/modules/$(uname -r)/extra/
rm -fr /tmp/$pkgname /tmp/$pkgname.apk
depmod

# Create an init script to remount the /.modloop overlay on next boot
cat > /etc/init.d/modloopoverlay <<EOF
#!/sbin/openrc-run

depend() {
    before networking
    need modules
}

start() {
    ebegin "Starting modloop overlay"
    modprobe overlay
    mkdir -p /.modloop.lower /.modloop.upper /.modloop.workdir
    if [ ! -d /.modloop.lower/modules ]; then
        mount /dev/loop0 /.modloop.lower
    fi
    umount /.modloop
    mount -t overlay -o lowerdir=/.modloop.lower,upperdir=/.modloop.upper,workdir=/.modloop.workdir none /.modloop
    eend 0
}
EOF
chmod +x /etc/init.d/modloopoverlay
/etc/init.d/modloopoverlay restart
rc-update add modloopoverlay boot
lbu include /etc/init.d/modloopoverlay

apk add wireguard-tools

# Create an init script to start wg0
cat > /etc/init.d/wg0 <<EOF
#!/sbin/openrc-run

depend() {
    need networking ntpd modloopoverlay
}

start() {
    ebegin "Starting Wireguard tunnel wg0"
    ntpd -n -q -p pool.ntp.org
    date
    wg-quick up wg0
    eend $?
}

stop() {
    ebegin "Stopping Wireguard tunnel wg0"
    wg-quick down wg0
    eend 0
}
EOF
chmod +x /etc/init.d/wg0
/etc/init.d/wg0 restart
rc-update add wg0 default
lbu include /etc/init.d/wg0

lbu commit -d

# inspect the overlay file
tar -tvf /media/mmcblk0p1/localhost.apkovl.tar.gz