cron: add scalingo_pgdump script to dump the database and upload it to scaleway s3. add cron.json to schedule pgdump everyday at noon. (#270)

bin/scalingo_pgdump.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -eu
+
+mkdir -p /tmp/pgdump
+cd /tmp/pgdump
+
+# Download awscli and verify the integrity
+cat > aws.gpg << _EOF
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG
+ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx
+PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G
+TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz
+gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk
+C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG
+94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO
+lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG
+fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG
+EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX
+XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB
+tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4CGwMF
+CwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQT7Xbd/1cEYuAURraimMQrMRnJHXAUC
+ZqFYbwUJCv/cOgAKCRCmMQrMRnJHXKYuEAC+wtZ611qQtOl0t5spM9SWZuszbcyA
+0xBAJq2pncnp6wdCOkuAPu4/R3UCIoD2C49MkLj9Y0Yvue8CCF6OIJ8L+fKBv2DI
+yWZGmHL0p9wa/X8NCKQrKxK1gq5PuCzi3f3SqwfbZuZGeK/ubnmtttWXpUtuU/Iz
+VR0u/0sAy3j4uTGKh2cX7XnZbSqgJhUk9H324mIJiSwzvw1Ker6xtH/LwdBeJCck
+bVBdh3LZis4zuD4IZeBO1vRvjot3Oq4xadUv5RSPATg7T1kivrtLCnwvqc6L4LnF
+0OkNysk94L3LQSHyQW2kQS1cVwr+yGUSiSp+VvMbAobAapmMJWP6e/dKyAUGIX6+
+2waLdbBs2U7MXznx/2ayCLPH7qCY9cenbdj5JhG9ibVvFWqqhSo22B/URQE/CMrG
++3xXwtHEBoMyWEATr1tWwn2yyQGbkUGANneSDFiTFeoQvKNyyCFTFO1F2XKCcuDs
+19nj34PE2TJilTG2QRlMr4D0NgwLLAMg2Los1CK6nXWnImYHKuaKS9LVaCoC8vu7
+IRBik1NX6SjrQnftk0M9dY+s0ZbAN1gbdjZ8H3qlbl/4TxMdr87m8LP4FZIIo261
+Eycv34pVkCePZiP+dgamEiQJ7IL4ZArio9mv6HbDGV6mLY45+l6/0EzCwkI5IyIf
+BfWC9s/USgxchg==
+=ptgS
+-----END PGP PUBLIC KEY BLOCK-----
+_EOF
+
+gpg --import aws.gpg
+curl -fsS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+curl -fsS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig" -o "awscliv2.sig"
+gpg --verify awscliv2.sig awscliv2.zip
+
+unzip -qq awscliv2.zip
+
+# Configure awscli for scaleway s3
+mkdir -p ~/.aws
+cat > ~/.aws/config << _EOF
+[default]
+region = fr-par
+output = json
+services = scw-fr-par
+
+[services scw-fr-par]
+s3 =
+  endpoint_url = https://s3.fr-par.scw.cloud
+  max_concurrent_requests = 100
+  max_queue_size = 1000
+  multipart_threshold = 50 MB
+  multipart_chunksize = 10 MB
+s3api =
+  endpoint_url = https://s3.fr-par.scw.cloud
+_EOF
+
+# Download postgresql client binaries
+dbclient-fetcher pgsql
+
+# Actually dump and upload to scaleway s3
+DATE=$(date +%Y-%m-%d)
+FILENAME="dump-${DATE}.sql"
+
+pg_dump --clean --if-exists --format c --dbname "${SCALINGO_POSTGRESQL_URL}" --no-owner --no-privileges --no-comments --exclude-schema 'information_schema' --exclude-schema '^pg_*' --file "${FILENAME}"
+
+export AWS_ACCESS_KEY_ID=${BACKUP_PGSQL_S3_KEY}
+export AWS_SECRET_ACCESS_KEY=${BACKUP_PGSQL_S3_SECRET}
+./aws/dist/aws s3 cp ${FILENAME} s3://${BACKUP_PGSQL_S3_BUCKET}/${APP}/

cron.json

@@ -0,0 +1,7 @@
+{
+  "jobs": [
+    {
+      "command": "0 0 * * * bin/scalingo_pgdump.sh"
+    }
+  ]
+}