GNOME.org
 

[damned-lies] feat: buildah scripts to build runtime and production env

commit f876a4be448031a8ebe02c3f4589f15bbbcfb3b4
Author: Guillaume Bernard <associations guillaume-bernard fr>
Date:   Mon May 9 15:06:21 2022 +0200

    feat: buildah scripts to build runtime and production env

 containers/build_buildah_runtime.sh               | 32 +++++++++
 containers/production/.containerignore            | 14 ++++
 containers/production/build_buildah_production.sh | 84 +++++++++++++++++++++++
 containers/production/entrypoint.sh               | 32 +++++++++
 containers/production/l10n.gnome.org.conf         | 55 +++++++++++++++
 containers/production/local_settings.py           | 61 ++++++++++++++++
 6 files changed, 278 insertions(+)
---
diff --git a/containers/build_buildah_runtime.sh b/containers/build_buildah_runtime.sh
new file mode 100755
index 00000000..87563645
--- /dev/null
+++ b/containers/build_buildah_runtime.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# FEDORA_BASE_IMAGE: the image tag to use to target a specific fedora version
+declare -r FEDORA_BASE_IMAGE=34
+
+# DAMNED_LIES_IMAGE_NAME: the target image name (without any tag)
+declare -r DAMNED_LIES_IMAGE_NAME="damned-lies-runtime"
+
+# CURRENT_VCS_REF_NAME: the name of the current branch, that is used to name the created image
+if [[ -z ${CURRENT_VCS_REF_NAME+x}  ]]; then
+    CURRENT_VCS_REF_NAME="$(git branch --show-current)"
+fi
+declare -r CURRENT_VCS_REF_NAME
+
+#######################################################################################################################
+
+# Stop the shell script if at least one command fails
+set -e
+
+# TODO: remove all non necessary dependencies installed in the image. Why is a g++ compiler required?
+declare -r PACKAGES_TO_INSTALL="python python-devel python-setuptools python-pip python-lxml python 
python3-mysqlclient python3-pillow python3-pyicu mod_wsgi httpd httpd-devel mod_ldap mod_session atlas-devel 
gcc-c++ gcc-gfortran libffi-devel libtool-ltdl enchant wget git gettext gnome-doc-utils intltool itstool 
libicu-devel mariadb-devel subversion yelp-tools automake autoconf make diffutils glibc-langpack-en"
+
+container=$(buildah from fedora:${FEDORA_BASE_IMAGE})
+
+# Install dependencies
+buildah run "${container}" /bin/sh -c  "dnf -y --disablerepo=fedora-modular --disablerepo=updates-modular 
--disablerepo=fedora-cisco-openh264 update"
+buildah run "${container}" /bin/sh -c  "dnf -y --disablerepo=fedora-modular --disablerepo=updates-modular 
--disablerepo=fedora-cisco-openh264 --setopt=tsflags=nodocs install ${PACKAGES_TO_INSTALL}"
+buildah run "${container}" -- dnf -y clean all --enablerepo="*"
+
+# Commit image
+buildah commit "${container}" "${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}-${FEDORA_BASE_IMAGE}"
+buildah tag "${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}-${FEDORA_BASE_IMAGE}" 
"${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}-latest"
diff --git a/containers/production/.containerignore b/containers/production/.containerignore
new file mode 100644
index 00000000..c3cf174c
--- /dev/null
+++ b/containers/production/.containerignore
@@ -0,0 +1,14 @@
+# Generic files: hidden, python cache
+**/.*
+**/*.pyc
+**/__pycache__
+venv*
+
+# Container specific files
+containers
+
+# Project files
+*.md
+setup.*
+
+
diff --git a/containers/production/build_buildah_production.sh 
b/containers/production/build_buildah_production.sh
new file mode 100755
index 00000000..d22c1ff9
--- /dev/null
+++ b/containers/production/build_buildah_production.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# PROCESS_USER: the user that will run the httpd server and be authorised to access application files
+declare -r PROCESS_USER="l10n"
+
+# DAMNED_LIES_CONTAINER_CODE_DIRECTORY: where is located project code
+declare -r DAMNED_LIES_CONTAINER_CODE_DIRECTORY="/l10n"
+
+# THIS_SCRIPT_DIRECTORY: the current script directory name
+THIS_SCRIPT_DIRECTORY="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+declare -r THIS_SCRIPT_DIRECTORY
+
+# DAMNED_LIES_LOCAL_CODE_DIRECTORY: there is located the source code on the local system that build the 
container
+DAMNED_LIES_LOCAL_CODE_DIRECTORY="$(realpath "$(dirname "${BASH_SOURCE[0]}")"/../..)"
+declare -r DAMNED_LIES_LOCAL_CODE_DIRECTORY
+
+# DAMNED_LIES_IMAGE_NAME: the target image name (without any tag)
+if [[ -z ${DAMNED_LIES_IMAGE_NAME+x}  ]]; then
+    declare -r DAMNED_LIES_IMAGE_NAME="damned-lies-production"
+fi
+
+# CURRENT_VCS_REF_NAME: the name of the current branch, that is used to name the created image
+if [[ -z ${CURRENT_VCS_REF_NAME+x}  ]]; then
+    CURRENT_VCS_REF_NAME="$(git branch --show-current)"
+fi
+declare -r CURRENT_VCS_REF_NAME
+
+# RUNTIME_IMAGE_NAME: the runtime image to use as a base in order to build the production image
+if [[ -z ${RUNTIME_IMAGE_NAME+x}  ]]; then
+    declare -r RUNTIME_IMAGE_NAME="damned-lies-runtime:${CURRENT_VCS_REF_NAME}-latest"
+fi
+
+#######################################################################################################################
+
+# Stop the shell script if at least one command fails
+set -e
+
+container=$(buildah from "${RUNTIME_IMAGE_NAME}")
+
+buildah config --author "Guillaume Bernard" "${container}"
+buildah config --label 'maintainer="Guillaume Bernard <associations guillaume-bernard fr>"' "${container}"
+
+# Create PROCESS_USER
+buildah run "${container}" -- groupadd "${PROCESS_USER}" -g 1000660000
+buildah run "${container}" -- useradd "${PROCESS_USER}" -g 1000660000 -u 1000660000 -G apache -r -l -m
+
+# Prepare HTTPD environment to accept our configuration: redirect output to console
+buildah run "${container}" -- sed -ri 's!^(\s*CustomLog)\s+\S+!\1 /proc/self/fd/1!g; 
s!^(\s*ErrorLog)\s+\S+!\1 /proc/self/fd/2!g;' /etc/httpd/conf/httpd.conf
+buildah run "${container}" -- sed -i 's/Listen\ 80/Listen\ 8080/' /etc/httpd/conf/httpd.conf
+buildah run "${container}" -- rm -f /etc/httpd/conf.d/mod_security.conf
+buildah run "${container}" -- chown "${PROCESS_USER}:${PROCESS_USER}" /var/run/httpd
+
+# Install the project in the container
+buildah run "${container}" -- mkdir -p "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"
+buildah add \
+    --chown "${PROCESS_USER}:${PROCESS_USER}" \
+    --contextdir "${DAMNED_LIES_LOCAL_CODE_DIRECTORY}" \
+    --ignorefile "${THIS_SCRIPT_DIRECTORY}/.containerignore" \
+    "${container}" \
+    "${DAMNED_LIES_LOCAL_CODE_DIRECTORY}" "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies
+buildah config --workingdir "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies "${container}"
+
+# Copy project settings files
+buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 660 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/local_settings.py" 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/damnedlies/damnedlies/local_settings.py
+buildah add "${container}" "${THIS_SCRIPT_DIRECTORY}/l10n*.gnome.org.conf" 
/etc/httpd/conf.d/l10n.gnome.org.conf
+
+# Install dependencies in the virtual environment
+buildah run "${container}" -- python3 -m venv --system-site-packages 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv
+buildah run "${container}" -- "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv/bin/pip install --upgrade pip
+buildah run "${container}" -- "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"/venv/bin/pip install -r 
requirements.txt
+
+# Set the image entrypoint
+buildah add --chown "${PROCESS_USER}:${PROCESS_USER}" --chmod 770 "${container}" 
"${THIS_SCRIPT_DIRECTORY}/entrypoint.sh" "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}/entrypoint.sh"
+buildah config --entrypoint "${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}/entrypoint.sh" "${container}"
+
+# Change ownership of HTTPd server and project to the user in the container
+buildah run "${container}" -- chown -R "${PROCESS_USER}":"${PROCESS_USER}" 
"${DAMNED_LIES_CONTAINER_CODE_DIRECTORY}"
+
+# Set the user that will run the process from inside the container
+buildah config --user "${PROCESS_USER}" "${container}"
+
+# Commit container to image
+buildah commit "${container}" "${DAMNED_LIES_IMAGE_NAME}:${CURRENT_VCS_REF_NAME}"
+
diff --git a/containers/production/entrypoint.sh b/containers/production/entrypoint.sh
new file mode 100755
index 00000000..8a1641d3
--- /dev/null
+++ b/containers/production/entrypoint.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+
+sed -i "s/\$DB_NAME/${DB_NAME}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$DB_USER/${DB_USER}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$DB_PASSWORD/${DB_PASSWORD}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$DB_HOST/${DB_HOST}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$DB_PORT/${DB_PORT}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$SECRET_KEY/${SECRET_KEY}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$SENTRY_DSN/${SENTRY_DSN}/" /l10n/damnedlies/damnedlies/local_settings.py
+sed -i "s/\$GITLAB_TOKEN/${GITLAB_TOKEN}/" /l10n/damnedlies/damnedlies/local_settings.py
+
+echo 'Populating .ssh for the l10n user'
+mkdir -p /home/l10n/.ssh
+ln -s /home/l10n/.ssh_secret/ssh-privatekey /home/l10n/.ssh/ssh-privatekey
+ln -s /home/l10n/.ssh_secret/.gitconfig /home/l10n/.gitconfig
+wget https://gitlab.gnome.org/Infrastructure/damned-lies/-/raw/oscp/httpd/ssh_config -O 
/home/l10n/.ssh/config
+
+echo 'Applying migrations'
+/l10n/venv/bin/python manage.py migrate
+
+echo 'Generating static files'
+/l10n/venv/bin/python manage.py collectstatic
+
+echo 'Compiling translations'
+/l10n/venv/bin/python manage.py compile-trans
+
+if [ $# -eq 0 ]; then
+  exec httpd -DFOREGROUND
+else
+  exec $*
+fi
diff --git a/containers/production/l10n.gnome.org.conf b/containers/production/l10n.gnome.org.conf
new file mode 100644
index 00000000..11208d87
--- /dev/null
+++ b/containers/production/l10n.gnome.org.conf
@@ -0,0 +1,55 @@
+<VirtualHost *:8080>
+    ServerName l10n.gnome.org
+
+    Alias /HTML /var/www/djamnedlies/data/scratchdir/HTML
+    Alias /POT /var/www/djamnedlies/data/scratchdir/POT
+    Alias /static /l10n/damnedlies/static
+    Alias /media /var/www/djamnedlies/data/media
+    Alias /robots.txt /l10n/damnedlies/static/robots.txt
+
+    # mod_wsgi
+    WSGIDaemonProcess l10n display-name=%{GROUP} processes=4 home=/l10n python-home=/l10n/venv 
python-path=/l10n/damnedlies user=l10n group=l10n
+    WSGIProcessGroup l10n
+    WSGIScriptAlias / /l10n/damnedlies/damnedlies/wsgi.py
+
+    # Map .po and .pot files accordingly to the correct Content-Type,
+    # and also make sure they are rendered as utf-8.
+    AddCharset UTF-8 .po
+    AddCharset UTF-8 .pot
+    AddType text/plain .po
+    AddType text/plain .pot
+
+<Directory "/l10n/damnedlies/">
+    # Avoid sending request to Django when host is not correct.
+    Require expr %{HTTP_HOST} == "l10n.gnome.org"
+</Directory>
+
+<DirectoryMatch "/l10n/damnedlies/(static|damnedlies)">
+    Options Indexes
+    Require all granted
+</DirectoryMatch> 
+
+<Directory "/var/www/djamnedlies/data/media">
+    Options Indexes
+    Require all granted
+</Directory> 
+
+<DirectoryMatch "^/var/www/djamnedlies/data/scratchdir/(POT|HTML)">
+    Options Indexes
+    <RequireAll>
+      Require all granted
+      Require not ip 95.108.249.30
+      Require not ip 218.30.103.149
+      Require not ip 65.55.24.217
+      Require not ip 180.76.5.0/24
+      Require not ip 218.30.103.0/24
+      Require not ip 1.202.219.0/24
+    </RequireAll>
+</DirectoryMatch>
+</VirtualHost>
+
+<LocationMatch "^/vertimus">
+  <If "%{HTTP_USER_AGENT} =~ /Sogou web spider/">
+    Require all denied
+  </If>
+</LocationMatch>
diff --git a/containers/production/local_settings.py b/containers/production/local_settings.py
new file mode 100644
index 00000000..ab604a05
--- /dev/null
+++ b/containers/production/local_settings.py
@@ -0,0 +1,61 @@
+from pathlib import Path
+
+DEBUG = False
+STATIC_SERVE = False
+USE_DJANGO_OPENID = False
+
+ADMINS = (
+    ('Guillaume Bernard', 'associations guillaume-bernard fr'),
+)
+
+MANAGERS = ADMINS
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME'  : '$DB_NAME',
+        'USER'  : '$DB_USER',
+        'PASSWORD': '$DB_PASSWORD',
+        'HOST'  : '$DB_HOST',
+        'PORT'  : '$DB_PORT',
+        'CONN_MAX_AGE': 300,
+        'OPTIONS': {
+            'charset': 'utf8mb4',
+        }
+    }
+}
+SECRET_KEY = '$SECRET_KEY'
+ALLOWED_HOSTS=['l10n.gnome.org', '8.43.85.13', '8.43.85.14', '8.43.85.29']
+SESSION_COOKIE_SECURE = True
+CSRF_COOKIE_SECURE = True
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+
+EMAIL_HOST = 'smtp-int.gnome.org'
+EMAIL_SUBJECT_PREFIX = '[DL]'
+DEFAULT_FROM_EMAIL = 'noreply gnome org'
+SERVER_EMAIL = 'gnomeweb gnome org'
+
+TIME_ZONE = 'UTC'
+
+DATADIR = Path('/var/www/djamnedlies/data/')
+MEDIA_ROOT = DATADIR / 'media'
+
+# Local directory path for VCS checkout
+SCRATCHDIR = DATADIR / "scratchdir"
+POTDIR = SCRATCHDIR / "POT"
+LOCK_DIR = DATADIR / "locks"
+
+ADMIN_GROUP = "coordination_team"
+
+UPLOAD_ARCHIVED_DIR = 'upload-backup'
+FILE_UPLOAD_PERMISSIONS = 0o644
+
+#ITSTOOL_PATH = '/usr/local/www/gnomeweb/local/bin/'
+
+# Sentry configuration (needs pip install raven)
+RAVEN_CONFIG = {
+    "dsn": "$SENTRY_DSN"
+}
+
+GITLAB_TOKEN = "$GITLAB_TOKEN"
+VCS_HOME_REGEX = "gitlab\.gnome\.org"
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]