GNOME.org
 

[pango/ci-improved: 3/3] Improve the Pango CI pipeline on Linux

commit 7b66f9604ddb4743276f38b6a17f10bf4d446583
Author: Emmanuele Bassi <ebassi gnome org>
Date:   Tue Apr 14 11:48:55 2020 +0100

    Improve the Pango CI pipeline on Linux
    
    Use a scaled down version of the GTK and GLib CI set ups, with a custom
    Docker image and multiple stages. The CI pipeline now generates a JUnit
    report, which integrates with the GitLab merge requests; builds the
    documentation as a separate stage; and publishes the API reference for
    the master branch at the end.

 .gitlab-ci.yml                   |  91 +++++++++++++++-----------
 .gitlab-ci/README.md             |  38 +++++++++++
 .gitlab-ci/fedora.Dockerfile     |  49 ++++++++++++++
 .gitlab-ci/meson-junit-report.py | 112 ++++++++++++++++++++++++++++++++
 .gitlab-ci/run-docker.sh         | 135 +++++++++++++++++++++++++++++++++++++++
 .gitlab-ci/run-tests.sh          |  29 +++++++++
 6 files changed, 416 insertions(+), 38 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d7a8e629..ee379f24 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,44 +1,35 @@
-image: registry.gitlab.gnome.org/gnome/gnome-runtime-images/base:latest
-
 stages:
   - build
+  - docs
+  - deploy
 
-#gnome-runtime-linux:
-#  stage: build
-#  before_script:
-#    - >
-#      dnf install -y \
-#        glibc-langpack-en \
-#        gcc \
-#        gcc-c++ \
-#        meson \
-#        redhat-rpm-config \
-#        glib2-devel \
-#        harfbuzz-devel \
-#        fribidi-devel \
-#        cairo-devel \
-#        libthai-devel \
-#        gobject-introspection-devel \
-#        cairo-gobject-devel \
-#        abattis-cantarell-fonts \
-#        google-droid-sans-fonts \
-#        thai-scalable-waree-fonts \
-#        desktop-file-utils \
-#        diffutils
-#  script:
-#    - meson _build
-#    - ninja -C _build
-#    - meson test -C_build --suite pango
-#    - _build/utils/pango-list --verbose > _build/fontlist.txt
-#    - _build/tests/test-font -p /pango/font/metrics --verbose
-#    - _build/utils/pango-view --no-display --output _build/hello.png utils/HELLO.txt
-#  artifacts:
-#    name: "%CI_JOB_NAME%-%CI_COMMIT_REF_NAME%"
-#    when: always
-#    paths:
-#      - _build/meson-logs
-#      - _build/hello.png
-#      - _build/fontlist.txt
+# Common variables
+variables:
+  COMMON_MESON_FLAGS: "--fatal-meson-warnings --werror"
+  MESON_TEST_TIMEOUT_MULTIPLIER: 2
+
+linux-fedora:
+  image: registry.gitlab.gnome.org/gnome/pango/fedora:v1
+  stage: build
+  variables:
+    EXTRA_MESON_FLAGS: "--buildtype=debug --default-library=both"
+  script:
+    - meson ${COMMON_MESON_FLAGS} ${EXTRA_MESON_FLAGS}
+            _build
+    - meson _build
+    - ninja -C _build
+    - .gitlab-ci/run-tests.sh _build
+  artifacts:
+    when: always
+    reports:
+      junit:
+        - "${CI_PROJECT_DIR}/_build/report.xml"
+    name: "gtk-${CI_COMMIT_REF_NAME}"
+    paths:
+      - "${CI_PROJECT_DIR}/_build/meson-logs"
+      - "${CI_PROJECT_DIR}/_build/report.xml"
+      - "${CI_PROJECT_DIR}/_build/hello.png"
+      - "${CI_PROJECT_DIR}/_build/fontlist.txt"
 
 msys2-mingw64:
   stage: build
@@ -59,3 +50,27 @@ msys2-mingw64:
       - _build/meson-logs
       - _build/hello.png
       - _build/fontlist.txt
+
+reference:
+  image: registry.gitlab.gnome.org/gnome/pango/fedora:v1
+  stage: docs
+  variables:
+    EXTRA_MESON_FLAGS: ""
+  script:
+    - meson ${COMMON_MESON_FLAGS} ${EXTRA_MESON_FLAGS} -Dgtk_doc=true _build
+    - ninja -C _build pango-doc
+    - mkdir -p _reference/
+    - mv _build/docs/html/ _reference/
+  artifacts:
+    paths:
+      - _reference
+
+pages:
+  stage: deploy
+  script:
+    - mv _reference/ public/
+  artifacts:
+    paths:
+      - public
+  only:
+    - master
diff --git a/.gitlab-ci/README.md b/.gitlab-ci/README.md
new file mode 100644
index 00000000..a5c7736d
--- /dev/null
+++ b/.gitlab-ci/README.md
@@ -0,0 +1,38 @@
+## Pango CI infrastructure
+
+Pango uses different CI images depending on platform and jobs.
+
+The CI images are Docker containers, generated either using `docker` or
+`podman`, and pushed to the GitLab [container registry][registry].
+
+Each Docker image has a tag composed of two parts:
+
+ - `${image}`: the base image for a given platform, like "fedora" or
+   "debian-stable"
+ - `${number}`: an incremental version number, or `latest`
+
+See the [container registry][registry] for the available images for each
+branch, as well as their available versions.
+
+### Checklist for Updating a CI image
+
+ - [ ] Update the `${image}.Dockerfile` file with the dependencies
+ - [ ] Run `./run-docker.sh build --base ${image} --base-version ${number}`
+ - [ ] Run `./run-docker.sh push --base ${image} --base-version ${number}`
+   once the Docker image is built; you may need to log in by using
+   `docker login` or `podman login`
+ - [ ] Update the `image` keys in the `.gitlab-ci.yml` file with the new
+   image tag
+ - [ ] Open a merge request with your changes and let it run
+
+### Checklist for Adding a new CI image
+
+ - [ ] Write a new `${image}.Dockerfile` with the instructions to set up
+   a build environment
+ - [ ] Add the `pip3 install meson` incantation
+ - [ ] Run `./run-docker.sh build --base ${image} --base-version ${number}`
+ - [ ] Run `./run-docker.sh push --base ${image} --base-version ${number}`
+ - [ ] Add the new job to `.gitlab-ci.yml` referencing the image
+ - [ ] Open a merge request with your changes and let it run
+
+[registry]: https://gitlab.gnome.org/GNOME/pango/container_registry
diff --git a/.gitlab-ci/fedora.Dockerfile b/.gitlab-ci/fedora.Dockerfile
new file mode 100644
index 00000000..c1aadc93
--- /dev/null
+++ b/.gitlab-ci/fedora.Dockerfile
@@ -0,0 +1,49 @@
+FROM fedora:31
+
+RUN dnf -y install \
+    abattis-cantarell-fonts \
+    cairo-devel \
+    cairo-gobject-devel \
+    ccache \
+    clang \
+    clang-analyzer \
+    desktop-file-utils \
+    diffutils \
+    fribidi-devel \
+    gcc \
+    gcc-c++ \
+    gettext \
+    git \
+    glib2-devel \
+    glib2-static \
+    glibc-devel \
+    glibc-headers \
+    glibc-langpack-en \
+    gobject-introspection-devel \
+    google-droid-sans-fonts \
+    gtk-doc \
+    harfbuzz-devel \
+    hicolor-icon-theme \
+    itstool \
+    lcov \
+    libthai-devel \
+    ninja-build \
+    python3 \
+    python3-jinja2 \
+    python3-pip \
+    python3-pygments \
+    python3-wheel \
+    redhat-rpm-config \
+    thai-scalable-waree-fonts \
+ && dnf clean all
+
+RUN pip3 install meson==0.53.1
+
+ARG HOST_USER_ID=5555
+ENV HOST_USER_ID ${HOST_USER_ID}
+RUN useradd -u $HOST_USER_ID -ms /bin/bash user
+
+USER user
+WORKDIR /home/user
+
+ENV LANG C.UTF-8
diff --git a/.gitlab-ci/meson-junit-report.py b/.gitlab-ci/meson-junit-report.py
new file mode 100755
index 00000000..fc96efba
--- /dev/null
+++ b/.gitlab-ci/meson-junit-report.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+# Turns a Meson testlog.json file into a JUnit XML report
+#
+# Copyright 2019  GNOME Foundation
+#
+# SPDX-License-Identifier: LGPL-2.1-or-later
+#
+# Original author: Emmanuele Bassi
+
+import argparse
+import datetime
+import json
+import os
+import sys
+import xml.etree.ElementTree as ET
+
+aparser = argparse.ArgumentParser(description='Turns a Meson test log into a JUnit report')
+aparser.add_argument('--project-name', metavar='NAME',
+                     help='The project name',
+                     default='unknown')
+aparser.add_argument('--job-id', metavar='ID',
+                     help='The job ID for the report',
+                     default='Unknown')
+aparser.add_argument('--branch', metavar='NAME',
+                     help='Branch of the project being tested',
+                     default='master')
+aparser.add_argument('--output', metavar='FILE',
+                     help='The output file, stdout by default',
+                     type=argparse.FileType('w', encoding='UTF-8'),
+                     default=sys.stdout)
+aparser.add_argument('infile', metavar='FILE',
+                     help='The input testlog.json, stdin by default',
+                     type=argparse.FileType('r', encoding='UTF-8'),
+                     default=sys.stdin)
+
+args = aparser.parse_args()
+
+outfile = args.output
+
+testsuites = ET.Element('testsuites')
+testsuites.set('id', '{}/{}'.format(args.job_id, args.branch))
+testsuites.set('package', args.project_name)
+testsuites.set('timestamp', datetime.datetime.utcnow().isoformat(timespec='minutes'))
+
+suites = {}
+for line in args.infile:
+    data = json.loads(line)
+    unit_name = data['name']
+    suite_name = args.project_name
+    project_name = args.project_name
+
+    duration = data['duration']
+    return_code = data['returncode']
+    result = data['result']
+    log = data['stdout']
+
+    unit = {
+        'suite': suite_name,
+        'name': unit_name,
+        'duration': duration,
+        'returncode': return_code,
+        'result': result,
+        'stdout': log,
+    }
+
+    units = suites.setdefault(suite_name, [])
+    units.append(unit)
+
+for name, units in suites.items():
+    print('Processing suite {} (units: {})'.format(name, len(units)))
+
+    def if_failed(unit):
+        if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT']:
+            return True
+        return False
+
+    def if_succeded(unit):
+        if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']:
+            return True
+        return False
+
+    successes = list(filter(if_succeded, units))
+    failures = list(filter(if_failed, units))
+    print(' - {}: {} pass, {} fail'.format(name, len(successes), len(failures)))
+
+    testsuite = ET.SubElement(testsuites, 'testsuite')
+    testsuite.set('name', '{}/{}'.format(args.project_name, name))
+    testsuite.set('tests', str(len(units)))
+    testsuite.set('errors', str(len(failures)))
+    testsuite.set('failures', str(len(failures)))
+
+    for unit in successes:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+    for unit in failures:
+        testcase = ET.SubElement(testsuite, 'testcase')
+        testcase.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        testcase.set('name', unit['name'])
+        testcase.set('time', str(unit['duration']))
+
+        failure = ET.SubElement(testcase, 'failure')
+        failure.set('classname', '{}/{}'.format(args.project_name, unit['suite']))
+        failure.set('name', unit['name'])
+        failure.set('type', 'error')
+        failure.text = unit['stdout']
+
+output = ET.tostring(testsuites, encoding='unicode')
+outfile.write(output)
diff --git a/.gitlab-ci/run-docker.sh b/.gitlab-ci/run-docker.sh
new file mode 100755
index 00000000..06c4f63d
--- /dev/null
+++ b/.gitlab-ci/run-docker.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+read_arg() {
+    # $1 = arg name
+    # $2 = arg value
+    # $3 = arg parameter
+    local rematch='^[^=]*=(.*)$'
+    if [[ $2 =~ $rematch ]]; then
+        read "$1" <<< "${BASH_REMATCH[1]}"
+    else
+        read "$1" <<< "$3"
+        # There is no way to shift our callers args, so
+        # return 1 to indicate they should do it instead.
+        return 1
+    fi
+}
+
+set -e
+
+build=0
+run=0
+push=0
+list=0
+print_help=0
+no_login=0
+
+while (($# > 0)); do
+        case "${1%%=*}" in
+                build) build=1;;
+                run) run=1;;
+                push) push=1;;
+                list) list=1;;
+                help) print_help=1;;
+                --base|-b) read_arg base "$@" || shift;;
+                --base-version) read_arg base_version "$@" || shift;;
+                --no-login) no_login=1;;
+                *) echo -e "\e[1;31mERROR\e[0m: Unknown option '$1'"; exit 1;;
+        esac
+        shift
+done
+
+if [ $print_help == 1 ]; then
+        echo "$0 - Build and run Docker images"
+        echo ""
+        echo "Usage: $0 <command> [options] [basename]"
+        echo ""
+        echo "Available commands"
+        echo ""
+        echo "  build --base=<BASENAME> - Build Docker image <BASENAME>.Dockerfile"
+        echo "  run --base=<BASENAME>   - Run Docker image <BASENAME>"
+        echo "  push --base=<BASENAME>  - Push Docker image <BASENAME> to the registry"
+        echo "  list                    - List available images"
+        echo "  help                    - This help message"
+        echo ""
+        exit 0
+fi
+
+cd "$(dirname "$0")"
+
+if [ $list == 1 ]; then
+        echo "Available Docker images:"
+        for f in *.Dockerfile; do
+                filename=$( basename -- "$f" )
+                basename="${filename%.*}"
+
+                echo -e "  \e[1;39m$basename\e[0m"
+        done
+        exit 0
+fi
+
+# All commands after this require --base to be set
+if [ -z $base ]; then
+        echo "Usage: $0 <command>"
+        exit 1
+fi
+
+if [ ! -f "$base.Dockerfile" ]; then
+        echo -e "\e[1;31mERROR\e[0m: Dockerfile for '$base' not found"
+        exit 1
+fi
+
+if [ -z $base_version ]; then
+        base_version="latest"
+else
+        base_version="v$base_version"
+fi
+
+if [ ! -x "$(command -v docker)" ] || [ docker --help |& grep -q podman ]; then
+        # Docker is actually implemented by podman, and its OCI output
+        # is incompatible with some of the dockerd instances on GitLab
+        # CI runners.
+        echo "Using: Podman"
+        format="--format docker"
+        CMD="podman"
+else
+        echo "Using: Docker"
+        format=""
+        CMD="sudo socker"
+fi
+
+REGISTRY="registry.gitlab.gnome.org"
+TAG="${REGISTRY}/gnome/pango/${base}:${base_version}"
+
+if [ $build == 1 ]; then
+        echo -e "\e[1;32mBUILDING\e[0m: ${base} as ${TAG}"
+        ${CMD} build \
+                ${format} \
+                --build-arg HOST_USER_ID="$UID" \
+                --tag "${TAG}" \
+                --file "${base}.Dockerfile" .
+        exit $?
+fi
+
+if [ $push == 1 ]; then
+        echo -e "\e[1;32mPUSHING\e[0m: ${base} as ${TAG}"
+
+        if [ $no_login == 0 ]; then
+                ${CMD} login ${REGISTRY}
+        fi
+
+        ${CMD} push ${TAG}
+        exit $?
+fi
+
+if [ $run == 1 ]; then
+        echo -e "\e[1;32mRUNNING\e[0m: ${base} as ${TAG}"
+        ${CMD} run \
+                --rm \
+                --volume "$(pwd)/..:/home/user/app" \
+                --workdir "/home/user/app" \
+                --tty \
+                --interactive "${TAG}" \
+                bash
+        exit $?
+fi
diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh
new file mode 100755
index 00000000..cca589b4
--- /dev/null
+++ b/.gitlab-ci/run-tests.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set +x
+set +e
+
+srcdir=$( pwd )
+builddir=$1
+
+meson test -C ${builddir} \
+        --print-errorlogs \
+        --suite=pango 
+
+# Store the exit code for the CI run, but always
+# generate the reports
+exit_code=$?
+
+cd ${builddir}
+
+./utils/pango-list --verbose > fontlist.txt
+./tests/test-font -p /pango/font/metrics --verbose
+./utils/pango-view --no-display --output hello.png ${srcdir}/utils/HELLO.txt
+
+$srcdir/.gitlab-ci/meson-junit-report.py \
+        --project-name=pango \
+        --job-id="${CI_JOB_NAME}" \
+        --output=report.xml \
+        meson-logs/testlog.json
+
+exit $exit_code
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]