Commit 21ce4cd6 authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Updating jenkins configuration scripts

parent f94efcc6
......@@ -10,14 +10,6 @@ curl [!platform:gentoo]
net-misc/curl [platform:gentoo]
dev-libs/cyrus-sasl [platform:gentoo]
cyrus-sasl-devel [platform:rpm]
app-text/docbook-xsl-stylesheets [platform:gentoo]
docbook-style-xsl [platform:rpm]
app-text/docbook-xml-dtd [platform:gentoo]
docbook-xml [platform:dpkg]
docbook-xsl [platform:dpkg]
docbook5-schemas [platform:rpm]
docbook5-style-xsl [platform:rpm]
docbook5-xml [platform:dpkg]
media-fonts/nanumfont [platform:gentoo]
fonts-nanum [platform:dpkg]
media-fonts/takao-fonts [platform:gentoo]
......@@ -38,7 +30,7 @@ dev-libs/libevent [platform:gentoo]
libffi-dev [platform:dpkg]
libffi-devel [platform:rpm]
virtual/libffi [platform:gentoo]
libjerasure-dev [platform:ubuntu !platform:ubuntu-precise]
libjerasure-dev [platform:ubuntu]
dev-libs/jerasure [platform:gentoo]
libjpeg-dev [platform:dpkg]
libjpeg-turbo-devel [platform:rpm]
......@@ -88,29 +80,31 @@ postgresql
postgresql-client [platform:dpkg]
postgresql-devel [platform:rpm]
postgresql-server [platform:rpm]
pypy [platform:ubuntu !platform:ubuntu-precise]
pypy-dev [platform:ubuntu !platform:ubuntu-precise]
pypy [platform:ubuntu]
pypy-dev [platform:ubuntu]
python-dev [platform:dpkg]
python-devel [platform:rpm]
dev-lang/python [platform:gentoo]
python-libvirt [platform:dpkg]
python-lxml [!platform:gentoo]
python-lxml [!platform:gentoo !platform:fedora]
python2-lxml [platform:fedora]
dev-python/lxml [platform:gentoo]
python-zmq [!platform:gentoo]
python-zmq [!platform:gentoo !platform:fedora]
python2-zmq [platform:fedora]
dev-python/pyzmq [platform:gentoo]
# Note that python3-all-dev includes python3-all, added
# both here for documentary purpose.
python3-all [platform:dpkg !platform:ubuntu-precise]
python3-all-dev [platform:dpkg !platform:ubuntu-precise]
python3-all [platform:dpkg]
python3-all-dev [platform:dpkg]
python3-devel [platform:fedora]
python34-devel [platform:centos]
redis [platform:rpm]
redis-server [platform:dpkg]
dev-db/redis [platform:gentoo]
rrdtool-devel [platform:rpm]
ruby [platform:ubuntu !platform:ubuntu-precise]
ruby [platform:ubuntu]
dev-lang/ruby [platform:gentoo]
rubygems [platform:rpm platform:ubuntu-precise]
rubygems [platform:rpm]
virtual/rubygems [platform:gentoo]
sqlite [platform:rpm]
sqlite-devel [platform:rpm]
- job-template:
name: 'noop-check-communication'
node: '{node}'
- shell: |
#!/bin/bash -xe
echo "Hello world, this is the {vendor} Testing System"
- link-logs
- devstack-logs
- console-log
#!/bin/bash -xe
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Mapping of language codes to language names
declare -A LANG_NAME=(
["en_AU"]="English (Australian)"
["en_GB"]="English (United Kingdom)"
["ko_KR"]="Korean (South Korea)"
["pt_BR"]="Portuguese (Brazil)"
["tr_TR"]="Turkish (Turkey)"
["zh_CN"]="Chinese (China)"
# This file always exists in OpenStack CI jobs, check for it so that
# it can be used manually as well.
if [ -e "$(pwd)/upper-constraints.txt" ]; then
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
if [ ! -e ${DIRECTORY}/source/locale/ ]; then
echo "No translations found, only building normal release notes"
$script_path/ releasenotes
exit 0
# Check that locale_dirs is really set, otherwise translations
# will not work.
if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/; then
echo "Translations exist and locale_dirs missing in source/"
exit 1
trap "rm -f -- '$REFERENCES'" EXIT
# Extract translations
tox -e venv -- sphinx-build -b gettext \
-d ${DIRECTORY}/build/doctrees.gettext \
${DIRECTORY}/source/ \
# Add links for translations to index file
cat <<EOF >> ${REFERENCES}
Translated Release Notes
# Check all language translation resources
for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
# Skip if it is not a valid language translation resource.
if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
language=$(basename $locale)
echo "Building $language translation"
# Prepare all translation resources
for pot in ${DIRECTORY}/source/locale/*.pot ; do
# Get filename
resname=$(basename ${pot} .pot)
# Merge all translation resources. Note this is done the same
# way as done in where we merge
# all strings together in a single file.
msgmerge --silent -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
# Compile all translation resources
msgfmt -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
# Build translated document
tox -e venv -- sphinx-build -b html -D language=${language} \
-d "${DIRECTORY}/build/doctrees.${language}" \
${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
# Reference translated document from index file
if [ ${LANG_NAME["${language}"]+_} ] ; then
name+=" (${language})"
echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
# Remove newly created files
git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
# revert changes to po file
git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
# Now append our references to the index file. We cannot do this
# earlier since the sphinx commands will read this file.
cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
# Remove newly created pot files
rm -f ${DIRECTORY}/source/locale/*.pot
# Now build releasenotes with reference to translations
$script_path/ releasenotes
# Revert any changes to the index file.
git checkout -- ${DIRECTORY}/source/index.rst
#!/bin/bash -xe
#!/bin/bash -xe
#!/bin/bash -xe
# This script is used to publish project-specific install-guide
# documents to the proper place. Master will be published to a draft
# directory, stable/X will be published to the X directory. For
# example stable/newton documents will life in the newton directory.
# This script is used to publish project-specific deploy-guide and
# install-guide documents to the proper place. Master will be
# published to a draft directory, stable/X will be published to the X
# directory. For example stable/newton documents will life in the
# newton directory.
# You need to pass in the following variables:
# VENV - virtual env to use
# GUIDEDIR - directory for building
if [ -z "$VENV" ] ; then
echo "The variable VENV is not set."
exit 1
if [ -z "$GUIDEDIR" ] ; then
echo "The variable GUIDEDIR is not set."
exit 1
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
tox -e $venv
tox -e $VENV
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
[ -e .tox/$VENV/bin/pbr ] && freezecmd=pbr || freezecmd=pip
echo "Begin pbr freeze output from test virtualenv:"
echo "======================================================================"
.tox/${venv}/bin/${freezecmd} freeze
.tox/${VENV}/bin/${freezecmd} freeze
echo "======================================================================"
echo $MARKER_TEXT > install-guide/build/html/.root-marker
echo $MARKER_TEXT > $GUIDEDIR/build/html/.root-marker
if [ -z "$ZUUL_REFNAME" ]; then
......@@ -43,9 +54,9 @@ fi
if [ ! -z $TARGET ] ; then
# Move the docs into subdir based on branch
TOP=`dirname $TARGET`
mv install-guide/build/html install-guide/build/tmp
mkdir -p install-guide/build/html/$TOP
mv install-guide/build/tmp install-guide/build/html/$TARGET
mv $GUIDEDIR/build/html $GUIDEDIR/build/tmp
mkdir -p $GUIDEDIR/build/html/$TOP
mv $GUIDEDIR/build/tmp $GUIDEDIR/build/html/$TARGET
exit $result
......@@ -28,7 +28,7 @@ except ImportError:
from urlparse import urljoin
class IniConfig:
class IniConfig(object):
"""Object that stores zanata.ini configuration
Read zanata.ini and make its values available.
......@@ -61,12 +61,13 @@ class IniConfig:
setattr(self, item_type, item[1])
class ZanataRestService:
def __init__(self, zconfig, content_type='application/xml', verify=True):
class ZanataRestService(object):
def __init__(self, zconfig, accept='application/xml',
content_type='application/xml', verify=True):
self.url = zconfig.url
if "charset" not in content_type:
content_type = "%s;charset=utf8" % content_type
self.headers = {'Accept': content_type,
self.headers = {'Accept': accept,
'Content-Type': content_type,
'X-Auth-User': zconfig.username,
'X-Auth-Token': zconfig.key}
......@@ -98,7 +99,7 @@ class ZanataRestService:
raise ValueError('Connection error')
class ProjectConfig:
class ProjectConfig(object):
"""Object that stores zanata.xml per-project configuration.
Write out a zanata.xml file for the project given the supplied values.
......@@ -162,8 +163,7 @@ class ProjectConfig:
return xml
def _add_configuration(self, xml):
Insert additional configuration
"""Insert additional configuration
Add locale mapping rules to the base zanata.xml retrieved from
the server.
#!/bin/bash -xe
# This script store release meta information in the git repository for
# a project. It does so on an isolated, hidden branch called
# refs/meta/openstack/release. Because it's not under refs/heads, a
# standard clone won't retrieve it or cause it to show up in the list
# of remote branches. The branch shares no history with the project
# itself; it starts with its own root commit. Jenkins is permitted to
# push directly to refs/meta/openstack/*.
if [[ ! -e ${PROJECT} ]]; then
git checkout master
# Get the list of meta refs
git fetch origin +refs/meta/*:refs/remotes/meta/*
# Checkout or create the meta/openstack/release branch
if ! { git branch -a |grep ^[[:space:]]*remotes/meta/openstack/release$; }
git checkout --orphan release
# Delete everything so the first commit is truly empty:
git rm -rf .
# git rm -rf leaves submodule directories:
find -maxdepth 1 -not -regex '\./\.git\(/.*\)?' -not -name . \
-exec rm -fr {} \;
ls -la
git branch -D release || /bin/true
git checkout -b release remotes/meta/openstack/release
# Normally a branch name will just be a file, but we can have branches
# like stable/diablo, so in that case, make the "stable/" directory
# if needed:
mkdir -p $(dirname $BRANCH)
# Read and update the value for the branch
if [ -e "$BRANCH" ]; then
echo "Current contents of ${BRANCH}:"
cat "${BRANCH}"
echo "${BRANCH} does not exist. Creating it."
echo "Updating ${BRANCH} to read $VALUE"
echo "$VALUE" > ${BRANCH}
git add ${BRANCH}
git commit -m "Milestone ${BRANCH} set to $VALUE"
git push origin HEAD:refs/meta/openstack/release
......@@ -37,8 +37,8 @@ venv=$(pwd)/$venv_name
install_from_source python-openstackclient
install_from_source python-aodhclient
install_from_source python-barbicanclient
install_from_source python-cloudkittyclient
install_from_source python-congressclient
install_from_source python-cueclient
install_from_source python-designateclient
install_from_source python-gnocchiclient
install_from_source python-heatclient
......@@ -47,10 +47,14 @@ install_from_source python-ironic-inspector-client
install_from_source python-mistralclient
install_from_source python-muranoclient
install_from_source python-neutronclient
install_from_source python-octaviaclient
install_from_source python-pankoclient
install_from_source python-saharaclient
install_from_source python-searchlightclient
install_from_source python-senlinclient
install_from_source python-tripleoclient
install_from_source python-troveclient
install_from_source python-vitrageclient
install_from_source python-watcherclient
install_from_source python-zaqarclient
......@@ -11,10 +11,16 @@
# under the License.
# Setup git so that git review works
function setup_git {
function configure_git_review {
git config "OpenStack Proposal Bot"
git config ""
git config gitreview.username "proposal-bot"
# Setup git so that git review works, including creating a branch in
# case the repo started with a detached head.
function setup_git {
# Initial state of repository is detached, create a branch to work
# from. Otherwise git review will complain.
......@@ -112,6 +112,17 @@ function setup_venv {
$VENV/bin/pip install -U os-testr
# Setup nodejs within the python venv. Match the nodejs version with
# the one used in the nodejs6-npm jobs.
function setup_nodeenv {
$VENV/bin/pip install -U nodeenv
$VENV/bin/nodeenv --node 6.9.4 $NODE_VENV
source $NODE_VENV/bin/activate
# Setup a project for Zanata. This is used by both Python and Django projects.
# syntax: setup_project <project> <zanata_version> <modulename> [<modulename> ...]
function setup_project {
......@@ -236,6 +247,27 @@ function setup_training_guides {
-f zanata.xml
# Setup a ReactJS project for Zanata
function setup_reactjs_project {
local project=$1
local version=$2
local exclude='node_modules/**'
# Extract messages
npm install
npm run build
# Transform them into .pot files
npm run json2pot
/usr/local/jenkins/slave_scripts/ \
-p $project -v $version --srcdir . --txdir . \
-r '**/*.pot' '{path}/{locale}.po' \
-e "$exclude" -f zanata.xml
# Setup project so that git review works, sets global variable
function setup_review {
......@@ -248,7 +280,7 @@ function setup_review {
Imported Translations from Zanata
For more information about this automatic import see:
set -e
git review -s
......@@ -294,17 +326,6 @@ EOF
return $success
# Setup global variables LEVELS and LKEYWORDS
function setup_loglevel_vars {
# Strings for various log levels
LEVELS="info warning error critical"
# Keywords for each log level:
declare -g -A LKEYWORD
# Delete empty pot files
function check_empty_pot {
......@@ -314,13 +335,14 @@ function check_empty_pot {
trans=$(msgfmt --statistics -o /dev/null ${pot} 2>&1)
if [ "$trans" = "0 translated messages." ] ; then
rm $pot
# Remove file from git if it's under version control.
# Remove file from git if it's under version control. We previously
# had all pot files under version control, so remove file also
# from git if needed.
git rm --ignore-unmatch $pot
# Run extract_messages for python projects.
# Needs variables setup via setup_loglevel_vars.
function extract_messages_python {
local modulename=$1
......@@ -341,18 +363,6 @@ function extract_messages_python {
-k "_C:1c,2" -k "_P:1,2" \
-o ${pot} ${modulename}
check_empty_pot ${pot}
# Update the log level .pot files
for level in $LEVELS ; do
$VENV/bin/pybabel ${QUIET} extract --no-default-keywords \
--add-comments Translators: \
--msgid-bugs-address="" \
--project=${PROJECT} --version=${VERSION} \
-k ${LKEYWORD[$level]} \
-o ${pot} ${modulename}
check_empty_pot ${pot}
# Django projects need horizon installed for extraction, install it in
......@@ -402,6 +412,8 @@ function extract_messages_django {
# Extract releasenotes messages
function extract_messages_releasenotes {
local keep_workdir=$1
# Extract messages
tox -e venv -- sphinx-build -b gettext -d releasenotes/build/doctrees \
releasenotes/source releasenotes/work
......@@ -410,7 +422,65 @@ function extract_messages_releasenotes {
mkdir -p releasenotes/source/locale/
msgcat --sort-by-file releasenotes/work/*.pot \
> releasenotes/source/locale/releasenotes.pot
rm -rf releasenotes/work
if [ ! -n "$keep_workdir" ]; then
rm -rf releasenotes/work
# Check releasenote translation progress per language.
# It checks the progress per release. Add the release note translation
# if at least one release is well translated (>= 75%).
# Keep the release note translation in the git repository
# if at least one release is translated >= 40%.
# Otherwise (< 40%) the translation are removed.
# NOTE: this function assume POT files in releasenotes/work
# extracted by extract_messages_releasenotes().
# The workdir should be clean up by the caller.
function check_releasenotes_per_language {
local lang_po=$1
# The expected PO location is
# releasenotes/source/locale/<lang>/LC_MESSAGES/releasenotes.po.
# Extract language name from 4th component.
local lang
lang=$(echo $lang_po | cut -d / -f 4)
local release_pot
local release_name
local workdir=releasenotes/work
local has_high_thresh=0
local has_low_thresh=0
mkdir -p $workdir/$lang
for release_pot in $(find $workdir -name '*.pot'); do
release_name=$(basename $release_pot .pot)
# The index file usually contains small number of words,
# so we skip to check it.
if [ $release_name = "index" ]; then
msgmerge --quiet -o $workdir/$lang/$release_name.po $lang_po $release_pot
check_po_file $workdir/$lang/$release_name.po
if [ $RATIO -ge 75 ]; then
if [ $RATIO -ge 40 ]; then
if ! git ls-files | grep -xq $lang_po; then
if [ $has_high_thresh -eq 0 ]; then
rm -f $lang_po
if [ $has_low_thresh -eq 0 ]; then
git rm -f --ignore-unmatch $lang_po
# Filter out files that we do not want to commit.
......@@ -421,11 +491,28 @@ function filter_commits {
# Don't add new empty files.
for f in $(git diff --cached --name-only --diff-filter=A); do
# Files should have at least one non-empty msgid string.
if ! grep -q 'msgid "[^"]' "$f" ; then