Commit 50ffcaa5 authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Merge branch 'updating_configs' into 'master'

Fix Jenkins slaves communication

See merge request !12
parents a1777b4b ce3d4462
# This is the fallback list for packages to install. Do not add # This is the fallback list for packages to install. Do not add
# additional packages here. Repositories should use bindep and create # additional packages here. Repositories should use bindep and create
# their own other-requirements.txt files if the list below is not # their own bindep.txt files if the list below is not
# working for them. # working for them.
ant ant
...@@ -10,14 +10,6 @@ curl [!platform:gentoo] ...@@ -10,14 +10,6 @@ curl [!platform:gentoo]
net-misc/curl [platform:gentoo] net-misc/curl [platform:gentoo]
dev-libs/cyrus-sasl [platform:gentoo] dev-libs/cyrus-sasl [platform:gentoo]
cyrus-sasl-devel [platform:rpm] cyrus-sasl-devel [platform:rpm]
app-text/docbook-xsl-stylesheets [platform:gentoo]
docbook-style-xsl [platform:rpm]
app-text/docbook-xml-dtd [platform:gentoo]
docbook-xml [platform:dpkg]
docbook-xsl [platform:dpkg]
docbook5-schemas [platform:rpm]
docbook5-style-xsl [platform:rpm]
docbook5-xml [platform:dpkg]
media-fonts/nanumfont [platform:gentoo] media-fonts/nanumfont [platform:gentoo]
fonts-nanum [platform:dpkg] fonts-nanum [platform:dpkg]
media-fonts/takao-fonts [platform:gentoo] media-fonts/takao-fonts [platform:gentoo]
...@@ -38,7 +30,7 @@ dev-libs/libevent [platform:gentoo] ...@@ -38,7 +30,7 @@ dev-libs/libevent [platform:gentoo]
libffi-dev [platform:dpkg] libffi-dev [platform:dpkg]
libffi-devel [platform:rpm] libffi-devel [platform:rpm]
virtual/libffi [platform:gentoo] virtual/libffi [platform:gentoo]
libjerasure-dev [platform:ubuntu !platform:ubuntu-precise] libjerasure-dev [platform:ubuntu]
dev-libs/jerasure [platform:gentoo] dev-libs/jerasure [platform:gentoo]
libjpeg-dev [platform:dpkg] libjpeg-dev [platform:dpkg]
libjpeg-turbo-devel [platform:rpm] libjpeg-turbo-devel [platform:rpm]
...@@ -88,29 +80,31 @@ postgresql ...@@ -88,29 +80,31 @@ postgresql
postgresql-client [platform:dpkg] postgresql-client [platform:dpkg]
postgresql-devel [platform:rpm] postgresql-devel [platform:rpm]
postgresql-server [platform:rpm] postgresql-server [platform:rpm]
pypy [platform:ubuntu !platform:ubuntu-precise] pypy [platform:ubuntu]
pypy-dev [platform:ubuntu !platform:ubuntu-precise] pypy-dev [platform:ubuntu]
python-dev [platform:dpkg] python-dev [platform:dpkg]
python-devel [platform:rpm] python-devel [platform:rpm]
dev-lang/python [platform:gentoo] dev-lang/python [platform:gentoo]
python-libvirt [platform:dpkg] python-libvirt [platform:dpkg]
python-lxml [!platform:gentoo] python-lxml [!platform:gentoo !platform:fedora]
python2-lxml [platform:fedora]
dev-python/lxml [platform:gentoo] dev-python/lxml [platform:gentoo]
python-zmq [!platform:gentoo] python-zmq [!platform:gentoo !platform:fedora]
python2-zmq [platform:fedora]
dev-python/pyzmq [platform:gentoo] dev-python/pyzmq [platform:gentoo]
# Note that python3-all-dev includes python3-all, added # Note that python3-all-dev includes python3-all, added
# both here for documentary purpose. # both here for documentary purpose.
python3-all [platform:dpkg !platform:ubuntu-precise] python3-all [platform:dpkg]
python3-all-dev [platform:dpkg !platform:ubuntu-precise] python3-all-dev [platform:dpkg]
python3-devel [platform:fedora] python3-devel [platform:fedora]
python34-devel [platform:centos] python34-devel [platform:centos]
redis [platform:rpm] redis [platform:rpm]
redis-server [platform:dpkg] redis-server [platform:dpkg]
dev-db/redis [platform:gentoo] dev-db/redis [platform:gentoo]
rrdtool-devel [platform:rpm] rrdtool-devel [platform:rpm]
ruby [platform:ubuntu !platform:ubuntu-precise] ruby [platform:ubuntu]
dev-lang/ruby [platform:gentoo] dev-lang/ruby [platform:gentoo]
rubygems [platform:rpm platform:ubuntu-precise] rubygems [platform:rpm]
virtual/rubygems [platform:gentoo] virtual/rubygems [platform:gentoo]
sqlite [platform:rpm] sqlite [platform:rpm]
sqlite-devel [platform:rpm] sqlite-devel [platform:rpm]
......
- job-template:
name: 'noop-check-communication'
node: '{node}'
builders:
- shell: |
#!/bin/bash -xe
echo "Hello world, this is the {vendor} Testing System"
- link-logs
publishers:
- devstack-logs
- console-log
#!/bin/bash -xe
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
DOCNAME=releasenotes
DIRECTORY=releasenotes
script_path=/usr/local/jenkins/slave_scripts
# Mapping of language codes to language names
declare -A LANG_NAME=(
["de"]="German"
["en_AU"]="English (Australian)"
["en_GB"]="English (United Kingdom)"
["es"]="Spanish"
["fr"]="French"
["id"]="Indonesian"
["it"]="Italian"
["ja"]="Japanese"
["ko_KR"]="Korean (South Korea)"
["pt_BR"]="Portuguese (Brazil)"
["ru"]="Russian"
["tr_TR"]="Turkish (Turkey)"
["zh_CN"]="Chinese (China)"
)
# This file always exists in OpenStack CI jobs, check for it so that
# it can be used manually as well.
if [ -e "$(pwd)/upper-constraints.txt" ]; then
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
fi
if [ ! -e ${DIRECTORY}/source/locale/ ]; then
echo "No translations found, only building normal release notes"
$script_path/run-tox.sh releasenotes
exit 0
fi
# Check that locale_dirs is really set, otherwise translations
# will not work.
if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
echo "Translations exist and locale_dirs missing in source/conf.py"
exit 1
fi
REFERENCES=`mktemp`
trap "rm -f -- '$REFERENCES'" EXIT
# Extract translations
tox -e venv -- sphinx-build -b gettext \
-d ${DIRECTORY}/build/doctrees.gettext \
${DIRECTORY}/source/ \
${DIRECTORY}/source/locale/
# Add links for translations to index file
cat <<EOF >> ${REFERENCES}
Translated Release Notes
========================
EOF
# Check all language translation resources
for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
# Skip if it is not a valid language translation resource.
if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
continue
fi
language=$(basename $locale)
echo "Building $language translation"
# Prepare all translation resources
for pot in ${DIRECTORY}/source/locale/*.pot ; do
# Get filename
resname=$(basename ${pot} .pot)
# Merge all translation resources. Note this is done the same
# way as done in common_translation_update.sh where we merge
# all strings together in a single file.
msgmerge --silent -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
${pot}
# Compile all translation resources
msgfmt -o \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
done
# Build translated document
tox -e venv -- sphinx-build -b html -D language=${language} \
-d "${DIRECTORY}/build/doctrees.${language}" \
${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
# Reference translated document from index file
if [ ${LANG_NAME["${language}"]+_} ] ; then
name=${LANG_NAME["${language}"]}
name+=" (${language})"
echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
else
echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
fi
# Remove newly created files
git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
# revert changes to po file
git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
done
# Now append our references to the index file. We cannot do this
# earlier since the sphinx commands will read this file.
cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
# Remove newly created pot files
rm -f ${DIRECTORY}/source/locale/*.pot
# Now build releasenotes with reference to translations
$script_path/run-tox.sh releasenotes
# Revert any changes to the index file.
git checkout -- ${DIRECTORY}/source/index.rst
#!/bin/bash -xe
VENV="deploy-guide"
GUIDEDIR="deploy-guide"
#!/bin/bash -xe
VENV="install-guide"
GUIDEDIR="install-guide"
#!/bin/bash -xe #!/bin/bash -xe
# This script is used to publish project-specific install-guide # This script is used to publish project-specific deploy-guide and
# documents to the proper place. Master will be published to a draft # install-guide documents to the proper place. Master will be
# directory, stable/X will be published to the X directory. For # published to a draft directory, stable/X will be published to the X
# example stable/newton documents will life in the newton directory. # directory. For example stable/newton documents will life in the
# newton directory.
venv=install-guide
# You need to pass in the following variables:
# VENV - virtual env to use
# GUIDEDIR - directory for building
if [ -z "$VENV" ] ; then
echo "The variable VENV is not set."
exit 1
fi
if [ -z "$GUIDEDIR" ] ; then
echo "The variable GUIDEDIR is not set."
exit 1
fi
export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
tox -e $venv tox -e $VENV
result=$?
[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip [ -e .tox/$VENV/bin/pbr ] && freezecmd=pbr || freezecmd=pip
echo "Begin pbr freeze output from test virtualenv:" echo "Begin pbr freeze output from test virtualenv:"
echo "======================================================================" echo "======================================================================"
.tox/${venv}/bin/${freezecmd} freeze .tox/${VENV}/bin/${freezecmd} freeze
echo "======================================================================" echo "======================================================================"
MARKER_TEXT="Project: $ZUUL_PROJECT Ref: $ZUUL_REFNAME Build: $ZUUL_UUID" MARKER_TEXT="Project: $ZUUL_PROJECT Ref: $ZUUL_REFNAME Build: $ZUUL_UUID Revision: $ZUUL_NEWREV"
echo $MARKER_TEXT > install-guide/build/html/.root-marker echo $MARKER_TEXT > $GUIDEDIR/build/html/.root-marker
if [ -z "$ZUUL_REFNAME" ]; then if [ -z "$ZUUL_REFNAME" ]; then
TARGET="" TARGET=""
...@@ -43,9 +54,9 @@ fi ...@@ -43,9 +54,9 @@ fi
if [ ! -z $TARGET ] ; then if [ ! -z $TARGET ] ; then
# Move the docs into subdir based on branch # Move the docs into subdir based on branch
TOP=`dirname $TARGET` TOP=`dirname $TARGET`
mv install-guide/build/html install-guide/build/tmp mv $GUIDEDIR/build/html $GUIDEDIR/build/tmp
mkdir -p install-guide/build/html/$TOP mkdir -p $GUIDEDIR/build/html/$TOP
mv install-guide/build/tmp install-guide/build/html/$TARGET mv $GUIDEDIR/build/tmp $GUIDEDIR/build/html/$TARGET
fi fi
exit $result exit
...@@ -28,7 +28,7 @@ except ImportError: ...@@ -28,7 +28,7 @@ except ImportError:
from urlparse import urljoin from urlparse import urljoin
class IniConfig: class IniConfig(object):
"""Object that stores zanata.ini configuration """Object that stores zanata.ini configuration
Read zanata.ini and make its values available. Read zanata.ini and make its values available.
...@@ -61,12 +61,13 @@ class IniConfig: ...@@ -61,12 +61,13 @@ class IniConfig:
setattr(self, item_type, item[1]) setattr(self, item_type, item[1])
class ZanataRestService: class ZanataRestService(object):
def __init__(self, zconfig, content_type='application/xml', verify=True): def __init__(self, zconfig, accept='application/xml',
content_type='application/xml', verify=True):
self.url = zconfig.url self.url = zconfig.url
if "charset" not in content_type: if "charset" not in content_type:
content_type = "%s;charset=utf8" % content_type content_type = "%s;charset=utf8" % content_type
self.headers = {'Accept': content_type, self.headers = {'Accept': accept,
'Content-Type': content_type, 'Content-Type': content_type,
'X-Auth-User': zconfig.username, 'X-Auth-User': zconfig.username,
'X-Auth-Token': zconfig.key} 'X-Auth-Token': zconfig.key}
...@@ -98,7 +99,7 @@ class ZanataRestService: ...@@ -98,7 +99,7 @@ class ZanataRestService:
raise ValueError('Connection error') raise ValueError('Connection error')
class ProjectConfig: class ProjectConfig(object):
"""Object that stores zanata.xml per-project configuration. """Object that stores zanata.xml per-project configuration.
Write out a zanata.xml file for the project given the supplied values. Write out a zanata.xml file for the project given the supplied values.
...@@ -162,8 +163,7 @@ class ProjectConfig: ...@@ -162,8 +163,7 @@ class ProjectConfig:
return xml return xml
def _add_configuration(self, xml): def _add_configuration(self, xml):
""" """Insert additional configuration
Insert additional configuration
Add locale mapping rules to the base zanata.xml retrieved from Add locale mapping rules to the base zanata.xml retrieved from
the server. the server.
......
#!/bin/bash -xe
# This script store release meta information in the git repository for
# a project. It does so on an isolated, hidden branch called
# refs/meta/openstack/release. Because it's not under refs/heads, a
# standard clone won't retrieve it or cause it to show up in the list
# of remote branches. The branch shares no history with the project
# itself; it starts with its own root commit. Jenkins is permitted to
# push directly to refs/meta/openstack/*.
GIT_HOST="review.openstack.org:29418"
PROJECT_PREFIX="openstack"
if [[ ! -e ${PROJECT} ]]; then
git clone ssh://$GIT_HOST/$PROJECT_PREFIX/$PROJECT
fi
cd $PROJECT
git checkout master
# Get the list of meta refs
git fetch origin +refs/meta/*:refs/remotes/meta/*
# Checkout or create the meta/openstack/release branch
if ! { git branch -a |grep ^[[:space:]]*remotes/meta/openstack/release$; }
then
git checkout --orphan release
# Delete everything so the first commit is truly empty:
git rm -rf .
# git rm -rf leaves submodule directories:
find -maxdepth 1 -not -regex '\./\.git\(/.*\)?' -not -name . \
-exec rm -fr {} \;
ls -la
else
git branch -D release || /bin/true
git checkout -b release remotes/meta/openstack/release
fi
# Normally a branch name will just be a file, but we can have branches
# like stable/diablo, so in that case, make the "stable/" directory
# if needed:
mkdir -p $(dirname $BRANCH)
# Read and update the value for the branch
if [ -e "$BRANCH" ]; then
echo "Current contents of ${BRANCH}:"
cat "${BRANCH}"
else
echo "${BRANCH} does not exist. Creating it."
fi
echo "Updating ${BRANCH} to read $VALUE"
echo "$VALUE" > ${BRANCH}
git add ${BRANCH}
git commit -m "Milestone ${BRANCH} set to $VALUE"
git push origin HEAD:refs/meta/openstack/release
...@@ -37,8 +37,8 @@ venv=$(pwd)/$venv_name ...@@ -37,8 +37,8 @@ venv=$(pwd)/$venv_name
install_from_source python-openstackclient install_from_source python-openstackclient
install_from_source python-aodhclient install_from_source python-aodhclient
install_from_source python-barbicanclient install_from_source python-barbicanclient
install_from_source python-cloudkittyclient
install_from_source python-congressclient install_from_source python-congressclient
install_from_source python-cueclient
install_from_source python-designateclient install_from_source python-designateclient
install_from_source python-gnocchiclient install_from_source python-gnocchiclient
install_from_source python-heatclient install_from_source python-heatclient
...@@ -47,10 +47,14 @@ install_from_source python-ironic-inspector-client ...@@ -47,10 +47,14 @@ install_from_source python-ironic-inspector-client
install_from_source python-mistralclient install_from_source python-mistralclient
install_from_source python-muranoclient install_from_source python-muranoclient
install_from_source python-neutronclient install_from_source python-neutronclient
install_from_source python-octaviaclient
install_from_source python-pankoclient
install_from_source python-saharaclient install_from_source python-saharaclient
install_from_source python-searchlightclient install_from_source python-searchlightclient
install_from_source python-senlinclient install_from_source python-senlinclient
install_from_source python-tripleoclient install_from_source python-tripleoclient
install_from_source python-troveclient
install_from_source python-vitrageclient
install_from_source python-watcherclient install_from_source python-watcherclient
install_from_source python-zaqarclient install_from_source python-zaqarclient
......
...@@ -11,10 +11,16 @@ ...@@ -11,10 +11,16 @@
# under the License. # under the License.
# Setup git so that git review works # Setup git so that git review works
function setup_git { function configure_git_review {
git config user.name "OpenStack Proposal Bot" git config user.name "OpenStack Proposal Bot"
git config user.email "openstack-infra@lists.openstack.org" git config user.email "openstack-infra@lists.openstack.org"
git config gitreview.username "proposal-bot" git config gitreview.username "proposal-bot"
}
# Setup git so that git review works, including creating a branch in
# case the repo started with a detached head.
function setup_git {
configure_git_review
# Initial state of repository is detached, create a branch to work # Initial state of repository is detached, create a branch to work
# from. Otherwise git review will complain. # from. Otherwise git review will complain.
......
...@@ -112,6 +112,17 @@ function setup_venv { ...@@ -112,6 +112,17 @@ function setup_venv {
$VENV/bin/pip install -U os-testr $VENV/bin/pip install -U os-testr
} }
# Setup nodejs within the python venv. Match the nodejs version with
# the one used in the nodejs6-npm jobs.
function setup_nodeenv {
$VENV/bin/pip install -U nodeenv
NODE_VENV=$VENV/node_venv
$VENV/bin/nodeenv --node 6.9.4 $NODE_VENV
source $NODE_VENV/bin/activate
}
# Setup a project for Zanata. This is used by both Python and Django projects. # Setup a project for Zanata. This is used by both Python and Django projects.
# syntax: setup_project <project> <zanata_version> <modulename> [<modulename> ...] # syntax: setup_project <project> <zanata_version> <modulename> [<modulename> ...]
function setup_project { function setup_project {
...@@ -236,6 +247,27 @@ function setup_training_guides { ...@@ -236,6 +247,27 @@ function setup_training_guides {
-f zanata.xml -f zanata.xml
} }
# Setup a ReactJS project for Zanata
function setup_reactjs_project {
local project=$1
local version=$2
local exclude='node_modules/**'
setup_nodeenv
# Extract messages
npm install
npm run build
# Transform them into .pot files
npm run json2pot
/usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-p $project -v $version --srcdir . --txdir . \
-r '**/*.pot' '{path}/{locale}.po' \
-e "$exclude" -f zanata.xml
}
# Setup project so that git review works, sets global variable # Setup project so that git review works, sets global variable
# COMMIT_MSG. # COMMIT_MSG.
function setup_review { function setup_review {
...@@ -248,7 +280,7 @@ function setup_review { ...@@ -248,7 +280,7 @@ function setup_review {
Imported Translations from Zanata Imported Translations from Zanata
For more information about this automatic import see: For more information about this automatic import see:
https://wiki.openstack.org/wiki/Translations/Infrastructure http://docs.openstack.org/developer/i18n/reviewing-translation-import.html
EOF EOF
set -e set -e
git review -s git review -s
...@@ -294,17 +326,6 @@ EOF ...@@ -294,17 +326,6 @@ EOF
return $success return $success
} }
# Setup global variables LEVELS and LKEYWORDS
function setup_loglevel_vars {
# Strings for various log levels
LEVELS="info warning error critical"
# Keywords for each log level:
declare -g -A LKEYWORD
LKEYWORD['info']='_LI'
LKEYWORD['warning']='_LW'
LKEYWORD['error']='_LE'
LKEYWORD['critical']='_LC'
}
# Delete empty pot files # Delete empty pot files
function check_empty_pot { function check_empty_pot {
...@@ -314,13 +335,14 @@ function check_empty_pot { ...@@ -314,13 +335,14 @@ function check_empty_pot {
trans=$(msgfmt --statistics -o /dev/null ${pot} 2>&1) trans=$(msgfmt --statistics -o /dev/null ${pot} 2>&1)
if [ "$trans" = "0 translated messages." ] ; then if [ "$trans" = "0 translated messages." ] ; then
rm $pot rm $pot
# Remove file from git if it's under version control. # Remove file from git if it's under version control. We previously
# had all pot files under version control, so remove file also
# from git if needed.
git rm --ignore-unmatch $pot