Merge branch 'updating_configs' into 'master'

Fix Jenkins slaves communication

See merge request !12

jenkins/data/bindep-fallback.txt

 # This is the fallback list for packages to install. Do not add
 # additional packages here. Repositories should use bindep and create
-# their own other-requirements.txt files if the list below is not
+# their own bindep.txt files if the list below is not
 # working for them.
 
 ant
@@ -10,14 +10,6 @@ curl [!platform:gentoo]
 net-misc/curl [platform:gentoo]
 dev-libs/cyrus-sasl [platform:gentoo]
 cyrus-sasl-devel [platform:rpm]
-app-text/docbook-xsl-stylesheets [platform:gentoo]
-docbook-style-xsl [platform:rpm]
-app-text/docbook-xml-dtd [platform:gentoo]
-docbook-xml [platform:dpkg]
-docbook-xsl [platform:dpkg]
-docbook5-schemas [platform:rpm]
-docbook5-style-xsl [platform:rpm]
-docbook5-xml [platform:dpkg]
 media-fonts/nanumfont [platform:gentoo]
 fonts-nanum [platform:dpkg]
 media-fonts/takao-fonts [platform:gentoo]
@@ -38,7 +30,7 @@ dev-libs/libevent [platform:gentoo]
 libffi-dev [platform:dpkg]
 libffi-devel [platform:rpm]
 virtual/libffi [platform:gentoo]
-libjerasure-dev [platform:ubuntu !platform:ubuntu-precise]
+libjerasure-dev [platform:ubuntu]
 dev-libs/jerasure [platform:gentoo]
 libjpeg-dev [platform:dpkg]
 libjpeg-turbo-devel [platform:rpm]
@@ -88,29 +80,31 @@ postgresql
 postgresql-client [platform:dpkg]
 postgresql-devel [platform:rpm]
 postgresql-server [platform:rpm]
-pypy [platform:ubuntu !platform:ubuntu-precise]
-pypy-dev [platform:ubuntu !platform:ubuntu-precise]
+pypy [platform:ubuntu]
+pypy-dev [platform:ubuntu]
 python-dev [platform:dpkg]
 python-devel [platform:rpm]
 dev-lang/python [platform:gentoo]
 python-libvirt [platform:dpkg]
-python-lxml [!platform:gentoo]
+python-lxml [!platform:gentoo !platform:fedora]
+python2-lxml [platform:fedora]
 dev-python/lxml [platform:gentoo]
-python-zmq [!platform:gentoo]
+python-zmq [!platform:gentoo !platform:fedora]
+python2-zmq [platform:fedora]
 dev-python/pyzmq [platform:gentoo]
 # Note that python3-all-dev includes python3-all, added
 # both here for documentary purpose.
-python3-all [platform:dpkg !platform:ubuntu-precise]
-python3-all-dev [platform:dpkg !platform:ubuntu-precise]
+python3-all [platform:dpkg]
+python3-all-dev [platform:dpkg]
 python3-devel [platform:fedora]
 python34-devel [platform:centos]
 redis [platform:rpm]
 redis-server [platform:dpkg]
 dev-db/redis [platform:gentoo]
 rrdtool-devel [platform:rpm]
-ruby [platform:ubuntu !platform:ubuntu-precise]
+ruby [platform:ubuntu]
 dev-lang/ruby [platform:gentoo]
-rubygems [platform:rpm platform:ubuntu-precise]
+rubygems [platform:rpm]
 virtual/rubygems [platform:gentoo]
 sqlite [platform:rpm]
 sqlite-devel [platform:rpm]

jenkins/jobs/examples.yaml

-- job-template:
-    name: 'noop-check-communication'
-    node: '{node}'
-
-    builders:
-      - shell: |
-          #!/bin/bash -xe
-          echo "Hello world, this is the {vendor} Testing System"
-      - link-logs
-
-    publishers:
-      - devstack-logs
-      - console-log

jenkins/jobs/include/build-releasenotes.sh

+#!/bin/bash -xe
+
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+DOCNAME=releasenotes
+DIRECTORY=releasenotes
+
+script_path=/usr/local/jenkins/slave_scripts
+
+# Mapping of language codes to language names
+declare -A LANG_NAME=(
+    ["de"]="German"
+    ["en_AU"]="English (Australian)"
+    ["en_GB"]="English (United Kingdom)"
+    ["es"]="Spanish"
+    ["fr"]="French"
+    ["id"]="Indonesian"
+    ["it"]="Italian"
+    ["ja"]="Japanese"
+    ["ko_KR"]="Korean (South Korea)"
+    ["pt_BR"]="Portuguese (Brazil)"
+    ["ru"]="Russian"
+    ["tr_TR"]="Turkish (Turkey)"
+    ["zh_CN"]="Chinese (China)"
+)
+
+# This file always exists in OpenStack CI jobs, check for it so that
+# it can be used manually as well.
+if [ -e "$(pwd)/upper-constraints.txt" ]; then
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+fi
+
+if [ ! -e ${DIRECTORY}/source/locale/ ]; then
+    echo "No translations found, only building normal release notes"
+    $script_path/run-tox.sh releasenotes
+    exit 0
+fi
+
+# Check that locale_dirs is really set, otherwise translations
+# will not work.
+if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then
+    echo "Translations exist and locale_dirs missing in source/conf.py"
+    exit 1
+fi
+
+
+REFERENCES=`mktemp`
+trap "rm -f -- '$REFERENCES'" EXIT
+
+# Extract translations
+tox -e venv -- sphinx-build -b gettext \
+    -d ${DIRECTORY}/build/doctrees.gettext \
+    ${DIRECTORY}/source/ \
+    ${DIRECTORY}/source/locale/
+
+# Add links for translations to index file
+cat <<EOF >> ${REFERENCES}
+
+Translated Release Notes
+========================
+
+EOF
+
+# Check all language translation resources
+for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do
+    # Skip if it is not a valid language translation resource.
+    if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then
+        continue
+    fi
+    language=$(basename $locale)
+
+    echo "Building $language translation"
+
+    # Prepare all translation resources
+    for pot in ${DIRECTORY}/source/locale/*.pot ; do
+        # Get filename
+        resname=$(basename ${pot} .pot)
+
+        # Merge all translation resources. Note this is done the same
+        # way as done in common_translation_update.sh where we merge
+        # all strings together in a single file.
+        msgmerge --silent -o \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \
+            ${pot}
+        # Compile all translation resources
+        msgfmt -o \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \
+            ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po
+    done
+
+    # Build translated document
+    tox -e venv -- sphinx-build -b html -D language=${language} \
+        -d "${DIRECTORY}/build/doctrees.${language}" \
+        ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language}
+
+    # Reference translated document from index file
+    if [ ${LANG_NAME["${language}"]+_} ] ; then
+        name=${LANG_NAME["${language}"]}
+        name+=" (${language})"
+        echo "* \`$name <${language}/index.html>\`__" >> ${REFERENCES}
+    else
+        echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES}
+    fi
+
+    # Remove newly created files
+    git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po
+    git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo
+    # revert changes to po file
+    git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+    git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po
+done
+
+# Now append our references to the index file. We cannot do this
+# earlier since the sphinx commands will read this file.
+cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst
+
+# Remove newly created pot files
+rm -f ${DIRECTORY}/source/locale/*.pot
+
+# Now build releasenotes with reference to translations
+$script_path/run-tox.sh releasenotes
+
+# Revert any changes to the index file.
+git checkout -- ${DIRECTORY}/source/index.rst

jenkins/jobs/include/deploy-guide-pre.sh

+#!/bin/bash -xe
+VENV="deploy-guide"
+GUIDEDIR="deploy-guide"

jenkins/jobs/include/install-guide-pre.sh

+#!/bin/bash -xe
+VENV="install-guide"
+GUIDEDIR="install-guide"

jenkins/scripts/run-install-guide.sh → jenkins/jobs/include/run-project-guide.sh

 #!/bin/bash -xe
 
-# This script is used to publish project-specific install-guide
-# documents to the proper place. Master will be published to a draft
-# directory, stable/X will be published to the X directory. For
-# example stable/newton documents will life in the newton directory.
-
-venv=install-guide
+# This script is used to publish project-specific deploy-guide and
+# install-guide documents to the proper place. Master will be
+# published to a draft directory, stable/X will be published to the X
+# directory. For example stable/newton documents will life in the
+# newton directory.
+
+# You need to pass in the following variables:
+# VENV - virtual env to use
+# GUIDEDIR - directory for building
+
+if [ -z "$VENV" ] ; then
+    echo "The variable VENV is not set."
+    exit 1
+fi
+if [ -z "$GUIDEDIR" ] ; then
+    echo "The variable GUIDEDIR is not set."
+    exit 1
+fi
 
 export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
 
-tox -e $venv
-result=$?
+tox -e $VENV
 
-[ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
+[ -e .tox/$VENV/bin/pbr ] && freezecmd=pbr || freezecmd=pip
 
 echo "Begin pbr freeze output from test virtualenv:"
 echo "======================================================================"
-.tox/${venv}/bin/${freezecmd} freeze
+.tox/${VENV}/bin/${freezecmd} freeze
 echo "======================================================================"
 
-MARKER_TEXT="Project: $ZUUL_PROJECT Ref: $ZUUL_REFNAME Build: $ZUUL_UUID"
-echo $MARKER_TEXT > install-guide/build/html/.root-marker
+MARKER_TEXT="Project: $ZUUL_PROJECT Ref: $ZUUL_REFNAME Build: $ZUUL_UUID Revision: $ZUUL_NEWREV"
+echo $MARKER_TEXT > $GUIDEDIR/build/html/.root-marker
 
 if [ -z "$ZUUL_REFNAME" ]; then
     TARGET=""
@@ -43,9 +54,9 @@ fi
 if [ ! -z $TARGET ] ; then
     # Move the docs into subdir based on branch
     TOP=`dirname $TARGET`
-    mv install-guide/build/html install-guide/build/tmp
-    mkdir -p install-guide/build/html/$TOP
-    mv install-guide/build/tmp install-guide/build/html/$TARGET
+    mv $GUIDEDIR/build/html $GUIDEDIR/build/tmp
+    mkdir -p $GUIDEDIR/build/html/$TOP
+    mv $GUIDEDIR/build/tmp $GUIDEDIR/build/html/$TARGET
 fi
 
-exit $result
+exit

jenkins/scripts/ZanataUtils.py

@@ -28,7 +28,7 @@ except ImportError:
     from urlparse import urljoin
 
 
-class IniConfig:
+class IniConfig(object):
     """Object that stores zanata.ini configuration
 
     Read zanata.ini and make its values available.
@@ -61,12 +61,13 @@ class IniConfig:
                 setattr(self, item_type, item[1])
 
 
-class ZanataRestService:
-    def __init__(self, zconfig, content_type='application/xml', verify=True):
+class ZanataRestService(object):
+    def __init__(self, zconfig, accept='application/xml',
+                 content_type='application/xml', verify=True):
         self.url = zconfig.url
         if "charset" not in content_type:
             content_type = "%s;charset=utf8" % content_type
-        self.headers = {'Accept': content_type,
+        self.headers = {'Accept': accept,
                         'Content-Type': content_type,
                         'X-Auth-User': zconfig.username,
                         'X-Auth-Token': zconfig.key}
@@ -98,7 +99,7 @@ class ZanataRestService:
             raise ValueError('Connection error')
 
 
-class ProjectConfig:
+class ProjectConfig(object):
     """Object that stores zanata.xml per-project configuration.
 
     Write out a zanata.xml file for the project given the supplied values.
@@ -162,8 +163,7 @@ class ProjectConfig:
         return xml
 
     def _add_configuration(self, xml):
-        """
-        Insert additional configuration
+        """Insert additional configuration
 
         Add locale mapping rules to the base zanata.xml retrieved from
         the server.

jenkins/scripts/bump-milestone.sh

-#!/bin/bash -xe
-
-# This script store release meta information in the git repository for
-# a project.  It does so on an isolated, hidden branch called
-# refs/meta/openstack/release.  Because it's not under refs/heads, a
-# standard clone won't retrieve it or cause it to show up in the list
-# of remote branches.  The branch shares no history with the project
-# itself; it starts with its own root commit.  Jenkins is permitted to
-# push directly to refs/meta/openstack/*.
-
-GIT_HOST="review.openstack.org:29418"
-PROJECT_PREFIX="openstack"
-
-if [[ ! -e ${PROJECT} ]]; then
-    git clone ssh://$GIT_HOST/$PROJECT_PREFIX/$PROJECT
-fi
-cd $PROJECT
-git checkout master
-
-# Get the list of meta refs
-git fetch origin +refs/meta/*:refs/remotes/meta/*
-
-# Checkout or create the meta/openstack/release branch
-if ! { git branch -a |grep ^[[:space:]]*remotes/meta/openstack/release$; }
-then
-    git checkout --orphan release
-    # Delete everything so the first commit is truly empty:
-    git rm -rf .
-    # git rm -rf leaves submodule directories:
-    find -maxdepth 1 -not -regex '\./\.git\(/.*\)?' -not -name . \
-        -exec rm -fr {} \;
-    ls -la
-else
-    git branch -D release || /bin/true
-    git checkout -b release remotes/meta/openstack/release
-fi
-
-# Normally a branch name will just be a file, but we can have branches
-# like stable/diablo, so in that case, make the "stable/" directory
-# if needed:
-mkdir -p $(dirname $BRANCH)
-
-# Read and update the value for the branch
-if [ -e "$BRANCH" ]; then
-    echo "Current contents of ${BRANCH}:"
-    cat "${BRANCH}"
-else
-    echo "${BRANCH} does not exist. Creating it."
-fi
-
-echo "Updating ${BRANCH} to read $VALUE"
-echo "$VALUE" > ${BRANCH}
-git add ${BRANCH}
-
-git commit -m "Milestone ${BRANCH} set to $VALUE"
-git push origin HEAD:refs/meta/openstack/release

jenkins/scripts/check-osc-plugins.sh

@@ -37,8 +37,8 @@ venv=$(pwd)/$venv_name
 install_from_source python-openstackclient
 install_from_source python-aodhclient
 install_from_source python-barbicanclient
+install_from_source python-cloudkittyclient
 install_from_source python-congressclient
-install_from_source python-cueclient
 install_from_source python-designateclient
 install_from_source python-gnocchiclient
 install_from_source python-heatclient
@@ -47,10 +47,14 @@ install_from_source python-ironic-inspector-client
 install_from_source python-mistralclient
 install_from_source python-muranoclient
 install_from_source python-neutronclient
+install_from_source python-octaviaclient
+install_from_source python-pankoclient
 install_from_source python-saharaclient
 install_from_source python-searchlightclient
 install_from_source python-senlinclient
 install_from_source python-tripleoclient
+install_from_source python-troveclient
+install_from_source python-vitrageclient
 install_from_source python-watcherclient
 install_from_source python-zaqarclient
 

jenkins/scripts/common.sh

@@ -11,10 +11,16 @@
 # under the License.
 
 # Setup git so that git review works
-function setup_git {
+function configure_git_review {
     git config user.name "OpenStack Proposal Bot"
     git config user.email "openstack-infra@lists.openstack.org"
     git config gitreview.username "proposal-bot"
+}
+
+# Setup git so that git review works, including creating a branch in
+# case the repo started with a detached head.
+function setup_git {
+    configure_git_review
 
     # Initial state of repository is detached, create a branch to work
     # from. Otherwise git review will complain.

jenkins/scripts/common_translation_update.sh

@@ -112,6 +112,17 @@ function setup_venv {
     $VENV/bin/pip install -U os-testr
 }
 
+# Setup nodejs within the python venv. Match the nodejs version with
+# the one used in the nodejs6-npm jobs.
+function setup_nodeenv {
+
+    $VENV/bin/pip install -U nodeenv
+    NODE_VENV=$VENV/node_venv
+    $VENV/bin/nodeenv --node 6.9.4 $NODE_VENV
+    source $NODE_VENV/bin/activate
+
+}
+
 # Setup a project for Zanata. This is used by both Python and Django projects.
 # syntax: setup_project <project> <zanata_version> <modulename> [<modulename> ...]
 function setup_project {
@@ -236,6 +247,27 @@ function setup_training_guides {
         -f zanata.xml
 }
 
+# Setup a ReactJS project for Zanata
+function setup_reactjs_project {
+    local project=$1
+    local version=$2
+
+    local exclude='node_modules/**'
+
+    setup_nodeenv
+
+    # Extract messages
+    npm install
+    npm run build
+    # Transform them into .pot files
+    npm run json2pot
+
+    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+        -p $project -v $version --srcdir . --txdir . \
+        -r '**/*.pot' '{path}/{locale}.po' \
+        -e "$exclude" -f zanata.xml
+}
+
 # Setup project so that git review works, sets global variable
 # COMMIT_MSG.
 function setup_review {
@@ -248,7 +280,7 @@ function setup_review {
 Imported Translations from Zanata
 
 For more information about this automatic import see:
-https://wiki.openstack.org/wiki/Translations/Infrastructure
+http://docs.openstack.org/developer/i18n/reviewing-translation-import.html
 EOF
     set -e
     git review -s
@@ -294,17 +326,6 @@ EOF
     return $success
 }
 
-# Setup global variables LEVELS and LKEYWORDS
-function setup_loglevel_vars {
-    # Strings for various log levels
-    LEVELS="info warning error critical"
-    # Keywords for each log level:
-    declare -g -A LKEYWORD
-    LKEYWORD['info']='_LI'
-    LKEYWORD['warning']='_LW'
-    LKEYWORD['error']='_LE'
-    LKEYWORD['critical']='_LC'
-}
 
 # Delete empty pot files
 function check_empty_pot {
@@ -314,13 +335,14 @@ function check_empty_pot {
     trans=$(msgfmt --statistics -o /dev/null ${pot} 2>&1)
     if [ "$trans" = "0 translated messages." ] ; then
         rm $pot
-        # Remove file from git if it's under version control.
+        # Remove file from git if it's under version control. We previously
+        # had all pot files under version control, so remove file also
+        # from git if needed.
         git rm --ignore-unmatch $pot
     fi
 }
 
 # Run extract_messages for python projects.
-# Needs variables setup via setup_loglevel_vars.
 function extract_messages_python {
     local modulename=$1
 
@@ -341,18 +363,6 @@ function extract_messages_python {
         -k "_C:1c,2" -k "_P:1,2" \
         -o ${pot} ${modulename}
     check_empty_pot ${pot}
-
-    # Update the log level .pot files
-    for level in $LEVELS ; do
-        pot=${modulename}/locale/${modulename}-log-${level}.pot
-        $VENV/bin/pybabel ${QUIET} extract --no-default-keywords \
-            --add-comments Translators: \
-            --msgid-bugs-address="https://bugs.launchpad.net/openstack-i18n/" \
-            --project=${PROJECT} --version=${VERSION} \
-            -k ${LKEYWORD[$level]} \
-            -o ${pot} ${modulename}
-        check_empty_pot ${pot}
-    done
 }
 
 # Django projects need horizon installed for extraction, install it in
@@ -402,6 +412,8 @@ function extract_messages_django {
 
 # Extract releasenotes messages
 function extract_messages_releasenotes {
+    local keep_workdir=$1
+
     # Extract messages
     tox -e venv -- sphinx-build -b gettext -d releasenotes/build/doctrees \
         releasenotes/source releasenotes/work
@@ -410,7 +422,65 @@ function extract_messages_releasenotes {
     mkdir -p releasenotes/source/locale/
     msgcat --sort-by-file releasenotes/work/*.pot \
         > releasenotes/source/locale/releasenotes.pot
-    rm -rf releasenotes/work
+    if [ ! -n "$keep_workdir" ]; then
+        rm -rf releasenotes/work
+    fi
+}
+
+# Check releasenote translation progress per language.
+# It checks the progress per release. Add the release note translation
+# if at least one release is well translated (>= 75%).
+# Keep the release note translation in the git repository
+# if at least one release is translated >= 40%.
+# Otherwise (< 40%) the translation are removed.
+#
+# NOTE: this function assume POT files in releasenotes/work
+# extracted by extract_messages_releasenotes().
+# The workdir should be clean up by the caller.
+function check_releasenotes_per_language {
+    local lang_po=$1
+
+    # The expected PO location is
+    # releasenotes/source/locale/<lang>/LC_MESSAGES/releasenotes.po.
+    # Extract language name from 4th component.
+    local lang
+    lang=$(echo $lang_po | cut -d / -f 4)
+
+    local release_pot
+    local release_name
+    local workdir=releasenotes/work
+
+    local has_high_thresh=0
+    local has_low_thresh=0
+
+    mkdir -p $workdir/$lang
+    for release_pot in $(find $workdir -name '*.pot'); do
+        release_name=$(basename $release_pot .pot)
+        # The index file usually contains small number of words,
+        # so we skip to check it.
+        if [ $release_name = "index" ]; then
+            continue
+        fi
+        msgmerge --quiet -o $workdir/$lang/$release_name.po $lang_po $release_pot
+        check_po_file $workdir/$lang/$release_name.po
+        if [ $RATIO -ge 75 ]; then
+            has_high_thresh=1
+            has_low_thresh=1
+        fi
+        if [ $RATIO -ge 40 ]; then
+            has_low_thresh=1
+        fi
+    done
+
+    if ! git ls-files | grep -xq $lang_po; then
+        if [ $has_high_thresh -eq 0 ]; then
+            rm -f $lang_po
+        fi
+    else
+        if [ $has_low_thresh -eq 0 ]; then
+            git rm -f --ignore-unmatch $lang_po
+        fi
+    fi
 }