Commit 62547883 authored by Ricardo Araujo's avatar Ricardo Araujo Committed by Ricardo Araujo
Browse files

Update jenkins scripts

parent ba2ad877
......@@ -3,8 +3,6 @@
# their own bindep.txt files if the list below is not
# working for them.
ant
asciidoc
build-essential [platform:dpkg]
curl [!platform:gentoo]
net-misc/curl [platform:gentoo]
......@@ -15,25 +13,28 @@ fonts-nanum [platform:dpkg]
media-fonts/takao-fonts [platform:gentoo]
fonts-takao [platform:dpkg]
gawk
gettext
gettext [!platform:suse]
gettext-runtime [platform:suse]
graphviz [!platform:gentoo]
media-gfx/graphviz [platform:gentoo]
language-pack-en [platform:ubuntu]
libcurl-devel [platform:rpm]
libcurl4-gnutls-dev [platform:dpkg]
liberasurecode-dev [platform:dpkg]
liberasurecode-devel [platform:rpm]
liberasurecode-devel [platform:rpm !platform:opensuseproject]
dev-libs/liberasurecode [platform:gentoo]
libevent-dev [platform:dpkg]
libevent-devel [platform:rpm]
dev-libs/libevent [platform:gentoo]
libffi-dev [platform:dpkg]
libffi-devel [platform:rpm]
libffi-devel [platform:redhat]
libffi48-devel [platform:suse]
virtual/libffi [platform:gentoo]
libjerasure-dev [platform:ubuntu]
dev-libs/jerasure [platform:gentoo]
libjpeg-dev [platform:dpkg]
libjpeg-turbo-devel [platform:rpm]
libjpeg8-devel [platform:suse]
libjpeg-turbo-devel [platform:redhat]
media-libs/libjpeg-turbo [platform:gentoo]
libldap2-dev [platform:dpkg]
net-nds/openldap [platform:gentoo]
......@@ -45,7 +46,8 @@ libpq-dev [platform:dpkg]
librrd-dev [platform:dpkg]
net-analyzer/rrdtool [platform:gentoo]
libsasl2-dev [platform:dpkg]
libselinux-python [platform:rpm]
libselinux-python [platform:redhat]
python-selinux [platform:suse]
sys-libs/libselinux [platform:gentoo]
libsqlite3-dev [platform:dpkg]
libuuid-devel [platform:rpm]
......@@ -63,18 +65,20 @@ libxslt1-dev [platform:dpkg]
dev-libs/libxslt [platform:gentoo]
locales [platform:debian]
mariadb [platform:rpm]
mariadb-devel [platform:rpm]
mariadb-server [platform:rpm]
mariadb-server [platform:redhat]
mariadb-devel [platform:redhat]
libmysqlclient-devel [platform:suse]
dev-db/mariadb [platform:gentoo]
memcached
mongodb [platform:dpkg]
mongodb-server [platform:rpm]
mongodb-server [platform:rpm !platform:opensuseproject]
dev-db/mongodb [platform:gentoo]
mysql-client [platform:dpkg]
mysql-server [platform:dpkg]
openldap-devel [platform:rpm]
pkg-config [platform:dpkg]
pkgconfig [platform:rpm]
openldap-devel [platform:redhat]
openldap2-devel [platform:suse]
pkg-config [platform:dpkg platform:suse]
pkgconfig [platform:redhat]
virtual/pkgconfig [platform:gentoo]
postgresql
postgresql-client [platform:dpkg]
......@@ -89,25 +93,33 @@ python-libvirt [platform:dpkg]
python-lxml [!platform:gentoo !platform:fedora]
python2-lxml [platform:fedora]
dev-python/lxml [platform:gentoo]
python-zmq [!platform:gentoo !platform:fedora]
python-zmq [!platform:gentoo !platform:fedora !platform:suse]
python2-zmq [platform:fedora]
dev-python/pyzmq [platform:gentoo]
# Note that python3-all-dev includes python3-all, added
# both here for documentary purpose.
python3-all [platform:dpkg]
python3-all-dev [platform:dpkg]
python3-devel [platform:fedora]
python34-devel [platform:centos]
python3-devel [platform:fedora platform:suse]
# python3-devel does not pull in the python3 package on openSUSE so
# we need to be explicit. The python3 package contains the XML module
# which is required by a python3 virtualenv. Similarly, in python2,
# the XML module is located in python-xml which is not pulled in
# by python-devel as well. See https://bugzilla.suse.com/show_bug.cgi?id=1046990
python3 [platform:suse]
python-xml [platform:suse]
redis [platform:rpm]
redis-server [platform:dpkg]
dev-db/redis [platform:gentoo]
rrdtool-devel [platform:rpm]
ruby [platform:ubuntu]
dev-lang/ruby [platform:gentoo]
rubygems [platform:rpm]
rubygems [platform:redhat]
ruby2.3-devel [platform:suse]
virtual/rubygems [platform:gentoo]
sqlite [platform:rpm]
sqlite-devel [platform:rpm]
sqlite [platform:redhat]
sqlite-devel [platform:redhat]
sqlite3-devel [platform:suse]
sqlite3 [platform:dpkg]
dev-db/sqlite [platform:gentoo]
swig
......
#!/bin/bash -xe
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
cp /var/log/orchestra/rsyslog/$host/syslog $WORKSPACE/logs/$host-syslog.txt
done
#!/bin/bash -xe
WORKSPACE=$(pwd)
mkdir -p logs
rm -f logs/*
cd $(dirname "$0")
echo "Jenkins: resetting hosts..."
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
scp lvm-kexec-reset.sh root@$host:/var/tmp/
ssh root@$host /var/tmp/lvm-kexec-reset.sh
sudo rm -f /var/log/orchestra/rsyslog/$host/syslog
done
# Have rsyslog reopen log files we rm'd from under it
sudo restart rsyslog
# wait for the host to come up (2 ping responses or timeout after 5 minutes)
echo "Jenkins: Waiting for head host to return after reset..."
sleep 10
if ! timeout 300 ./ping.py $HEAD_HOST; then
echo "Jenkins: ERROR: Head node did not come back up after reset"
exit 1
fi
echo "Jenkins: Pre-populating PIP cache"
for host in $HEAD_HOST ${COMPUTE_HOSTS//,/ }; do
scp -r ~/cache/pip root@$host:/var/cache/pip
done
echo "Jenkins: Caching images."
cd ~/devstack
source stackrc
for image_url in ${IMAGE_URLS//,/ }; do
# Downloads the image (uec ami+aki style), then extracts it.
IMAGE_FNAME=$(echo "$image_url" | python -c "import sys; print sys.stdin.read().split('/')[-1]")
IMAGE_NAME=$(echo "$IMAGE_FNAME" | python -c "import sys; print sys.stdin.read().split('.tar.gz')[0].split('.tgz')[0]")
if [ ! -f files/$IMAGE_FNAME ]; then
wget -c $image_url -O files/$IMAGE_FNAME
fi
done
echo "Jenkins: Executing build_bm_multi.sh."
./tools/build_bm_multi.sh
#!/bin/bash -xe
set -x
sudo cobbler sync
sudo cobbler system edit --netboot-enabled=Y --name=baremetal1
sudo cobbler system edit --netboot-enabled=Y --name=baremetal2
sudo cobbler system edit --netboot-enabled=Y --name=baremetal3
sudo cobbler system edit --netboot-enabled=Y --name=baremetal4
sudo cobbler system edit --netboot-enabled=Y --name=baremetal5
sudo cobbler system edit --netboot-enabled=Y --name=baremetal6
sudo cobbler system edit --netboot-enabled=Y --name=baremetal7
sudo cobbler system edit --netboot-enabled=Y --name=baremetal8
sudo cobbler system edit --netboot-enabled=Y --name=baremetal9
sudo cobbler system reboot --name=baremetal1
sudo cobbler system reboot --name=baremetal2
sudo cobbler system reboot --name=baremetal3
sudo cobbler system reboot --name=baremetal4
sudo cobbler system reboot --name=baremetal5
sudo cobbler system reboot --name=baremetal6
sudo cobbler system reboot --name=baremetal7
sudo cobbler system reboot --name=baremetal8
sudo cobbler system reboot --name=baremetal9
......@@ -40,10 +40,10 @@ install_from_source python-barbicanclient
install_from_source python-cloudkittyclient
install_from_source python-congressclient
install_from_source python-designateclient
install_from_source python-gnocchiclient
install_from_source python-heatclient
install_from_source python-ironicclient
install_from_source python-ironic-inspector-client
install_from_source python-karborclient
install_from_source python-mistralclient
install_from_source python-muranoclient
install_from_source python-neutronclient
......
......@@ -133,23 +133,10 @@ function setup_project {
local exclude='.tox/**'
# For projects with one module on stable/mitaka, we use "old" setup.
# Note that stable/mitaka is only stable translated branch for
# projects.
# TODO(jaegerandi): Remove once mitaka translation ends.
if [ $# -eq 1 ] && [ "$version" == "stable-mitaka" ] ; then
local modulename=$1
/usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-p $project -v $version --srcdir $modulename/locale \
--txdir $modulename/locale \
-r '**/*.pot' '{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
-f zanata.xml
else
/usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-p $project -v $version --srcdir . --txdir . \
-r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
-e "$exclude" -f zanata.xml
fi
}
......@@ -247,6 +234,21 @@ function setup_training_guides {
-f zanata.xml
}
# Setup a i18n project for Zanata
function setup_i18n {
local project=i18n
local version=${1:-master}
# Update the .pot file
tox -e generatepot
/usr/local/jenkins/slave_scripts/create-zanata-xml.py \
-p $project -v $version \
--srcdir doc/source/locale \
--txdir doc/source/locale \
-f zanata.xml
}
# Setup a ReactJS project for Zanata
function setup_reactjs_project {
local project=$1
......@@ -280,7 +282,7 @@ function setup_review {
Imported Translations from Zanata
For more information about this automatic import see:
http://docs.openstack.org/developer/i18n/reviewing-translation-import.html
https://docs.openstack.org/i18n/latest/reviewing-translation-import.html
EOF
set -e
git review -s
......
#!/bin/bash -xe
if [ -z "$PROJECT" ]; then
echo '$PROJECT not set.'
exit 1
fi
case "$ZUUL_REFNAME" in
master)
export PPAS="ppa:openstack-ppa/bleeding-edge"
;;
milestone-proposed)
export PPAS="ppa:openstack-ppa/milestone-proposed"
;;
*)
echo "No PPA defined for branch $ZUUL_REFNAME"
exit 0
esac
HUDSON=http://localhost:8080/
# We keep packaging for openstack trunk in lp:~o-u-p/$project/ubuntu
# For a release (diablo, essex), it's in lp:~o-u-p/$project/$release
OPENSTACK_RELEASE=${OPENSTACK_RELEASE:-ubuntu}
BZR_BRANCH=${BZR_BRANCH:-lp:~openstack-ubuntu-packagers/$PROJECT/${OPENSTACK_RELEASE}}
PPAS=${PPAS:-ppa:$PROJECT-core/trunk}
PACKAGING_REVNO=${PACKAGING_REVNO:--1}
series=${series:-lucid}
cd build
tarball="$(echo dist/$PROJECT*.tar.gz)"
version="${tarball%.tar.gz}"
version="${version#*$PROJECT-}"
base_version=$version
if [ -n "${EXTRAVERSION}" ]; then
version="${version%~*}${EXTRAVERSION}~${version#*~}"
fi
tar xvzf "${tarball}"
echo ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
ln -s "${tarball}" "${PROJECT}_${version}.orig.tar.gz"
# Overlay packaging
# (Intentionally using the natty branch. For these PPA builds, we don't need to diverge
# (yet, at least), so it makes the branch management easier this way.
# Note: Doing a checkout and deleting .bzr afterwards instead of just doing an export,
# because export refuses to overlay over an existing directory, so this was easier.
# (We need to not have the .bzr in there, otherwise vcsversion.py might get overwritten)
echo bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-*
bzr checkout -r ${PACKAGING_REVNO} --lightweight $BZR_BRANCH $PROJECT-*
cd $PROJECT-*
if [ -d .git ]; then
PACKAGING_REVNO="$(git log --oneline | wc -l)"
rm -rf .git
else
PACKAGING_REVNO="$(bzr revno --tree)"
rm -rf .bzr
fi
# Please don't change this. It's the only way I'll get notified
# if an upload fails.
export DEBFULLNAME="Soren Hansen"
export DEBEMAIL="soren@openstack.org"
buildno=$BUILD_NUMBER
pkgversion="${version}-0ubuntu0~${series}${buildno}"
dch -b --force-distribution --v "${pkgversion}" \
"Automated PPA build. Packaging revision: ${PACKAGING_REVNO}." -D $series
dpkg-buildpackage -rfakeroot -S -sa -nc -k32EE128C
if ! [ "$DO_UPLOAD" = "no" ]; then
for ppa in $PPAS; do
dput --force $ppa "../${PROJECT}_${pkgversion}_source.changes"
done
fi
cd ..
#!/bin/bash -xe
RETRY_LIMIT=20
# Keep fetching until this uuid appears in the logs before uploading
END_UUID=$(cat /proc/sys/kernel/random/uuid)
echo "Grabbing consoleLog ($END_UUID)"
# Since we are appending to fetched logs, clear any possibly old runs
# Don't add a newline so we end up with a 0 byte file.
echo -n > /tmp/console.html
# Grab the HTML version of the log (includes timestamps)
TRIES=0
console_log_path='logText/progressiveHtml'
while ! grep -q "$END_UUID" /tmp/console.html; do
TRIES=$((TRIES+1))
if [ $TRIES -gt $RETRY_LIMIT ]; then
echo "Failed grabbing consoleLog within $RETRY_LIMIT retries."
break
fi
sleep 3
# -X POST because Jenkins doesn't do partial gets properly when
# job is running.
# --data start=X instructs Jenkins to mimic a partial get using
# POST. We determine how much data we need based on
# how much we already have.
# --fail will cause curl to not output data if the request
# fails. This allows us to retry when we have Jenkins proxy
# errors without polluting the output document.
# --insecure because our Jenkins masters use self signed SSL certs.
curl -X POST --data "start=$(stat -c %s /tmp/console.html || echo 0)" \
--fail --insecure $BUILD_URL$console_log_path \
>> /tmp/console.html || true
done
# We need to add <pre> tags around the output for log-osanalyze to not escape
# the content
sed -i '1s/^/<pre>\n/' /tmp/console.html
echo "</pre>" >> /tmp/console.html
......@@ -49,7 +49,10 @@ until $BINDEP -b -f $PACKAGES test; do
elif emerge --version >/dev/null 2>&1 ; then
sudo emerge -uDNq --jobs=4 @world
sudo PATH=/usr/sbin:/sbin:$PATH emerge -q --jobs=4 \
`$BINDEP -b -f $PACKAGES`
`$BINDEP -b -f $PACKAGES test`
elif zypper --version >/dev/null 2>&1 ; then
sudo PATH=/usr/sbin:/sbin:$PATH zypper --non-interactive install \
`$BINDEP -b -f $PACKAGES test`
else
sudo PATH=/usr/sbin:/sbin:$PATH $YUM install -y \
`$BINDEP -b -f $PACKAGES test`
......
......@@ -16,25 +16,28 @@
# under the License.
# Find out if jenkins has attempted to run any sudo commands by checking
# the auth.log or secure log files before and after a test run.
# the auth.log or secure log or messages files before and after a test run.
PATTERN="sudo.*jenkins.*:.*\(incorrect password attempts\|command not allowed\)"
case $( facter osfamily ) in
Debian)
if [ -f /var/log/auth.log ]; then
OLDLOGFILE=/var/log/auth.log.1
LOGFILE=/var/log/auth.log
;;
RedHat)
elif [ -f /var/log/secure ]; then
OLDLOGFILE=$( ls /var/log/secure-* | sort | tail -n1 )
LOGFILE=/var/log/secure
;;
esac
elif [ -f /var/log/messages ]; then
OLDLOGFILE=$( ls /var/log/messages-* | sort | tail -n1 )
LOGFILE=/var/log/messages
else
echo "*** Could not find auth.log/secure/messages log for sudo tracing"
exit 1
fi
case "$1" in
pre)
rm -fr /tmp/jenkins-sudo-log
mkdir /tmp/jenkins-sudo-log
if [ -f $OLDLOGFILE ]; then
if [ -f "$OLDLOGFILE" ]; then
stat -c %Y $OLDLOGFILE > /tmp/jenkins-sudo-log/mtime-pre
else
echo "0" > /tmp/jenkins-sudo-log/mtime-pre
......@@ -43,7 +46,7 @@ case "$1" in
exit 0
;;
post)
if [ -f $OLDLOGFILE ]; then
if [ -f "$OLDLOGFILE" ]; then
stat -c %Y $OLDLOGFILE > /tmp/jenkins-sudo-log/mtime-post
else
echo "0" > /tmp/jenkins-sudo-log/mtime-post
......
......@@ -179,6 +179,8 @@ def main():
for k, entries in global_reqs.items():
# Discard the lines: we don't need them.
global_reqs[k] = set(r for (r, line) in entries)
blacklist = requirement.parse(
open(reqdir + '/blacklist.txt', 'rt').read())
cwd = os.getcwd()
# build a list of requirements in the proposed change,
# and check them for style violations while doing so
......@@ -221,6 +223,11 @@ def main():
reqs == branch_reqs.reqs[name]):
# Unchanged [or a change that preserves a current value]
continue
if name in blacklist:
# Blacklisted items are not synced and are managed
# by project teams as they see fit, so no further
# testing is needed.
continue
if name not in global_reqs:
failed = True
print("Requirement %s not in openstack/requirements" %
......
......@@ -106,6 +106,19 @@ function propose_training_guides {
git_add_po_files doc/upstream-training/source/locale
}
# Propose updates for i18n
function propose_i18n {
# Pull updated translations from Zanata.
pull_from_zanata "$PROJECT"
# Cleanup po and pot files
cleanup_module "doc"
# Add all changed files to git
git_add_po_files doc/source/locale
}
# Propose updates for python and django projects
function propose_python_django {
......@@ -264,6 +277,10 @@ case "$PROJECT" in
setup_training_guides "$ZANATA_VERSION"
propose_training_guides
;;
i18n)
setup_i18n "$ZANATA_VERSION"
propose_i18n
;;
tripleo-ui)
setup_reactjs_project "$PROJECT" "$ZANATA_VERSION"
propose_reactjs
......
......@@ -65,6 +65,20 @@ elif [ "$OWN_PROJECT" == "puppet-openstack-constraints" ] ; then
function update {
bash /usr/local/jenkins/slave_scripts/generate_puppetfile.sh
}
elif [ "$OWN_PROJECT" == "openstack-ansible-tests" ] ; then
INITIAL_COMMIT_MSG="Updated from OpenStack Ansible Tests"
TOPIC="openstack/openstack-ansible-tests/sync-tests"
PROJECTS=$(./gen-projects-list.sh)
function update {
bash /usr/local/jenkins/slave_scripts/sync_openstack_ansible_common_files.sh $1
}
elif [ "$OWN_PROJECT" == "os-service-types" ] ; then
INITIAL_COMMIT_MSG="Updated from OpenStack Service Type Authority"
TOPIC="openstack/os-service-types/sync-service-types-authority"
PROJECTS="openstack/os-service-types"
function update {
curl https://service-types.openstack.org/service-types.json > os_service_types/data/service-types.json
}
else
echo "Unknown project $1" >2
exit 1
......
......@@ -44,7 +44,7 @@ cat - > releasenotes/source/${SERIES}.rst <<EOF
===================================
.. release-notes::
:branch: origin/$NEW_BRANCH
:branch: $NEW_BRANCH
EOF
# Look at the indentation of the existing entries and reuse the same
......
#!/usr/bin/env python
#!/usr/bin/env python3
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
......@@ -15,8 +15,6 @@
"""Lists the branches in the modified deliverable files.
"""
from __future__ import print_function
import argparse
import os.path
import re
......@@ -35,7 +33,7 @@ def find_modified_deliverable_files(reporoot):
results = subprocess.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'],
cwd=reporoot,
)
).decode('utf-8')
filenames = [
l.strip()
for l in results.splitlines()
......@@ -65,7 +63,7 @@ def get_modified_deliverable_file_content(reporoot, filenames):
if not os.path.exists(filename):
# The file must have been deleted, skip it.
continue
with open(filename, 'r') as f:
with open(filename, 'r', encoding='utf-8') as f:
deliverable_data = yaml.load(f.read())
# Map the release version to the release contents so we can
......@@ -76,17 +74,18 @@ def get_modified_deliverable_file_content(reporoot, filenames):
}
for branch in deliverable_data.get('branches', []):
branch_type = branch['name'].split('/')[0]
location = branch['location']
if branch_type == 'stable':
for proj in releases_by_version[location]['projects']:
yield (proj['repo'], branch['name'], branch['location'])
else:
if isinstance(location, dict):
for repo, sha in sorted(location.items()):
yield (repo, branch['name'], sha)
else:
# Assume a single location string that is a valid
# reference in the git repository.
for proj in releases_by_version[location]['projects']:
yield (proj['repo'], branch['name'], branch['location'])
def main():
parser = argparse.ArgumentParser()
......
#!/usr/bin/env python
#!/usr/bin/env python3
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
......@@ -15,8 +15,6 @@
"""Lists the most recent versions in the deliverable files.
"""
from __future__ import print_function
import argparse
import os.path
import re
......@@ -35,7 +33,7 @@ def find_modified_deliverable_files(reporoot):
results = subprocess.check_output(
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^'],
cwd=reporoot,
)
).decode('utf-8')
filenames = [
l.strip()
for l in results.splitlines()
......@@ -66,7 +64,7 @@ def get_modified_deliverable_file_content(reporoot, filenames):
if not os.path.exists(filename):
# The file must have been deleted, skip it.
continue
with open(filename, 'r') as f:
with open(filename, 'r', encoding='utf-8') as f:
deliverable_data = yaml.load(f.read())
# If there are no releases listed in this file, skip it.
......
pbr>=1.4
requests>=1.1
PyYAML>=3.1.0
zuul
simplejson>=2.2.0