Commit 4fb0c7d7 authored by Ricardo Araujo's avatar Ricardo Araujo Committed by Ricardo Araujo
Browse files

Update nodepool scripts

parent 62547883
Empty elements. Use these elements as a starting point:
[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/elements)
Pre-cache all of the things bindep can identify in our repos
......@@ -16,6 +16,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
import os
import subprocess
import sys
......@@ -44,7 +47,7 @@ if 'DISTRO_NAME' in os.environ:
# centos7 matches as rhel7 in devstack
RELEASE='rhel7'
if not RELEASE:
print "Can not determine RELEASE"
print("Can not determine RELEASE")
sys.exit(1)
DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
......@@ -53,11 +56,12 @@ IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
print("Running:", cmd)
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
stderr=subprocess.STDOUT, env=newenv)
stderr=subprocess.STDOUT, env=newenv,
universal_newlines=True)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
......@@ -106,8 +110,8 @@ def _find_images(basedir):
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
print("%s failed" % image_tool)
print("Exit: %s, Output: %s" % (returncode, out))
# reset images so we'll fall back
images = []
else:
......@@ -126,7 +130,7 @@ def local_prep(distribution):
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
print('Branch: ', branch)
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
......@@ -160,7 +164,7 @@ def main():
image_filenames.append(fname)
args = dict(
name=fname,
location=os.path.join('/home/jenkins/cache/files', fname),
location=os.path.join('/opt/cache/files', fname),
url=url)
images_list.write(line_template % args)
......
get-pip-py file /home/jenkins/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py
get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py
stackviz file /opt/cache/files/stackviz-latest.tar.gz http://tarballs.openstack.org/package-stackviz-element/stackviz-latest.tar.gz
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -21,10 +20,6 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi
set -e
# Unset the download cache for this invocation to prevent bleed from build host
unset PIP_DOWNLOAD_CACHE
/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
/bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
if [ "$DISTRO_NAME" == "centos" ]; then
yum -y install --enablerepo=epel haveged
fi
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -21,12 +20,8 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi
set -e
export SUDO='true'
export THIN='true'
packages='tox'
prepare-node
mkdir -p ~jenkins/cache/files
mkdir -p ~jenkins/cache/pip
DEBIAN_FRONTEND=noninteractive sudo apt-get update -y && DEBIAN_FRONTEND=noninteractive sudo apt-get install default-jdk -y
for package in $packages ; do
pip install $package
done
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,25 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail sete
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
STACKVIZ_NPM_TIMEOUT="15m"
STACKVIZ_DIR=/opt/stackviz
cp -r /opt/git/openstack/stackviz $STACKVIZ_DIR
cd $STACKVIZ_DIR
timeout $STACKVIZ_NPM_TIMEOUT npm install
ret=$?
if [[ $ret -eq 0 ]]; then
npm run prod && rm -rf node_modules/
else
echo "stackviz build aborted due to npm error"
rm -rf $STACKVIZ_DIR
if [ "$DISTRO_NAME" == "ubuntu" ]; then
rsyslog_dir="$(dirname $0)/../rsyslog.d"
cp -RP $rsyslog_dir/* /etc/rsyslog.d/
fi
exit 0
acpid:
coreutils:
cron:
util-linux:
build-essential:
python-xml:
phase: pre-install.d
python-dev:
python3-dev:
python3:
phase: pre-install.d
uuid-runtime:
traceroute:
ntp:
......@@ -17,7 +22,13 @@ rsyslog:
git:
rsync:
parted:
puppet:
wget:
iputils-ping:
iproute2:
dnsutils:
haveged:
iptables:
centos-release-openstack-ocata:
redhat-rpm-config:
redhat-lsb-core:
curl:
{
"distro": {
"fedora": {
"python3-dev": "python3-devel"
"python3-dev": "python3-devel",
"iptables": "iptables-services",
"haveged": "haveged",
"redhat-rpm-config": "redhat-rpm-config",
"redhat-lsb-core": "redhat-lsb-core"
},
"gentoo": {
"build-essential": "",
"cron": "sys-process/cronie",
"curl": "net-misc/curl",
"ntp": "net-misc/ntp",
"python-dev": "",
"python3-dev": "",
"traceroute": "net-analyzer/traceroute",
"uuid-runtime": ""
},
"centos": {
"centos-release-openstack-ocata": "centos-release-openstack-ocata"
}
},
"family": {
......@@ -18,24 +26,37 @@
"cron": "cronie",
"build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils",
"iproute2": "iproute",
"iputils-ping": "iputils",
"ntp": "ntp ntp-perl",
"python-dev": "python-devel",
"python3-dev": "",
"uuid-runtime": ""
"uuid-runtime": "",
"iptables": "iptables-services",
"haveged": ""
},
"suse": {
"build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils",
"git": "git-core",
"iputils-ping": "iputils",
"ntpdate": "",
"puppet": "ruby2.1-rubygem-puppet",
"python-xml": "python-xml",
"python-dev": "python-devel",
"python3-dev": "python3-devel",
"python3": "python3",
"iptables": "iptables SuSEfirewall2",
"uuid-runtime": "uuidd"
}
},
"default": {
"gentoolkit": ""
"curl": "curl",
"iptables": "iptables-persistent",
"gentoolkit": "",
"centos-release-openstack-ocata": "",
"redhat-rpm-config": "",
"redhat-lsb-core": "",
"python-xml": "",
"python3": ""
}
}
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
systemctl enable haveged.service
;;
openrc)
# TODO(pabelanger): Make sure we support gentoo if we bring images
# online for nodepool.o.o.
exit 0
;;
sysv)
exit 0
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
# Default rules for rsyslog.
#
# For more information see rsyslog.conf(5) and /etc/rsyslog.conf
#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
#daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
#lpr.* -/var/log/lpr.log
mail.* -/var/log/mail.log
#user.* -/var/log/user.log
#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
#mail.info -/var/log/mail.info
#mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err
#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice
#
# Some "catch-all" log files.
#
#*.=debug;\
# auth,authpriv.none;\
# news.none;mail.none -/var/log/debug
#*.=info;*.=notice;*.=warn;\
# auth,authpriv.none;\
# cron,daemon.none;\
# mail,news.none -/var/log/messages
#
# Emergencies are sent to everybody logged in.
#
*.emerg :omusrmsg:*
#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
#daemon.*;mail.*;\
# news.err;\
# *.=debug;*.=info;\
# *.=notice;*.=warn |/dev/xconsole
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# TODO(pabelanger): Once we complete remove puppet from our diskimages, we
# should also remove our default SSH key. Since only 3rdparty CI system would be
# using this element moving forward, it doesn't make sense to bake in our
# default key.
NODEPOOL_SSH_KEY=${NODEPOOL_SSH_KEY:-AAAAB3NzaC1yc2EAAAADAQABAAABAQC6WutNHfM+YdnjeNFeaIpvxqt+9aDn95Ykpmc+fASSjlDZJtOrueH3ch/v08wkE4WQKg03i+t8VonqEwMGmApYA3VzFsURUQbxzlSz5kHlBQSqgz5JTwUmnt1RH5sePL5pkuJ6JgqJ8PxJod6fiD7YDjaKJW/wBzXGnGg2EkgqrkBQXYL4hyaPuSwsQF0Gdwg3QFqXl+R/GrM6FscUkkJzbjqGKI2GhLT8mf2BIMEAiMFhF5Wl4FFrbvhTfPfW+9VdcsiMxCXaxp00n1x1+Y7OqR5AZ/id0Lkz9ZoFVGS901OB/L4xXrvUtI2y+kIYeF6hxfmAl/zhY0eWzwo9lDPz}
if [ -z $NODEPOOL_SSH_KEY ]; then
die "Can not find public key for jenkins user!"
fi
# save the public key inside the chroot
echo "ssh-rsa $NODEPOOL_SSH_KEY" > $TMP_HOOKS_PATH/jenkins-user-ssh-public-key
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# Add jenkins user and group. Note we don't want to rely on
# "useradd"'s group adding behaviour, because it might differ across
# distros.
groupadd jenkins
useradd -g jenkins -m jenkins -s /bin/bash
# a lot of caching happens in extra-data.d (for "historical" reasons).
# We've put the cache stuff into /opt/cache/files, but again, for
# "historical" reasons, ensure this is available in /home/jenkins
#
# Check if the cache exists as we don't have a strict dependency on the
# devstack-cache element. This allows you to build an image without
# incurring the cost of caching all the things.
if [ -d /opt/cache/files ] ; then
mkdir -p /home/jenkins/cache
ln -sf /opt/cache/files /home/jenkins/cache/files
chown -R jenkins:jenkins /opt/cache/files
fi
# this was copied from outside the chroot by extras.d
_pub_key=/tmp/in_target.d/jenkins-user-ssh-public-key
if [ ! -f $_pub_key ]; then
die "Can not find Jenkins public key!"
fi
mkdir /home/jenkins/.ssh
chmod 700 /home/jenkins/.ssh
cp $_pub_key /home/jenkins/.ssh/authorized_keys
chmod 644 /home/jenkins/.ssh/authorized_keys
cat > /home/jenkins/.gitconfig <<EOF
[user]
name = OpenStack Jenkins
email = jenkins@openstack.org
signingkey = jenkins@openstack.org
[gitreview]
rebase = false
username = jenkins
EOF
# cleanup everything to the right owner
chown -R jenkins:jenkins /home/jenkins
......@@ -13,5 +13,7 @@ are. Default:
The image should have the unbound DNS resolver package installed, the
nodepool-base element then configures it to forward DNS queries to:
`NODEPOOL_STATIC_NAMESERVER_V6`, default: `2001:4860:4860::8888`
`NODEPOOL_STATIC_NAMESERVER_V4`, default: `8.8.8.8`.
`NODEPOOL_STATIC_NAMESERVER_V6`, default: `2620:0:ccc::2`
`NODEPOOL_STATIC_NAMESERVER_V4`, default: `208.67.222.222`
`NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK`, default: `2001:4860:4860::8888`
`NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK`, default: `8.8.8.8`.
#!/usr/bin/env python2
#!/bin/bash
#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
......@@ -13,40 +13,11 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
import os
import sys
from subunit2sql.db import api
from subunit2sql import shell
from subunit2sql import write_subunit
DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
if len(sys.argv) == 2:
TEMPEST_PATH = sys.argv[1]
elif len(sys.argv) > 2:
TEMPEST_PATH = sys.argv[1]
DB_URI = sys.argv[2]
else:
TEMPEST_PATH = '/opt/stack/new/tempest'
def main():
shell.parse_args([])
shell.CONF.set_override('connection', DB_URI, group='database')
session = api.get_session()
runs = api.get_recent_successful_runs_by_run_metadata(
'build_name', 'gate-tempest-dsvm-neutron-full',
num_runs=10, session=session)
session.close()
preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams')
if not os.path.isdir(preseed_path):
os.mkdir(preseed_path)
for run in runs:
with open(os.path.join(preseed_path, run.uuid + '.subunit'), 'w') as fd:
write_subunit.sql2subunit(run.uuid, fd)
if __name__ == '__main__':
main()
sudo rm -rf $TARGET_ROOT/root/.cache
jenkins-slave
package-installs
pip-and-virtualenv
puppet
zuul-worker
export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2001:4860:4860::8888}
export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-8.8.8.8}
export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2620:0:ccc::2}
export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-208.67.222.222}
export NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888}
export NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment