Commit 4fb0c7d7 authored by Ricardo Araujo's avatar Ricardo Araujo Committed by Ricardo Araujo
Browse files

Update nodepool scripts

parent 62547883
Empty elements. Use these elements as a starting point:
[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/elements)
Pre-cache all of the things bindep can identify in our repos
...@@ -16,6 +16,9 @@ ...@@ -16,6 +16,9 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import print_function
from __future__ import unicode_literals
import os import os
import subprocess import subprocess
import sys import sys
...@@ -44,7 +47,7 @@ if 'DISTRO_NAME' in os.environ: ...@@ -44,7 +47,7 @@ if 'DISTRO_NAME' in os.environ:
# centos7 matches as rhel7 in devstack # centos7 matches as rhel7 in devstack
RELEASE='rhel7' RELEASE='rhel7'
if not RELEASE: if not RELEASE:
print "Can not determine RELEASE" print("Can not determine RELEASE")
sys.exit(1) sys.exit(1)
DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack') DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
...@@ -53,11 +56,12 @@ IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images') ...@@ -53,11 +56,12 @@ IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
def run_local(cmd, status=False, cwd='.', env={}): def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd print("Running:", cmd)
newenv = os.environ newenv = os.environ
newenv.update(env) newenv.update(env)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd, p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
stderr=subprocess.STDOUT, env=newenv) stderr=subprocess.STDOUT, env=newenv,
universal_newlines=True)
(out, nothing) = p.communicate() (out, nothing) = p.communicate()
if status: if status:
return (p.returncode, out.strip()) return (p.returncode, out.strip())
...@@ -106,8 +110,8 @@ def _find_images(basedir): ...@@ -106,8 +110,8 @@ def _find_images(basedir):
if os.path.exists(image_tool): if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True) returncode, out = run_local(image_tool, status=True)
if returncode: if returncode:
print "%s failed" % image_tool print("%s failed" % image_tool)
print "Exit: %s, Output: %s" % (returncode, out) print("Exit: %s, Output: %s" % (returncode, out))
# reset images so we'll fall back # reset images so we'll fall back
images = [] images = []
else: else:
...@@ -126,7 +130,7 @@ def local_prep(distribution): ...@@ -126,7 +130,7 @@ def local_prep(distribution):
if ' -> ' in branch: if ' -> ' in branch:
continue continue
branch_data = {'name': branch} branch_data = {'name': branch}
print 'Branch: ', branch print('Branch: ', branch)
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK) run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
...@@ -160,7 +164,7 @@ def main(): ...@@ -160,7 +164,7 @@ def main():
image_filenames.append(fname) image_filenames.append(fname)
args = dict( args = dict(
name=fname, name=fname,
location=os.path.join('/home/jenkins/cache/files', fname), location=os.path.join('/opt/cache/files', fname),
url=url) url=url)
images_list.write(line_template % args) images_list.write(line_template % args)
......
get-pip-py file /home/jenkins/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py
stackviz file /opt/cache/files/stackviz-latest.tar.gz http://tarballs.openstack.org/package-stackviz-element/stackviz-latest.tar.gz
#!/bin/bash #!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -21,10 +20,6 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then ...@@ -21,10 +20,6 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi fi
set -e set -e
# Unset the download cache for this invocation to prevent bleed from build host if [ "$DISTRO_NAME" == "centos" ]; then
unset PIP_DOWNLOAD_CACHE yum -y install --enablerepo=epel haveged
fi
/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
/bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
#!/bin/bash #!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -21,12 +20,8 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then ...@@ -21,12 +20,8 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi fi
set -e set -e
export SUDO='true' packages='tox'
export THIN='true'
prepare-node for package in $packages ; do
pip install $package
mkdir -p ~jenkins/cache/files done
mkdir -p ~jenkins/cache/pip
DEBIAN_FRONTEND=noninteractive sudo apt-get update -y && DEBIAN_FRONTEND=noninteractive sudo apt-get install default-jdk -y
#!/bin/bash #!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -15,25 +14,13 @@ ...@@ -15,25 +14,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# dib-lint: disable=setu setpipefail sete # dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x set -x
fi fi
set -e
STACKVIZ_NPM_TIMEOUT="15m" if [ "$DISTRO_NAME" == "ubuntu" ]; then
STACKVIZ_DIR=/opt/stackviz rsyslog_dir="$(dirname $0)/../rsyslog.d"
cp -r /opt/git/openstack/stackviz $STACKVIZ_DIR cp -RP $rsyslog_dir/* /etc/rsyslog.d/
cd $STACKVIZ_DIR
timeout $STACKVIZ_NPM_TIMEOUT npm install
ret=$?
if [[ $ret -eq 0 ]]; then
npm run prod && rm -rf node_modules/
else
echo "stackviz build aborted due to npm error"
rm -rf $STACKVIZ_DIR
fi fi
exit 0
acpid:
coreutils: coreutils:
cron: cron:
util-linux: util-linux:
build-essential: build-essential:
python-xml:
phase: pre-install.d
python-dev: python-dev:
python3-dev: python3-dev:
python3:
phase: pre-install.d
uuid-runtime: uuid-runtime:
traceroute: traceroute:
ntp: ntp:
...@@ -17,7 +22,13 @@ rsyslog: ...@@ -17,7 +22,13 @@ rsyslog:
git: git:
rsync: rsync:
parted: parted:
puppet:
wget: wget:
iputils-ping: iputils-ping:
iproute2:
dnsutils: dnsutils:
haveged:
iptables:
centos-release-openstack-ocata:
redhat-rpm-config:
redhat-lsb-core:
curl:
{ {
"distro": { "distro": {
"fedora": { "fedora": {
"python3-dev": "python3-devel" "python3-dev": "python3-devel",
"iptables": "iptables-services",
"haveged": "haveged",
"redhat-rpm-config": "redhat-rpm-config",
"redhat-lsb-core": "redhat-lsb-core"
}, },
"gentoo": { "gentoo": {
"build-essential": "", "build-essential": "",
"cron": "sys-process/cronie", "cron": "sys-process/cronie",
"curl": "net-misc/curl",
"ntp": "net-misc/ntp", "ntp": "net-misc/ntp",
"python-dev": "", "python-dev": "",
"python3-dev": "", "python3-dev": "",
"traceroute": "net-analyzer/traceroute", "traceroute": "net-analyzer/traceroute",
"uuid-runtime": "" "uuid-runtime": ""
},
"centos": {
"centos-release-openstack-ocata": "centos-release-openstack-ocata"
} }
}, },
"family": { "family": {
...@@ -18,24 +26,37 @@ ...@@ -18,24 +26,37 @@
"cron": "cronie", "cron": "cronie",
"build-essential": "glibc-devel gcc make", "build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils", "dnsutils": "bind-utils",
"iproute2": "iproute",
"iputils-ping": "iputils", "iputils-ping": "iputils",
"ntp": "ntp ntp-perl", "ntp": "ntp ntp-perl",
"python-dev": "python-devel", "python-dev": "python-devel",
"python3-dev": "", "python3-dev": "",
"uuid-runtime": "" "uuid-runtime": "",
"iptables": "iptables-services",
"haveged": ""
}, },
"suse": { "suse": {
"build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils", "dnsutils": "bind-utils",
"git": "git-core", "git": "git-core",
"iputils-ping": "iputils", "iputils-ping": "iputils",
"ntpdate": "", "ntpdate": "",
"puppet": "ruby2.1-rubygem-puppet", "python-xml": "python-xml",
"python-dev": "python-devel", "python-dev": "python-devel",
"python3-dev": "python3-devel", "python3-dev": "python3-devel",
"python3": "python3",
"iptables": "iptables SuSEfirewall2",
"uuid-runtime": "uuidd" "uuid-runtime": "uuidd"
} }
}, },
"default": { "default": {
"gentoolkit": "" "curl": "curl",
"iptables": "iptables-persistent",
"gentoolkit": "",
"centos-release-openstack-ocata": "",
"redhat-rpm-config": "",
"redhat-lsb-core": "",
"python-xml": "",
"python3": ""
} }
} }
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
systemctl enable haveged.service
;;
openrc)
# TODO(pabelanger): Make sure we support gentoo if we bring images
# online for nodepool.o.o.
exit 0
;;
sysv)
exit 0
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
# Default rules for rsyslog.
#
# For more information see rsyslog.conf(5) and /etc/rsyslog.conf
#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
#daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
#lpr.* -/var/log/lpr.log
mail.* -/var/log/mail.log
#user.* -/var/log/user.log
#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
#mail.info -/var/log/mail.info
#mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err
#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice
#
# Some "catch-all" log files.
#
#*.=debug;\
# auth,authpriv.none;\
# news.none;mail.none -/var/log/debug
#*.=info;*.=notice;*.=warn;\
# auth,authpriv.none;\
# cron,daemon.none;\
# mail,news.none -/var/log/messages
#
# Emergencies are sent to everybody logged in.
#
*.emerg :omusrmsg:*
#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
#daemon.*;mail.*;\
# news.err;\
# *.=debug;*.=info;\
# *.=notice;*.=warn |/dev/xconsole
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# TODO(pabelanger): Once we complete remove puppet from our diskimages, we
# should also remove our default SSH key. Since only 3rdparty CI system would be
# using this element moving forward, it doesn't make sense to bake in our
# default key.
NODEPOOL_SSH_KEY=${NODEPOOL_SSH_KEY:-AAAAB3NzaC1yc2EAAAADAQABAAABAQC6WutNHfM+YdnjeNFeaIpvxqt+9aDn95Ykpmc+fASSjlDZJtOrueH3ch/v08wkE4WQKg03i+t8VonqEwMGmApYA3VzFsURUQbxzlSz5kHlBQSqgz5JTwUmnt1RH5sePL5pkuJ6JgqJ8PxJod6fiD7YDjaKJW/wBzXGnGg2EkgqrkBQXYL4hyaPuSwsQF0Gdwg3QFqXl+R/GrM6FscUkkJzbjqGKI2GhLT8mf2BIMEAiMFhF5Wl4FFrbvhTfPfW+9VdcsiMxCXaxp00n1x1+Y7OqR5AZ/id0Lkz9ZoFVGS901OB/L4xXrvUtI2y+kIYeF6hxfmAl/zhY0eWzwo9lDPz}
if [ -z $NODEPOOL_SSH_KEY ]; then
die "Can not find public key for jenkins user!"
fi
# save the public key inside the chroot
echo "ssh-rsa $NODEPOOL_SSH_KEY" > $TMP_HOOKS_PATH/jenkins-user-ssh-public-key
#!/bin/bash
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
# Add jenkins user and group. Note we don't want to rely on
# "useradd"'s group adding behaviour, because it might differ across
# distros.
groupadd jenkins
useradd -g jenkins -m jenkins -s /bin/bash
# a lot of caching happens in extra-data.d (for "historical" reasons).
# We've put the cache stuff into /opt/cache/files, but again, for
# "historical" reasons, ensure this is available in /home/jenkins
#
# Check if the cache exists as we don't have a strict dependency on the
# devstack-cache element. This allows you to build an image without
# incurring the cost of caching all the things.
if [ -d /opt/cache/files ] ; then
mkdir -p /home/jenkins/cache
ln -sf /opt/cache/files /home/jenkins/cache/files
chown -R jenkins:jenkins /opt/cache/files
fi
# this was copied from outside the chroot by extras.d
_pub_key=/tmp/in_target.d/jenkins-user-ssh-public-key
if [ ! -f $_pub_key ]; then
die "Can not find Jenkins public key!"
fi
mkdir /home/jenkins/.ssh
chmod 700 /home/jenkins/.ssh
cp $_pub_key /home/jenkins/.ssh/authorized_keys
chmod 644 /home/jenkins/.ssh/authorized_keys
cat > /home/jenkins/.gitconfig <<EOF
[user]
name = OpenStack Jenkins
email = jenkins@openstack.org
signingkey = jenkins@openstack.org
[gitreview]
rebase = false
username = jenkins
EOF
# cleanup everything to the right owner
chown -R jenkins:jenkins /home/jenkins
...@@ -13,5 +13,7 @@ are. Default: ...@@ -13,5 +13,7 @@ are. Default:
The image should have the unbound DNS resolver package installed, the The image should have the unbound DNS resolver package installed, the
nodepool-base element then configures it to forward DNS queries to: nodepool-base element then configures it to forward DNS queries to:
`NODEPOOL_STATIC_NAMESERVER_V6`, default: `2001:4860:4860::8888` `NODEPOOL_STATIC_NAMESERVER_V6`, default: `2620:0:ccc::2`
`NODEPOOL_STATIC_NAMESERVER_V4`, default: `8.8.8.8`. `NODEPOOL_STATIC_NAMESERVER_V4`, default: `208.67.222.222`
`NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK`, default: `2001:4860:4860::8888`
`NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK`, default: `8.8.8.8`.
#!/usr/bin/env python2 #!/bin/bash
# #
# Copyright 2014 Hewlett-Packard Development Company, L.P. # Copyright 2017 Red Hat, Inc.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may # Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain # not use this file except in compliance with the License. You may obtain
# a copy of the License at # a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
#
if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
import os sudo rm -rf $TARGET_ROOT/root/.cache
import sys
from subunit2sql.db import api
from subunit2sql import shell
from subunit2sql import write_subunit
DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
if len(sys.argv) == 2:
TEMPEST_PATH = sys.argv[1]
elif len(sys.argv) > 2:
TEMPEST_PATH = sys.argv[1]
DB_URI = sys.argv[2]
else:
TEMPEST_PATH = '/opt/stack/new/tempest'
def main():
shell.parse_args([])
shell.CONF.set_override('connection', DB_URI, group='database')
session = api.get_session()
runs = api.get_recent_successful_runs_by_run_metadata(
'build_name', 'gate-tempest-dsvm-neutron-full',
num_runs=10, session=session)
session.close()
preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams')
if not os.path.isdir(preseed_path):
os.mkdir(preseed_path)
for run in runs:
with open(os.path.join(preseed_path, run.uuid + '.subunit'), 'w') as fd:
write_subunit.sql2subunit(run.uuid, fd)
if __name__ == '__main__':
main()