Commit a1777b4b authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11
parents 3e077606 f94efcc6
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -12,7 +12,7 @@ Install the dependencies:
::
sudo apt-get install kpartx qemu-utils curl python-yaml
sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap
Install diskimage-builder:
......@@ -28,11 +28,14 @@ Building an image is simple, we have a script!
::
DISTRO="ubuntu" bash tools/build-image.sh
bash tools/build-image.sh
See the script for environment variables to set distribution, etc.
You should be left with a .qcow2 image file of your selected
distribution.
See the script for environment variables to set distribution, etc. By default
it builds an ubuntu-minimal based image. You should be left with a .qcow2
image file of your selected distribution.
Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For
example: ubuntu-minimal.
It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space
to avoid the disappointment of a full-disk mid-way through the script
......
#!/bin/bash
# Copyright (C) 2011-2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Build up a list of packages to install by concatenating bindep lists
FALLBACKREPO=/opt/git/openstack-infra/project-config
FALLBACKFILE=jenkins/data/bindep-fallback.txt
PACKAGES=`git --git-dir=$FALLBACKREPO/.git show master:$FALLBACKFILE`
# NOTE(pabelanger): We currently don't gate on bindep.txt/other-requirements.txt
# for centos and fedora nodes. As such, we have no way to ensure this will
# not break DIB process. So we only run this for ubuntu-trusty.
# bindep.txt is new filename, other-requirements.txt is the old one. We should
# handle both for now.
if [ -f /usr/bin/apt-get ] ; then
for REPO in /opt/git/*/* ; do
BRANCHES=`git --git-dir=$REPO/.git branch -r | grep '^ origin/[^H]'`
for BRANCH in $BRANCHES ; do
for reqsfile in bindep.txt other-requirements.txt ; do
REQS=`git --git-dir=$REPO/.git show $BRANCH:$reqsfile 2>/dev/null \
|| true`
if [ -n "$REQS" ] ; then
PACKAGES=`echo -e "$PACKAGES\n$REQS" | sort -u`
fi
done
done
done
fi
PACKAGELIST=`echo "$PACKAGES" | /usr/bindep-env/bin/bindep -b -f - || true`
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $PACKAGELIST || true
else
${YUM:-yum} install -y --downloadonly $PACKAGELIST
fi
......@@ -74,19 +74,6 @@ def git_branches():
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.find(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
......@@ -143,24 +130,6 @@ def local_prep(distribution):
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/yum')):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
......@@ -174,15 +143,6 @@ def main():
branches = local_prep(RELEASE)
with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
for branch_data in branches:
if branch_data.get('debs'):
pkgs.write(" ".join(branch_data['debs']) + "\n")
elif branch_data.get('rpms'):
pkgs.write(" ".join(branch_data['rpms']) + "\n")
else:
sys.exit('No supported package data found.')
image_filenames = []
line_template = "%(name)s file %(location)s %(url)s\n"
with open(IMAGES, 'w') as images_list:
......
......@@ -27,7 +27,7 @@ TEMPEST_DIR=/opt/git/openstack/tempest
sudo virtualenv /opt/git/subunit2sql-env
sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository PyMySQL
sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python2 /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
......
coreutils:
cron:
util-linux:
openssh-server:
build-essential:
python-dev:
python3-dev:
uuid-runtime:
traceroute:
ntp:
ntpdate:
gentoolkit:
at:
lvm2:
strace:
tcpdump:
rsyslog:
git:
rsync:
parted:
puppet:
wget:
iputils-ping:
dnsutils:
......@@ -2,15 +2,40 @@
"distro": {
"fedora": {
"python3-dev": "python3-devel"
},
"gentoo": {
"build-essential": "",
"cron": "sys-process/cronie",
"ntp": "net-misc/ntp",
"python-dev": "",
"python3-dev": "",
"traceroute": "net-analyzer/traceroute",
"uuid-runtime": ""
}
},
"family": {
"redhat": {
"cron": "cronie",
"build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils",
"iputils-ping": "iputils",
"ntp": "ntp ntp-perl",
"python-dev": "python-devel",
"python3-dev": "",
"uuid-runtime": ""
},
"suse": {
"dnsutils": "bind-utils",
"git": "git-core",
"iputils-ping": "iputils",
"ntpdate": "",
"puppet": "ruby2.1-rubygem-puppet",
"python-dev": "python-devel",
"python3-dev": "python3-devel",
"uuid-runtime": "uuidd"
}
},
"default": {
"gentoolkit": ""
}
}
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
systemctl enable ntp.service
else
systemctl enable ntpd.service
fi
;;
openrc)
# let dib-init-system's postinstall handle enabling init scripts
exit 0
;;
sysv)
# ntp is enabled by default, nothing to do
exit 0
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,16 +16,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
set -eu
set -o pipefail
export SUDO='true'
export THIN='true'
prepare-node
mkdir -p ~jenkins/cache/files
mkdir -p ~jenkins/cache/pip
# NOTE(pabelanger): Glean configures access for root user, so allow us to
# properly login.
sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
&& echo "PermitRootLogin Yes" >> /etc/ssh/sshd_config
......@@ -223,7 +223,7 @@ class Pump(object):
r = self.getrandom(1, nonblock=True)
if len(r) != 1:
raise Exception("No data returned from getrandom")
print("Nonblocking pool initilaized")
print("Nonblocking pool initialized")
return True
except GeneratorNotInitializedError:
return False
......
Prepare a node to be a devstack slave
cache-devstack
nodepool-base
prepare-node
puppet
package-installs
pip-and-virtualenv
puppet
zuul-worker
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright (C) 2017 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -21,21 +21,11 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi
set -e
if [[ ${YUM:-yum} == "dnf" ]]; then
echo "*** caching disabled for dnf"
echo "*** see https://etherpad.openstack.org/p/infra-f22"
exit 0
# Journald default is to not persist logs to disk if /var/log/journal is
# not present. Update the configuration to set storage to persistent which
# will create /var/log/journal if necessary and store logs on disk. This
# avoids the situation where test runs can fill the journald ring buffer
# deleting older logs that may be important to the job.
if [ -f /etc/systemd/journald.conf ] ; then
sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
fi
while read line ; do
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $line || true
else
yum install -y --downloadonly $line
fi
done < /tmp/pkgs-to-install
rm /tmp/pkgs-to-install
......@@ -35,7 +35,35 @@ mv /tmp/forwarding.conf /etc/unbound/
chown root:root /etc/unbound/forwarding.conf
chmod a+r /etc/unbound/forwarding.conf
# Overwrite /etc/resolv.conf at boot
# You'd think rc.local would be simple ...
#
# On Redhat systems, systemd's rc-local service looks for an
# executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the
# eqivalent file is /etc/rc.local.
#
# Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local
# correctly. Fedora, however, does not come with an rc.local file at
# all. Thus if we have a rc.d directory, but no rc.local file, we
# need to create it (if you don't have an rc.d directory, and don't
# have /etc/rc.local, then it's not clear what platform you are on).
#
# Bug [1] is filed to bring Fedora in-line with Centos, and has more
# details on all this. As at 2016-10-18 is unresolved.
#
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052
if [[ ! -e /etc/rc.local ]]; then
if [[ ! -d /etc/rc.d ]]; then
echo "No rc.local and no rc.d directory! See comments in 89-unbound"
exit 1
fi
touch /etc/rc.d/rc.local
ln -sf /etc/rc.d/rc.local /etc/rc.local
# permissions added below. selinux context will be fixed up at
# end of build.
fi
# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using
# rc.local...)
dd of=/etc/rc.local <<EOF
#!/bin/bash
set -o xtrace
......@@ -51,6 +79,12 @@ echo 'nameserver 127.0.0.1' > /etc/resolv.conf
exit 0
EOF
# Debian/Ubuntu ship their dummy rc.local with +x permissions, but
# CentOS (and the file we created for Fedora) do not ... this is to
# avoid it holding up the boot as it relies on the network being up).
# Add +x so it runs.
chmod a+x /etc/rc.local
echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
# Disable DNSSEC
......
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Set grub timeout to zero to save boot time.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' /boot/grub/grub.conf
fi
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Limit all test slaves to 8GB of memory so that larger flavors with more
# cpu resources can be used without the risk of becoming dependent on more
# memory.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=8G /g' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
fi
#!/bin/bash
# Copyright (C) 2015 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Create a virtualenv for zuul-swift-logs
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
sudo -H virtualenv /usr/zuul-swift-logs-env
sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
requests glob2 requestsexceptions
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment