Commit a1777b4b authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11
parents 3e077606 f94efcc6
This diff is collapsed.
...@@ -12,7 +12,7 @@ Install the dependencies: ...@@ -12,7 +12,7 @@ Install the dependencies:
:: ::
sudo apt-get install kpartx qemu-utils curl python-yaml sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap
Install diskimage-builder: Install diskimage-builder:
...@@ -28,11 +28,14 @@ Building an image is simple, we have a script! ...@@ -28,11 +28,14 @@ Building an image is simple, we have a script!
:: ::
DISTRO="ubuntu" bash tools/build-image.sh bash tools/build-image.sh
See the script for environment variables to set distribution, etc. See the script for environment variables to set distribution, etc. By default
You should be left with a .qcow2 image file of your selected it builds an ubuntu-minimal based image. You should be left with a .qcow2
distribution. image file of your selected distribution.
Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For
example: ubuntu-minimal.
It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space
to avoid the disappointment of a full-disk mid-way through the script to avoid the disappointment of a full-disk mid-way through the script
......
#!/bin/bash
# Copyright (C) 2011-2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Build up a list of packages to install by concatenating bindep lists
FALLBACKREPO=/opt/git/openstack-infra/project-config
FALLBACKFILE=jenkins/data/bindep-fallback.txt
PACKAGES=`git --git-dir=$FALLBACKREPO/.git show master:$FALLBACKFILE`
# NOTE(pabelanger): We currently don't gate on bindep.txt/other-requirements.txt
# for centos and fedora nodes. As such, we have no way to ensure this will
# not break DIB process. So we only run this for ubuntu-trusty.
# bindep.txt is new filename, other-requirements.txt is the old one. We should
# handle both for now.
if [ -f /usr/bin/apt-get ] ; then
for REPO in /opt/git/*/* ; do
BRANCHES=`git --git-dir=$REPO/.git branch -r | grep '^ origin/[^H]'`
for BRANCH in $BRANCHES ; do
for reqsfile in bindep.txt other-requirements.txt ; do
REQS=`git --git-dir=$REPO/.git show $BRANCH:$reqsfile 2>/dev/null \
|| true`
if [ -n "$REQS" ] ; then
PACKAGES=`echo -e "$PACKAGES\n$REQS" | sort -u`
fi
done
done
done
fi
PACKAGELIST=`echo "$PACKAGES" | /usr/bindep-env/bin/bindep -b -f - || true`
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $PACKAGELIST || true
else
${YUM:-yum} install -y --downloadonly $PACKAGELIST
fi
...@@ -74,19 +74,6 @@ def git_branches(): ...@@ -74,19 +74,6 @@ def git_branches():
return branches return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.find(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir): def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc.""" """Divine what images we should use based on parsing stackrc."""
images = [] images = []
...@@ -143,24 +130,6 @@ def local_prep(distribution): ...@@ -143,24 +130,6 @@ def local_prep(distribution):
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK) run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK) run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/yum')):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK) images = _find_images(DEVSTACK)
if not images: if not images:
images = _legacy_find_images(DEVSTACK) images = _legacy_find_images(DEVSTACK)
...@@ -174,15 +143,6 @@ def main(): ...@@ -174,15 +143,6 @@ def main():
branches = local_prep(RELEASE) branches = local_prep(RELEASE)
with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
for branch_data in branches:
if branch_data.get('debs'):
pkgs.write(" ".join(branch_data['debs']) + "\n")
elif branch_data.get('rpms'):
pkgs.write(" ".join(branch_data['rpms']) + "\n")
else:
sys.exit('No supported package data found.')
image_filenames = [] image_filenames = []
line_template = "%(name)s file %(location)s %(url)s\n" line_template = "%(name)s file %(location)s %(url)s\n"
with open(IMAGES, 'w') as images_list: with open(IMAGES, 'w') as images_list:
......
...@@ -27,7 +27,7 @@ TEMPEST_DIR=/opt/git/openstack/tempest ...@@ -27,7 +27,7 @@ TEMPEST_DIR=/opt/git/openstack/tempest
sudo virtualenv /opt/git/subunit2sql-env sudo virtualenv /opt/git/subunit2sql-env
sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository PyMySQL sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository PyMySQL
sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python2 /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
......
openssh-server
package-installs package-installs
coreutils: coreutils:
cron: cron:
util-linux: util-linux:
openssh-server:
build-essential: build-essential:
python-dev: python-dev:
python3-dev: python3-dev:
uuid-runtime: uuid-runtime:
traceroute: traceroute:
ntp:
ntpdate: ntpdate:
gentoolkit:
at:
lvm2:
strace:
tcpdump:
rsyslog:
git:
rsync:
parted:
puppet:
wget:
iputils-ping:
dnsutils:
...@@ -2,15 +2,40 @@ ...@@ -2,15 +2,40 @@
"distro": { "distro": {
"fedora": { "fedora": {
"python3-dev": "python3-devel" "python3-dev": "python3-devel"
},
"gentoo": {
"build-essential": "",
"cron": "sys-process/cronie",
"ntp": "net-misc/ntp",
"python-dev": "",
"python3-dev": "",
"traceroute": "net-analyzer/traceroute",
"uuid-runtime": ""
} }
}, },
"family": { "family": {
"redhat": { "redhat": {
"cron": "cronie", "cron": "cronie",
"build-essential": "glibc-devel gcc make", "build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils",
"iputils-ping": "iputils",
"ntp": "ntp ntp-perl",
"python-dev": "python-devel", "python-dev": "python-devel",
"python3-dev": "", "python3-dev": "",
"uuid-runtime": "" "uuid-runtime": ""
},
"suse": {
"dnsutils": "bind-utils",
"git": "git-core",
"iputils-ping": "iputils",
"ntpdate": "",
"puppet": "ruby2.1-rubygem-puppet",
"python-dev": "python-devel",
"python3-dev": "python3-devel",
"uuid-runtime": "uuidd"
} }
},
"default": {
"gentoolkit": ""
} }
} }
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
systemctl enable ntp.service
else
systemctl enable ntpd.service
fi
;;
openrc)
# let dib-init-system's postinstall handle enabling init scripts
exit 0
;;
sysv)
# ntp is enabled by default, nothing to do
exit 0
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
#!/bin/bash #!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation # Copyright (C) 2011-2013 OpenStack Foundation
# Copyright 2016 Red Hat, Inc.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -15,16 +16,13 @@ ...@@ -15,16 +16,13 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x set -x
fi fi
set -e set -eu
set -o pipefail
export SUDO='true' # NOTE(pabelanger): Glean configures access for root user, so allow us to
export THIN='true' # properly login.
sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
prepare-node && echo "PermitRootLogin Yes" >> /etc/ssh/sshd_config
mkdir -p ~jenkins/cache/files
mkdir -p ~jenkins/cache/pip
...@@ -223,7 +223,7 @@ class Pump(object): ...@@ -223,7 +223,7 @@ class Pump(object):
r = self.getrandom(1, nonblock=True) r = self.getrandom(1, nonblock=True)
if len(r) != 1: if len(r) != 1:
raise Exception("No data returned from getrandom") raise Exception("No data returned from getrandom")
print("Nonblocking pool initilaized") print("Nonblocking pool initialized")
return True return True
except GeneratorNotInitializedError: except GeneratorNotInitializedError:
return False return False
......
Prepare a node to be a devstack slave
cache-devstack
nodepool-base
prepare-node
puppet
package-installs
pip-and-virtualenv
puppet puppet
zuul-worker zuul-worker
#!/bin/bash #!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation # Copyright (C) 2017 OpenStack Foundation
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
...@@ -21,21 +21,11 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then ...@@ -21,21 +21,11 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi fi
set -e set -e
if [[ ${YUM:-yum} == "dnf" ]]; then # Journald default is to not persist logs to disk if /var/log/journal is
echo "*** caching disabled for dnf" # not present. Update the configuration to set storage to persistent which
echo "*** see https://etherpad.openstack.org/p/infra-f22" # will create /var/log/journal if necessary and store logs on disk. This
exit 0 # avoids the situation where test runs can fill the journald ring buffer
# deleting older logs that may be important to the job.
if [ -f /etc/systemd/journald.conf ] ; then
sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
fi fi
while read line ; do
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $line || true
else
yum install -y --downloadonly $line
fi
done < /tmp/pkgs-to-install
rm /tmp/pkgs-to-install
...@@ -35,7 +35,35 @@ mv /tmp/forwarding.conf /etc/unbound/ ...@@ -35,7 +35,35 @@ mv /tmp/forwarding.conf /etc/unbound/
chown root:root /etc/unbound/forwarding.conf chown root:root /etc/unbound/forwarding.conf
chmod a+r /etc/unbound/forwarding.conf chmod a+r /etc/unbound/forwarding.conf
# Overwrite /etc/resolv.conf at boot # You'd think rc.local would be simple ...
#
# On Redhat systems, systemd's rc-local service looks for an
# executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the
# eqivalent file is /etc/rc.local.
#
# Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local
# correctly. Fedora, however, does not come with an rc.local file at
# all. Thus if we have a rc.d directory, but no rc.local file, we
# need to create it (if you don't have an rc.d directory, and don't
# have /etc/rc.local, then it's not clear what platform you are on).
#
# Bug [1] is filed to bring Fedora in-line with Centos, and has more
# details on all this. As at 2016-10-18 is unresolved.
#
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052
if [[ ! -e /etc/rc.local ]]; then
if [[ ! -d /etc/rc.d ]]; then
echo "No rc.local and no rc.d directory! See comments in 89-unbound"
exit 1
fi
touch /etc/rc.d/rc.local
ln -sf /etc/rc.d/rc.local /etc/rc.local
# permissions added below. selinux context will be fixed up at
# end of build.
fi
# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using
# rc.local...)
dd of=/etc/rc.local <<EOF dd of=/etc/rc.local <<EOF
#!/bin/bash #!/bin/bash
set -o xtrace set -o xtrace
...@@ -51,6 +79,12 @@ echo 'nameserver 127.0.0.1' > /etc/resolv.conf ...@@ -51,6 +79,12 @@ echo 'nameserver 127.0.0.1' > /etc/resolv.conf
exit 0 exit 0
EOF EOF
# Debian/Ubuntu ship their dummy rc.local with +x permissions, but
# CentOS (and the file we created for Fedora) do not ... this is to
# avoid it holding up the boot as it relies on the network being up).
# Add +x so it runs.
chmod a+x /etc/rc.local
echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
# Disable DNSSEC # Disable DNSSEC
......
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Set grub timeout to zero to save boot time.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' /boot/grub/grub.conf
fi
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Limit all test slaves to 8GB of memory so that larger flavors with more
# cpu resources can be used without the risk of becoming dependent on more
# memory.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=8G /g' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
fi
#!/bin/bash
# Copyright (C) 2015 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Create a virtualenv for zuul-swift-logs
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
sudo -H virtualenv /usr/zuul-swift-logs-env
sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
requests glob2 requestsexceptions
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment