Commit a1777b4b authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11
parents 3e077606 f94efcc6
#!/bin/bash -xe
# Copyright (C) 2014 - Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Install the virt-preview repository that includes bleeding edge versions of
# libvirt and qemu. The packages from this repo will get installed via
# prepare_devstack.sh.
wget http://fedorapeople.org/groups/virt/virt-preview/fedora-virt-preview.repo
sudo mv fedora-virt-preview.repo /etc/yum.repos.d/
/opt/nodepool-scripts/prepare_devstack.sh
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
SUDO=${SUDO:-true}
THIN=${THIN:-true}
ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false}
GIT_BASE=${GIT_BASE:-git://git.openstack.org}
sudo hostname $HOSTNAME
if [ -n "$HOSTNAME" ] && ! grep -q $HOSTNAME /etc/hosts ; then
echo "127.0.1.1 $HOSTNAME" | sudo tee -a /etc/hosts
fi
echo $HOSTNAME > /tmp/image-hostname.txt
sudo mv /tmp/image-hostname.txt /etc/image-hostname.txt
if [ ! -f /etc/redhat-release ]; then
# Cloud provider apt repos break us - so stop using them
LSBDISTID=$(lsb_release -is)
LSBDISTCODENAME=$(lsb_release -cs)
if [ "$LSBDISTID" == "Ubuntu" ] ; then
sudo dd of=/etc/apt/sources.list <<EOF
# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME main restricted
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME main restricted
## Major bug fix updates produced after the final release of the
## distribution.
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates main restricted
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates main restricted
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME universe
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME universe
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates universe
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates universe
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team, and may not be under a free licence. Please satisfy yourself as to
## your rights to use the software. Also, please note that software in
## multiverse WILL NOT receive any review or updates from the Ubuntu
## security team.
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME multiverse
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-updates multiverse
## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
deb http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-backports main restricted universe multiverse
deb-src http://us.archive.ubuntu.com/ubuntu/ $LSBDISTCODENAME-backports main restricted universe multiverse
deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security main restricted
deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security main restricted
deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security universe
deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security universe
deb http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security multiverse
deb-src http://security.ubuntu.com/ubuntu $LSBDISTCODENAME-security multiverse
EOF
fi
fi
# Fedora image doesn't come with wget
if [ -f /usr/bin/yum ]; then
sudo yum -y install wget
fi
wget https://git.openstack.org/cgit/openstack-infra/system-config/plain/install_puppet.sh
sudo bash -xe install_puppet.sh
sudo git clone --depth=1 $GIT_BASE/openstack-infra/system-config.git \
/root/system-config
sudo /bin/bash /root/system-config/install_modules.sh
set +e
if [ -z "$NODEPOOL_SSH_KEY" ] ; then
sudo puppet apply --detailed-exitcodes --color=false \
--modulepath=/root/system-config/modules:/etc/puppet/modules \
-e "class {'openstack_project::single_use_slave':
sudo => $SUDO,
thin => $THIN,
all_mysql_privs => $ALL_MYSQL_PRIVS,
}"
PUPPET_RET_CODE=$?
else
sudo puppet apply --detailed-exitcodes --color=false \
--modulepath=/root/system-config/modules:/etc/puppet/modules \
-e "class {'openstack_project::single_use_slave':
install_users => false,
sudo => $SUDO,
thin => $THIN,
all_mysql_privs => $ALL_MYSQL_PRIVS,
ssh_key => '$NODEPOOL_SSH_KEY',
}"
PUPPET_RET_CODE=$?
fi
# Puppet doesn't properly return exit codes. Check here the values that
# indicate failure of some sort happened. 0 and 2 indicate success.
if [ "$PUPPET_RET_CODE" -eq "4" ] || [ "$PUPPET_RET_CODE" -eq "6" ] ; then
exit $PUPPET_RET_CODE
fi
set -e
# The puppet modules should install unbound. Set up some nameservers.
cat >/tmp/forwarding.conf <<EOF
forward-zone:
name: "."
forward-addr: 8.8.8.8
EOF
sudo mv /tmp/forwarding.conf /etc/unbound/
sudo chown root:root /etc/unbound/forwarding.conf
sudo chmod a+r /etc/unbound/forwarding.conf
# HPCloud has selinux enabled by default, Rackspace apparently not.
# Regardless, apply the correct context.
if [ -x /sbin/restorecon ] ; then
sudo chcon system_u:object_r:named_conf_t:s0 /etc/unbound/forwarding.conf
fi
# Overwrite /etc/resolv.conf at boot
sudo dd of=/etc/rc.local <<EOF
#!/bin/bash
set -o xtrace
# Some providers inject dynamic network config statically. Work around this
# for DNS nameservers. This is expected to fail on some nodes so remove -e.
set +e
sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
set -e
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
exit 0
EOF
# hpcloud has started mounting ephemeral /dev/vdb at /mnt.
# devstack-gate wants to partition the ephemeral disk, add some swap
# and mount it at /opt. get rid of the mount.
#
# note this comes down from the cloud-init metadata; which we setup to
# ignore below.
sudo sed -i '/^\/dev\/vdb/d' /etc/fstab
# Make all cloud-init data sources match rackspace- only attempt to look
# at ConfigDrive, not at metadata service. This is not needed if there
# is no cloud-init
if [ -d /etc/cloud/cloud.cfg.d ] ; then
sudo dd of=/etc/cloud/cloud.cfg.d/95_real_datasources.cfg <<EOF
datasource_list: [ ConfigDrive, None ]
EOF
fi
# reset cloud-init
sudo rm -rf /var/lib/cloud/instances
sudo bash -c "echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf"
if [ -e /etc/init.d/unbound ] ; then
sudo /etc/init.d/unbound restart
elif [ -e /usr/lib/systemd/system/unbound.service ] ; then
sudo systemctl restart unbound
else
echo "Can't discover a method to restart \"unbound\""
exit 1
fi
# Make sure DNS works.
dig git.openstack.org
# Cache all currently known gerrit repos.
sudo mkdir -p /opt/git
sudo -i python /opt/nodepool-scripts/cache_git_repos.py $GIT_BASE
# We don't always get ext4 from our clouds, mount ext3 as ext4 on the next
# boot (eg when this image is used for testing).
sudo sed -i 's/ext3/ext4/g' /etc/fstab
# Remove additional sources used to install puppet or special version of pypi.
# We do this because leaving these sources in place causes every test that
# does an apt-get update to hit those servers which may not have the uptime
# of our local mirrors.
OS_FAMILY=$(facter osfamily)
if [ "$OS_FAMILY" == "Debian" ] ; then
sudo rm -f /etc/apt/sources.list.d/*
sudo apt-get update
elif [ "$OS_FAMILY" == "RedHat" ] ; then
# Can't delete * in yum.repos.d since all of the repos are listed there.
# Be specific instead.
if [ -f /etc/yum.repos.d/puppetlabs.repo ] ; then
sudo rm -f /etc/yum.repos.d/puppetlabs.repo
fi
fi
# Remove cron jobs
# We create fresh servers for these hosts, and they are used once. They don't
# need to do things like update the locatedb or the mandb or rotate logs
# or really any of those things. We only want code running here that we want
# here.
sudo rm -f /etc/cron.{monthly,weekly,daily,hourly,d}/*
# Install Zuul into a virtualenv
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
git clone /opt/git/openstack-infra/zuul /tmp/zuul
sudo virtualenv /usr/zuul-env
sudo -H /usr/zuul-env/bin/pip install /tmp/zuul
sudo rm -fr /tmp/zuul
# Create a virtualenv for zuul-swift-logs
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
sudo -H virtualenv /usr/zuul-swift-logs-env
sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
requests glob2 requestsexceptions
# Create a virtualenv for os-testr (which contains subunit2html)
# this is in /usr instead of /usr/loca/ due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
sudo -H virtualenv /usr/os-testr-env
sudo -H /usr/os-testr-env/bin/pip install os-testr
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
export SUDO='true'
export THIN='false'
export ALL_MYSQL_PRIVS='true'
./prepare_node.sh "$HOSTNAME"
./fix_grub_timeout.sh
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
export SUDO='true'
export THIN='true'
TEMPEST_DIR=${TEMPEST_DIR:-/opt/git/openstack/tempest}
./prepare_node.sh "$HOSTNAME"
sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack.sh "$HOSTNAME"
# Setup venv and install deps for prepare_tempest_testrepository.py
sudo virtualenv -p python2 /opt/git/subunit2sql-env
sudo -H /opt/git/subunit2sql-env/bin/pip install -U testrepository \
subunit2sql PyMySQL
# Pre-seed tempest testrepository with data from subunit2sql
sudo -i env PATH=/opt/git/subunit2sql-env/bin:$PATH \
/opt/git/subunit2sql-env/bin/python2 \
/opt/nodepool-scripts/prepare_tempest_testrepository.py \
$TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
# Delete the venv after the script is called
sudo rm -rf /opt/git/subunit2sql-env
./fix_grub_timeout.sh
#!/bin/bash -xe
# Copyright (C) 2014 - Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
export SUDO='true'
export THIN='true'
./prepare_node.sh "$HOSTNAME"
sudo -u jenkins -i /opt/nodepool-scripts/prepare_devstack_virt_preview.sh \
"$HOSTNAME"
./fix_grub_timeout.sh
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
export SUDO='true'
export THIN='true'
# Workaround bug 1270646 during node bootstrapping.
sudo ip link set mtu 1458 dev eth0
./prepare_node.sh "$HOSTNAME"
sudo -u jenkins -i /opt/nodepool-scripts/prepare_tripleo.sh "$HOSTNAME"
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
HOSTNAME=$1
function tsfilter {
$@ 2>&1 | awk '
{
cmd ="date +\"%Y-%m-%d %H:%M:%S.%3N | \""
cmd | getline now
close("date +\"%Y-%m-%d %H:%M:%S.%3N | \"")
sub(/^/, now)
print
fflush()
}'
}
set -o pipefail
tsfilter ./prepare_node_devstack.sh "$HOSTNAME"
# After the node has been prepared, the hypervisor needs to be halted to make
# sure that the filesystem is in a consistent state.
sudo -u domzero \
ssh \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
root@192.168.33.2 \
halt -p </dev/null
#!/bin/bash -xe
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Enable precise-backports so we can install jq
if [ -f /usr/bin/apt-get ]; then
sudo sed -i -e 's/# \(deb .*precise-backports main \)/\1/g' \
/etc/apt/sources.list
sudo apt-get update
fi
cd /opt/nodepool-scripts/
./install_devstack_dependencies.sh
# toci scripts use both of these
sudo -H pip install gear os-apply-config
# tripleo-gate runs with two networks - the public access network and eth1
# pointing at the in-datacentre L2 network where we can talk to the test
# environments directly. We need to enable DHCP on eth1 though.
# Note that we don't bring it up during prepare - it's only needed to run
# tests.
if [ -d /etc/sysconfig/network-scripts ]; then
sudo dd of=/etc/sysconfig/network-scripts/ifcfg-eth1 << EOF
DEVICE="eth1"
BOOTPROTO="dhcp"
ONBOOT="yes"
TYPE="Ethernet"
PEERDNS="no"
EOF
elif [ -f /etc/network/interfaces ]; then
sudo dd of=/etc/network/interfaces oflag=append conv=notrunc << EOF
auto eth1
iface eth1 inet dhcp
EOF
# Workaround bug 1270646 for actual slaves
sudo dd of=/etc/network/interfaces.d/eth0.cfg oflag=append conv=notrunc << EOF
post-up ip link set mtu 1458 dev eth0
EOF
else
echo "Unsupported distro."
exit 1
fi
......@@ -14,6 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Read through this script. There are a lot of comments that might
# help you!
#
# To run on Ubuntu 14.04, this depends on:
# diskimage-builder
# qemu-utils
......@@ -21,15 +26,75 @@
set -e
export ELEMENTS_PATH=${ELEMENTS_PATH:-nodepool/elements}
## Other options
# export DISTRO=${DISTRO:-centos-minimal}
# export DISTRO=${DISTRO:-fedora-minimal}
export DISTRO=${DISTRO:-ubuntu-minimal}
## Overrite the release
# export DIB_RELEASE=${DIB_RELEASE:-trusty}
# export DIB_RELEASE=${DIB_RELEASE:-25} # fedora
export ELEMENTS_PATH=${ELEMENTS_PATH:-nodepool/elements}
export IMAGE_NAME=${IMAGE_NAME:-devstack-gate}
export NODEPOOL_SCRIPTDIR=${NODEPOOL_SCRIPTDIR:-nodepool/scripts}
export CONFIG_SOURCE=${CONFIG_SOURCE:-https://git.openstack.org/openstack-infra/system-config}
export CONFIG_REF=${CONFIG_REF:-master}
export EXTRA_ELEMENTS=${EXTRA_ELEMENTS:-}
# The list of elements here should match nodepool/nodepool.yaml
## Test your changes to system-config by overriding this; note you can
## get a CONFIG_REF from gerrit should you have uploaded a change
# export CONFIG_SOURCE=${CONFIG_SOURCE:-https://git.openstack.org/openstack-infra/system-config}
# export CONFIG_REF=${CONFIG_REF:-refs/changes/12/123456/1
ZUUL_USER_SSH_PUBLIC_KEY=${ZUUL_USER_SSH_PUBLIC_KEY:-$HOME/.ssh/id_rsa.pub}
if [ ! -f ${ZUUL_USER_SSH_PUBLIC_KEY} ]; then
echo "Error: There is no SSH public key at: ${ZUUL_USER_SSH_PUBLIC_KEY}"
echo "Error: Image build will fail. Exiting now."
exit 1
fi
## If your firewall won't allow outbound DNS connections, you'll want
## to set these to local resolvers
# NODEPOOL_STATIC_NAMESERVER_V4=192.168.0.1
# NODEPOOL_STATIC_NAMESERVER_V6=2000::...
## This will get dib to drop you into a shell on error, useful for debugging
# export break="after-error"
## If you need to debug the boot, setting this longer might help you
## break into the grub console. Or set it to 0 for fast boot.
# export DIB_GRUB_TIMEOUT=10
## The openstack-repos element caches every git repo, wihch can take
## quite some time. We can override this, but some minimal repos are
## required for a successful build. For speeding up builds when
## you're testing the following should work, but be very careful
## stripping things out when generating real images.
## add to /tmp/custom_projects.yaml:
##
## - project: openstack-infra/project-config
## - project: openstack-infra/system-config
## - project: openstack-dev/devstack
# export DIB_CUSTOM_PROJECTS_LIST_URL='file:///tmp/custom_projects.yaml'
## If you are building test images, or dealing with networking issues,
## you will want to have a local login with password available (as
## opposed to key-based ssh only). You can use the "devuser" element
## from dib to set this up. Don't forget to enable sudo and set the
## password.
## defaults
## DIB_DEV_USER_USERNAME=devuser
## DIB_DEV_USER_AUTHORIZED_KEYS=$HOME/.ssh/id_rsa.pub
# EXTRA_ELEMENTS+=devuser
# DIB_DEV_USER_PWDLESS_SUDO=1
# DIB_DEV_USER_PASSWORD=devuser
## The list of elements below should match those configured
## in nodepool/nodepool.yaml
disk-image-create -x --no-tmpfs -o $IMAGE_NAME \
$DISTRO \
vm \
......
......@@ -9,4 +9,5 @@ channels_file=${1:-gerritbot/channels.yaml}
(printf "# This file is sorted alphabetically by channel name.\n"; sed '/^[[:space:]]*#.*$/d;s/[[:space:]]*#.*$//' gerritbot/channels.yaml) > $T/comments-removed
./tools/normalize_channels_yaml.py >$T/regenned
echo "Checking whether entries are sorted alphabetically"
diff -u $T/comments-removed $T/regenned
......@@ -14,3 +14,5 @@ git checkout $GITHEAD
python tools/check_gerrit_projects_changed.py gerrit/projects-old.yaml \
gerrit/projects.yaml
rm gerrit/projects-old.yaml
......@@ -59,17 +59,6 @@ def access_gerrit_check():
print("ERROR: %s is missing from accessbot" % channel)
errors = True
# IRC has a limit of 120 channels that we unfortunately hit with
# gerritbot. If we try connect to more, it will not connect to
# all. Avoid this situation.
if len(gerrit_config) > 120:
print("ERROR: gerritbot can only handle 120 channels but found %s."
% len(gerrit_config))