Commit 4fb0c7d7 authored by Ricardo Araujo's avatar Ricardo Araujo Committed by Ricardo Araujo
Browse files

Update nodepool scripts

parent 62547883
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=set setu setpipefail indent
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
case "$DIB_INIT_SYSTEM" in
systemd)
glean_path_dib="/etc/systemd/system/glean@.service.d"
mkdir -p $glean_path_dib
nodepool_base="$(dirname $0)/../glean@.service.d"
cp -RP $nodepool_base/override.conf $glean_path_dib/override.conf
;;
*)
echo "Skipping glean systemd configuration"
;;
esac
......@@ -21,25 +21,50 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi
set -e
NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2001:4860:4860::8888}
NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-8.8.8.8}
# The puppet modules should install unbound.
NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2620:0:ccc::2}
NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-208.67.222.222}
NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888}
NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8}
dd of=/tmp/forwarding.conf <<EOF
forward-zone:
name: "."
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V6
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4
forward-addr: $NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK
EOF
mv /tmp/forwarding.conf /etc/unbound/
chown root:root /etc/unbound/forwarding.conf
chmod a+r /etc/unbound/forwarding.conf
dd of=/tmp/unbound-logging.conf <<EOF
server:
logfile: "/var/log/unbound.log"
# Log only errors
verbosity: 0
EOF
if [[ "$DISTRO_NAME" =~ (centos|fedora|opensuse) ]] ; then
UNBOUND_CONFD=/etc/unbound/conf.d
else
UNBOUND_CONFD=/etc/unbound/unbound.conf.d
fi
mv /tmp/unbound-logging.conf $UNBOUND_CONFD
chown root:root $UNBOUND_CONFD/unbound-logging.conf
chmod a+r $UNBOUND_CONFD/unbound-logging.conf
touch /var/log/unbound.log
chown unbound /var/log/unbound.log
chmod 0644 /var/log/unbound.log
# You'd think rc.local would be simple ...
#
# On Redhat systems, systemd's rc-local service looks for an
# executable /etc/rc.d/rc.local file to run. On Debian/Ubuntu, the
# eqivalent file is /etc/rc.local.
# eqivalent file is /etc/rc.local, which is missing on Debian stretch.
#
# Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local
# correctly. Fedora, however, does not come with an rc.local file at
......@@ -51,7 +76,7 @@ chmod a+r /etc/unbound/forwarding.conf
# details on all this. As at 2016-10-18 is unresolved.
#
# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052
if [[ ! -e /etc/rc.local ]]; then
if [[ ! -e /etc/rc.local ]] && [[ ! -e /etc/debian_version ]]; then
if [[ ! -d /etc/rc.d ]]; then
echo "No rc.local and no rc.d directory! See comments in 89-unbound"
exit 1
......@@ -62,9 +87,13 @@ if [[ ! -e /etc/rc.local ]]; then
# end of build.
fi
# And furthermore on SUSE systems rc.local is called /etc/init.d/boot.local
rclocal=/etc/rc.local
[ -f /etc/init.d/boot.local ] && rclocal=/etc/init.d/boot.local
# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using
# rc.local...)
dd of=/etc/rc.local <<EOF
dd of=$rclocal <<EOF
#!/bin/bash
set -o xtrace
......@@ -72,6 +101,7 @@ set -o xtrace
# for DNS nameservers. This is expected to fail on some nodes so remove -e.
set +e
sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-*
sed -i -e 's/^NETCONFIG_DNS_POLICY=.*/NETCONFIG_DNS_POLICY=""/g' /etc/sysconfig/network/config
set -e
echo 'nameserver 127.0.0.1' > /etc/resolv.conf
......@@ -83,9 +113,60 @@ EOF
# CentOS (and the file we created for Fedora) do not ... this is to
# avoid it holding up the boot as it relies on the network being up).
# Add +x so it runs.
chmod a+x /etc/rc.local
chmod a+x $rclocal
echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
# Disable DNSSEC
# This defaults file must be in place to work around this bug:
# https://bugs.launchpad.net/ubuntu/+source/unbound/+bug/988513
# The issue is fixed for Xenial so this is only required for Trusty.
if [ "$DIB_RELEASE" == "trusty" ] ; then
cat > /etc/default/unbound <<EOF
UNBOUND_ENABLE=true
ROOT_TRUST_ANCHOR_UPDATE=true
ROOT_TRUST_ANCHOR_FILE=/var/lib/unbound/root.key
RESOLVCONF=true
RESOLVCONF_FORWARDERS=false
EOF
fi
# Tripleo uses dhcp
dhcp_file=''
if [[ "$DISTRO_NAME" =~ (centos|rhel7|fedora) ]] ; then
dhcp_file=/etc/dhcp/dhclient-eth0.conf
elif [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then
dhcp_file=/etc/dhcp/dhclient.conf
fi
if [ -n "$dhcp_file" ] ; then
cat > $dhcp_file <<EOF
option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;
send host-name "<hostname>";
request subnet-mask, broadcast-address, routers,
interface-mtu, rfc3442-classless-static-routes;
supersede domain-name-servers 127.0.0.1;
supersede domain-search "";
supersede domain-name "";
EOF
fi
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
;;
systemd)
systemctl enable unbound.service
;;
openrc)
# let dib-init-system's postinstall handle enabling init scripts
;;
sysv)
# nothing to do
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
# ricardoas: Disable DNSSEC
rm -f /etc/unbound/unbound.conf.d/root-auto-trust-anchor-file.conf
[Service]
Environment="ARGS=--interface %I --skip-dns"
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then
rules_dir=/etc/iptables
ipv4_rules=${rules_dir}/rules.v4
ipv6_rules=${rules_dir}/rules.v6
elif [[ "$DISTRO_NAME" =~ (centos|fedora) ]] ; then
rules_dir=/etc/sysconfig
ipv4_rules=${rules_dir}/iptables
ipv6_rules=${rules_dir}/ip6tables
elif [[ "$DISTRO_NAME" =~ 'opensuse' ]] ; then
rules_dir=/etc/sysconfig
ipv4_rules=${rules_dir}/iptables
ipv6_rules=${rules_dir}/ip6tables
else
echo "Unsupported operating system $DISTRO_NAME"
exit 1
fi
mkdir -p $rules_dir
cat > $ipv4_rules << EOF
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:openstack-INPUT - [0:0]
-A INPUT -j openstack-INPUT
-A openstack-INPUT -i lo -j ACCEPT
-A openstack-INPUT -p icmp --icmp-type any -j ACCEPT
#-A openstack-INPUT -p udp --dport 5353 -d 224.0.0.251 -j ACCEPT
-A openstack-INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
# SSH from anywhere
-A openstack-INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
# Public TCP ports
-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT
# Ports 69 and 6385 allow to allow ironic VM nodes to reach tftp and
# the ironic API from the neutron public net
-A openstack-INPUT -s 172.24.4.0/23 -p udp -m udp --dport 69 -j ACCEPT
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 6385 -j ACCEPT
# Ports 80, 8000, 8003, 8004 from the devstack neutron public net to allow
# nova servers to reach heat-api-cfn, heat-api-cloudwatch, heat-api
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 80 -j ACCEPT
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8000 -j ACCEPT
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8003 -j ACCEPT
-A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 8004 -j ACCEPT
-A openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: "
-A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited
COMMIT
EOF
cat > $ipv6_rules << EOF
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:openstack-INPUT - [0:0]
-A INPUT -j openstack-INPUT
-A openstack-INPUT -i lo -j ACCEPT
-A openstack-INPUT -p ipv6-icmp -j ACCEPT
-A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
# SSH from anywhere
-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
# Public TCP ports
-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 19885 -j ACCEPT
-A openstack-INPUT -j REJECT --reject-with icmp6-adm-prohibited
COMMIT
EOF
if [[ "$DISTRO_NAME" =~ 'opensuse' ]] ; then
sed -i -e 's,^FW_CUSTOMRULES=.*$,FW_CUSTOMRULES="/etc/sysconfig/scripts/SuSEfirewall2-custom",' /etc/sysconfig/SuSEfirewall2
cat > /etc/sysconfig/scripts/SuSEfirewall2-custom <<EOF
fw_custom_after_finished() {
/usr/sbin/iptables-restore $ipv4_rules
/usr/sbin/ip6tables-restore $ipv6_rules
}
EOF
fi
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,21 +15,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
TEMPEST_DIR=/opt/git/openstack/tempest
cat > /etc/sudoers.d/jenkins-sudo << EOF
jenkins ALL=(ALL) NOPASSWD:ALL
EOF
chmod 0440 /etc/sudoers.d/jenkins-sudo
# Setup venv and Install python deps from pip
sudo virtualenv /opt/git/subunit2sql-env
sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository PyMySQL
cat > /etc/sudoers.d/jenkins-sudo-grep <<EOF
jenkins ALL = NOPASSWD:/usr/local/jenkins/slave_scripts/jenkins-sudo-grep.sh
EOF
chmod 0440 /etc/sudoers.d/jenkins-sudo-grep
sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
# Delete the venv after the script
sudo rm -rf /opt/git/subunit2sql-env
visudo -c || die "Error setting jenkins sudo!"
#!/bin/bash
# Copyright (C) 2015 OpenStack Foundation
# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
virtualenv /usr/zuul-env
/usr/zuul-env/bin/pip install git+https://git.openstack.org/openstack-infra/zuul.git
#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
if [[ "$DISTRO_NAME" =~ (debian|ubuntu) ]] ; then
if [[ "$DIB_RELEASE" == 'trusty' ]] ; then
service_name=iptables-persistent
else
service_name=netfilter-persistent
fi
elif [[ "$DISTRO_NAME" =~ (centos|fedora) ]] ; then
service_name=iptables
elif [[ "$DISTRO_NAME" == 'opensuse' ]] ; then
service_name=SuSEfirewall2
else
echo "Unsupported operating system $DISTRO_NAME"
exit 1
fi
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
;;
systemd)
systemctl enable ${service_name}.service
if [[ "$DISTRO_NAME" =~ (centos|fedora) ]] ; then
systemctl enable ip6tables.service
fi
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
cache-url
prepare-node
puppet
source-repositories
......@@ -16,11 +16,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import os
import urllib2
from urllib2 import URLError
import yaml
from six.moves.urllib.request import urlopen
from six.moves.urllib.error import URLError
URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
'plain/gerrit/projects.yaml')
......@@ -32,10 +35,25 @@ CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
def get_project_list(url):
try:
projects = [f['project'] for f in yaml.load(urllib2.urlopen(url))]
projects = []
for f in yaml.load(urlopen(url)):
# Skip repos that are inactive
project = f['project']
dirname = os.path.dirname(project)
if 'attic' in dirname or dirname == 'stackforge':
continue
# Skip the /deb- git repo copies
if '/deb-' in project:
continue
acl = f.get('acl-config')
# Ignore retired repositories
if acl and os.path.basename(acl) == 'retired.config':
continue
projects.append(project)
return projects
except URLError:
print "Could not open project list url: '%s'" % url
print("Could not open project list url: '%s'" % url)
raise
def main():
......@@ -48,14 +66,6 @@ def main():
with open(PROJECTS_REPOS, 'w') as projects_list:
for project in projects:
# Skip repos that are inactive
dirname = os.path.dirname(project)
if ('attic' in dirname or dirname == 'stackforge'):
continue
# Skip the /deb- git repo copies
if ('/deb-' in project):
continue
args = dict(
name=os.path.basename(project),
location=os.path.join('/opt/git', project),
......@@ -64,18 +74,6 @@ def main():
projects_list.write("%(name)s git %(location)s "
"%(url)s %(ref)s\n" % args)
# Clone openstack-infra/system-config again so that we can use it to
# build the image without interferring with the slave repo cache.
project = 'openstack-infra/system-config'
args = dict(
name='config_tmp',
location=os.path.join('/opt/build_git', project),
url=os.environ.get('CONFIG_SOURCE',
'%s/%s.git' % (GIT_BASE, project)),
ref=os.environ.get('CONFIG_REF', '*'))
projects_list.write(
"%(name)s git %(location)s %(url)s %(ref)s\n" % args)
if __name__ == '__main__':
main()
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail dibdebugtrace indent
set -e
SUDO=${SUDO:-true}
THIN=${THIN:-true}
ALL_MYSQL_PRIVS=${ALL_MYSQL_PRIVS:-false}
if [ -n "$NODEPOOL_SSH_KEY" ] ; then
puppet_install_users="install_users => false,
ssh_key => '$NODEPOOL_SSH_KEY',"
else
puppet_install_users=""
fi
export FACTER_in_chroot=true
cat >/tmp/local.pp <<EOF
Service {
start => '/bin/true',
stop => '/bin/true',
status => '/bin/true',
restart => '/bin/true'
}
class {'openstack_project::single_use_slave':
sudo => $SUDO,
thin => $THIN,
all_mysql_privs => $ALL_MYSQL_PRIVS,
install_resolv_conf => false,
$puppet_install_users
}
EOF
# Puppet doesn't return nonzero if some things fail by default.
# Use detailed exit codes to get that info and determine whether
# the return code indicates failure.
set +e
puppet apply --detailed-exitcodes --color=false \
--modulepath=/opt/build_git/openstack-infra/system-config/modules:/etc/puppet/modules \
/tmp/local.pp
PUPPET_RETURN=$?
if [ "$PUPPET_RETURN" -eq 4 ] || [ "$PUPPET_RETURN" -eq 6 ] ; then
exit $PUPPET_RETURN
fi
set -e
# Make sure resolv.conf settings don't break dib
if [ -n "$NODEPOOL_STATIC_NAMESERVER_V4" ] ; then
echo "nameserver $NODEPOOL_STATIC_NAMESERVER_V4"> /etc/resolv.conf
fi
cache-url
openstack-repos
package-installs
source-repositories
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# Remove additional sources used to install puppet or special version of pypi.
# We do this because leaving these sources in place causes every test that
# does an apt-get update to hit those servers which may not have the uptime
# of our local mirrors.
# dib-lint: disable=sete setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
OS_FAMILY=$(facter osfamily)