Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11

nodepool/elements/README.rst

@@ -12,7 +12,7 @@ Install the dependencies:
 
 ::
 
-  sudo apt-get install kpartx qemu-utils curl python-yaml
+  sudo apt-get install kpartx qemu-utils curl python-yaml debootstrap
 
 Install diskimage-builder:
 
@@ -28,11 +28,14 @@ Building an image is simple, we have a script!
 
 ::
 
-  DISTRO="ubuntu" bash tools/build-image.sh
+  bash tools/build-image.sh
 
-See the script for environment variables to set distribution, etc.
-You should be left with a .qcow2 image file of your selected
-distribution.
+See the script for environment variables to set distribution, etc. By default
+it builds an ubuntu-minimal based image.  You should be left with a .qcow2
+image file of your selected distribution.
+
+Infra uses the -minimal build type for building Ubuntu/CentOS/Fedora. For
+example: ubuntu-minimal.
 
 It is a good idea to set ``TMP_DIR`` to somewhere with plenty of space
 to avoid the disappointment of a full-disk mid-way through the script

nodepool/elements/cache-bindep/install.d/50-cache-pkgs

-#!/bin/bash
-# Copyright (C) 2011-2015 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# dib-lint: disable=setu setpipefail
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -e
-
-# Build up a list of packages to install by concatenating bindep lists
-FALLBACKREPO=/opt/git/openstack-infra/project-config
-FALLBACKFILE=jenkins/data/bindep-fallback.txt
-PACKAGES=`git --git-dir=$FALLBACKREPO/.git show master:$FALLBACKFILE`
-
-# NOTE(pabelanger): We currently don't gate on bindep.txt/other-requirements.txt
-# for centos and fedora nodes. As such, we have no way to ensure this will
-# not break DIB process. So we only run this for ubuntu-trusty.
-# bindep.txt is new filename, other-requirements.txt is the old one. We should
-# handle both for now.
-if [ -f /usr/bin/apt-get ] ; then
-    for REPO in /opt/git/*/* ; do
-        BRANCHES=`git --git-dir=$REPO/.git branch -r | grep '^  origin/[^H]'`
-        for BRANCH in $BRANCHES ; do
-            for reqsfile in bindep.txt other-requirements.txt ; do
-                REQS=`git --git-dir=$REPO/.git show $BRANCH:$reqsfile 2>/dev/null \
-                    || true`
-                if [ -n "$REQS" ] ; then
-                    PACKAGES=`echo -e "$PACKAGES\n$REQS" | sort -u`
-                fi
-            done
-        done
-    done
-fi
-
-PACKAGELIST=`echo "$PACKAGES" | /usr/bindep-env/bin/bindep -b -f - || true`
-
-if [ -f /usr/bin/apt-get ] ; then
-    # --ignore-missing conflicts with set -e, so force it to be ok
-    apt-get -y --ignore-missing -d install $PACKAGELIST || true
-else
-    ${YUM:-yum} install -y --downloadonly $PACKAGELIST
-fi

nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos

@@ -74,19 +74,6 @@ def git_branches():
     return branches
 
 
-def tokenize(fn, tokens, distribution, comment=None):
-    for line in open(fn):
-        if 'dist:' in line and ('dist:%s' % distribution not in line):
-            continue
-        if 'qpid' in line:
-            continue  # TODO: explain why this is here
-        if comment and comment in line:
-            line = line[:line.find(comment)]
-        line = line.strip()
-        if line and line not in tokens:
-            tokens.append(line)
-
-
 def _legacy_find_images(basedir):
     """Divine what images we should use based on parsing stackrc."""
     images = []
@@ -143,24 +130,6 @@ def local_prep(distribution):
         run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
         run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
 
-        if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/apt-get')):
-            debs = []
-            debdir = os.path.join(DEVSTACK, 'files', 'debs')
-            if not os.path.exists(debdir):
-                debdir = os.path.join(DEVSTACK, 'files', 'apts')
-            for fn in os.listdir(debdir):
-                fn = os.path.join(debdir, fn)
-                tokenize(fn, debs, distribution, comment='#')
-            branch_data['debs'] = debs
-
-        if os.path.exists(os.path.join(TMP_MOUNT_PATH, 'usr/bin/yum')):
-            rpms = []
-            rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
-            for fn in os.listdir(rpmdir):
-                fn = os.path.join(rpmdir, fn)
-                tokenize(fn, rpms, distribution, comment='#')
-            branch_data['rpms'] = rpms
-
         images = _find_images(DEVSTACK)
         if not images:
             images = _legacy_find_images(DEVSTACK)
@@ -174,15 +143,6 @@ def main():
 
     branches = local_prep(RELEASE)
 
-    with open(os.path.join(CACHEDIR, 'pkgs-to-install'), 'w') as pkgs:
-        for branch_data in branches:
-            if branch_data.get('debs'):
-                pkgs.write(" ".join(branch_data['debs']) + "\n")
-            elif branch_data.get('rpms'):
-                pkgs.write(" ".join(branch_data['rpms']) + "\n")
-            else:
-                sys.exit('No supported package data found.')
-
     image_filenames = []
     line_template = "%(name)s file %(location)s %(url)s\n"
     with open(IMAGES, 'w') as images_list:

nodepool/elements/cache-devstack/install.d/99-cache-testrepository-db

@@ -27,7 +27,7 @@ TEMPEST_DIR=/opt/git/openstack/tempest
 sudo virtualenv /opt/git/subunit2sql-env
 sudo -H /opt/git/subunit2sql-env/bin/pip install -U subunit2sql testrepository PyMySQL
 
-sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python2 /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
+sudo env PATH=/opt/git/subunit2sql-env/bin:$PATH /opt/git/subunit2sql-env/bin/python /opt/nodepool-scripts/prepare_tempest_testrepository.py $TEMPEST_DIR
 
 sudo chown -R jenkins:jenkins $TEMPEST_DIR/preseed-streams
 

nodepool/elements/infra-package-needs/element-deps

+openssh-server
 package-installs

nodepool/elements/infra-package-needs/package-installs.yaml

 coreutils:
 cron:
 util-linux:
-openssh-server:
 build-essential:
 python-dev:
 python3-dev:
 uuid-runtime:
 traceroute:
+ntp:
 ntpdate:
+gentoolkit:
+at:
+lvm2:
+strace:
+tcpdump:
+rsyslog:
+git:
+rsync:
+parted:
+puppet:
+wget:
+iputils-ping:
+dnsutils:

nodepool/elements/infra-package-needs/pkg-map

@@ -2,15 +2,40 @@
   "distro": {
     "fedora": {
       "python3-dev": "python3-devel"
+    },
+    "gentoo": {
+      "build-essential": "",
+      "cron": "sys-process/cronie",
+      "ntp": "net-misc/ntp",
+      "python-dev": "",
+      "python3-dev": "",
+      "traceroute": "net-analyzer/traceroute",
+      "uuid-runtime": ""
     }
   },
   "family": {
     "redhat": {
       "cron": "cronie",
       "build-essential": "glibc-devel gcc make",
+      "dnsutils": "bind-utils",
+      "iputils-ping": "iputils",
+      "ntp": "ntp ntp-perl",
       "python-dev": "python-devel",
       "python3-dev": "",
       "uuid-runtime": ""
+    },
+    "suse": {
+      "dnsutils": "bind-utils",
+      "git": "git-core",
+      "iputils-ping": "iputils",
+      "ntpdate": "",
+      "puppet": "ruby2.1-rubygem-puppet",
+      "python-dev": "python-devel",
+      "python3-dev": "python3-devel",
+      "uuid-runtime": "uuidd"
     }
+  },
+  "default": {
+    "gentoolkit": ""
   }
 }

nodepool/elements/infra-package-needs/post-install.d/80-enable-infra-services

+#!/bin/bash
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+case "$DIB_INIT_SYSTEM" in
+    upstart)
+        # nothing to do
+        exit 0
+        ;;
+    systemd)
+        if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
+            systemctl enable ntp.service
+        else
+            systemctl enable ntpd.service
+        fi
+        ;;
+    openrc)
+        # let dib-init-system's postinstall handle enabling init scripts
+        exit 0
+        ;;
+    sysv)
+        # ntp is enabled by default, nothing to do
+        exit 0
+        ;;
+    *)
+        echo "Unsupported init system $DIB_INIT_SYSTEM"
+        exit 1
+        ;;
+esac

nodepool/elements/node-devstack/install.d/20-prepare-node → nodepool/elements/infra-package-needs/post-install.d/89-sshd

 #!/bin/bash
 # Copyright (C) 2011-2013 OpenStack Foundation
+# Copyright 2016 Red Hat, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,16 +16,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# dib-lint: disable=setu setpipefail
 if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
     set -x
 fi
-set -e
+set -eu
+set -o pipefail
 
-export SUDO='true'
-export THIN='true'
-
-prepare-node
-
-mkdir -p ~jenkins/cache/files
-mkdir -p ~jenkins/cache/pip
+# NOTE(pabelanger): Glean configures access for root user, so allow us to
+# properly login.
+sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
+    && echo "PermitRootLogin Yes" >> /etc/ssh/sshd_config

nodepool/elements/initialize-urandom/static/usr/local/bin/initialize-urandom.py

@@ -223,7 +223,7 @@ class Pump(object):
             r = self.getrandom(1, nonblock=True)
             if len(r) != 1:
                 raise Exception("No data returned from getrandom")
-            print("Nonblocking pool initilaized")
+            print("Nonblocking pool initialized")
             return True
         except GeneratorNotInitializedError:
             return False

nodepool/elements/node-devstack/README.rst

-Prepare a node to be a devstack slave

nodepool/elements/node-devstack/element-deps

-cache-devstack
-nodepool-base
-prepare-node
-puppet

nodepool/elements/node-devstack/package-installs.yaml

-build-essential:
-python-dev:
-python3-dev:

nodepool/elements/nodepool-base/element-deps

+package-installs
+pip-and-virtualenv
 puppet
 zuul-worker

nodepool/elements/cache-devstack/install.d/50-download-pkgs → nodepool/elements/nodepool-base/finalise.d/89-journald-persistent

 #!/bin/bash
-# Copyright (C) 2011-2013 OpenStack Foundation
+# Copyright (C) 2017 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,21 +21,11 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
 fi
 set -e
 
-if [[ ${YUM:-yum} == "dnf" ]]; then
-    echo "*** caching disabled for dnf"
-    echo "*** see https://etherpad.openstack.org/p/infra-f22"
-    exit 0
+# Journald default is to not persist logs to disk if /var/log/journal is
+# not present. Update the configuration to set storage to persistent which
+# will create /var/log/journal if necessary and store logs on disk. This
+# avoids the situation where test runs can fill the journald ring buffer
+# deleting older logs that may be important to the job.
+if [ -f /etc/systemd/journald.conf ] ; then
+    sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
 fi
-
-while read line ; do
-
-    if [ -f /usr/bin/apt-get ] ; then
-        # --ignore-missing conflicts with set -e, so force it to be ok
-        apt-get -y --ignore-missing -d install $line || true
-    else
-        yum install -y --downloadonly $line
-    fi
-
-done < /tmp/pkgs-to-install
-
-rm /tmp/pkgs-to-install

nodepool/elements/nodepool-base/finalise.d/89-unbound

@@ -35,7 +35,35 @@ mv /tmp/forwarding.conf /etc/unbound/
 chown root:root /etc/unbound/forwarding.conf
 chmod a+r /etc/unbound/forwarding.conf
 
-# Overwrite /etc/resolv.conf at boot
+# You'd think rc.local would be simple ...
+#
+# On Redhat systems, systemd's rc-local service looks for an
+# executable /etc/rc.d/rc.local file to run.  On Debian/Ubuntu, the
+# eqivalent file is /etc/rc.local.
+#
+# Centos' systemd package symlinks /etc/rc.local to /etc/rc.d/rc.local
+# correctly. Fedora, however, does not come with an rc.local file at
+# all.  Thus if we have a rc.d directory, but no rc.local file, we
+# need to create it (if you don't have an rc.d directory, and don't
+# have /etc/rc.local, then it's not clear what platform you are on).
+#
+# Bug [1] is filed to bring Fedora in-line with Centos, and has more
+# details on all this.  As at 2016-10-18 is unresolved.
+#
+# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1386052
+if [[ ! -e /etc/rc.local ]]; then
+    if [[ ! -d /etc/rc.d ]]; then
+        echo "No rc.local and no rc.d directory!  See comments in 89-unbound"
+        exit 1
+    fi
+    touch /etc/rc.d/rc.local
+    ln -sf /etc/rc.d/rc.local /etc/rc.local
+    # permissions added below.  selinux context will be fixed up at
+    # end of build.
+fi
+
+# Overwrite /etc/resolv.conf at boot (let's hope nothing else is using
+# rc.local...)
 dd of=/etc/rc.local <<EOF
 #!/bin/bash
 set -o xtrace
@@ -51,6 +79,12 @@ echo 'nameserver 127.0.0.1' > /etc/resolv.conf
 exit 0
 EOF
 
+# Debian/Ubuntu ship their dummy rc.local with +x permissions, but
+# CentOS (and the file we created for Fedora) do not ... this is to
+# avoid it holding up the boot as it relies on the network being up).
+# Add +x so it runs.
+chmod a+x /etc/rc.local
+
 echo 'include: /etc/unbound/forwarding.conf' >> /etc/unbound/unbound.conf
 
 # Disable DNSSEC

nodepool/elements/nodepool-base/finalise.d/99-fix-grub-timeout

-#!/bin/bash
-# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is done in 50 because the vm element has, at 51, an bunch of code
-# for properly handling grub. All we need to do here is get our values
-# in to the files.
-
-# dib-lint: disable=setu setpipefail
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -e
-
-# Set grub timeout to zero to save boot time.
-if [ -f /etc/default/grub ] ; then
-    sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' /etc/default/grub
-    if which update-grub &> /dev/null ; then
-        update-grub
-    else
-        # If update-grub isn't available, use grub2-mkconfig directly
-        /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
-    fi
-elif [ -f /boot/grub/grub.conf ] ; then
-    sed -i -e 's/^timeout=[0-9]\+/timeout=0/' /boot/grub/grub.conf
-fi

nodepool/elements/nodepool-base/finalise.d/99-restrict-memory

-#!/bin/bash -xe
-# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is done in 50 because the vm element has, at 51, an bunch of code
-# for properly handling grub. All we need to do here is get our values
-# in to the files.
-
-# dib-lint: disable=setu setpipefail
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -e
-
-# Limit all test slaves to 8GB of memory so that larger flavors with more
-# cpu resources can be used without the risk of becoming dependent on more
-# memory.
-if [ -f /etc/default/grub ] ; then
-    sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=8G /g' /etc/default/grub
-    if which update-grub &> /dev/null ; then
-        update-grub
-    else
-        # If update-grub isn't available, use grub2-mkconfig directly
-        /usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
-    fi
-elif [ -f /boot/grub/grub.conf ] ; then
-    sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
-fi

nodepool/elements/nodepool-base/install.d/90-venv-swift-logs

-#!/bin/bash
-# Copyright (C) 2015 Rackspace Australia
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-
-# dib-lint: disable=setu setpipefail
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -e
-
-# Create a virtualenv for zuul-swift-logs
-# This is in /usr instead of /usr/local due to this bug on precise:
-# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
-sudo -H virtualenv /usr/zuul-swift-logs-env
-sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
-    requests glob2 requestsexceptions