Update nodepool scripts

nodepool/elements/README.md

-Empty elements. Use these elements as a starting point:
-[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/elements)
-

nodepool/elements/cache-bindep/README.rst

-Pre-cache all of the things bindep can identify in our repos

nodepool/elements/cache-bindep/element-deps

-openstack-repos

nodepool/elements/cache-devstack/extra-data.d/55-cache-devstack-repos

@@ -16,6 +16,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import os
 import subprocess
 import sys
@@ -44,7 +47,7 @@ if 'DISTRO_NAME' in os.environ:
         # centos7 matches as rhel7 in devstack
         RELEASE='rhel7'
 if not RELEASE:
-    print "Can not determine RELEASE"
+    print("Can not determine RELEASE")
     sys.exit(1)
 
 DEVSTACK = os.path.join(TMP_MOUNT_PATH, 'opt/git/openstack-dev/devstack')
@@ -53,11 +56,12 @@ IMAGES=os.path.join(TMP_HOOKS_PATH, 'source-repository-images')
 
 
 def run_local(cmd, status=False, cwd='.', env={}):
-    print "Running:", cmd
+    print("Running:", cmd)
     newenv = os.environ
     newenv.update(env)
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
-                         stderr=subprocess.STDOUT, env=newenv)
+                         stderr=subprocess.STDOUT, env=newenv,
+                         universal_newlines=True)
     (out, nothing) = p.communicate()
     if status:
         return (p.returncode, out.strip())
@@ -106,8 +110,8 @@ def _find_images(basedir):
     if os.path.exists(image_tool):
         returncode, out = run_local(image_tool, status=True)
         if returncode:
-            print "%s failed" % image_tool
-            print "Exit: %s, Output: %s" % (returncode, out)
+            print("%s failed" % image_tool)
+            print("Exit: %s, Output: %s" % (returncode, out))
             # reset images so we'll fall back
             images = []
         else:
@@ -126,7 +130,7 @@ def local_prep(distribution):
         if ' -> ' in branch:
             continue
         branch_data = {'name': branch}
-        print 'Branch: ', branch
+        print('Branch: ', branch)
         run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
         run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
 
@@ -160,7 +164,7 @@ def main():
                 image_filenames.append(fname)
                 args = dict(
                     name=fname,
-                    location=os.path.join('/home/jenkins/cache/files', fname),
+                    location=os.path.join('/opt/cache/files', fname),
                     url=url)
 
                 images_list.write(line_template % args)

nodepool/elements/cache-devstack/source-repository-pip

-get-pip-py file /home/jenkins/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py
+get-pip-py file /opt/cache/files/get-pip.py https://bootstrap.pypa.io/get-pip.py

nodepool/elements/cache-devstack/source-repository-stackviz

+stackviz file /opt/cache/files/stackviz-latest.tar.gz http://tarballs.openstack.org/package-stackviz-element/stackviz-latest.tar.gz

nodepool/elements/puppet/install.d/05-puppet → nodepool/elements/infra-package-needs/install.d/10-packages

 #!/bin/bash
-# Copyright (C) 2011-2013 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,10 +20,6 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
 fi
 set -e
 
-# Unset the download cache for this invocation to prevent bleed from build host
-unset PIP_DOWNLOAD_CACHE
-
-/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
-/bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
-
-install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
+if [ "$DISTRO_NAME" == "centos" ]; then
+    yum -y install --enablerepo=epel haveged
+fi

nodepool/elements/prepare-node/install.d/20-prepare-node → nodepool/elements/infra-package-needs/install.d/10-pip-packages

 #!/bin/bash
-# Copyright (C) 2011-2013 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,12 +20,8 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
 fi
 set -e
 
-export SUDO='true'
-export THIN='true'
+packages='tox'
 
-prepare-node
-
-mkdir -p ~jenkins/cache/files
-mkdir -p ~jenkins/cache/pip
-
-DEBIAN_FRONTEND=noninteractive sudo apt-get update -y && DEBIAN_FRONTEND=noninteractive sudo apt-get install default-jdk -y
+for package in $packages ; do
+    pip install $package
+done

nodepool/elements/stackviz/install.d/89-prepare-stackviz → nodepool/elements/infra-package-needs/install.d/89-rsyslog

 #!/bin/bash
-# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,25 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# dib-lint: disable=setu setpipefail sete
+# dib-lint: disable=setu setpipefail
 if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
     set -x
 fi
+set -e
 
-STACKVIZ_NPM_TIMEOUT="15m"
-STACKVIZ_DIR=/opt/stackviz
-cp -r /opt/git/openstack/stackviz $STACKVIZ_DIR
-
-cd $STACKVIZ_DIR
-
-timeout $STACKVIZ_NPM_TIMEOUT npm install
-ret=$?
-
-if [[ $ret -eq 0 ]]; then
-    npm run prod && rm -rf node_modules/
-else
-    echo "stackviz build aborted due to npm error"
-    rm -rf $STACKVIZ_DIR
+if [ "$DISTRO_NAME" == "ubuntu" ]; then
+    rsyslog_dir="$(dirname $0)/../rsyslog.d"
+    cp -RP $rsyslog_dir/* /etc/rsyslog.d/
 fi
-
-exit 0

nodepool/elements/infra-package-needs/package-installs.yaml

+acpid:
 coreutils:
 cron:
 util-linux:
 build-essential:
+python-xml:
+  phase: pre-install.d
 python-dev:
 python3-dev:
+python3:
+  phase: pre-install.d
 uuid-runtime:
 traceroute:
 ntp:
@@ -17,7 +22,13 @@ rsyslog:
 git:
 rsync:
 parted:
-puppet:
 wget:
 iputils-ping:
+iproute2:
 dnsutils:
+haveged:
+iptables:
+centos-release-openstack-ocata:
+redhat-rpm-config:
+redhat-lsb-core:
+curl:

nodepool/elements/infra-package-needs/pkg-map

 {
   "distro": {
     "fedora": {
-      "python3-dev": "python3-devel"
+      "python3-dev": "python3-devel",
+      "iptables": "iptables-services",
+      "haveged": "haveged",
+      "redhat-rpm-config": "redhat-rpm-config",
+      "redhat-lsb-core": "redhat-lsb-core"
     },
     "gentoo": {
       "build-essential": "",
       "cron": "sys-process/cronie",
+      "curl": "net-misc/curl",
       "ntp": "net-misc/ntp",
       "python-dev": "",
       "python3-dev": "",
       "traceroute": "net-analyzer/traceroute",
       "uuid-runtime": ""
+    },
+    "centos": {
+      "centos-release-openstack-ocata": "centos-release-openstack-ocata"
     }
   },
   "family": {
@@ -18,24 +26,37 @@
       "cron": "cronie",
       "build-essential": "glibc-devel gcc make",
       "dnsutils": "bind-utils",
+      "iproute2": "iproute",
       "iputils-ping": "iputils",
       "ntp": "ntp ntp-perl",
       "python-dev": "python-devel",
       "python3-dev": "",
-      "uuid-runtime": ""
+      "uuid-runtime": "",
+      "iptables": "iptables-services",
+      "haveged": ""
     },
     "suse": {
+      "build-essential": "glibc-devel gcc make",
       "dnsutils": "bind-utils",
       "git": "git-core",
       "iputils-ping": "iputils",
       "ntpdate": "",
-      "puppet": "ruby2.1-rubygem-puppet",
+      "python-xml": "python-xml",
       "python-dev": "python-devel",
       "python3-dev": "python3-devel",
+      "python3": "python3",
+      "iptables": "iptables SuSEfirewall2",
       "uuid-runtime": "uuidd"
     }
   },
   "default": {
-    "gentoolkit": ""
+    "curl": "curl",
+    "iptables": "iptables-persistent",
+    "gentoolkit": "",
+    "centos-release-openstack-ocata": "",
+    "redhat-rpm-config": "",
+    "redhat-lsb-core": "",
+    "python-xml": "",
+    "python3": ""
   }
 }

nodepool/elements/infra-package-needs/post-install.d/80-enable-haveged

+#!/bin/bash
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+case "$DIB_INIT_SYSTEM" in
+    upstart)
+        # nothing to do
+        exit 0
+        ;;
+    systemd)
+        systemctl enable haveged.service
+        ;;
+    openrc)
+        # TODO(pabelanger): Make sure we support gentoo if we bring images
+        # online for nodepool.o.o.
+        exit 0
+        ;;
+    sysv)
+        exit 0
+        ;;
+    *)
+        echo "Unsupported init system $DIB_INIT_SYSTEM"
+        exit 1
+        ;;
+esac

nodepool/elements/infra-package-needs/rsyslog.d/50-default.conf

+#  Default rules for rsyslog.
+#
+#			For more information see rsyslog.conf(5) and /etc/rsyslog.conf
+
+#
+# First some standard log files.  Log by facility.
+#
+auth,authpriv.*			/var/log/auth.log
+*.*;auth,authpriv.none		-/var/log/syslog
+#cron.*				/var/log/cron.log
+#daemon.*			-/var/log/daemon.log
+kern.*				-/var/log/kern.log
+#lpr.*				-/var/log/lpr.log
+mail.*				-/var/log/mail.log
+#user.*				-/var/log/user.log
+
+#
+# Logging for the mail system.  Split it up so that
+# it is easy to write scripts to parse these files.
+#
+#mail.info			-/var/log/mail.info
+#mail.warn			-/var/log/mail.warn
+mail.err			/var/log/mail.err
+
+#
+# Logging for INN news system.
+#
+news.crit			/var/log/news/news.crit
+news.err			/var/log/news/news.err
+news.notice			-/var/log/news/news.notice
+
+#
+# Some "catch-all" log files.
+#
+#*.=debug;\
+#	auth,authpriv.none;\
+#	news.none;mail.none	-/var/log/debug
+#*.=info;*.=notice;*.=warn;\
+#	auth,authpriv.none;\
+#	cron,daemon.none;\
+#	mail,news.none		-/var/log/messages
+
+#
+# Emergencies are sent to everybody logged in.
+#
+*.emerg                                :omusrmsg:*
+
+#
+# I like to have messages displayed on the console, but only on a virtual
+# console I usually leave idle.
+#
+#daemon,mail.*;\
+#	news.=crit;news.=err;news.=notice;\
+#	*.=debug;*.=info;\
+#	*.=notice;*.=warn	/dev/tty8
+
+# The named pipe /dev/xconsole is for the `xconsole' utility.  To use it,
+# you must invoke `xconsole' with the `-file' option:
+#
+#    $ xconsole -file /dev/xconsole [...]
+#
+# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
+#      busy site..
+#
+#daemon.*;mail.*;\
+#	news.err;\
+#	*.=debug;*.=info;\
+#	*.=notice;*.=warn	|/dev/xconsole

nodepool/elements/jenkins-slave/extra-data.d/20-jenkins-user

+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# TODO(pabelanger): Once we complete remove puppet from our diskimages, we
+# should also remove our default SSH key. Since only 3rdparty CI system would be
+# using this element moving forward, it doesn't make sense to bake in our
+# default key.
+NODEPOOL_SSH_KEY=${NODEPOOL_SSH_KEY:-AAAAB3NzaC1yc2EAAAADAQABAAABAQC6WutNHfM+YdnjeNFeaIpvxqt+9aDn95Ykpmc+fASSjlDZJtOrueH3ch/v08wkE4WQKg03i+t8VonqEwMGmApYA3VzFsURUQbxzlSz5kHlBQSqgz5JTwUmnt1RH5sePL5pkuJ6JgqJ8PxJod6fiD7YDjaKJW/wBzXGnGg2EkgqrkBQXYL4hyaPuSwsQF0Gdwg3QFqXl+R/GrM6FscUkkJzbjqGKI2GhLT8mf2BIMEAiMFhF5Wl4FFrbvhTfPfW+9VdcsiMxCXaxp00n1x1+Y7OqR5AZ/id0Lkz9ZoFVGS901OB/L4xXrvUtI2y+kIYeF6hxfmAl/zhY0eWzwo9lDPz}
+
+if [ -z $NODEPOOL_SSH_KEY ]; then
+    die "Can not find public key for jenkins user!"
+fi
+
+# save the public key inside the chroot
+echo "ssh-rsa $NODEPOOL_SSH_KEY" > $TMP_HOOKS_PATH/jenkins-user-ssh-public-key

nodepool/elements/jenkins-slave/install.d/20-jenkins-slave

+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# Add jenkins user and group.  Note we don't want to rely on
+# "useradd"'s group adding behaviour, because it might differ across
+# distros.
+groupadd jenkins
+useradd -g jenkins -m jenkins -s /bin/bash
+
+# a lot of caching happens in extra-data.d (for "historical" reasons).
+# We've put the cache stuff into /opt/cache/files, but again, for
+# "historical" reasons, ensure this is available in /home/jenkins
+#
+# Check if the cache exists as we don't have a strict dependency on the
+# devstack-cache element. This allows you to build an image without
+# incurring the cost of caching all the things.
+if [ -d /opt/cache/files ] ; then
+    mkdir -p /home/jenkins/cache
+    ln -sf /opt/cache/files /home/jenkins/cache/files
+    chown -R jenkins:jenkins /opt/cache/files
+fi
+
+# this was copied from outside the chroot by extras.d
+_pub_key=/tmp/in_target.d/jenkins-user-ssh-public-key
+if [ ! -f $_pub_key ]; then
+    die "Can not find Jenkins public key!"
+fi
+
+mkdir /home/jenkins/.ssh
+chmod 700 /home/jenkins/.ssh
+
+cp $_pub_key /home/jenkins/.ssh/authorized_keys
+chmod 644 /home/jenkins/.ssh/authorized_keys
+
+cat > /home/jenkins/.gitconfig <<EOF
+[user]
+    name = OpenStack Jenkins
+    email = jenkins@openstack.org
+    signingkey = jenkins@openstack.org
+[gitreview]
+    rebase = false
+    username = jenkins
+EOF
+
+# cleanup everything to the right owner
+chown -R jenkins:jenkins /home/jenkins

nodepool/elements/nodepool-base/README.rst

@@ -13,5 +13,7 @@ are. Default:
 
 The image should have the unbound DNS resolver package installed, the
 nodepool-base element then configures it to forward DNS queries to:
-  `NODEPOOL_STATIC_NAMESERVER_V6`, default: `2001:4860:4860::8888`
-  `NODEPOOL_STATIC_NAMESERVER_V4`, default: `8.8.8.8`.
+  `NODEPOOL_STATIC_NAMESERVER_V6`, default: `2620:0:ccc::2`
+  `NODEPOOL_STATIC_NAMESERVER_V4`, default: `208.67.222.222`
+  `NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK`, default: `2001:4860:4860::8888`
+  `NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK`, default: `8.8.8.8`.

nodepool/scripts/prepare_tempest_testrepository.py → nodepool/elements/nodepool-base/cleanup.d/50-root

-#!/usr/bin/env python2
+#!/bin/bash
 #
-# Copyright 2014 Hewlett-Packard Development Company, L.P.
+# Copyright 2017 Red Hat, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+#
+if [ ${DIB_DEBUG_TRACE:-1} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
 
-import os
-import sys
-
-from subunit2sql.db import api
-from subunit2sql import shell
-from subunit2sql import write_subunit
-
-
-DB_URI = 'mysql+pymysql://query:query@logstash.openstack.org/subunit2sql'
-
-if len(sys.argv) == 2:
-    TEMPEST_PATH = sys.argv[1]
-elif len(sys.argv) > 2:
-    TEMPEST_PATH = sys.argv[1]
-    DB_URI = sys.argv[2]
-else:
-    TEMPEST_PATH = '/opt/stack/new/tempest'
-
-
-def main():
-    shell.parse_args([])
-    shell.CONF.set_override('connection', DB_URI, group='database')
-    session = api.get_session()
-    runs = api.get_recent_successful_runs_by_run_metadata(
-        'build_name', 'gate-tempest-dsvm-neutron-full',
-        num_runs=10, session=session)
-    session.close()
-    preseed_path = os.path.join(TEMPEST_PATH, 'preseed-streams')
-    if not os.path.isdir(preseed_path):
-        os.mkdir(preseed_path)
-    for run in runs:
-        with open(os.path.join(preseed_path, run.uuid + '.subunit'), 'w') as fd:
-            write_subunit.sql2subunit(run.uuid, fd)
-
-if __name__ == '__main__':
-    main()
+sudo rm -rf $TARGET_ROOT/root/.cache

nodepool/elements/nodepool-base/element-deps

+jenkins-slave
 package-installs
 pip-and-virtualenv
-puppet
 zuul-worker

nodepool/elements/nodepool-base/environment.d/75-nodepool-base-env

-export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2001:4860:4860::8888}
-export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-8.8.8.8}
+export NODEPOOL_STATIC_NAMESERVER_V6=${NODEPOOL_STATIC_NAMESERVER_V6:-2620:0:ccc::2}
+export NODEPOOL_STATIC_NAMESERVER_V4=${NODEPOOL_STATIC_NAMESERVER_V4:-208.67.222.222}
+export NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V6_FALLBACK:-2001:4860:4860::8888}
+export NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK=${NODEPOOL_STATIC_NAMESERVER_V4_FALLBACK:-8.8.8.8}