Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11

nodepool/elements/nodepool-base/install.d/99-install-zuul

-#!/bin/bash
-# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# Install Zuul into a virtualenv
-# This is in /usr instead of /usr/local due to this bug on precise:
-# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
-
-# dib-lint: disable=setu setpipefail
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
-    set -x
-fi
-set -e
-
-git clone /opt/git/openstack-infra/zuul /tmp/zuul
-sudo virtualenv /usr/zuul-env
-sudo -H /usr/zuul-env/bin/pip install /tmp/zuul
-sudo rm -fr /tmp/zuul

nodepool/elements/nodepool-base/package-installs.yaml

+unbound:

nodepool/scripts/common.py → nodepool/elements/nodepool-base/post-install.d/89-jenkins-scripts

-#!/usr/bin/env python
-
+#!/bin/bash
 # Copyright (C) 2011-2013 OpenStack Foundation
+# Copyright 2016 Red Hat, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,17 +16,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import subprocess
-
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
 
-def run_local(cmd, status=False, cwd='.', env={}):
-    print "Running:", cmd
-    newenv = os.environ
-    newenv.update(env)
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
-                         stderr=subprocess.STDOUT, env=newenv)
-    (out, nothing) = p.communicate()
-    if status:
-        return (p.returncode, out.strip())
-    return out.strip()
+# NOTE(pabelanger): Until zuulv3 lands, manually copy scripts / data into
+# /usr/local/jenkins.
+mkdir -p /usr/local/jenkins
+cp -a /opt/git/openstack-infra/project-config/jenkins/data \
+    /usr/local/jenkins/common_data
+cp -a /opt/git/openstack-infra/project-config/jenkins/scripts \
+    /usr/local/jenkins/slave_scripts
+# TODO(pabelanger): Stop changing permissions, we really should be setting them
+# within git, not at runtime.
+chmod 0755 -R /usr/local/jenkins

nodepool/elements/openstack-repos/README.rst

-Download all repos and packages that a devstack run might need.
+===============
+openstack-repos
+===============
+
+Download all repos and packages that might be needed.
+
+Environment variables:
+----------------------
+
+DIB_CUSTOM_PROJECTS_LIST_URL
+  :Required: No
+  :Default: None
+  :Description: Url to a yaml file contains custom list of repos.
+    The custom yaml file has the same structure as the default file:
+    'https://git.openstack.org/cgit/openstack-infra/project-config/plain/gerrit/projects.yaml'
+    Download only the repos that appear in the custom file rather than
+    downloading all openstack repos that appear in the default file.
+  :Example:
+    DIB_CUSTOM_PROJECTS_LIST_URL='file:///etc//project-config//gerrit//custom_projects.yaml'

nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list

@@ -18,6 +18,7 @@
 
 import os
 import urllib2
+from urllib2 import URLError
 import yaml
 
 URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
@@ -27,22 +28,42 @@ TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
 PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
 GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
 
+CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
+
+def get_project_list(url):
+    try:
+        projects = [f['project'] for f in yaml.load(urllib2.urlopen(url))]
+        return projects
+    except URLError:
+        print "Could not open project list url: '%s'" % url
+        raise
 
 def main():
-    projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
+    projects = []
+    if CUSTOM_PROJECTS_LIST_URL:
+        projects = get_project_list(CUSTOM_PROJECTS_LIST_URL)
+
+    if not projects:
+        projects = get_project_list(URL)
+
     with open(PROJECTS_REPOS, 'w') as projects_list:
         for project in projects:
             # Skip repos that are inactive
             dirname = os.path.dirname(project)
-            if not ('attic' in dirname or dirname == 'stackforge'):
-                args = dict(
-                    name=os.path.basename(project),
-                    location=os.path.join('/opt/git', project),
-                    url='%s/%s.git' % (GIT_BASE, project),
-                    ref='*')
-
-                projects_list.write("%(name)s git %(location)s "
-                                    "%(url)s %(ref)s\n" % args)
+            if ('attic' in dirname or dirname == 'stackforge'):
+                continue
+            # Skip the /deb- git repo copies
+            if ('/deb-' in project):
+                continue
+
+            args = dict(
+                name=os.path.basename(project),
+                location=os.path.join('/opt/git', project),
+                url='%s/%s.git' % (GIT_BASE, project),
+                ref='*')
+
+            projects_list.write("%(name)s git %(location)s "
+                                "%(url)s %(ref)s\n" % args)
         # Clone openstack-infra/system-config again so that we can use it to
         # build the image without interferring with the slave repo cache.
         project = 'openstack-infra/system-config'

nodepool/elements/puppet/bin/prepare-node

@@ -60,4 +60,6 @@ fi
 set -e
 
 # Make sure resolv.conf settings don't break dib
-echo "nameserver $NODEPOOL_STATIC_NAMESERVER_V4"> /etc/resolv.conf
+if [ -n "$NODEPOOL_STATIC_NAMESERVER_V4" ] ; then
+    echo "nameserver $NODEPOOL_STATIC_NAMESERVER_V4"> /etc/resolv.conf
+fi

nodepool/elements/puppet/install.d/05-puppet

@@ -24,7 +24,7 @@ set -e
 # Unset the download cache for this invocation to prevent bleed from build host
 unset PIP_DOWNLOAD_CACHE
 
-/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
+SETUP_PIP=false /bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
 /bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
 
 install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin

nodepool/elements/puppet/install.d/95-clean-repos

@@ -39,6 +39,16 @@ case $OS_FAMILY in
         # time updating.
         update=""
         ;;
+    "Gentoo")
+        # make a fake repo to satisfy the removal below
+        touch /tmp/fakerepo
+        repo=/tmp/fakerepo
+        update=""
+        ;;
+    "Suse")
+        repo=/etc/zypp/repos.d/systemsmanagement_puppet.repo
+        update=""
+        ;;
     *)
         die "Don't know how to cleanup!"
         ;;

nodepool/elements/slave-db/README.rst

-Install databases on slave nodes

nodepool/elements/stackviz/pkg-map

@@ -4,6 +4,12 @@
       "24": {
         "npm": ""
       }
+    },
+    "opensuse": {
+      "42.2": {
+        "nodejs": "nodejs4",
+        "npm": "npm4"
+      }
     }
   },
   "family": {
@@ -11,7 +17,8 @@
       "nodejs": "nodejs-legacy"
     },
     "suse": {
-      "npm": "nodejs-npm"
+      "nodejs": "nodejs6",
+      "npm": "npm6"
     }
   },
   "default": {

nodepool/elements/zuul-worker/pkg-map

@@ -2,6 +2,12 @@
   "family": {
     "debian": {
       "libselinux-python": ""
+    },
+    "gentoo": {
+      "libselinux-python": "sys-libs/libselinux"
+    },
+    "suse": {
+      "libselinux-python": ""
     }
   }
 }

nodepool/nodepool.yaml

-script-dir: /etc/nodepool/scripts
 elements-dir: /etc/nodepool/elements
 images-dir: /opt/nodepool_dib
 
@@ -47,6 +46,7 @@ diskimages:
         - openstack-repos
         - nodepool-base
         - cache-devstack
+        - initialize-urandom
         - cache-bindep
         - growroot
         - infra-package-needs
@@ -54,9 +54,15 @@ diskimages:
       release: xenial
       env-vars:
         TMPDIR: /opt/dib_tmp
+#        DIB_CHECKSUM: '1'
         DIB_IMAGE_CACHE: /opt/dib_cache
+#        DIB_APT_LOCAL_CACHE: '0'
+#        DIB_DISABLE_APT_CLEANUP: '1'
+#        DIB_GRUB_TIMEOUT: '0'
+#        DIB_DEBIAN_COMPONENTS: 'main,universe'
         NODEPOOL_STATIC_NAMESERVER_V4: 10.20.6.1
 
+
 providers:
   - name: local_01
     cloud: cloud4

nodepool/nodepoolv3.yaml

+elements-dir: /etc/nodepool/elements
+images-dir: /opt/nodepool_dib
+
+cron:
+  cleanup: '*/1 * * * *'
+  check: '*/15 * * * *'
+
+zookeeper-servers:
+  - host: nodepool.openstack.org
+    port: 2181
+
+labels:
+  - name: ubuntu-xenial
+    min-ready: 2
+
+providers:
+  - name: infracloud-chocolate
+    region-name: 'RegionOne'
+    cloud: infracloud-chocolate
+    boot-timeout: 120
+    rate: 0.25
+    image-name-format: '{image.name}-{timestamp}'
+    diskimages:
+      - name: ubuntu-xenial
+        config-drive: true
+    pools:
+      - name: main
+        max-servers: 5
+        labels:
+          - name: ubuntu-xenial
+            min-ram: 8000
+            name-filter: 'nodepool'
+            diskimage: ubuntu-xenial
+
+  - name: infracloud-vanilla
+    region-name: 'RegionOne'
+    cloud: infracloud-vanilla
+    boot-timeout: 120
+    rate: 0.25
+    image-name-format: '{image.name}-{timestamp}'
+    diskimages:
+      - name: ubuntu-xenial
+        config-drive: true
+    pools:
+      - name: main
+        max-servers: 5
+        labels:
+          - name: ubuntu-xenial
+            min-ram: 8000
+            name-filter: 'nodepool'
+            diskimage: ubuntu-xenial
+
+diskimages:
+  - name: ubuntu-xenial
+    elements:
+      - ubuntu-minimal
+      - vm
+      - simple-init
+      - openstack-repos
+      - nodepool-base
+      - cache-devstack
+      - initialize-urandom
+      - cache-bindep
+      - growroot
+      - infra-package-needs
+      - stackviz
+    release: xenial
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_CHECKSUM: '1'
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_APT_LOCAL_CACHE: '0'
+      DIB_DISABLE_APT_CLEANUP: '1'
+      DIB_GRUB_TIMEOUT: '0'
+      DIB_DEBIAN_COMPONENTS: 'main,universe'

nodepool/scripts/README.md

-These scripts will be copied to the nodepool slave image's
-/etc/nodepool/scripts folder.
-
-The are optional. You can reference these:
-[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/scripts)

nodepool/scripts/cache_devstack.py

-#!/usr/bin/env python
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-
-from common import run_local
-
-DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
-CACHEDIR = os.path.expanduser('~/cache/files')
-
-# Some jobs might require newer distro packages, so we can pre-cache
-# deb packages from specified Ubuntu Cloud Archive pockets.
-UCA_POCKETS = []
-
-
-def git_branches():
-    branches = []
-    for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
-        branch = branch.strip()
-        if not branch.startswith('remotes/origin'):
-            continue
-        branches.append(branch)
-    return branches
-
-
-def tokenize(fn, tokens, distribution, comment=None):
-    for line in open(fn):
-        if 'dist:' in line and ('dist:%s' % distribution not in line):
-            continue
-        if 'qpid' in line:
-            continue  # TODO: explain why this is here
-        if comment and comment in line:
-            line = line[:line.find(comment)]
-        line = line.strip()
-        if line and line not in tokens:
-            tokens.append(line)
-
-
-def _legacy_find_images(basedir):
-    """Divine what images we should use based on parsing stackrc."""
-    images = []
-    for line in open(os.path.join(basedir, 'stackrc')):
-        line = line.strip()
-        if line.startswith('IMAGE_URLS'):
-            if '#' in line:
-                line = line[:line.find('#')]
-            if line.endswith(';;'):
-                line = line[:-2]
-            line = line.split('=', 1)[1].strip()
-            if line.startswith('${IMAGE_URLS:-'):
-                line = line[len('${IMAGE_URLS:-'):]
-            if line.endswith('}'):
-                line = line[:-1]
-            if not line:
-                continue
-            if line[0] == line[-1] == '"':
-                line = line[1:-1]
-            # Add image to the list to be downloaded, but
-            # skip downloading giant vmware images
-            images += [x.strip() for x in line.split(',')
-                       if not x.strip().endswith('vmdk')]
-    return images
-
-
-def _find_images(basedir):
-    images = []
-    image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
-    if os.path.exists(image_tool):
-        returncode, out = run_local(image_tool, status=True)
-        if returncode:
-            print "%s failed" % image_tool
-            print "Exit: %s, Output: %s" % (returncode, out)
-            # reset images so we'll fall back
-            images = []
-        else:
-            images = out.split('\n')
-    return images
-
-
-def local_prep(distribution):
-    branches = []
-    for branch in git_branches():
-        # Ignore branches of the form 'somestring -> someotherstring'
-        # as this denotes a symbolic reference and the entire string
-        # as is cannot be checked out. We can do this safely as the
-        # reference will refer to one of the other branches returned
-        # by git_branches.
-        if ' -> ' in branch:
-            continue
-        branch_data = {'name': branch}
-        print 'Branch: ', branch
-        run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
-        run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
-
-        if os.path.exists('/usr/bin/apt-get'):
-            debs = []
-            debdir = os.path.join(DEVSTACK, 'files', 'debs')
-            if not os.path.exists(debdir):
-                debdir = os.path.join(DEVSTACK, 'files', 'apts')
-            for fn in os.listdir(debdir):
-                fn = os.path.join(debdir, fn)
-                tokenize(fn, debs, distribution, comment='#')
-            branch_data['debs'] = debs
-
-        if os.path.exists('/usr/bin/yum'):
-            rpms = []
-            rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
-            for fn in os.listdir(rpmdir):
-                fn = os.path.join(rpmdir, fn)
-                tokenize(fn, rpms, distribution, comment='#')
-            branch_data['rpms'] = rpms
-
-        images = _find_images(DEVSTACK)
-        if not images:
-            images = _legacy_find_images(DEVSTACK)
-
-        branch_data['images'] = images
-        branches.append(branch_data)
-    return branches
-
-
-def download(url, fname):
-    run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
-
-
-def cache_debs(debs, uca_pocket=None):
-    """Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
-    Archive pocket.  If a UCA pocket is specified, it is enabled temporarily
-    for caching only.
-    """
-    if uca_pocket:
-        # Note this will install the ubuntu-cloud-keyring package which
-        # contains the required GPG key.
-        run_local(['sudo', 'add-apt-repository', '-y',
-                   'cloud-archive:%s' % uca_pocket])
-        run_local(['sudo', 'apt-get', 'update'])
-    run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
-    if uca_pocket:
-        run_local(['sudo', 'rm', '-f',
-                  '/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
-        run_local(['sudo', 'apt-get', 'update'])
-
-
-def main():
-    distribution = sys.argv[1]
-
-    branches = local_prep(distribution)
-    image_filenames = []
-    for branch_data in branches:
-        if branch_data.get('debs'):
-            cache_debs(branch_data['debs'])
-            for uca in sorted(UCA_POCKETS):
-                cache_debs(branch_data['debs'], uca)
-        elif branch_data.get('rpms'):
-            run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
-                      branch_data['rpms'])
-        else:
-            sys.exit('No supported package data found.')
-
-        for url in branch_data['images']:
-            fname = url.split('/')[-1]
-            if fname in image_filenames:
-                continue
-            image_filenames.append(fname)
-            download(url, fname)
-
-    # cache get-pip, because upstream network connection fails more
-    # often than you might imagine.
-    download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
-
-
-if __name__ == '__main__':
-    main()

nodepool/scripts/cache_git_repos.py

-#!/usr/bin/env python
-
-# Copyright (C) 2011-2013 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path
-import re
-import shutil
-import sys
-import urllib2
-
-from common import run_local
-
-URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
-       'plain/gerrit/projects.yaml')
-PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
-
-# Not using an arg libraries in order to avoid module imports that
-# are not available across all python versions
-if len(sys.argv) > 1:
-    GIT_BASE = sys.argv[1]
-else:
-    GIT_BASE = 'git://git.openstack.org'
-
-
-def clone_repo(project):
-    remote = '%s/%s.git' % (GIT_BASE, project)
-
-    # Clear out any existing target directory first, in case of a retry.
-    try:
-        shutil.rmtree(os.path.join('/opt/git', project))
-    except OSError:
-        pass
-
-    # Try to clone the requested git repository.
-    (status, out) = run_local(['git', 'clone', remote, project],
-                              status=True, cwd='/opt/git')
-
-    # If it claims to have worked, make sure we can list branches.
-    if status == 0:
-        (status, moreout) = run_local(['git', 'branch', '-a'], status=True,
-                                      cwd=os.path.join('/opt/git', project))
-        out = '\n'.join((out, moreout))
-
-    # If that worked, try resetting to HEAD to make sure it's there.
-    if status == 0:
-        (status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
-                                      status=True,
-                                      cwd=os.path.join('/opt/git', project))
-        out = '\n'.join((out, moreout))
-
-    # Status of 0 imples all the above worked, 1 means something failed.
-    return (status, out)
-
-
-def main():
-    # TODO(jeblair): use gerrit rest api when available