Commit a1777b4b authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Merge branch 'updating_configs' into 'master'

Updating configuration to fix image generation

See merge request !11
parents 3e077606 f94efcc6
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# Install Zuul into a virtualenv
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
git clone /opt/git/openstack-infra/zuul /tmp/zuul
sudo virtualenv /usr/zuul-env
sudo -H /usr/zuul-env/bin/pip install /tmp/zuul
sudo rm -fr /tmp/zuul
#!/usr/bin/env python
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -16,17 +16,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
def run_local(cmd, status=False, cwd='.', env={}):
print "Running:", cmd
newenv = os.environ
newenv.update(env)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
stderr=subprocess.STDOUT, env=newenv)
(out, nothing) = p.communicate()
if status:
return (p.returncode, out.strip())
return out.strip()
# NOTE(pabelanger): Until zuulv3 lands, manually copy scripts / data into
# /usr/local/jenkins.
mkdir -p /usr/local/jenkins
cp -a /opt/git/openstack-infra/project-config/jenkins/data \
/usr/local/jenkins/common_data
cp -a /opt/git/openstack-infra/project-config/jenkins/scripts \
/usr/local/jenkins/slave_scripts
# TODO(pabelanger): Stop changing permissions, we really should be setting them
# within git, not at runtime.
chmod 0755 -R /usr/local/jenkins
Download all repos and packages that a devstack run might need.
===============
openstack-repos
===============
Download all repos and packages that might be needed.
Environment variables:
----------------------
DIB_CUSTOM_PROJECTS_LIST_URL
:Required: No
:Default: None
:Description: Url to a yaml file contains custom list of repos.
The custom yaml file has the same structure as the default file:
'https://git.openstack.org/cgit/openstack-infra/project-config/plain/gerrit/projects.yaml'
Download only the repos that appear in the custom file rather than
downloading all openstack repos that appear in the default file.
:Example:
DIB_CUSTOM_PROJECTS_LIST_URL='file:///etc//project-config//gerrit//custom_projects.yaml'
......@@ -18,6 +18,7 @@
import os
import urllib2
from urllib2 import URLError
import yaml
URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
......@@ -27,22 +28,42 @@ TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
def get_project_list(url):
try:
projects = [f['project'] for f in yaml.load(urllib2.urlopen(url))]
return projects
except URLError:
print "Could not open project list url: '%s'" % url
raise
def main():
projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
projects = []
if CUSTOM_PROJECTS_LIST_URL:
projects = get_project_list(CUSTOM_PROJECTS_LIST_URL)
if not projects:
projects = get_project_list(URL)
with open(PROJECTS_REPOS, 'w') as projects_list:
for project in projects:
# Skip repos that are inactive
dirname = os.path.dirname(project)
if not ('attic' in dirname or dirname == 'stackforge'):
args = dict(
name=os.path.basename(project),
location=os.path.join('/opt/git', project),
url='%s/%s.git' % (GIT_BASE, project),
ref='*')
projects_list.write("%(name)s git %(location)s "
"%(url)s %(ref)s\n" % args)
if ('attic' in dirname or dirname == 'stackforge'):
continue
# Skip the /deb- git repo copies
if ('/deb-' in project):
continue
args = dict(
name=os.path.basename(project),
location=os.path.join('/opt/git', project),
url='%s/%s.git' % (GIT_BASE, project),
ref='*')
projects_list.write("%(name)s git %(location)s "
"%(url)s %(ref)s\n" % args)
# Clone openstack-infra/system-config again so that we can use it to
# build the image without interferring with the slave repo cache.
project = 'openstack-infra/system-config'
......
......@@ -60,4 +60,6 @@ fi
set -e
# Make sure resolv.conf settings don't break dib
echo "nameserver $NODEPOOL_STATIC_NAMESERVER_V4"> /etc/resolv.conf
if [ -n "$NODEPOOL_STATIC_NAMESERVER_V4" ] ; then
echo "nameserver $NODEPOOL_STATIC_NAMESERVER_V4"> /etc/resolv.conf
fi
......@@ -24,7 +24,7 @@ set -e
# Unset the download cache for this invocation to prevent bleed from build host
unset PIP_DOWNLOAD_CACHE
/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
SETUP_PIP=false /bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
/bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
......@@ -39,6 +39,16 @@ case $OS_FAMILY in
# time updating.
update=""
;;
"Gentoo")
# make a fake repo to satisfy the removal below
touch /tmp/fakerepo
repo=/tmp/fakerepo
update=""
;;
"Suse")
repo=/etc/zypp/repos.d/systemsmanagement_puppet.repo
update=""
;;
*)
die "Don't know how to cleanup!"
;;
......
Install databases on slave nodes
......@@ -4,6 +4,12 @@
"24": {
"npm": ""
}
},
"opensuse": {
"42.2": {
"nodejs": "nodejs4",
"npm": "npm4"
}
}
},
"family": {
......@@ -11,7 +17,8 @@
"nodejs": "nodejs-legacy"
},
"suse": {
"npm": "nodejs-npm"
"nodejs": "nodejs6",
"npm": "npm6"
}
},
"default": {
......
......@@ -2,6 +2,12 @@
"family": {
"debian": {
"libselinux-python": ""
},
"gentoo": {
"libselinux-python": "sys-libs/libselinux"
},
"suse": {
"libselinux-python": ""
}
}
}
script-dir: /etc/nodepool/scripts
elements-dir: /etc/nodepool/elements
images-dir: /opt/nodepool_dib
......@@ -47,6 +46,7 @@ diskimages:
- openstack-repos
- nodepool-base
- cache-devstack
- initialize-urandom
- cache-bindep
- growroot
- infra-package-needs
......@@ -54,9 +54,15 @@ diskimages:
release: xenial
env-vars:
TMPDIR: /opt/dib_tmp
# DIB_CHECKSUM: '1'
DIB_IMAGE_CACHE: /opt/dib_cache
# DIB_APT_LOCAL_CACHE: '0'
# DIB_DISABLE_APT_CLEANUP: '1'
# DIB_GRUB_TIMEOUT: '0'
# DIB_DEBIAN_COMPONENTS: 'main,universe'
NODEPOOL_STATIC_NAMESERVER_V4: 10.20.6.1
providers:
- name: local_01
cloud: cloud4
......
elements-dir: /etc/nodepool/elements
images-dir: /opt/nodepool_dib
cron:
cleanup: '*/1 * * * *'
check: '*/15 * * * *'
zookeeper-servers:
- host: nodepool.openstack.org
port: 2181
labels:
- name: ubuntu-xenial
min-ready: 2
providers:
- name: infracloud-chocolate
region-name: 'RegionOne'
cloud: infracloud-chocolate
boot-timeout: 120
rate: 0.25
image-name-format: '{image.name}-{timestamp}'
diskimages:
- name: ubuntu-xenial
config-drive: true
pools:
- name: main
max-servers: 5
labels:
- name: ubuntu-xenial
min-ram: 8000
name-filter: 'nodepool'
diskimage: ubuntu-xenial
- name: infracloud-vanilla
region-name: 'RegionOne'
cloud: infracloud-vanilla
boot-timeout: 120
rate: 0.25
image-name-format: '{image.name}-{timestamp}'
diskimages:
- name: ubuntu-xenial
config-drive: true
pools:
- name: main
max-servers: 5
labels:
- name: ubuntu-xenial
min-ram: 8000
name-filter: 'nodepool'
diskimage: ubuntu-xenial
diskimages:
- name: ubuntu-xenial
elements:
- ubuntu-minimal
- vm
- simple-init
- openstack-repos
- nodepool-base
- cache-devstack
- initialize-urandom
- cache-bindep
- growroot
- infra-package-needs
- stackviz
release: xenial
env-vars:
TMPDIR: /opt/dib_tmp
DIB_CHECKSUM: '1'
DIB_IMAGE_CACHE: /opt/dib_cache
DIB_APT_LOCAL_CACHE: '0'
DIB_DISABLE_APT_CLEANUP: '1'
DIB_GRUB_TIMEOUT: '0'
DIB_DEBIAN_COMPONENTS: 'main,universe'
These scripts will be copied to the nodepool slave image's
/etc/nodepool/scripts folder.
The are optional. You can reference these:
[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/scripts)
#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from common import run_local
DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
CACHEDIR = os.path.expanduser('~/cache/files')
# Some jobs might require newer distro packages, so we can pre-cache
# deb packages from specified Ubuntu Cloud Archive pockets.
UCA_POCKETS = []
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.find(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
for line in open(os.path.join(basedir, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.find('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
return images
def _find_images(basedir):
images = []
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
# reset images so we'll fall back
images = []
else:
images = out.split('\n')
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/yum'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
return branches
def download(url, fname):
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
def cache_debs(debs, uca_pocket=None):
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
for caching only.
"""
if uca_pocket:
# Note this will install the ubuntu-cloud-keyring package which
# contains the required GPG key.
run_local(['sudo', 'add-apt-repository', '-y',
'cloud-archive:%s' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
if uca_pocket:
run_local(['sudo', 'rm', '-f',
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
if __name__ == '__main__':
main()
#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os.path
import re
import shutil
import sys
import urllib2
from common import run_local
URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
'plain/gerrit/projects.yaml')
PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
# Not using an arg libraries in order to avoid module imports that
# are not available across all python versions
if len(sys.argv) > 1:
GIT_BASE = sys.argv[1]
else:
GIT_BASE = 'git://git.openstack.org'
def clone_repo(project):
remote = '%s/%s.git' % (GIT_BASE, project)
# Clear out any existing target directory first, in case of a retry.
try:
shutil.rmtree(os.path.join('/opt/git', project))
except OSError:
pass
# Try to clone the requested git repository.
(status, out) = run_local(['git', 'clone', remote, project],
status=True, cwd='/opt/git')
# If it claims to have worked, make sure we can list branches.
if status == 0:
(status, moreout) = run_local(['git', 'branch', '-a'], status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# If that worked, try resetting to HEAD to make sure it's there.
if status == 0:
(status, moreout) = run_local(['git', 'reset', '--hard', 'HEAD'],
status=True,
cwd=os.path.join('/opt/git', project))
out = '\n'.join((out, moreout))
# Status of 0 imples all the above worked, 1 means something failed.
return (status, out)
def main():
# TODO(jeblair): use gerrit rest api when available