Skip to content

GitLab

Commit 27a03a8a authored by Ricardo Araujo's avatar Ricardo Araujo
Browse files

Remove nodepool old scripts

parent 3cab158a
Hide whitespace changes
Inline Side-by-side
nodepool/elements/cache-bindep/install.d/50-cache-pkgs deleted 100755 → 0
View file @ 3cab158a
#!/bin/bash
# Copyright (C) 2011-2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Build up a list of packages to install by concatenating bindep lists
FALLBACKREPO=/opt/git/openstack-infra/project-config
FALLBACKFILE=jenkins/data/bindep-fallback.txt
PACKAGES=`git --git-dir=$FALLBACKREPO/.git show master:$FALLBACKFILE`
# NOTE(pabelanger): We currently don't gate on bindep.txt/other-requirements.txt
# for centos and fedora nodes. As such, we have no way to ensure this will
# not break DIB process. So we only run this for ubuntu-trusty.
# bindep.txt is new filename, other-requirements.txt is the old one. We should
# handle both for now.
if [ -f /usr/bin/apt-get ] ; then
for REPO in /opt/git/*/* ; do
BRANCHES=`git --git-dir=$REPO/.git branch -r | grep '^ origin/[^H]'`
for BRANCH in $BRANCHES ; do
for reqsfile in bindep.txt other-requirements.txt ; do
REQS=`git --git-dir=$REPO/.git show $BRANCH:$reqsfile 2>/dev/null \
|| true`
if [ -n "$REQS" ] ; then
PACKAGES=`echo -e "$PACKAGES\n$REQS" | sort -u`
fi
done
done
done
fi
PACKAGELIST=`echo "$PACKAGES" | /usr/bindep-env/bin/bindep -b -f - || true`
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $PACKAGELIST || true
else
${YUM:-yum} install -y --downloadonly $PACKAGELIST
fi
nodepool/elements/infra-package-needs/package-installs.yaml
View file @ 27a03a8a
coreutils:
cron:
util-linux:
openssh-server:
build-essential:
python-dev:
python3-dev:
uuid-runtime:
traceroute:
ntp:
ntpdate:
gentoolkit:
at:
......
nodepool/elements/infra-package-needs/pkg-map
View file @ 27a03a8a
......@@ -6,6 +6,7 @@
"gentoo": {
"build-essential": "",
"cron": "sys-process/cronie",
"ntp": "net-misc/ntp",
"python-dev": "",
"python3-dev": "",
"traceroute": "net-analyzer/traceroute",
......@@ -18,6 +19,7 @@
"build-essential": "glibc-devel gcc make",
"dnsutils": "bind-utils",
"iputils-ping": "iputils",
"ntp": "ntp ntp-perl",
"python-dev": "python-devel",
"python3-dev": "",
"uuid-runtime": ""
......
nodepool/elements/infra-package-needs/post-install.d/80-enable-infra-services 0 → 100755
View file @ 27a03a8a
#!/bin/bash
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
set -x
fi
set -eu
set -o pipefail
case "$DIB_INIT_SYSTEM" in
upstart)
# nothing to do
exit 0
;;
systemd)
if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then
systemctl enable ntp.service
else
systemctl enable ntpd.service
fi
;;
openrc)
# let dib-init-system's postinstall handle enabling init scripts
exit 0
;;
sysv)
# ntp is enabled by default, nothing to do
exit 0
;;
*)
echo "Unsupported init system $DIB_INIT_SYSTEM"
exit 1
;;
esac
nodepool/elements/node-devstack/install.d/20-prepare-node → nodepool/elements/infra-package-needs/post-install.d/89-sshd
View file @ 27a03a8a
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -15,16 +16,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
set -eu
set -o pipefail
export SUDO='true'
export THIN='true'
prepare-node
mkdir -p ~jenkins/cache/files
mkdir -p ~jenkins/cache/pip
# NOTE(pabelanger): Glean configures access for root user, so allow us to
# properly login.
sed -i -e'/PermitRootLogin/d' /etc/ssh/sshd_config \
&& echo "PermitRootLogin Yes" >> /etc/ssh/sshd_config
nodepool/elements/node-devstack/README.rst deleted 100644 → 0
View file @ 3cab158a
Prepare a node to be a devstack slave
nodepool/elements/node-devstack/element-deps deleted 100644 → 0
View file @ 3cab158a
cache-devstack
nodepool-base
prepare-node
puppet
nodepool/elements/nodepool-base/element-deps
View file @ 27a03a8a
package-installs
pip-and-virtualenv
puppet
zuul-worker
nodepool/elements/cache-devstack/install.d/50-download-pkgs → nodepool/elements/nodepool-base/finalise.d/89-journald-persistent
View file @ 27a03a8a
#!/bin/bash
# Copyright (C) 2011-2013 OpenStack Foundation
# Copyright (C) 2017 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
......@@ -21,21 +21,11 @@ if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
fi
set -e
if [[ ${YUM:-yum} == "dnf" ]]; then
echo "*** caching disabled for dnf"
echo "*** see https://etherpad.openstack.org/p/infra-f22"
exit 0
# Journald default is to not persist logs to disk if /var/log/journal is
# not present. Update the configuration to set storage to persistent which
# will create /var/log/journal if necessary and store logs on disk. This
# avoids the situation where test runs can fill the journald ring buffer
# deleting older logs that may be important to the job.
if [ -f /etc/systemd/journald.conf ] ; then
sed -i -e 's/#Storage=auto/Storage=persistent/' /etc/systemd/journald.conf
fi
while read line ; do
if [ -f /usr/bin/apt-get ] ; then
# --ignore-missing conflicts with set -e, so force it to be ok
apt-get -y --ignore-missing -d install $line || true
else
yum install -y --downloadonly $line
fi
done < /tmp/pkgs-to-install
rm /tmp/pkgs-to-install
nodepool/elements/nodepool-base/finalise.d/99-fix-grub-timeout deleted 100755 → 0
View file @ 3cab158a
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Set grub timeout to zero to save boot time.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' /boot/grub/grub.conf
fi
nodepool/elements/nodepool-base/finalise.d/99-restrict-memory deleted 100755 → 0
View file @ 3cab158a
#!/bin/bash -xe
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
# This is done in 50 because the vm element has, at 51, an bunch of code
# for properly handling grub. All we need to do here is get our values
# in to the files.
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
# Limit all test slaves to 8GB of memory so that larger flavors with more
# cpu resources can be used without the risk of becoming dependent on more
# memory.
if [ -f /etc/default/grub ] ; then
sed -i -e 's/^GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/' -e 's/#\?GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="mem=8G /g' /etc/default/grub
if which update-grub &> /dev/null ; then
update-grub
else
# If update-grub isn't available, use grub2-mkconfig directly
/usr/sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
fi
elif [ -f /boot/grub/grub.conf ] ; then
sed -i -e 's/^timeout=[0-9]\+/timeout=0/' -e 's/\(^\s\+kernel.*\)/\1 mem=8G/' /boot/grub/grub.conf
fi
nodepool/elements/nodepool-base/install.d/99-install-zuul deleted 100755 → 0
View file @ 3cab158a
#!/bin/bash
# Copyright (C) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# Install Zuul into a virtualenv
# This is in /usr instead of /usr/local due to this bug on precise:
# https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
# dib-lint: disable=setu setpipefail
if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
set -x
fi
set -e
git clone /opt/git/openstack-infra/zuul /tmp/zuul
sudo virtualenv /usr/zuul-env
sudo -H /usr/zuul-env/bin/pip install /tmp/zuul
sudo rm -fr /tmp/zuul
nodepool/elements/puppet/install.d/05-puppet
View file @ 27a03a8a
......@@ -24,7 +24,7 @@ set -e
# Unset the download cache for this invocation to prevent bleed from build host
unset PIP_DOWNLOAD_CACHE
/bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
SETUP_PIP=false /bin/bash /opt/build_git/openstack-infra/system-config/install_puppet.sh
/bin/bash /opt/build_git/openstack-infra/system-config/install_modules.sh
install -m 0755 -o root -g root $(dirname $0)/../bin/prepare-node /usr/local/bin
nodepool/nodepoolv3.yaml 0 → 100644
View file @ 27a03a8a
elements-dir: /etc/nodepool/elements
images-dir: /opt/nodepool_dib
cron:
cleanup: '*/1 * * * *'
check: '*/15 * * * *'
zookeeper-servers:
- host: nodepool.openstack.org
port: 2181
labels:
- name: ubuntu-xenial
min-ready: 2
providers:
- name: infracloud-chocolate
region-name: 'RegionOne'
cloud: infracloud-chocolate
boot-timeout: 120
rate: 0.25
image-name-format: '{image.name}-{timestamp}'
diskimages:
- name: ubuntu-xenial
config-drive: true
pools:
- name: main
max-servers: 5
labels:
- name: ubuntu-xenial
min-ram: 8000
name-filter: 'nodepool'
diskimage: ubuntu-xenial
- name: infracloud-vanilla
region-name: 'RegionOne'
cloud: infracloud-vanilla
boot-timeout: 120
rate: 0.25
image-name-format: '{image.name}-{timestamp}'
diskimages:
- name: ubuntu-xenial
config-drive: true
pools:
- name: main
max-servers: 5
labels:
- name: ubuntu-xenial
min-ram: 8000
name-filter: 'nodepool'
diskimage: ubuntu-xenial
diskimages:
- name: ubuntu-xenial
elements:
- ubuntu-minimal
- vm
- simple-init
- openstack-repos
- nodepool-base
- cache-devstack
- initialize-urandom
- cache-bindep
- growroot
- infra-package-needs
- stackviz
release: xenial
env-vars:
TMPDIR: /opt/dib_tmp
DIB_CHECKSUM: '1'
DIB_IMAGE_CACHE: /opt/dib_cache
DIB_APT_LOCAL_CACHE: '0'
DIB_DISABLE_APT_CLEANUP: '1'
DIB_GRUB_TIMEOUT: '0'
DIB_DEBIAN_COMPONENTS: 'main,universe'
nodepool/scripts/README.md deleted 100644 → 0
View file @ 3cab158a
These scripts will be copied to the nodepool slave image's
/etc/nodepool/scripts folder.
The are optional. You can reference these:
[openstack-infra/project-config](http://git.openstack.org/cgit/openstack-infra/project-config/tree/nodepool/scripts)
nodepool/scripts/cache_devstack.py deleted 100755 → 0
View file @ 3cab158a
#!/usr/bin/env python
# Copyright (C) 2011-2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
from common import run_local
DEVSTACK = os.path.expanduser('/opt/git/openstack-dev/devstack')
CACHEDIR = os.path.expanduser('~/cache/files')
# Some jobs might require newer distro packages, so we can pre-cache
# deb packages from specified Ubuntu Cloud Archive pockets.
UCA_POCKETS = []
def git_branches():
branches = []
for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
branch = branch.strip()
if not branch.startswith('remotes/origin'):
continue
branches.append(branch)
return branches
def tokenize(fn, tokens, distribution, comment=None):
for line in open(fn):
if 'dist:' in line and ('dist:%s' % distribution not in line):
continue
if 'qpid' in line:
continue # TODO: explain why this is here
if comment and comment in line:
line = line[:line.find(comment)]
line = line.strip()
if line and line not in tokens:
tokens.append(line)
def _legacy_find_images(basedir):
"""Divine what images we should use based on parsing stackrc."""
images = []
for line in open(os.path.join(basedir, 'stackrc')):
line = line.strip()
if line.startswith('IMAGE_URLS'):
if '#' in line:
line = line[:line.find('#')]
if line.endswith(';;'):
line = line[:-2]
line = line.split('=', 1)[1].strip()
if line.startswith('${IMAGE_URLS:-'):
line = line[len('${IMAGE_URLS:-'):]
if line.endswith('}'):
line = line[:-1]
if not line:
continue
if line[0] == line[-1] == '"':
line = line[1:-1]
# Add image to the list to be downloaded, but
# skip downloading giant vmware images
images += [x.strip() for x in line.split(',')
if not x.strip().endswith('vmdk')]
return images
def _find_images(basedir):
images = []
image_tool = os.path.join(basedir, 'tools', 'image_list.sh')
if os.path.exists(image_tool):
returncode, out = run_local(image_tool, status=True)
if returncode:
print "%s failed" % image_tool
print "Exit: %s, Output: %s" % (returncode, out)
# reset images so we'll fall back
images = []
else:
images = out.split('\n')
return images
def local_prep(distribution):
branches = []
for branch in git_branches():
# Ignore branches of the form 'somestring -> someotherstring'
# as this denotes a symbolic reference and the entire string
# as is cannot be checked out. We can do this safely as the
# reference will refer to one of the other branches returned
# by git_branches.
if ' -> ' in branch:
continue
branch_data = {'name': branch}
print 'Branch: ', branch
run_local(['sudo', 'git', 'checkout', branch], cwd=DEVSTACK)
run_local(['sudo', 'git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)
if os.path.exists('/usr/bin/apt-get'):
debs = []
debdir = os.path.join(DEVSTACK, 'files', 'debs')
if not os.path.exists(debdir):
debdir = os.path.join(DEVSTACK, 'files', 'apts')
for fn in os.listdir(debdir):
fn = os.path.join(debdir, fn)
tokenize(fn, debs, distribution, comment='#')
branch_data['debs'] = debs
if os.path.exists('/usr/bin/yum'):
rpms = []
rpmdir = os.path.join(DEVSTACK, 'files', 'rpms')
for fn in os.listdir(rpmdir):
fn = os.path.join(rpmdir, fn)
tokenize(fn, rpms, distribution, comment='#')
branch_data['rpms'] = rpms
images = _find_images(DEVSTACK)
if not images:
images = _legacy_find_images(DEVSTACK)
branch_data['images'] = images
branches.append(branch_data)
return branches
def download(url, fname):
run_local(['wget', '-nv', '-c', url, '-O', os.path.join(CACHEDIR, fname)])
def cache_debs(debs, uca_pocket=None):
"""Cache a list of deb packages, optionally pulling from an Ubuntu Cloud
Archive pocket. If a UCA pocket is specified, it is enabled temporarily
for caching only.
"""
if uca_pocket:
# Note this will install the ubuntu-cloud-keyring package which
# contains the required GPG key.
run_local(['sudo', 'add-apt-repository', '-y',
'cloud-archive:%s' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
run_local(['sudo', 'apt-get', '-y', '-d', 'install'] + debs)
if uca_pocket:
run_local(['sudo', 'rm', '-f',
'/etc/apt/sources.list.d/cloudarchive-%s.list' % uca_pocket])
run_local(['sudo', 'apt-get', 'update'])
def main():
distribution = sys.argv[1]
branches = local_prep(distribution)
image_filenames = []
for branch_data in branches:
if branch_data.get('debs'):
cache_debs(branch_data['debs'])
for uca in sorted(UCA_POCKETS):
cache_debs(branch_data['debs'], uca)
elif branch_data.get('rpms'):
run_local(['sudo', 'yum', 'install', '-y', '--downloadonly'] +
branch_data['rpms'])
else:
sys.exit('No supported package data found.')
for url in branch_data['images']:
fname = url.split('/')[-1]
if fname in image_filenames:
continue
image_filenames.append(fname)
download(url, fname)
# cache get-pip, because upstream network connection fails more
# often than you might imagine.
download('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
if __name__ == '__main__':
main()