diff --git a/ReadME.MD b/ReadME.MD index 4a9a2f1..c9a7fa1 100644 --- a/ReadME.MD +++ b/ReadME.MD @@ -13,4 +13,7 @@ charm pull cs:~containers/calico-812 charm pull cs:~containers/kubeapi-load-balancer-786 charm pull cs:~containers/keepalived-85 charm pull cs:~containers/coredns-20 +charm pull cs:~containers/ubuntu-20 +charm pull cs:~containers/nrpe-75 ``` + diff --git a/nrpe/Makefile b/nrpe/Makefile new file mode 100644 index 0000000..5cbc30f --- /dev/null +++ b/nrpe/Makefile @@ -0,0 +1,75 @@ +PYTHON := /usr/bin/python3 + +PROJECTPATH=$(dir $(realpath $(MAKEFILE_LIST))) +ifndef CHARM_BUILD_DIR + CHARM_BUILD_DIR=${PROJECTPATH}.build +endif +METADATA_FILE="metadata.yaml" +CHARM_NAME=$(shell cat ${PROJECTPATH}/${METADATA_FILE} | grep -E '^name:' | awk '{print $$2}') + +help: + @echo "This project supports the following targets" + @echo "" + @echo " make help - show this text" + @echo " make clean - remove unneeded files" + @echo " make submodules - make sure that the submodules are up-to-date" + @echo " make submodules-update - update submodules to latest changes on remote branch" + @echo " make build - build the charm" + @echo " make release - run clean and build targets" + @echo " make lint - run flake8 and black --check" + @echo " make black - run black and reformat files" + @echo " make proof - run charm proof" + @echo " make unittests - run the tests defined in the unittest subdirectory" + @echo " make functional - run the tests defined in the functional subdirectory" + @echo " make test - run lint, proof, unittests and functional targets" + @echo "" + +clean: + @echo "Cleaning files" + @git clean -ffXd -e '!.idea' + @echo "Cleaning existing build" + @rm -rf ${CHARM_BUILD_DIR}/${CHARM_NAME} + +submodules: + @echo "Cloning submodules" + @git submodule update --init --recursive + +submodules-update: + @echo "Pulling latest updates for submodules" + @git submodule update --init --recursive --remote --merge + +build: submodules-update + @echo "Building charm to base directory ${CHARM_BUILD_DIR}/${CHARM_NAME}" + @-git rev-parse --abbrev-ref HEAD > ./repo-info + @-git describe --always > ./version + @mkdir -p ${CHARM_BUILD_DIR}/${CHARM_NAME} + @cp -a ./* ${CHARM_BUILD_DIR}/${CHARM_NAME} + +release: clean build + @echo "Charm is built at ${CHARM_BUILD_DIR}/${CHARM_NAME}" + +lint: + @echo "Running lint checks" + @tox -e lint + +black: + @echo "Reformat files with black" + @tox -e black + +proof: + @echo "Running charm proof" + @-charm proof + +unittests: submodules-update + @echo "Running unit tests" + @tox -e unit + +functional: build + @echo "Executing functional tests in ${CHARM_BUILD_DIR}" + @CHARM_BUILD_DIR=${CHARM_BUILD_DIR} tox -e func + +test: lint proof unittests functional + @echo "Charm ${CHARM_NAME} has been tested" + +# The targets below don't depend on a file +.PHONY: help submodules submodules-update clean build release lint black proof unittests functional test diff --git a/nrpe/README.md b/nrpe/README.md new file mode 100644 index 0000000..a82afad --- /dev/null +++ b/nrpe/README.md @@ -0,0 +1,225 @@ +Introduction +============ + +This subordinate charm is used to configure nrpe (Nagios Remote Plugin +Executor). It can be related to the nagios charm via the monitors relation and +will pass a monitors yaml to nagios informing it of what checks to monitor. + +Principal Relations +=================== + +This charm can be attached to any principal charm (via the juju-info relation) +regardless of whether it has implemented the local-monitors or +nrpe-external-master relations. For example: + + juju deploy ubuntu + juju deploy nrpe + juju deploy nagios + juju add-relation ubuntu nrpe + juju add-relation nrpe:monitors nagios:monitors + +If joined via the juju-info relation the default checks are configured and +additional checks can be added via the monitors config option (see below). + +The local-monitors relations allows the principal to request checks to be setup +by passing a monitors yaml and listing them in the 'local' section. It can +also list checks that is has configured by listing them in the remote nrpe +section and finally it can request external monitors are setup by using one of +the other remote types. See "Monitors yaml" below. + +Other Subordinate Charms +======================== + +If another subordinate charm deployed to the same principal has a +local-monitors or nrpe-external-master relation then it can also be related to +the local nrpe charm. For example: + + echo -e "glance:\n vip: 10.5.106.1" > glance.yaml + juju deploy -n3 --config glance.yaml glance + juju deploy hacluster glance-hacluster + juju deploy nrpe glance-nrpe + juju deploy nagios + juju add-relation glance glance-hacluster + juju add-relation glance-nrpe:monitors nagios:monitors + juju add-relation glance glance-nrpe + juju add-relation glance-hacluster glance-nrpe + +The glance-hacluster charm will pass monitoring information to glance-nrpe +which will amalgamate all monitor definitions before passing them to nagios. + +Check sources +============= + +Check definitions can come from three places: + +Default Checks +-------------- + +This charm creates a base set of checks in /etc/nagios/nrpe.d, including +check\_load, check\_users, check\_disk\_root. All of the options for these are +configurable but sensible defaults have been set in config.yaml. +For example to increase the alert threshold for number of processes: + + juju config nrpe load="-w 10,10,10 -c 25,25,25" + +Default checks maybe disabled by setting them to the empty string. + +Principal Requested Checks +-------------------------- + +Monitors passed to this charm by the principal charm via the local-monitors +or nrpe-external-master relation. The principal charm can write its own +check definition into */etc/nagios/nrpe.d* and then inform this charm via the +monitors setting. It can also request a direct external check of a service +without using nrpe. See "Monitors yaml" below for examples. + +User Requested Checks +--------------------- + +This works in the same way as the Principal requested except the monitors yaml +is set by the user via the monitors config option. For example to add a monitor +for the rsyslog process: + + juju config nrpe monitors=" + monitors: + local: + procrunning: + rsyslogd: + min: 1 + max: 1 + executable: rsyslogd + " + + + +External Nagios +=============== + +If the nagios server is not deployed in the juju environment then the charm can +be configured, via the export\_nagios\_definitions, to write out nagios config +fragments to /var/lib/nagios/export. Rsync is then configured to allow a host +(specified by nagios\_master) to collect the fragments. An rsync stanza is created +allowing the Nagios server to pick up configs from /var/lib/nagios/export (as +a target called "external-nagios"), which will also be configured to allow +connections from the hostname or IP address as specified for the +"nagios\_master" variable. + +It is up to you to configure the Nagios master to pull the configs needed, which +will then cause it to connect back to the instances in question to run the nrpe +checks you have defined. + +Monitors yaml +============= + +The list of monitors past down the monitors relation is an amalgamation of the +lists provided via the principal, the user and the default checks. + +The monitors yaml is of the following form: + + + # Version of the spec, mostly ignored but 0.3 is the current one + version: '0.3' + # Dict with just 'local' and 'remote' as parts + monitors: + # local monitors need an agent to be handled. See nrpe charm for + # some example implementations + local: + # procrunning checks for a running process named X (no path) + procrunning: + # Multiple procrunning can be defined, this is the "name" of it + nagios3: + min: 1 + max: 1 + executable: nagios3 + # Remote monitors can be polled directly by a remote system + remote: + # do a request on the HTTP protocol + http: + nagios: + port: 80 + path: /nagios3/ + # expected status response (otherwise just look for 200) + status: 'HTTP/1.1 401' + # Use as the Host: header (the server address will still be used to connect() to) + host: www.fewbar.com + mysql: + # Named basic check + basic: + username: monitors + password: abcdefg123456 + nrpe: + apache2: + command: check_apache2 + + + +Before a monitor is added it is checked to see if it is in the 'local' section. +If it is this charm needs to convert it into an nrpe checks. Only a small +number of check types are currently supported (see below) .These checks can +then be called by the nagios charm via the nrpe service. So for each check +listed in the local section: + +1. The definition is read and a check definition it written /etc/nagios/nrpe.d +2. The check is defined as a remote nrpe check in the yaml passed to nagios + +In the example above a check\_proc\_nagios3\_user.cfg file would be written +out which contains: + + # Check process nagios3 is running (user) + command[check_proc_nagios3_user]=/usr/lib/nagios/plugins/check_procs -w 1 -c 1 -C nagios3 + +And the monitors yaml passed to nagios would include: + + monitors: + nrpe: + check_proc_nagios3_user: + command: check_proc_nagios3_user + +The principal charm, or the user via the monitors config option, can request an +external check by adding it to the remote section of the monitors yaml. In the +example above direct checks of a webserver and of mysql are being requested. +This charm passes those on to nagios unaltered. + +Local check types +----------------- + +Supported nrpe checks are: + + procrunning: + min: Minimum number of 'executable' processes + max: Maximum number of 'executable' processes + executable: Name of executable to look for in process list + processcount: + min: Minimum total number processes + max: Maximum total number processes + executable: Name of executable to look for in process list + disk: + path: Directory to monitor space usage of + custom: + check: the name of the check to execute + plugin_path: (optional) Absolute path to the directory containing the + custom plugin. Default value is /var/lib/nagios/plugins + description: (optional) Description of the check + params: (optional) Parameters to pass to the check on invocation + +Remote check types +------------------ + +Supported remote types: + http, mysql, nrpe, tcp, rpc, pgsql + (See Nagios charm for up-to-date list and options) + +Spaces +====== + +By defining 'monitors' binding, you can influence which nrpe's IP will be reported +back to Nagios. This can be very handy if nrpe is placed on machines with multiple +IPs/networks. + +Actions +======= + +The charm defines 2 actions, 'list-nrpe-checks' that gives a list of all the +nrpe checks defined for this unit and what commands they use. The other is +run-nrpe-check, which allows you to run a specified nrpe check and get the +output. This is useful to confirm if an alert is actually resolved. diff --git a/nrpe/actions.yaml b/nrpe/actions.yaml new file mode 100644 index 0000000..0024fba --- /dev/null +++ b/nrpe/actions.yaml @@ -0,0 +1,9 @@ +list-nrpe-checks: + description: Lists all NRPE checks defined on this unit +run-nrpe-check: + description: Run a specific NRPE check defined on this unit + params: + name: + type: string + description: Check name to run + required: [name] diff --git a/nrpe/actions/list-nrpe-checks b/nrpe/actions/list-nrpe-checks new file mode 100755 index 0000000..6eea894 --- /dev/null +++ b/nrpe/actions/list-nrpe-checks @@ -0,0 +1,16 @@ +#!/bin/bash + +nrpedir=/etc/nagios/nrpe.d + +if [ ! -d $nrpedir ]; then + action-fail "No $nrpedir exists" + exit 1 +else + for i in $nrpedir/*.cfg; do + check=$(grep command $i | awk -F "=" '{ print $1 }' | sed -e 's/command\[//' | sed -e 's/\]//' | sed -e 's/_/-/g'); + command=$(grep command $i | awk -F "=" '{ print $2 }'); + action-set checks.$check="$command"; + done +fi + +action-set timestamp="$(date)" diff --git a/nrpe/actions/run-nrpe-check b/nrpe/actions/run-nrpe-check new file mode 100755 index 0000000..3113b4a --- /dev/null +++ b/nrpe/actions/run-nrpe-check @@ -0,0 +1,15 @@ +#!/bin/bash + +check=$(action-get name | sed -e 's/-/_/g') + +nrpedir="/etc/nagios/nrpe.d" +checkfile="$nrpedir/${check}.cfg" + +if [ -f $checkfile ]; then + command=$(awk -F "=" '{ print $2 }' $checkfile) + output=$(sudo -u nagios $command) + action-set check-output="$output" +else + action-fail "$checkfile does not exist" +fi + diff --git a/nrpe/config.yaml b/nrpe/config.yaml new file mode 100644 index 0000000..44d82c6 --- /dev/null +++ b/nrpe/config.yaml @@ -0,0 +1,210 @@ +options: + nagios_master: + default: "None" + type: string + description: | + IP address of the nagios master from which to allow rsync access + server_port: + default: 5666 + type: int + description: | + Port on which nagios-nrpe-server will listen + nagios_address_type: + default: "private" + type: string + description: | + Determines whether the nagios host check should use the private + or public IP address of an instance. Can be "private" or "public". + nagios_host_context: + default: "juju" + type: string + description: | + A string which will be prepended to instance name to set the host name + in nagios. So for instance the hostname would be something like: + juju-postgresql-0 + If you're running multiple environments with the same services in them + this allows you to differentiate between them. + nagios_hostname_type: + default: "auto" + type: string + description: | + Determines whether a server is identified by its unit name or + host name. If you're in a virtual environment, "unit" is + probably best. If you're using MaaS, you may prefer "host". + Use "auto" to have nrpe automatically distinguish between + metal and non-metal hosts. + dont_blame_nrpe: + default: False + type: boolean + description: | + Setting dont_blame_nrpe to True sets dont_blame_nrpe=1 in nrpe.cfg + This config option which allows specifying arguments to nrpe scripts. + This can be a security risk so it is disabled by default. Nrpe is + compiled with --enable-command-args option by default, which this + option enables. + debug: + default: False + type: boolean + description: | + Setting debug to True enables debug=1 in nrpe.cfg + disk_root: + default: "-u GB -w 25% -c 20% -K 5%" + type: string + description: | + Root disk check. This can be made to also check non-root disk systems + as follows: + -u GB -w 20% -c 15% -r '/srv/juju/vol-' -C -u GB -w 25% -c 20% + The string '-p /' will be appended to this check, so you must finish + the string taking that into account. See the nagios check_disk plugin + help for further details. + . + Set to '' in order to disable this check. + zombies: + default: "" + type: string + description: | + Zombie processes check; defaults to disabled. To enable, set the desired + check_procs arguments pertaining to zombies, for example: "-w 3 -c 6 -s Z" + procs: + default: "" + type: string + description: | + Set thresholds for number of running processes. Defaults to disabled; + to enable, specify 'auto' for the charm to generate thresholds based + on processor count, or manually provide arguments for check_procs, for + example: "-k -w 250 -c 300" to set warning and critical levels + manually and exclude kernel threads. + load: + default: "auto" + type: string + description: | + Load check arguments (e.g. "-w 8,8,8 -c 15,15,15"); if 'auto' is set, + thresholds will be set to multipliers of processor count for 1m, 5m + and 15m thresholds, with warning as "(4, 2, 1)", and critical set to + "(8, 4, 2)". So if you have two processors, you'd get thresholds of + "-w 8,4,2 -c 16,8,4". + . + Set to '' in order to disable this check. + conntrack: + default: "-w 80 -c 90" + type: string + description: | + Check conntrack (net.netfilter.nf_conntrack_count) against thresholds. + . + Set to '' in order to disable this check. + users: + default: "" + type: string + description: | + Set thresholds for number of logged-in users. Defaults to disabled; + to enable, manually provide arguments for check_user, for example: + "-w 20 -c 25" + swap: + default: '' + type: string + description: | + Check swap utilisation. See the nagios check_swap plugin help for + further details. The format looks like "-w 40% -c 25%" + . + Set to '' in order to disable this check. + swap_activity: + default: "-i 5 -w 10240 -c 40960" + type: string + description: | + Swapout activity check. Thresholds are expressed in kB, interval in + seconds. + . + Set to '' in order to disable this check. + mem: + default: "-C -h -u -w 85 -c 90" + type: string + description: | + Check memory % used. + By default, thresholds are applied to the non-hugepages portion of the + memory. + . + Set to '' in order to disable this check. + lacp_bonds: + default: '' + type: string + description: | + LACP bond interfaces, space-delimited (ie. 'bond0 bond1') + netlinks: + default: '' + type: string + description: | + Network interfaces to monitor for correct link state, MTU size + and speed negotiated. The first argument is either an interface name or + a CIDR expression. Parsed keywords are "mtu", "speed", and "op". Other + keywords are ignored. + . + Note that CIDR expressions can match multiple devices. + . + For example (multi-line starts with pipe): + - 10.1.2.0/24 mtu:9000 speed:25000 + - eth0 mtu:9000 speed:25000 + - lo mtu:65536 op:unknown + - br0-mgmt mtu:9000 + - br0-sta mtu:9000 + - br0-stc mtu:9000 + - br0-api mtu:1500 + - bond0 mtu:9000 speed:50000 + - bond0.25 mtu:1500 speed:50000 + - ens3 mtu:1500 speed:-1 desc:openstack_iface + - ... + netlinks_skip_unfound_ifaces: + default: False + type: boolean + description: | + add --skip-unfound-ifaces to check_netlinks.py. + monitors: + default: '' + type: string + description: | + Additional monitors defined in the monitors yaml format (see README) + hostgroups: + default: "" + type: string + description: Comma separated list of hostgroups to add for these hosts + hostcheck_inherit: + default: "server" + type: string + description: Hostcheck to inherit + export_nagios_definitions: + default: False + type: boolean + description: | + If True nagios check definitions are written to + '/var/lib/nagios/export' and rync is configured to allow nagios_master + to collect them. Useful when Nagios is outside of the juju environment + sub_postfix: + default: "" + type: string + description: | + A string to be appended onto all the nrpe checks created by this charm + to avoid potential clashes with existing checks + xfs_errors: + default: "" + type: string + description: | + dmesg history length to check for xfs errors, in minutes + . + Defaults to disabled, set the time to enable. + ro_filesystem_excludes: + default: "/snap/,/sys/fs/cgroup,/run/containerd,/var/lib/docker" + type: string + description: | + Comma separated list of mount points to exclude from checks for readonly filesystem. + Can be a substring rather than the entire mount point, e.g. /sys will match all filesystems + beginning with the string /sys. + The check is disabled on all LXD units, and also for non-container units if this parameter is + set to ''. + cpu_governor: + default: "" + type: string + description: | + CPU governor check. The string value here will be checked against all CPUs in + /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor. The supported values are + 'ondemand', 'performance', 'powersave'. Unset value means the check will be disabled. + There is a relation key called requested_cpu_governor='string', but the charm config value + will take precedence over the relation data. diff --git a/nrpe/copyright b/nrpe/copyright new file mode 100644 index 0000000..d29545f --- /dev/null +++ b/nrpe/copyright @@ -0,0 +1,53 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2012, Canonical Ltd., All Rights Reserved. +License: GPL-3 + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Files: files/plugins/check_exit_status.pl +Copyright: Copyright (C) 2011 Chad Columbus +License: GPL-2 + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Files: files/plugins/check_mem.pl +Copyright: Copyright (c) 2011 justin@techadvise.com +License: MIT/X11 + Permission is hereby granted, free of charge, to any person obtaining a copy of this + software and associated documentation files (the "Software"), to deal in the Software + without restriction, including without limitation the rights to use, copy, modify, + merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in all copies + or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE + FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT + OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. diff --git a/nrpe/files/default_rsync b/nrpe/files/default_rsync new file mode 100644 index 0000000..04dc60e --- /dev/null +++ b/nrpe/files/default_rsync @@ -0,0 +1,7 @@ +#------------------------------------------------ +# This file is juju managed +#------------------------------------------------ + +RSYNC_ENABLE=true +RSYNC_NICE='' +RSYNC_OPTS='' diff --git a/nrpe/files/nagios_plugin.py b/nrpe/files/nagios_plugin.py new file mode 100644 index 0000000..d5a1ff2 --- /dev/null +++ b/nrpe/files/nagios_plugin.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +"""Nagios plugin for python2.7.""" +# Copyright (C) 2005, 2006, 2007, 2012 James Troup + +import os +import stat +import time +import traceback +import sys + + +################################################################################ + + +class CriticalError(Exception): + """This indicates a critical error.""" + + pass + + +class WarnError(Exception): + """This indicates a warning condition.""" + + pass + + +class UnknownError(Exception): + """This indicates a unknown error was encountered.""" + + pass + + +def try_check(function, *args, **kwargs): + """Perform a check with error/warn/unknown handling.""" + try: + function(*args, **kwargs) + except UnknownError, msg: # noqa: E999 + print msg + sys.exit(3) + except CriticalError, msg: # noqa: E999 + print msg + sys.exit(2) + except WarnError, msg: # noqa: E999 + print msg + sys.exit(1) + except: # noqa: E722 + print "%s raised unknown exception '%s'" % (function, sys.exc_info()[0]) + print "=" * 60 + traceback.print_exc(file=sys.stdout) + print "=" * 60 + sys.exit(3) + + +################################################################################ + + +def check_file_freshness(filename, newer_than=600): + """Check a file. + + It check that file exists, is readable and is newer than seconds (where + defaults to 600). + """ + # First check the file exists and is readable + if not os.path.exists(filename): + raise CriticalError("%s: does not exist." % (filename)) + if os.access(filename, os.R_OK) == 0: + raise CriticalError("%s: is not readable." % (filename)) + + # Then ensure the file is up-to-date enough + mtime = os.stat(filename)[stat.ST_MTIME] + last_modified = time.time() - mtime + if last_modified > newer_than: + raise CriticalError( + "%s: was last modified on %s and is too old (> %s seconds)." + % (filename, time.ctime(mtime), newer_than) + ) + if last_modified < 0: + raise CriticalError( + "%s: was last modified on %s which is in the future." + % (filename, time.ctime(mtime)) + ) + + +################################################################################ diff --git a/nrpe/files/nagios_plugin3.py b/nrpe/files/nagios_plugin3.py new file mode 100644 index 0000000..c7fb8d6 --- /dev/null +++ b/nrpe/files/nagios_plugin3.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Nagios plugin for python3.""" + +# Copyright (C) 2005, 2006, 2007, 2012, 2017 James Troup + +import os +import stat +import sys +import time +import traceback + + +############################################################################### + + +class CriticalError(Exception): + """This indicates a critical error.""" + + pass + + +class WarnError(Exception): + """This indicates a warning condition.""" + + pass + + +class UnknownError(Exception): + """This indicates a unknown error was encountered.""" + + pass + + +def try_check(function, *args, **kwargs): + """Perform a check with error/warn/unknown handling.""" + try: + function(*args, **kwargs) + except UnknownError as msg: + print(msg) + sys.exit(3) + except CriticalError as msg: + print(msg) + sys.exit(2) + except WarnError as msg: + print(msg) + sys.exit(1) + except: # noqa: E722 + print("{} raised unknown exception '{}'".format(function, sys.exc_info()[0])) + print("=" * 60) + traceback.print_exc(file=sys.stdout) + print("=" * 60) + sys.exit(3) + + +############################################################################### + + +def check_file_freshness(filename, newer_than=600): + """Check a file. + + It check that file exists, is readable and is newer than seconds (where + defaults to 600). + """ + # First check the file exists and is readable + if not os.path.exists(filename): + raise CriticalError("%s: does not exist." % (filename)) + if os.access(filename, os.R_OK) == 0: + raise CriticalError("%s: is not readable." % (filename)) + + # Then ensure the file is up-to-date enough + mtime = os.stat(filename)[stat.ST_MTIME] + last_modified = time.time() - mtime + if last_modified > newer_than: + raise CriticalError( + "%s: was last modified on %s and is too old (> %s " + "seconds)." % (filename, time.ctime(mtime), newer_than) + ) + if last_modified < 0: + raise CriticalError( + "%s: was last modified on %s which is in the " + "future." % (filename, time.ctime(mtime)) + ) + + +############################################################################### diff --git a/nrpe/files/plugins/check_arp_cache.py b/nrpe/files/plugins/check_arp_cache.py new file mode 100755 index 0000000..be11e02 --- /dev/null +++ b/nrpe/files/plugins/check_arp_cache.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Check arp cache usage and alert.""" +# -*- coding: us-ascii -*- + +# Copyright (C) 2019 Canonical +# All rights reserved + +import argparse +import os + +from nagios_plugin3 import ( + CriticalError, + UnknownError, + WarnError, + try_check, +) + + +def check_arp_cache(warn, crit): + """Check the usage of arp cache against gc_thresh. + + Alerts when the number of arp entries exceeds a threshold of gc_thresh3. + See https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt for + full details. + + :param warn: integer, % level of hard limit at which to raise Warning + :param crit: integer, % level of hard limit at which to raise Critical + """ + arp_table_entries = "/proc/net/arp" + gc_thresh_location = "/proc/sys/net/ipv4/neigh/default/gc_thresh3" + + if not os.path.exists(arp_table_entries): + raise UnknownError("No arp table found!") + if not os.path.exists(gc_thresh_location): + raise UnknownError("sysctl entry net.ipv4.neigh.default.gc_thresh3 not found!") + + with open(gc_thresh_location) as fd: + gc_thresh3 = int(fd.read()) + + with open(arp_table_entries) as fd: + arp_cache = fd.read().count("\n") - 1 # remove header + extra_info = "arp cache entries: {}".format(arp_cache) + + warn_threshold = gc_thresh3 * warn / 100 + crit_threshold = gc_thresh3 * crit / 100 + + if arp_cache >= crit_threshold: + message = "CRITICAL: arp cache is more than {} of limit, {}".format( + crit, extra_info + ) + raise CriticalError(message) + if arp_cache >= warn_threshold: + message = "WARNING: arp cache is more than {} of limit, {}".format( + warn, extra_info + ) + raise WarnError(message) + + print("OK: arp cache is healthy: {}".format(extra_info)) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description="Check bond status") + parser.add_argument( + "--warn", + "-w", + type=int, + help="% of gc_thresh3 to exceed for warning", + default=60, + ) + parser.add_argument( + "--crit", + "-c", + type=int, + help="% of gc_thresh3 to exceed for critical", + default=80, + ) + args = parser.parse_args() + return args + + +def main(): + """Parse args and check the arp cache.""" + args = parse_args() + try_check(check_arp_cache, args.warn, args.crit) + + +if __name__ == "__main__": + main() diff --git a/nrpe/files/plugins/check_conntrack.sh b/nrpe/files/plugins/check_conntrack.sh new file mode 100755 index 0000000..98ba54b --- /dev/null +++ b/nrpe/files/plugins/check_conntrack.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# This file is managed by juju. Do not make local changes. + +# Copyright (C) 2013, 2016 Canonical Ltd. +# Author: Haw Loeung +# Paul Gear + +# Alert when current conntrack entries exceeds certain percentage of max. to +# detect when we're about to fill it up and start dropping packets. + +set -eu + +STATE_OK=0 +STATE_WARNING=1 +STATE_CRITICAL=2 +STATE_UNKNOWN=3 + +if ! lsmod | grep -q conntrack; then + echo "OK: no conntrack modules present" + exit $STATE_OK +fi + +if ! [ -e /proc/sys/net/netfilter/nf_conntrack_max ]; then + echo "OK: conntrack not available" + exit $STATE_OK +fi + +max=$(sysctl net.netfilter.nf_conntrack_max 2>/dev/null | awk '{ print $3 }') +if [ -z "$max" ]; then + echo "UNKNOWN: unable to retrieve value of net.netfilter.nf_conntrack_max" + exit $STATE_UNKNOWN +fi +current=$(sysctl net.netfilter.nf_conntrack_count 2>/dev/null | awk '{ print $3 }') +if [ -z "$current" ]; then + echo "UNKNOWN: unable to retrieve value of net.netfilter.nf_conntrack_count" + exit $STATE_UNKNOWN +fi + +# default thresholds +crit=90 +warn=80 + +# parse command line +set +e +OPTIONS=$(getopt w:c: "$@") +if [ $? -ne 0 ]; then + echo "Usage: $0 [-w warningpercent] [-c criticalpercent]" >&2 + echo " Check nf_conntrack_count against nf_conntrack_max" >&2 + exit $STATE_UNKNOWN +fi +set -e + +set -- $OPTIONS +while true; do + case "$1" in + -w) warn=$2; shift 2 ;; + -c) crit=$2; shift 2 ;; + --) shift; break ;; + *) break ;; + esac +done + +percent=$((current * 100 / max)) +stats="| current=$current max=$max percent=$percent;$warn;$crit" + +threshold=$((max * crit / 100)) +if [ $current -gt $threshold ]; then + echo "CRITICAL: conntrack table nearly full. $stats" + exit $STATE_CRITICAL +fi + +threshold=$((max * warn / 100)) +if [ $current -gt $threshold ]; then + echo "WARNING: conntrack table filling. $stats" + exit $STATE_WARNING +fi + +echo "OK: conntrack table normal $stats" +exit $STATE_OK diff --git a/nrpe/files/plugins/check_cpu_governor.py b/nrpe/files/plugins/check_cpu_governor.py new file mode 100755 index 0000000..0e085b2 --- /dev/null +++ b/nrpe/files/plugins/check_cpu_governor.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Check CPU governor scaling and alert.""" + +import argparse +import os +import re + +from nagios_plugin3 import ( + CriticalError, + try_check, +) + + +def wanted_governor(governor): + """Check /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor.""" + cpu_path = os.listdir("/sys/devices/system/cpu") + regex = re.compile("(cpu[0-9][0-9]*)") + numcpus = sum(1 for x in cpu_path if regex.match(x)) + error_cpus = set() + for cpu in range(0, numcpus): + path = f"/sys/devices/system/cpu/cpu{cpu}/cpufreq/scaling_governor" + with open(path) as f: + out = f.readline().strip() + + if governor in out: + continue + else: + error_cpus.add(f"CPU{cpu}") + + if error_cpus: + error_cpus = ",".join(error_cpus) + raise CriticalError(f"CRITICAL: {error_cpus} not set to {governor}") + + print(f"OK: All CPUs set to {governor}.") + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description="Check CPU governor") + parser.add_argument( + "--governor", + "-g", + type=str, + help="The requested governor to check for each CPU", + default="performance", + ) + args = parser.parse_args() + return args + + +def main(): + """Check the CPU governors.""" + args = parse_args() + try_check(wanted_governor, args.governor) + + +if __name__ == "__main__": + main() diff --git a/nrpe/files/plugins/check_exit_status.pl b/nrpe/files/plugins/check_exit_status.pl new file mode 100755 index 0000000..49df22d --- /dev/null +++ b/nrpe/files/plugins/check_exit_status.pl @@ -0,0 +1,189 @@ +#!/usr/bin/perl +################################################################################ +# # +# Copyright (C) 2011 Chad Columbus # +# # +# This program is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 2 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program; if not, write to the Free Software # +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# # +################################################################################ + +use strict; +use Getopt::Std; +$| = 1; + +my %opts; +getopts('heronp:s:', \%opts); + +my $VERSION = "Version 1.0"; +my $AUTHOR = '(c) 2011 Chad Columbus '; + +# Default values: +my $script_to_check; +my $pattern = 'is running'; +my $cmd; +my $message; +my $error; + +# Exit codes +my $STATE_OK = 0; +my $STATE_WARNING = 1; +my $STATE_CRITICAL = 2; +my $STATE_UNKNOWN = 3; + +# Parse command line options +if ($opts{'h'} || scalar(%opts) == 0) { + &print_help(); + exit($STATE_OK); +} + +# Make sure scipt is provided: +if ($opts{'s'} eq '') { + # Script to run not provided + print "\nYou must provide a script to run. Example: -s /etc/init.d/httpd\n"; + exit($STATE_UNKNOWN); +} else { + $script_to_check = $opts{'s'}; +} + +# Make sure only a-z, 0-9, /, _, and - are used in the script. +if ($script_to_check =~ /[^a-z0-9\_\-\/\.]/) { + # Script contains illegal characters exit. + print "\nScript to check can only contain Letters, Numbers, Periods, Underscores, Hyphens, and/or Slashes\n"; + exit($STATE_UNKNOWN); +} + +# See if script is executable +if (! -x "$script_to_check") { + print "\nIt appears you can't execute $script_to_check, $!\n"; + exit($STATE_UNKNOWN); +} + +# If a pattern is provided use it: +if ($opts{'p'} ne '') { + $pattern = $opts{'p'}; +} + +# If -r run command via sudo as root: +if ($opts{'r'}) { + $cmd = "sudo -n $script_to_check status" . ' 2>&1'; +} else { + $cmd = "$script_to_check status" . ' 2>&1'; +} + +my $cmd_result = `$cmd`; +chomp($cmd_result); +if ($cmd_result =~ /sudo/i) { + # This means it could not run the sudo command + $message = "$script_to_check CRITICAL - Could not run: 'sudo -n $script_to_check status'. Result is $cmd_result"; + $error = $STATE_UNKNOWN; +} else { + # Check exitstatus instead of output: + if ($opts{'e'} == 1) { + if ($? != 0) { + # error + $message = "$script_to_check CRITICAL - Exit code: $?\."; + if ($opts{'o'} == 0) { + $message .= " $cmd_result"; + } + $error = $STATE_CRITICAL; + } else { + # success + $message = "$script_to_check OK - Exit code: $?\."; + if ($opts{'o'} == 0) { + $message .= " $cmd_result"; + } + $error = $STATE_OK; + } + } else { + my $not_check = 1; + if ($opts{'n'} == 1) { + $not_check = 0; + } + if (($cmd_result =~ /$pattern/i) == $not_check) { + $message = "$script_to_check OK"; + if ($opts{'o'} == 0) { + $message .= " - $cmd_result"; + } + $error = $STATE_OK; + } else { + $message = "$script_to_check CRITICAL"; + if ($opts{'o'} == 0) { + $message .= " - $cmd_result"; + } + $error = $STATE_CRITICAL; + } + } +} + +if ($message eq '') { + print "Error: program failed in an unknown way\n"; + exit($STATE_UNKNOWN); +} + +if ($error) { + print "$message\n"; + exit($error); +} else { + # If we get here we are OK + print "$message\n"; + exit($STATE_OK); +} + +#################################### +# Start Subs: +#################################### +sub print_help() { + print << "EOF"; +Check the output or exit status of a script. +$VERSION +$AUTHOR + +Options: +-h + Print detailed help screen + +-s + 'FULL PATH TO SCRIPT' (required) + This is the script to run, the script is designed to run scripts in the + /etc/init.d dir (but can run any script) and will call the script with + a 'status' argument. So if you use another script make sure it will + work with /path/script status, example: /etc/init.d/httpd status + +-e + This is the "exitstaus" flag, it means check the exit status + code instead of looking for a pattern in the output of the script. + +-p 'REGEX' + This is a pattern to look for in the output of the script to confirm it + is running, default is 'is running', but not all init.d scripts output + (iptables), so you can specify an arbitrary pattern. + All patterns are case insensitive. + +-n + This is the "NOT" flag, it means not the -p pattern, so if you want to + make sure the output of the script does NOT contain -p 'REGEX' + +-r + This is the "ROOT" flag, it means run as root via sudo. You will need a + line in your /etc/sudoers file like: + nagios ALL=(root) NOPASSWD: /etc/init.d/* status + +-o + This is the "SUPPRESS OUTPUT" flag. Some programs have a long output + (like iptables), this flag suppresses that output so it is not printed + as a part of the nagios message. +EOF +} + diff --git a/nrpe/files/plugins/check_lacp_bond.py b/nrpe/files/plugins/check_lacp_bond.py new file mode 100755 index 0000000..ac64b4f --- /dev/null +++ b/nrpe/files/plugins/check_lacp_bond.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Check lacp bonds and alert.""" +# -*- coding: us-ascii -*- + +# Copyright (C) 2017 Canonical +# All rights reserved +# Author: Alvaro Uria + +import argparse +import glob +import os +import sys + +from nagios_plugin3 import CriticalError, WarnError, try_check + +# LACPDU port states in binary +LACPDU_ACTIVE = 0b1 # 1 = Active, 0 = Passive +LACPDU_RATE = 0b10 # 1 = Short Timeout, 0 = Long Timeout +LACPDU_AGGREGATED = 0b100 # 1 = Yes, 0 = No (individual link) +LACPDU_SYNC = 0b1000 # 1 = In sync, 0 = Not in sync +LACPDU_COLLECT = 0b10000 # Mux is accepting traffic received on this port +LACPDU_DIST = 0b100000 # Mux is sending traffic using this port +LACPDU_DEFAULT = 0b1000000 # 1 = default settings, 0 = via LACP PDU +LACPDU_EXPIRED = 0b10000000 # In an expired state + + +def check_lacpdu_port(actor_port, partner_port): + """Return message for LACPDU port state mismatch.""" + diff = int(actor_port) ^ int(partner_port) + msg = [] + if diff & LACPDU_RATE: + msg.append("lacp rate mismatch") + if diff & LACPDU_AGGREGATED: + msg.append("not aggregated") + if diff & LACPDU_SYNC: + msg.append("not in sync") + if diff & LACPDU_COLLECT: + msg.append("not collecting") + return ", ".join(msg) + + +def check_lacp_bond(iface): + """Check LACP bonds are correctly configured (AD Aggregator IDs match).""" + bond_aggr_template = "/sys/class/net/{0}/bonding/ad_aggregator" + bond_slaves_template = "/sys/class/net/{0}/bonding/slaves" + bond_mode_template = "/sys/class/net/{0}/bonding/mode" + slave_template = "/sys/class/net/{0}/bonding_slave/ad_aggregator_id" + actor_port_state = "/sys/class/net/{0}/bonding_slave/ad_actor_oper_port_state" + partnet_port_state = "/sys/class/net/{0}/bonding_slave/ad_partner_oper_port_state" + + bond_aggr = bond_aggr_template.format(iface) + bond_slaves = bond_slaves_template.format(iface) + + if os.path.exists(bond_aggr): + with open(bond_mode_template.format(iface)) as fd: + bond_mode = fd.readline() + + if "802.3ad" not in bond_mode: + msg = "WARNING: {} is not in lacp mode".format(iface) + raise WarnError(msg) + + with open(bond_aggr) as fd: + bond_aggr_value = fd.readline().strip() + + d_bond = {iface: bond_aggr_value} + + with open(bond_slaves) as fd: + slaves = fd.readline().strip().split(" ") + for slave in slaves: + # Check aggregator ID + with open(slave_template.format(slave)) as fd: + slave_aggr_value = fd.readline().strip() + + d_bond[slave] = slave_aggr_value + + if slave_aggr_value != bond_aggr_value: + # If we can report then only 1/2 the bond is down + msg = "WARNING: aggregator_id mismatch " + msg += "({0}:{1} - {2}:{3})" + msg = msg.format(iface, bond_aggr_value, slave, slave_aggr_value) + raise WarnError(msg) + # Check LACPDU port state + with open(actor_port_state.format(slave)) as fd: + actor_port_value = fd.readline().strip() + with open(partnet_port_state.format(slave)) as fd: + partner_port_value = fd.readline().strip() + if actor_port_value != partner_port_value: + res = check_lacpdu_port(actor_port_value, partner_port_value) + msg = ( + "WARNING: LACPDU port state mismatch " + "({0}: {1} - actor_port_state={2}, " + "partner_port_state={3})".format( + res, slave, actor_port_value, partner_port_value + ) + ) + raise WarnError(msg) + + else: + msg = "CRITICAL: {} is not a bonding interface".format(iface) + raise CriticalError(msg) + + extra_info = "{0}:{1}".format(iface, d_bond[iface]) + for k_iface, v_aggrid in d_bond.items(): + if k_iface == iface: + continue + extra_info += ", {0}:{1}".format(k_iface, v_aggrid) + print("OK: bond config is healthy: {}".format(extra_info)) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description="Check bond status") + parser.add_argument("--iface", "-i", help="bond iface name") + args = parser.parse_args() + + if not args.iface: + ifaces = map(os.path.basename, glob.glob("/sys/class/net/bond?")) + print( + "UNKNOWN: Please specify one of these bond " + "ifaces: {}".format(",".join(ifaces)) + ) + sys.exit(1) + return args + + +def main(): + """Parse args and check the lacp bonds.""" + args = parse_args() + try_check(check_lacp_bond, args.iface) + + +if __name__ == "__main__": + main() diff --git a/nrpe/files/plugins/check_mem.pl b/nrpe/files/plugins/check_mem.pl new file mode 100755 index 0000000..044ed5c --- /dev/null +++ b/nrpe/files/plugins/check_mem.pl @@ -0,0 +1,412 @@ +#!/usr/bin/perl -w + +# Heavily based on the script from: +# check_mem.pl Copyright (C) 2000 Dan Larsson +# heavily modified by +# Justin Ellison +# +# The MIT License (MIT) +# Copyright (c) 2011 justin@techadvise.com + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this +# software and associated documentation files (the "Software"), to deal in the Software +# without restriction, including without limitation the rights to use, copy, modify, +# merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies +# or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT +# OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# Tell Perl what we need to use +use strict; +use Getopt::Std; + +#TODO - Convert to Nagios::Plugin +#TODO - Use an alarm + +# Predefined exit codes for Nagios +use vars qw($opt_c $opt_f $opt_u $opt_w $opt_C $opt_v $opt_h %exit_codes); +%exit_codes = ('UNKNOWN' , 3, + 'OK' , 0, + 'WARNING' , 1, + 'CRITICAL', 2, + ); + +# Get our variables, do our checking: +init(); + +# Get the numbers: +my ($free_memory_kb,$used_memory_kb,$caches_kb,$hugepages_kb) = get_memory_info(); +print "$free_memory_kb Free\n$used_memory_kb Used\n$caches_kb Cache\n" if ($opt_v); +print "$hugepages_kb Hugepages\n" if ($opt_v and $opt_h); + +if ($opt_C) { #Do we count caches as free? + $used_memory_kb -= $caches_kb; + $free_memory_kb += $caches_kb; +} + +if ($opt_h) { + $used_memory_kb -= $hugepages_kb; +} + +print "$used_memory_kb Used (after Hugepages)\n" if ($opt_v); + +# Round to the nearest KB +$free_memory_kb = sprintf('%d',$free_memory_kb); +$used_memory_kb = sprintf('%d',$used_memory_kb); +$caches_kb = sprintf('%d',$caches_kb); + +# Tell Nagios what we came up with +tell_nagios($used_memory_kb,$free_memory_kb,$caches_kb,$hugepages_kb); + + +sub tell_nagios { + my ($used,$free,$caches,$hugepages) = @_; + + # Calculate Total Memory + my $total = $free + $used; + print "$total Total\n" if ($opt_v); + + my $perf_warn; + my $perf_crit; + if ( $opt_u ) { + $perf_warn = int(${total} * $opt_w / 100); + $perf_crit = int(${total} * $opt_c / 100); + } else { + $perf_warn = int(${total} * ( 100 - $opt_w ) / 100); + $perf_crit = int(${total} * ( 100 - $opt_c ) / 100); + } + + my $perfdata = "|TOTAL=${total}KB;;;; USED=${used}KB;${perf_warn};${perf_crit};; FREE=${free}KB;;;; CACHES=${caches}KB;;;;"; + $perfdata .= " HUGEPAGES=${hugepages}KB;;;;" if ($opt_h); + + if ($opt_f) { + my $percent = sprintf "%.1f", ($free / $total * 100); + if ($percent <= $opt_c) { + finish("CRITICAL - $percent% ($free kB) free!$perfdata",$exit_codes{'CRITICAL'}); + } + elsif ($percent <= $opt_w) { + finish("WARNING - $percent% ($free kB) free!$perfdata",$exit_codes{'WARNING'}); + } + else { + finish("OK - $percent% ($free kB) free.$perfdata",$exit_codes{'OK'}); + } + } + elsif ($opt_u) { + my $percent = sprintf "%.1f", ($used / $total * 100); + if ($percent >= $opt_c) { + finish("CRITICAL - $percent% ($used kB) used!$perfdata",$exit_codes{'CRITICAL'}); + } + elsif ($percent >= $opt_w) { + finish("WARNING - $percent% ($used kB) used!$perfdata",$exit_codes{'WARNING'}); + } + else { + finish("OK - $percent% ($used kB) used.$perfdata",$exit_codes{'OK'}); + } + } +} + +# Show usage +sub usage() { + print "\ncheck_mem.pl v1.0 - Nagios Plugin\n\n"; + print "usage:\n"; + print " check_mem.pl - -w -c \n\n"; + print "options:\n"; + print " -f Check FREE memory\n"; + print " -u Check USED memory\n"; + print " -C Count OS caches as FREE memory\n"; + print " -h Remove hugepages from the total memory count\n"; + print " -w PERCENT Percent free/used when to warn\n"; + print " -c PERCENT Percent free/used when critical\n"; + print "\nCopyright (C) 2000 Dan Larsson \n"; + print "check_mem.pl comes with absolutely NO WARRANTY either implied or explicit\n"; + print "This program is licensed under the terms of the\n"; + print "MIT License (check source code for details)\n"; + exit $exit_codes{'UNKNOWN'}; +} + +sub get_memory_info { + my $used_memory_kb = 0; + my $free_memory_kb = 0; + my $total_memory_kb = 0; + my $caches_kb = 0; + my $hugepages_nr = 0; + my $hugepages_size = 0; + my $hugepages_kb = 0; + + my $uname; + if ( -e '/usr/bin/uname') { + $uname = `/usr/bin/uname -a`; + } + elsif ( -e '/bin/uname') { + $uname = `/bin/uname -a`; + } + else { + die "Unable to find uname in /usr/bin or /bin!\n"; + } + print "uname returns $uname" if ($opt_v); + if ( $uname =~ /Linux/ ) { + my @meminfo = `/bin/cat /proc/meminfo`; + foreach (@meminfo) { + chomp; + if (/^Mem(Total|Free):\s+(\d+) kB/) { + my $counter_name = $1; + if ($counter_name eq 'Free') { + $free_memory_kb = $2; + } + elsif ($counter_name eq 'Total') { + $total_memory_kb = $2; + } + } + elsif (/^MemAvailable:\s+(\d+) kB/) { + $caches_kb += $1; + } + elsif (/^(Buffers|Cached|SReclaimable):\s+(\d+) kB/) { + $caches_kb += $2; + } + elsif (/^Shmem:\s+(\d+) kB/) { + $caches_kb -= $1; + } + # These variables will most likely be overwritten once we look into + # /sys/kernel/mm/hugepages, unless we are running on linux <2.6.27 + # and have to rely on them + elsif (/^HugePages_Total:\s+(\d+)/) { + $hugepages_nr = $1; + } + elsif (/^Hugepagesize:\s+(\d+) kB/) { + $hugepages_size = $1; + } + } + $hugepages_kb = $hugepages_nr * $hugepages_size; + $used_memory_kb = $total_memory_kb - $free_memory_kb; + + # Read hugepages info from the newer sysfs interface if available + my $hugepages_sysfs_dir = '/sys/kernel/mm/hugepages'; + if ( -d $hugepages_sysfs_dir ) { + # Reset what we read from /proc/meminfo + $hugepages_kb = 0; + opendir(my $dh, $hugepages_sysfs_dir) + || die "Can't open $hugepages_sysfs_dir: $!"; + while (my $entry = readdir $dh) { + if ($entry =~ /^hugepages-(\d+)kB/) { + $hugepages_size = $1; + my $hugepages_nr_file = "$hugepages_sysfs_dir/$entry/nr_hugepages"; + open(my $fh, '<', $hugepages_nr_file) + || die "Can't open $hugepages_nr_file for reading: $!"; + $hugepages_nr = <$fh>; + close($fh); + $hugepages_kb += $hugepages_nr * $hugepages_size; + } + } + closedir($dh); + } + } + elsif ( $uname =~ /HP-UX/ ) { + # HP-UX, thanks to Christoph Fürstaller + my @meminfo = `/usr/bin/sudo /usr/local/bin/kmeminfo`; + foreach (@meminfo) { + chomp; + if (/^Physical memory\s\s+=\s+(\d+)\s+(\d+.\d)g/) { + $total_memory_kb = ($2 * 1024 * 1024); + } + elsif (/^Free memory\s\s+=\s+(\d+)\s+(\d+.\d)g/) { + $free_memory_kb = ($2 * 1024 * 1024); + } + } + $used_memory_kb = $total_memory_kb - $free_memory_kb; + } + elsif ( $uname =~ /FreeBSD/ ) { + # The FreeBSD case. 2013-03-19 www.claudiokuenzler.com + # free mem = Inactive*Page Size + Cache*Page Size + Free*Page Size + my $pagesize = `sysctl vm.stats.vm.v_page_size`; + $pagesize =~ s/[^0-9]//g; + my $mem_inactive = 0; + my $mem_cache = 0; + my $mem_free = 0; + my $mem_total = 0; + my $free_memory = 0; + my @meminfo = `/sbin/sysctl vm.stats.vm`; + foreach (@meminfo) { + chomp; + if (/^vm.stats.vm.v_inactive_count:\s+(\d+)/) { + $mem_inactive = ($1 * $pagesize); + } + elsif (/^vm.stats.vm.v_cache_count:\s+(\d+)/) { + $mem_cache = ($1 * $pagesize); + } + elsif (/^vm.stats.vm.v_free_count:\s+(\d+)/) { + $mem_free = ($1 * $pagesize); + } + elsif (/^vm.stats.vm.v_page_count:\s+(\d+)/) { + $mem_total = ($1 * $pagesize); + } + } + $free_memory = $mem_inactive + $mem_cache + $mem_free; + $free_memory_kb = ( $free_memory / 1024); + $total_memory_kb = ( $mem_total / 1024); + $used_memory_kb = $total_memory_kb - $free_memory_kb; + $caches_kb = ($mem_cache / 1024); + } + elsif ( $uname =~ /joyent/ ) { + # The SmartOS case. 2014-01-10 www.claudiokuenzler.com + # free mem = pagesfree * pagesize + my $pagesize = `pagesize`; + my $phys_pages = `kstat -p unix:0:system_pages:pagestotal | awk '{print \$NF}'`; + my $free_pages = `kstat -p unix:0:system_pages:pagesfree | awk '{print \$NF}'`; + my $arc_size = `kstat -p zfs:0:arcstats:size | awk '{print \$NF}'`; + my $arc_size_kb = $arc_size / 1024; + + print "Pagesize is $pagesize" if ($opt_v); + print "Total pages is $phys_pages" if ($opt_v); + print "Free pages is $free_pages" if ($opt_v); + print "Arc size is $arc_size" if ($opt_v); + + $caches_kb += $arc_size_kb; + + $total_memory_kb = $phys_pages * $pagesize / 1024; + $free_memory_kb = $free_pages * $pagesize / 1024; + $used_memory_kb = $total_memory_kb - $free_memory_kb; + } + elsif ( $uname =~ /SunOS/ ) { + eval "use Sun::Solaris::Kstat"; + if ($@) { #Kstat not available + if ($opt_C) { + print "You can't report on Solaris caches without Sun::Solaris::Kstat available!\n"; + exit $exit_codes{UNKNOWN}; + } + my @vmstat = `/usr/bin/vmstat 1 2`; + my $line; + foreach (@vmstat) { + chomp; + $line = $_; + } + $free_memory_kb = (split(/ /,$line))[5] / 1024; + my @prtconf = `/usr/sbin/prtconf`; + foreach (@prtconf) { + if (/^Memory size: (\d+) Megabytes/) { + $total_memory_kb = $1 * 1024; + } + } + $used_memory_kb = $total_memory_kb - $free_memory_kb; + + } + else { # We have kstat + my $kstat = Sun::Solaris::Kstat->new(); + my $phys_pages = ${kstat}->{unix}->{0}->{system_pages}->{physmem}; + my $free_pages = ${kstat}->{unix}->{0}->{system_pages}->{freemem}; + # We probably should account for UFS caching here, but it's unclear + # to me how to determine UFS's cache size. There's inode_cache, + # and maybe the physmem variable in the system_pages module?? + # In the real world, it looks to be so small as not to really matter, + # so we don't grab it. If someone can give me code that does this, + # I'd be glad to put it in. + my $arc_size = (exists ${kstat}->{zfs} && ${kstat}->{zfs}->{0}->{arcstats}->{size}) ? + ${kstat}->{zfs}->{0}->{arcstats}->{size} / 1024 + : 0; + $caches_kb += $arc_size; + my $pagesize = `pagesize`; + + $total_memory_kb = $phys_pages * $pagesize / 1024; + $free_memory_kb = $free_pages * $pagesize / 1024; + $used_memory_kb = $total_memory_kb - $free_memory_kb; + } + } + elsif ( $uname =~ /Darwin/ ) { + $total_memory_kb = (split(/ /,`/usr/sbin/sysctl hw.memsize`))[1]/1024; + my $pagesize = (split(/ /,`/usr/sbin/sysctl hw.pagesize`))[1]; + $caches_kb = 0; + my @vm_stat = `/usr/bin/vm_stat`; + foreach (@vm_stat) { + chomp; + if (/^(Pages free):\s+(\d+)\.$/) { + $free_memory_kb = $2*$pagesize/1024; + } + # 'caching' concept works different on MACH + # this should be a reasonable approximation + elsif (/^Pages (inactive|purgable):\s+(\d+).$/) { + $caches_kb += $2*$pagesize/1024; + } + } + $used_memory_kb = $total_memory_kb - $free_memory_kb; + } + elsif ( $uname =~ /AIX/ ) { + my @meminfo = `/usr/bin/vmstat -vh`; + foreach (@meminfo) { + chomp; + if (/^\s*([0-9.]+)\s+(.*)/) { + my $counter_name = $2; + if ($counter_name eq 'memory pages') { + $total_memory_kb = $1*4; + } + if ($counter_name eq 'free pages') { + $free_memory_kb = $1*4; + } + if ($counter_name eq 'file pages') { + $caches_kb = $1*4; + } + if ($counter_name eq 'Number of 4k page frames loaned') { + $free_memory_kb += $1*4; + } + } + } + $used_memory_kb = $total_memory_kb - $free_memory_kb; + } + else { + if ($opt_C) { + print "You can't report on $uname caches!\n"; + exit $exit_codes{UNKNOWN}; + } + my $command_line = `vmstat | tail -1 | awk '{print \$4,\$5}'`; + chomp $command_line; + my @memlist = split(/ /, $command_line); + + # Define the calculating scalars + $used_memory_kb = $memlist[0]/1024; + $free_memory_kb = $memlist[1]/1024; + $total_memory_kb = $used_memory_kb + $free_memory_kb; + } + return ($free_memory_kb,$used_memory_kb,$caches_kb,$hugepages_kb); +} + +sub init { + # Get the options + if ($#ARGV le 0) { + &usage; + } + else { + getopts('c:fuChvw:'); + } + + # Shortcircuit the switches + if (!$opt_w or $opt_w == 0 or !$opt_c or $opt_c == 0) { + print "*** You must define WARN and CRITICAL levels!\n"; + &usage; + } + elsif (!$opt_f and !$opt_u) { + print "*** You must select to monitor either USED or FREE memory!\n"; + &usage; + } + + # Check if levels are sane + if ($opt_w <= $opt_c and $opt_f) { + print "*** WARN level must not be less than CRITICAL when checking FREE memory!\n"; + &usage; + } + elsif ($opt_w >= $opt_c and $opt_u) { + print "*** WARN level must not be greater than CRITICAL when checking USED memory!\n"; + &usage; + } +} + +sub finish { + my ($msg,$state) = @_; + print "$msg\n"; + exit $state; +} diff --git a/nrpe/files/plugins/check_netlinks.py b/nrpe/files/plugins/check_netlinks.py new file mode 100755 index 0000000..c5d01d2 --- /dev/null +++ b/nrpe/files/plugins/check_netlinks.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Check netlinks and alert.""" +# -*- coding: us-ascii -*- + +# Copyright (C) 2017 Canonical +# All rights reserved +# Author: Alvaro Uria +# +# check_netlinks.py -i eth0 -o up -m 1500 -s 1000 + + +import argparse +import glob +import os +import sys + +from nagios_plugin3 import ( + CriticalError, + WarnError, + try_check, +) + +FILTER = ("operstate", "mtu", "speed") + + +def check_iface(iface, skiperror, crit_thr): + """Return /sys/class/net// values.""" + file_path = "/sys/class/net/{0}/{1}" + filter = ["operstate", "mtu"] + if not os.path.exists(file_path.format(iface, "bridge")) and iface != "lo": + filter.append("speed") + + for metric_key in filter: + try: + with open(file_path.format(iface, metric_key)) as fd: + metric_value = fd.readline().strip() + except FileNotFoundError: + if not skiperror: + raise WarnError("WARNING: {} iface does not exist".format(iface)) + return + except OSError as e: + if ( + metric_key == "speed" + and "Invalid argument" in str(e) + and crit_thr["operstate"] == "down" + ): + filter = [f for f in filter if f != "speed"] + continue + else: + raise CriticalError( + "CRITICAL: {} ({} returns " + "invalid argument)".format(iface, metric_key) + ) + + if metric_key == "operstate" and metric_value != "up": + if metric_value != crit_thr["operstate"]: + raise CriticalError( + "CRITICAL: {} link state is {}".format(iface, metric_value) + ) + + if metric_value != crit_thr[metric_key]: + raise CriticalError( + "CRITICAL: {}/{} is {} (target: " + "{})".format(iface, metric_key, metric_value, crit_thr[metric_key]) + ) + + for metric in crit_thr: + if metric not in filter: + crit_thr[metric] = "n/a" + crit_thr["iface"] = iface + print( + "OK: {iface} matches thresholds: " + "o:{operstate}, m:{mtu}, s:{speed}".format(**crit_thr) + ) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description="check ifaces status") + parser.add_argument( + "--iface", + "-i", + type=str, + help="interface to monitor; listed in /sys/class/net/*)", + ) + parser.add_argument( + "--skip-unfound-ifaces", + "-q", + default=False, + action="store_true", + help="ignores unfound ifaces; otherwise, alert will be triggered", + ) + parser.add_argument( + "--operstate", + "-o", + default="up", + type=str, + help="operstate: up, down, unknown (default: up)", + ) + parser.add_argument( + "--mtu", "-m", default="1500", type=str, help="mtu size (default: 1500)" + ) + parser.add_argument( + "--speed", + "-s", + default="10000", + type=str, + help="link speed in Mbps (default 10000)", + ) + args = parser.parse_args() + + if not args.iface: + ifaces = map(os.path.basename, glob.glob("/sys/class/net/*")) + print( + "UNKNOWN: Please specify one of these " + "ifaces: {}".format(",".join(ifaces)) + ) + sys.exit(1) + return args + + +def main(): + """Parse args and check the netlinks.""" + args = parse_args() + crit_thr = { + "operstate": args.operstate.lower(), + "mtu": args.mtu, + "speed": args.speed, + } + try_check(check_iface, args.iface, args.skip_unfound_ifaces, crit_thr) + + +if __name__ == "__main__": + main() diff --git a/nrpe/files/plugins/check_netns.sh b/nrpe/files/plugins/check_netns.sh new file mode 100755 index 0000000..2477bb5 --- /dev/null +++ b/nrpe/files/plugins/check_netns.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Copyright (c) 2014 Canonical, Ltd +# Author: Brad Marshall + +# Checks if a network namespace is responding by doing an ip a in each one. + +. /usr/lib/nagios/plugins/utils.sh + +check_ret_value() { + RET=$1 + if [[ $RET -ne 0 ]];then + echo "CRIT: $2" + exit $STATE_CRIT + fi +} + +check_netns_create() { + RET_VAL=$(ip netns add nrpe-check 2>&1) + check_ret_value $? "$RET_VAL" + RET_VAL=$(ip netns delete nrpe-check 2>&1) + check_ret_value $? "$RET_VAL" +} + + +netnsok=() +netnscrit=() + +for ns in $(ip netns list |awk '!/^nrpe-check$/ {print $1}'); do + output=$(ip netns exec $ns ip a 2>/dev/null) + err=$? + if [ $err -eq 0 ]; then + netnsok=("${netnsok[@]}" $ns) + else + netnscrit=("${netnscrit[@]}" $ns) + fi +done + +if [ ${#netnscrit[@]} -eq 0 ]; then + if [ ${#netnsok[@]} -eq 0 ]; then + check_netns_create + echo "OK: no namespaces defined" + exit $STATE_OK + else + echo "OK: ${netnsok[@]} are responding" + exit $STATE_OK + fi +else + echo "CRIT: ${netnscrit[@]} aren't responding" + exit $STATE_CRIT +fi + diff --git a/nrpe/files/plugins/check_ro_filesystem.py b/nrpe/files/plugins/check_ro_filesystem.py new file mode 100755 index 0000000..d1578a7 --- /dev/null +++ b/nrpe/files/plugins/check_ro_filesystem.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Check readonly filesystems and alert.""" +# -*- coding: us-ascii -*- + +# Copyright (C) 2020 Canonical +# All rights reserved +# + +import argparse + +from nagios_plugin3 import ( + CriticalError, + UnknownError, + try_check, +) + +EXCLUDE = {"/snap/", "/sys/fs/cgroup"} + + +def check_ro_filesystem(excludes=""): + """Loop /proc/mounts looking for readonly mounts. + + :param excludes: list of mount points to exclude from checks + """ + # read /proc/mounts, add each line to a list + try: + with open("/proc/mounts") as fd: + mounts = [mount.strip() for mount in fd.readlines()] + except Exception as e: + raise UnknownError("UNKNOWN: unable to read mounts with {}".format(e)) + + exclude_mounts = EXCLUDE + ro_filesystems = [] + # if excludes != "" and excludes is not None: + if excludes: + try: + exclude_mounts = EXCLUDE.union(set(excludes.split(","))) + except Exception as e: + msg = "UNKNOWN: unable to read list of mounts to exclude {}".format(e) + raise UnknownError(msg) + for mount in mounts: + # for each line in the list, split by space to a new list + split_mount = mount.split() + # if mount[1] matches EXCLUDE_FS then next, else check it's not readonly + if not any( + split_mount[1].startswith(exclusion.strip()) for exclusion in exclude_mounts + ): + mount_options = split_mount[3].split(",") + if "ro" in mount_options: + ro_filesystems.append(split_mount[1]) + if len(ro_filesystems) > 0: + msg = "CRITICAL: filesystem(s) {} readonly".format(",".join(ro_filesystems)) + raise CriticalError(msg) + + print("OK: no readonly filesystems found") + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description="Check for readonly filesystems") + parser.add_argument( + "--exclude", + "-e", + type=str, + help="""Comma separated list of mount points to exclude from checks for readonly filesystem. + Can be just a substring of the whole mount point.""", + default="", + ) + args = parser.parse_args() + return args + + +def main(): + """Parse args and check the readonly filesystem.""" + args = parse_args() + try_check(check_ro_filesystem, args.exclude) + + +if __name__ == "__main__": + main() diff --git a/nrpe/files/plugins/check_status_file.py b/nrpe/files/plugins/check_status_file.py new file mode 100755 index 0000000..1e15bbe --- /dev/null +++ b/nrpe/files/plugins/check_status_file.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Read file and return nagios status based on its content.""" +# -------------------------------------------------------- +# This file is managed by Juju +# -------------------------------------------------------- + +# +# Copyright 2014 Canonical Ltd. +# +# Author: Jacek Nykis +# + +import re + +import nagios_plugin3 as nagios_plugin + + +def parse_args(): + """Parse command-line options.""" + import argparse + + parser = argparse.ArgumentParser( + description="Read file and return nagios status based on its content", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("-f", "--status-file", required=True, help="Status file path") + parser.add_argument( + "-c", + "--critical-text", + default="CRITICAL", + help="String indicating critical status", + ) + parser.add_argument( + "-w", + "--warning-text", + default="WARNING", + help="String indicating warning status", + ) + parser.add_argument( + "-o", "--ok-text", default="OK", help="String indicating OK status" + ) + parser.add_argument( + "-u", + "--unknown-text", + default="UNKNOWN", + help="String indicating unknown status", + ) + return parser.parse_args() + + +def check_status(args): + """Return nagios status.""" + nagios_plugin.check_file_freshness(args.status_file, 43200) + + with open(args.status_file, "r") as f: + content = [line.strip() for line in f.readlines()] + + for line in content: + if re.search(args.critical_text, line): + raise nagios_plugin.CriticalError(line) + elif re.search(args.warning_text, line): + raise nagios_plugin.WarnError(line) + elif re.search(args.unknown_text, line): + raise nagios_plugin.UnknownError(line) + else: + print(line) + + +if __name__ == "__main__": + args = parse_args() + nagios_plugin.try_check(check_status, args) diff --git a/nrpe/files/plugins/check_swap_activity b/nrpe/files/plugins/check_swap_activity new file mode 100755 index 0000000..3fc49d4 --- /dev/null +++ b/nrpe/files/plugins/check_swap_activity @@ -0,0 +1,78 @@ +#!/bin/bash + +# This script checks swap pageouts and reports number of kbytes moved +# from physical ram to swap space in a given number of seconds +# +# Usage: "check_swap_activity -i interval -w warning_kbyts -c critical_kbytes +# +# + +set -eu + +. /usr/lib/nagios/plugins/utils.sh + + +help() { +cat << EOH +usage: $0 [ -i ## ] -w ## -c ## + +Measures page-outs to swap over a given interval, by default 5 seconds. + + -i time in seconds to monitor (defaults to 5 seconds) + -w warning Level in kbytes + -c critical Level in kbytes + +EOH +} + +TIMEWORD=seconds +WARN_LVL= +CRIT_LVL= +INTERVAL=5 +## FETCH ARGUMENTS +while getopts "i:w:c:" OPTION; do + case "${OPTION}" in + i) + INTERVAL=${OPTARG} + if [ $INTERVAL -eq 1 ]; then + TIMEWORD=second + fi + ;; + w) + WARN_LVL=${OPTARG} + ;; + c) + CRIT_LVL=${OPTARG} + ;; + ?) + help + exit 3 + ;; + esac +done + +if [ -z ${WARN_LVL} ] || [ -z ${CRIT_LVL} ] ; then + help + exit 3 +fi + +## Get swap pageouts over $INTERVAL +PAGEOUTS=$(vmstat -w ${INTERVAL} 2 | tail -n 1 | awk '{print $8}') + +SUMMARY="| swapout_size=${PAGEOUTS}KB;${WARN_LVL};${CRIT_LVL};" +if [ ${PAGEOUTS} -lt ${WARN_LVL} ]; then + # pageouts are below threshold + echo "OK - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY" + exit $STATE_OK +elif [ ${PAGEOUTS} -ge ${CRIT_LVL} ]; then + ## SWAP IS IN CRITICAL STATE + echo "CRITICAL - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY" + exit $STATE_CRITICAL +elif [ ${PAGEOUTS} -ge ${WARN_LVL} ] && [ ${PAGEOUTS} -lt ${CRIT_LVL} ]; then + ## SWAP IS IN WARNING STATE + echo "WARNING - ${PAGEOUTS} kb swapped out in last ${INTERVAL} ${TIMEWORD} $SUMMARY" + exit $STATE_WARNING +else + echo "CRITICAL: Failure to process pageout information $SUMMARY" + exit $STATE_UNKNOWN +fi diff --git a/nrpe/files/plugins/check_systemd.py b/nrpe/files/plugins/check_systemd.py new file mode 100755 index 0000000..ae8b601 --- /dev/null +++ b/nrpe/files/plugins/check_systemd.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 +"""Check systemd service and alert.""" +# +# Copyright 2016 Canonical Ltd +# +# Author: Brad Marshall +# +# Based on check_upstart_job and +# https://zignar.net/2014/09/08/getting-started-with-dbus-python-systemd/ +# +import sys + +import dbus + + +service_arg = sys.argv[1] +service_name = "%s.service" % service_arg + +try: + bus = dbus.SystemBus() + systemd = bus.get_object("org.freedesktop.systemd1", "/org/freedesktop/systemd1") + manager = dbus.Interface(systemd, dbus_interface="org.freedesktop.systemd1.Manager") + try: + service_unit = manager.LoadUnit(service_name) + service_proxy = bus.get_object("org.freedesktop.systemd1", str(service_unit)) + service = dbus.Interface( + service_proxy, dbus_interface="org.freedesktop.systemd1.Unit" + ) + service_res = service_proxy.Get( + "org.freedesktop.systemd1.Unit", + "SubState", + dbus_interface="org.freedesktop.DBus.Properties", + ) + + if service_res == "running": + print("OK: %s is running" % service_name) + sys.exit(0) + else: + print("CRITICAL: %s is not running" % service_name) + sys.exit(2) + + except dbus.DBusException: + print("CRITICAL: unable to find %s in systemd" % service_name) + sys.exit(2) + +except dbus.DBusException: + print("CRITICAL: unable to connect to system for %s" % service_name) + sys.exit(2) diff --git a/nrpe/files/plugins/check_upstart_job b/nrpe/files/plugins/check_upstart_job new file mode 100755 index 0000000..94efb95 --- /dev/null +++ b/nrpe/files/plugins/check_upstart_job @@ -0,0 +1,72 @@ +#!/usr/bin/python + +# +# Copyright 2012, 2013 Canonical Ltd. +# +# Author: Paul Collins +# +# Based on http://www.eurion.net/python-snippets/snippet/Upstart%20service%20status.html +# + +import sys + +import dbus + + +class Upstart(object): + def __init__(self): + self._bus = dbus.SystemBus() + self._upstart = self._bus.get_object('com.ubuntu.Upstart', + '/com/ubuntu/Upstart') + def get_job(self, job_name): + path = self._upstart.GetJobByName(job_name, + dbus_interface='com.ubuntu.Upstart0_6') + return self._bus.get_object('com.ubuntu.Upstart', path) + + def get_properties(self, job): + path = job.GetInstance([], dbus_interface='com.ubuntu.Upstart0_6.Job') + instance = self._bus.get_object('com.ubuntu.Upstart', path) + return instance.GetAll('com.ubuntu.Upstart0_6.Instance', + dbus_interface=dbus.PROPERTIES_IFACE) + + def get_job_instances(self, job_name): + job = self.get_job(job_name) + paths = job.GetAllInstances([], dbus_interface='com.ubuntu.Upstart0_6.Job') + return [self._bus.get_object('com.ubuntu.Upstart', path) for path in paths] + + def get_job_instance_properties(self, job): + return job.GetAll('com.ubuntu.Upstart0_6.Instance', + dbus_interface=dbus.PROPERTIES_IFACE) + +try: + upstart = Upstart() + try: + job = upstart.get_job(sys.argv[1]) + props = upstart.get_properties(job) + + if props['state'] == 'running': + print 'OK: %s is running' % sys.argv[1] + sys.exit(0) + else: + print 'CRITICAL: %s is not running' % sys.argv[1] + sys.exit(2) + + except dbus.DBusException as e: + instances = upstart.get_job_instances(sys.argv[1]) + propses = [upstart.get_job_instance_properties(instance) for instance in instances] + states = dict([(props['name'], props['state']) for props in propses]) + if len(states) != states.values().count('running'): + not_running = [] + for name in states.keys(): + if states[name] != 'running': + not_running.append(name) + print 'CRITICAL: %d instances of %s not running: %s' % \ + (len(not_running), sys.argv[1], not_running.join(', ')) + sys.exit(2) + else: + print 'OK: %d instances of %s running' % (len(states), sys.argv[1]) + +except dbus.DBusException as e: + print 'CRITICAL: failed to get properties of \'%s\' from upstart' % sys.argv[1] + sys.exit(2) + diff --git a/nrpe/files/plugins/check_xfs_errors.py b/nrpe/files/plugins/check_xfs_errors.py new file mode 100755 index 0000000..2450511 --- /dev/null +++ b/nrpe/files/plugins/check_xfs_errors.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Check for xfs errors and alert.""" +# +# Copyright 2017 Canonical Ltd +# +# Author: Jill Rouleau +# +# Check for xfs errors and alert +# +import re +import subprocess +import sys +from datetime import datetime, timedelta + + +# error messages commonly seen in dmesg on xfs errors +raw_xfs_errors = [ + "XFS_WANT_CORRUPTED_", + "xfs_error_report", + "corruption detected at xfs_", + "Unmount and run xfs_repair", +] + +xfs_regex = [re.compile(i) for i in raw_xfs_errors] + +# nagios can't read from kern.log, so we look at dmesg - this does present +# a known limitation if a node is rebooted or dmesg is otherwise cleared. +log_lines = [line for line in subprocess.getoutput(["dmesg -T"]).split("\n")] + +err_results = [line for line in log_lines for rgx in xfs_regex if re.search(rgx, line)] + +# Look for errors within the last N minutes, specified in the check definition +check_delta = int(sys.argv[1]) + +# dmesg -T formatted timestamps are inside [], so we need to add them +datetime_delta = datetime.now() - timedelta(minutes=check_delta) + +recent_logs = [ + i for i in err_results if datetime.strptime(i[1:25], "%c") >= datetime_delta +] + +if recent_logs: + print("CRITICAL: Recent XFS errors in kern.log." + "\n" + "{}".format(recent_logs)) + sys.exit(2) +else: + print("OK") + sys.exit(0) diff --git a/nrpe/files/rsyncd.conf b/nrpe/files/rsyncd.conf new file mode 100644 index 0000000..7b48f16 --- /dev/null +++ b/nrpe/files/rsyncd.conf @@ -0,0 +1,13 @@ +#------------------------------------------------ +# This file is juju managed +#------------------------------------------------ + +uid = nobody +gid = nogroup +pid file = /var/run/rsyncd.pid +syslog facility = daemon +socket options = SO_KEEPALIVE +timeout = 7200 + +&merge /etc/rsync-juju.d +&include /etc/rsync-juju.d diff --git a/nrpe/hooks/charmhelpers b/nrpe/hooks/charmhelpers new file mode 120000 index 0000000..488c89e --- /dev/null +++ b/nrpe/hooks/charmhelpers @@ -0,0 +1 @@ +../mod/charmhelpers/charmhelpers \ No newline at end of file diff --git a/nrpe/hooks/config-changed b/nrpe/hooks/config-changed new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/config-changed @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/general-info-relation-changed b/nrpe/hooks/general-info-relation-changed new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/general-info-relation-changed @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/general-info-relation-joined b/nrpe/hooks/general-info-relation-joined new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/general-info-relation-joined @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/install b/nrpe/hooks/install new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/install @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/local-monitors-relation-changed b/nrpe/hooks/local-monitors-relation-changed new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/local-monitors-relation-changed @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/local-monitors-relation-joined b/nrpe/hooks/local-monitors-relation-joined new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/local-monitors-relation-joined @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/monitors-relation-changed b/nrpe/hooks/monitors-relation-changed new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/monitors-relation-changed @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/monitors-relation-joined b/nrpe/hooks/monitors-relation-joined new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/monitors-relation-joined @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/nrpe-external-master-relation-changed b/nrpe/hooks/nrpe-external-master-relation-changed new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/nrpe-external-master-relation-changed @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/nrpe-external-master-relation-joined b/nrpe/hooks/nrpe-external-master-relation-joined new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/nrpe-external-master-relation-joined @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/nrpe_helpers.py b/nrpe/hooks/nrpe_helpers.py new file mode 100644 index 0000000..d1ddb84 --- /dev/null +++ b/nrpe/hooks/nrpe_helpers.py @@ -0,0 +1,696 @@ +"""Nrpe helpers module.""" +import glob +import ipaddress +import os +import socket +import subprocess + +from charmhelpers.core import hookenv +from charmhelpers.core.host import is_container +from charmhelpers.core.services import helpers + +import yaml + + +NETLINKS_ERROR = False + + +class InvalidCustomCheckException(Exception): + """Custom exception for Invalid nrpe check.""" + + pass + + +class Monitors(dict): + """List of checks that a remote Nagios can query.""" + + def __init__(self, version="0.3"): + """Build monitors structure.""" + self["monitors"] = {"remote": {"nrpe": {}}} + self["version"] = version + + def add_monitors(self, mdict, monitor_label="default"): + """Add monitors passed in mdict.""" + if not mdict or not mdict.get("monitors"): + return + + for checktype in mdict["monitors"].get("remote", []): + check_details = mdict["monitors"]["remote"][checktype] + if self["monitors"]["remote"].get(checktype): + self["monitors"]["remote"][checktype].update(check_details) + else: + self["monitors"]["remote"][checktype] = check_details + + for checktype in mdict["monitors"].get("local", []): + check_details = self.convert_local_checks( + mdict["monitors"]["local"], + monitor_label, + ) + self["monitors"]["remote"]["nrpe"].update(check_details) + + def add_nrpe_check(self, check_name, command): + """Add nrpe check to remote monitors.""" + self["monitors"]["remote"]["nrpe"][check_name] = command + + def convert_local_checks(self, monitors, monitor_src): + """Convert check from local checks to remote nrpe checks. + + monitors -- monitor dict + monitor_src -- Monitor source principal, subordinate or user + """ + mons = {} + for checktype in monitors.keys(): + for checkname in monitors[checktype]: + try: + check_def = NRPECheckCtxt( + checktype, + monitors[checktype][checkname], + monitor_src, + ) + mons[check_def["cmd_name"]] = {"command": check_def["cmd_name"]} + except InvalidCustomCheckException as e: + hookenv.log( + "Error encountered configuring local check " + '"{check}": {err}'.format(check=checkname, err=str(e)), + hookenv.ERROR, + ) + return mons + + +def get_ingress_address(binding, external=False): + """Get ingress IP address for a binding. + + Returns a local IP address for incoming requests to NRPE. + + :param binding: name of the binding, e.g. 'monitors' + :param external: bool, if True return the public address if charm config requests + otherwise return the local address which would be used for incoming + nrpe requests. + """ + # using network-get to retrieve the address details if available. + hookenv.log("Getting ingress IP address for binding %s" % binding) + if hookenv.config("nagios_address_type").lower() == "public" and external: + return hookenv.unit_get("public-address") + + ip_address = None + try: + network_info = hookenv.network_get(binding) + if network_info is not None and "ingress-addresses" in network_info: + try: + ip_address = network_info["bind-addresses"][0]["addresses"][0][ + "address" + ] + hookenv.log("Using ingress-addresses, found %s" % ip_address) + except KeyError: + hookenv.log("Using primary-addresses") + ip_address = hookenv.network_get_primary_address(binding) + + except (NotImplementedError, FileNotFoundError) as e: + hookenv.log( + "Unable to determine inbound IP address for binding {} with {}".format( + binding, e + ), + level=hookenv.ERROR, + ) + + return ip_address + + +class MonitorsRelation(helpers.RelationContext): + """Define a monitors relation.""" + + name = "monitors" + interface = "monitors" + + def __init__(self, *args, **kwargs): + """Build superclass and principal relation.""" + self.principal_relation = PrincipalRelation() + super(MonitorsRelation, self).__init__(*args, **kwargs) + + def is_ready(self): + """Return true if the principal relation is ready.""" + return self.principal_relation.is_ready() + + def get_subordinate_monitors(self): + """Return default monitors defined by this charm.""" + monitors = Monitors() + for check in SubordinateCheckDefinitions()["checks"]: + if check["cmd_params"]: + monitors.add_nrpe_check(check["cmd_name"], check["cmd_name"]) + return monitors + + def get_user_defined_monitors(self): + """Return monitors defined by monitors config option.""" + monitors = Monitors() + monitors.add_monitors(yaml.safe_load(hookenv.config("monitors")), "user") + return monitors + + def get_principal_monitors(self): + """Return monitors passed by relation with principal.""" + return self.principal_relation.get_monitors() + + def get_monitor_dicts(self): + """Return all monitor dicts.""" + monitor_dicts = { + "principal": self.get_principal_monitors(), + "subordinate": self.get_subordinate_monitors(), + "user": self.get_user_defined_monitors(), + } + return monitor_dicts + + def get_monitors(self): + """Return monitor dict. + + All monitors merged together and local + monitors converted to remote nrpe checks. + """ + all_monitors = Monitors() + monitors = [ + self.get_principal_monitors(), + self.get_subordinate_monitors(), + self.get_user_defined_monitors(), + ] + for mon in monitors: + all_monitors.add_monitors(mon) + return all_monitors + + def egress_subnets(self, relation_data): + """Return egress subnets. + + This behaves the same as charmhelpers.core.hookenv.egress_subnets(). + If it can't determine the egress subnets it will fall back to + ingress-address or finally private-address. + """ + if "egress-subnets" in relation_data: + return relation_data["egress-subnets"] + if "ingress-address" in relation_data: + return relation_data["ingress-address"] + return relation_data["private-address"] + + def get_data(self): + """Get relation data.""" + super(MonitorsRelation, self).get_data() + if not hookenv.relation_ids(self.name): + return + # self['monitors'] comes from the superclass helpers.RelationContext + # and contains relation data for each 'monitors' relation (to/from + # Nagios). + subnets = [self.egress_subnets(info) for info in self["monitors"]] + self["monitor_allowed_hosts"] = ",".join(subnets) + + def provide_data(self): + """Return relation info.""" + # get the address to send to Nagios for host definition + address = get_ingress_address("monitors", external=True) + + relation_info = { + "target-id": self.principal_relation.nagios_hostname(), + "monitors": self.get_monitors(), + "private-address": address, + "ingress-address": address, + "target-address": address, + "machine_id": os.environ["JUJU_MACHINE_ID"], + "model_id": hookenv.model_uuid(), + } + return relation_info + + +class PrincipalRelation(helpers.RelationContext): + """Define a principal relation.""" + + def __init__(self, *args, **kwargs): + """Set name and interface.""" + if hookenv.relations_of_type("nrpe-external-master"): + self.name = "nrpe-external-master" + self.interface = "nrpe-external-master" + elif hookenv.relations_of_type("general-info"): + self.name = "general-info" + self.interface = "juju-info" + elif hookenv.relations_of_type("local-monitors"): + self.name = "local-monitors" + self.interface = "local-monitors" + super(PrincipalRelation, self).__init__(*args, **kwargs) + + def is_ready(self): + """Return true if the relation is connected.""" + if self.name not in self: + return False + return "__unit__" in self[self.name][0] + + def nagios_hostname(self): + """Return the string that nagios will use to identify this host.""" + host_context = hookenv.config("nagios_host_context") + if host_context: + host_context += "-" + hostname_type = hookenv.config("nagios_hostname_type") + + # Detect bare metal hosts + if hostname_type == "auto": + is_metal = "none" in subprocess.getoutput("/usr/bin/systemd-detect-virt") + if is_metal: + hostname_type = "host" + else: + hostname_type = "unit" + + if hostname_type == "host" or not self.is_ready(): + nagios_hostname = "{}{}".format(host_context, socket.gethostname()) + return nagios_hostname + else: + principal_unitname = hookenv.principal_unit() + # Fallback to using "primary" if it exists. + if not principal_unitname: + for relunit in self[self.name]: + if relunit.get("primary", "False").lower() == "true": + principal_unitname = relunit["__unit__"] + break + nagios_hostname = "{}{}".format(host_context, principal_unitname) + nagios_hostname = nagios_hostname.replace("/", "-") + return nagios_hostname + + def get_monitors(self): + """Return monitors passed by services on the self.interface relation.""" + if not self.is_ready(): + return + monitors = Monitors() + for rel in self[self.name]: + if rel.get("monitors"): + monitors.add_monitors(yaml.load(rel["monitors"]), "principal") + return monitors + + def provide_data(self): + """Return nagios hostname and nagios host context.""" + # Provide this data to principals because get_nagios_hostname expects + # them in charmhelpers/contrib/charmsupport/nrpe when writing principal + # service__* files + return { + "nagios_hostname": self.nagios_hostname(), + "nagios_host_context": hookenv.config("nagios_host_context"), + } + + +class NagiosInfo(dict): + """Define a NagiosInfo dict.""" + + def __init__(self): + """Set principal relation and dict values.""" + self.principal_relation = PrincipalRelation() + self["external_nagios_master"] = "127.0.0.1" + if hookenv.config("nagios_master") != "None": + self["external_nagios_master"] = "{},{}".format( + self["external_nagios_master"], hookenv.config("nagios_master") + ) + self["nagios_hostname"] = self.principal_relation.nagios_hostname() + + # export_host.cfg.tmpl host definition for Nagios + self["nagios_ipaddress"] = get_ingress_address("monitors", external=True) + # Address configured for NRPE to listen on + self["nrpe_ipaddress"] = get_ingress_address("monitors") + + self["dont_blame_nrpe"] = "1" if hookenv.config("dont_blame_nrpe") else "0" + self["debug"] = "1" if hookenv.config("debug") else "0" + + +class RsyncEnabled(helpers.RelationContext): + """Define a relation context for rsync enabled relation.""" + + def __init__(self): + """Set export_nagios_definitions.""" + self["export_nagios_definitions"] = hookenv.config("export_nagios_definitions") + if ( + hookenv.config("nagios_master") + and hookenv.config("nagios_master") != "None" + ): + self["export_nagios_definitions"] = True + + def is_ready(self): + """Return true if relation is ready.""" + return self["export_nagios_definitions"] + + +class NRPECheckCtxt(dict): + """Convert a local monitor definition. + + Create a dict needed for writing the nrpe check definition. + """ + + def __init__(self, checktype, check_opts, monitor_src): + """Set dict values.""" + plugin_path = "/usr/lib/nagios/plugins" + if checktype == "procrunning": + self["cmd_exec"] = plugin_path + "/check_procs" + self["description"] = "Check process {executable} is running".format( + **check_opts + ) + self["cmd_name"] = "check_proc_" + check_opts["executable"] + self["cmd_params"] = "-w {min} -c {max} -C {executable}".format( + **check_opts + ) + elif checktype == "processcount": + self["cmd_exec"] = plugin_path + "/check_procs" + self["description"] = "Check process count" + self["cmd_name"] = "check_proc_principal" + if "min" in check_opts: + self["cmd_params"] = "-w {min} -c {max}".format(**check_opts) + else: + self["cmd_params"] = "-c {max}".format(**check_opts) + elif checktype == "disk": + self["cmd_exec"] = plugin_path + "/check_disk" + self["description"] = "Check disk usage " + check_opts["path"].replace( + "/", "_" + ) + self["cmd_name"] = "check_disk_principal" + self["cmd_params"] = "-w 20 -c 10 -p " + check_opts["path"] + elif checktype == "custom": + custom_path = check_opts.get("plugin_path", plugin_path) + if not custom_path.startswith(os.path.sep): + custom_path = os.path.join(os.path.sep, custom_path) + if not os.path.isdir(custom_path): + raise InvalidCustomCheckException( + 'Specified plugin_path "{}" does not exist or is not a ' + "directory.".format(custom_path) + ) + check = check_opts["check"] + self["cmd_exec"] = os.path.join(custom_path, check) + self["description"] = check_opts.get("desc", "Check %s" % check) + self["cmd_name"] = check + self["cmd_params"] = check_opts.get("params", "") or "" + self["description"] += " ({})".format(monitor_src) + self["cmd_name"] += "_" + monitor_src + + +class SubordinateCheckDefinitions(dict): + """Return dict of checks the charm configures.""" + + def __init__(self): + """Set dict values.""" + self.procs = self.proc_count() + load_thresholds = self._get_load_thresholds() + proc_thresholds = self._get_proc_thresholds() + disk_root_thresholds = self._get_disk_root_thresholds() + + pkg_plugin_dir = "/usr/lib/nagios/plugins/" + local_plugin_dir = "/usr/local/lib/nagios/plugins/" + checks = [ + { + "description": "Number of Zombie processes", + "cmd_name": "check_zombie_procs", + "cmd_exec": pkg_plugin_dir + "check_procs", + "cmd_params": hookenv.config("zombies"), + }, + { + "description": "Number of processes", + "cmd_name": "check_total_procs", + "cmd_exec": pkg_plugin_dir + "check_procs", + "cmd_params": proc_thresholds, + }, + { + "description": "Number of Users", + "cmd_name": "check_users", + "cmd_exec": pkg_plugin_dir + "check_users", + "cmd_params": hookenv.config("users"), + }, + { + "description": "Connnection tracking table", + "cmd_name": "check_conntrack", + "cmd_exec": local_plugin_dir + "check_conntrack.sh", + "cmd_params": hookenv.config("conntrack"), + }, + ] + + if not is_container(): + checks.extend( + [ + { + "description": "Root disk", + "cmd_name": "check_disk_root", + "cmd_exec": pkg_plugin_dir + "check_disk", + "cmd_params": disk_root_thresholds, + }, + { + "description": "System Load", + "cmd_name": "check_load", + "cmd_exec": pkg_plugin_dir + "check_load", + "cmd_params": load_thresholds, + }, + { + "description": "Swap", + "cmd_name": "check_swap", + "cmd_exec": pkg_plugin_dir + "check_swap", + "cmd_params": hookenv.config("swap").strip(), + }, + # Note: check_swap_activity *must* be listed after check_swap, else + # check_swap_activity will be removed during installation of + # check_swap. + { + "description": "Swap Activity", + "cmd_name": "check_swap_activity", + "cmd_exec": local_plugin_dir + "check_swap_activity", + "cmd_params": hookenv.config("swap_activity"), + }, + { + "description": "Memory", + "cmd_name": "check_mem", + "cmd_exec": local_plugin_dir + "check_mem.pl", + "cmd_params": hookenv.config("mem"), + }, + { + "description": "XFS Errors", + "cmd_name": "check_xfs_errors", + "cmd_exec": local_plugin_dir + "check_xfs_errors.py", + "cmd_params": hookenv.config("xfs_errors"), + }, + { + "description": "ARP cache entries", + "cmd_name": "check_arp_cache", + "cmd_exec": os.path.join( + local_plugin_dir, "check_arp_cache.py" + ), + "cmd_params": "-w 60 -c 80", + }, + ] + ) + + ro_filesystem_excludes = hookenv.config("ro_filesystem_excludes") + if ro_filesystem_excludes == "": + # specify cmd_params = '' to disable/remove the check from nrpe + check_ro_filesystem = { + "description": "Readonly filesystems", + "cmd_name": "check_ro_filesystem", + "cmd_exec": os.path.join( + local_plugin_dir, "check_ro_filesystem.py" + ), + "cmd_params": "", + } + else: + check_ro_filesystem = { + "description": "Readonly filesystems", + "cmd_name": "check_ro_filesystem", + "cmd_exec": os.path.join( + local_plugin_dir, "check_ro_filesystem.py" + ), + "cmd_params": "-e {}".format( + hookenv.config("ro_filesystem_excludes") + ), + } + checks.append(check_ro_filesystem) + + if hookenv.config("lacp_bonds").strip(): + for bond_iface in hookenv.config("lacp_bonds").strip().split(): + if os.path.exists("/sys/class/net/{}".format(bond_iface)): + description = "LACP Check {}".format(bond_iface) + cmd_name = "check_lacp_{}".format(bond_iface) + cmd_exec = local_plugin_dir + "check_lacp_bond.py" + cmd_params = "-i {}".format(bond_iface) + lacp_check = { + "description": description, + "cmd_name": cmd_name, + "cmd_exec": cmd_exec, + "cmd_params": cmd_params, + } + checks.append(lacp_check) + + if hookenv.config("netlinks"): + ifaces = yaml.safe_load(hookenv.config("netlinks")) + cmd_exec = local_plugin_dir + "check_netlinks.py" + if hookenv.config("netlinks_skip_unfound_ifaces"): + cmd_exec += " --skip-unfound-ifaces" + d_ifaces = self.parse_netlinks(ifaces) + for iface in d_ifaces: + description = "Netlinks status ({})".format(iface) + cmd_name = "check_netlinks_{}".format(iface) + cmd_params = d_ifaces[iface] + netlink_check = { + "description": description, + "cmd_name": cmd_name, + "cmd_exec": cmd_exec, + "cmd_params": cmd_params, + } + checks.append(netlink_check) + + # Checking if CPU governor is supported by the system and add nrpe check + cpu_governor_paths = "/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" + cpu_governor_supported = glob.glob(cpu_governor_paths) + requested_cpu_governor = hookenv.relation_get("requested_cpu_governor") + cpu_governor_config = hookenv.config("cpu_governor") + wanted_cpu_governor = cpu_governor_config or requested_cpu_governor + if wanted_cpu_governor and cpu_governor_supported: + description = "Check CPU governor scaler" + cmd_name = "check_cpu_governor" + cmd_exec = local_plugin_dir + "check_cpu_governor.py" + cmd_params = "--governor {}".format(wanted_cpu_governor) + cpu_governor_check = { + "description": description, + "cmd_name": cmd_name, + "cmd_exec": cmd_exec, + "cmd_params": cmd_params, + } + checks.append(cpu_governor_check) + + self["checks"] = [] + sub_postfix = str(hookenv.config("sub_postfix")) + # Automatically use _sub for checks shipped on a unit with the nagios + # charm. Mostly for backwards compatibility. + principal_unit = hookenv.principal_unit() + if sub_postfix == "" and principal_unit: + md = hookenv._metadata_unit(principal_unit) + if md and md.pop("name", None) == "nagios": + sub_postfix = "_sub" + nrpe_config_sub_tmpl = "/etc/nagios/nrpe.d/{}_*.cfg" + nrpe_config_tmpl = "/etc/nagios/nrpe.d/{}.cfg" + for check in checks: + # This can be used to clean up old files before rendering the new + # ones + nrpe_configfiles_sub = nrpe_config_sub_tmpl.format(check["cmd_name"]) + nrpe_configfiles = nrpe_config_tmpl.format(check["cmd_name"]) + check["matching_files"] = glob.glob(nrpe_configfiles_sub) + check["matching_files"].extend(glob.glob(nrpe_configfiles)) + check["description"] += " (sub)" + check["cmd_name"] += sub_postfix + self["checks"].append(check) + + def _get_proc_thresholds(self): + """Return suitable processor thresholds.""" + if hookenv.config("procs") == "auto": + proc_thresholds = "-k -w {} -c {}".format( + 25 * self.procs + 100, 50 * self.procs + 100 + ) + else: + proc_thresholds = hookenv.config("procs") + return proc_thresholds + + def _get_load_thresholds(self): + """Return suitable load thresholds.""" + if hookenv.config("load") == "auto": + # Give 1min load alerts higher thresholds than 15 min load alerts + warn_multipliers = (4, 2, 1) + crit_multipliers = (8, 4, 2) + load_thresholds = ("-w %s -c %s") % ( + ",".join([str(m * self.procs) for m in warn_multipliers]), + ",".join([str(m * self.procs) for m in crit_multipliers]), + ) + else: + load_thresholds = hookenv.config("load") + return load_thresholds + + def _get_disk_root_thresholds(self): + """Return suitable disk thresholds.""" + if hookenv.config("disk_root"): + disk_root_thresholds = hookenv.config("disk_root") + " -p / " + else: + disk_root_thresholds = "" + return disk_root_thresholds + + def proc_count(self): + """Return number number of processing units.""" + return int(subprocess.check_output(["nproc", "--all"])) + + def parse_netlinks(self, ifaces): + """Parse a list of strings, or a single string. + + Looks if the interfaces exist and configures extra parameters (or + properties) -> ie. ['mtu:9000', 'speed:1000', 'op:up'] + """ + iface_path = "/sys/class/net/{}" + props_dict = {"mtu": "-m {}", "speed": "-s {}", "op": "-o {}"} + if type(ifaces) == str: + ifaces = [ifaces] + + d_ifaces = {} + for iface in ifaces: + iface_props = iface.strip().split() + # no ifaces defined; SKIP + if len(iface_props) == 0: + continue + + target = iface_props[0] + try: + matches = match_cidr_to_ifaces(target) + except Exception as e: + # Log likely unintentional errors and set flag for blocked status, + # if appropriate. + if isinstance(e, ValueError) and "has host bits set" in e.args[0]: + hookenv.log( + "Error parsing netlinks: {}".format(e.args[0]), + level=hookenv.ERROR, + ) + set_netlinks_error() + # Treat target as explicit interface name + matches = [target] + + iface_devs = [ + target + for target in matches + if os.path.exists(iface_path.format(target)) + ] + # no ifaces found; SKIP + if not iface_devs: + continue + + # parse extra parameters (properties) + del iface_props[0] + extra_params = "" + for prop in iface_props: + # wrong format (key:value); SKIP + if prop.find(":") < 0: + continue + + # only one ':' expected + kv = prop.split(":") + if len(kv) == 2 and kv[0].lower() in props_dict: + extra_params += " " + extra_params += props_dict[kv[0].lower()].format(kv[1]) + + for iface_dev in iface_devs: + d_ifaces[iface_dev] = "-i {}{}".format(iface_dev, extra_params) + return d_ifaces + + +def match_cidr_to_ifaces(cidr): + """Use CIDR expression to search for matching network adapters. + + Returns a list of adapter names. + """ + import netifaces # Avoid import error before this dependency gets installed + + network = ipaddress.IPv4Network(cidr) + matches = [] + for adapter in netifaces.interfaces(): + ipv4_addr_structs = netifaces.ifaddresses(adapter).get(netifaces.AF_INET, []) + addrs = [ + ipaddress.IPv4Address(addr_struct["addr"]) + for addr_struct in ipv4_addr_structs + ] + if any(addr in network for addr in addrs): + matches.append(adapter) + return matches + + +def has_netlinks_error(): + """Return True in case of netlinks related errors.""" + return NETLINKS_ERROR + + +def set_netlinks_error(): + """Set the flag indicating a netlinks related error.""" + global NETLINKS_ERROR + NETLINKS_ERROR = True diff --git a/nrpe/hooks/nrpe_hooks.py b/nrpe/hooks/nrpe_hooks.py new file mode 100755 index 0000000..b09adc3 --- /dev/null +++ b/nrpe/hooks/nrpe_hooks.py @@ -0,0 +1,6 @@ +#!/usr/bin/python3 +"""Nrpe hooks module.""" + +import services + +services.manage() diff --git a/nrpe/hooks/nrpe_utils.py b/nrpe/hooks/nrpe_utils.py new file mode 100644 index 0000000..bc342b0 --- /dev/null +++ b/nrpe/hooks/nrpe_utils.py @@ -0,0 +1,275 @@ +"""Nrpe utils module.""" +import glob +import os +import shutil +import subprocess + +from charmhelpers import fetch +from charmhelpers.core import hookenv +from charmhelpers.core import host +from charmhelpers.core.services import helpers +from charmhelpers.core.services.base import ( + ManagerCallback, + PortManagerCallback, +) +from charmhelpers.core.templating import render + +import nrpe_helpers + +import yaml + + +def restart_rsync(service_name): + """Restart rsync.""" + host.service_restart("rsync") + + +def restart_nrpe(service_name): + """Restart nrpe.""" + host.service_restart("nagios-nrpe-server") + + +def determine_packages(): + """Return a list of packages this charm needs installed.""" + pkgs = [ + "nagios-nrpe-server", + "nagios-plugins-basic", + "nagios-plugins-standard", + "python3", + "python3-netifaces", + ] + if hookenv.config("export_nagios_definitions"): + pkgs.append("rsync") + if hookenv.config("nagios_master") not in ["None", "", None]: + pkgs.append("rsync") + return pkgs + + +def install_packages(service_name): + """Install packages.""" + fetch.apt_update() + apt_options = [ + # avoid installing rpcbind LP#1873171 + "--no-install-recommends", + # and retain the default option too + "--option=Dpkg::Options::=--force-confold", + ] + fetch.apt_install(determine_packages(), options=apt_options, fatal=True) + + +def remove_host_export_fragments(service_name): + """Remove nagios host config fragment.""" + for fname in glob.glob("/var/lib/nagios/export/host__*"): + os.unlink(fname) + + +def install_charm_files(service_name): + """Install files shipped with charm.""" + # The preinst script of nagios-nrpe-server deb package will add nagios user + # and create this dir as home + # ref: https://git.launchpad.net/ubuntu/+source/nagios-nrpe/tree/debian/nagios-nrpe-server.preinst#n28 # NOQA: E501 + nagios_home = "/var/lib/nagios" + + # it's possible dir owner be changed to root by other process, e.g.: LP1866382 + # here we ensure owner is nagios, but didn't apply it resursively intentionally. + shutil.chown(nagios_home, user="nagios", group="nagios") + + # the `2` in mode will setgid for group, set dir permission to `drwxr-sr-x`. + # the `s` (setgid) will ensure any file created in this dir inherits parent dir + # group `nagios`, regardless of the effective user, such as root. + os.chmod(nagios_home, 0o2755) # 2 will set the s flag for group + + nag_dirs = [ + "/etc/nagios/nrpe.d/", + "/usr/local/lib/nagios/plugins", + "/var/lib/nagios/export/", + ] + for nag_dir in nag_dirs: + if not os.path.exists(nag_dir): + host.mkdir(nag_dir, perms=0o755) + charm_file_dir = os.path.join(hookenv.charm_dir(), "files") + charm_plugin_dir = os.path.join(charm_file_dir, "plugins") + pkg_plugin_dir = "/usr/lib/nagios/plugins/" + local_plugin_dir = "/usr/local/lib/nagios/plugins/" + + shutil.copy2( + os.path.join(charm_file_dir, "nagios_plugin.py"), + pkg_plugin_dir + "/nagios_plugin.py", + ) + shutil.copy2( + os.path.join(charm_file_dir, "nagios_plugin3.py"), + pkg_plugin_dir + "/nagios_plugin3.py", + ) + shutil.copy2(os.path.join(charm_file_dir, "default_rsync"), "/etc/default/rsync") + shutil.copy2(os.path.join(charm_file_dir, "rsyncd.conf"), "/etc/rsyncd.conf") + host.mkdir("/etc/rsync-juju.d", perms=0o755) + host.rsync(charm_plugin_dir, "/usr/local/lib/nagios/", options=["--executability"]) + for nagios_plugin in ("nagios_plugin.py", "nagios_plugin3.py"): + if not os.path.exists(local_plugin_dir + nagios_plugin): + os.symlink(pkg_plugin_dir + nagios_plugin, local_plugin_dir + nagios_plugin) + + +def render_nrpe_check_config(checkctxt): + """Write nrpe check definition.""" + # Only render if we actually have cmd parameters + if checkctxt["cmd_params"]: + render( + "nrpe_command.tmpl", + "/etc/nagios/nrpe.d/{}.cfg".format(checkctxt["cmd_name"]), + checkctxt, + ) + + +def render_nrped_files(service_name): + """Render each of the predefined checks.""" + for checkctxt in nrpe_helpers.SubordinateCheckDefinitions()["checks"]: + # Clean up existing files + for fname in checkctxt["matching_files"]: + try: + os.unlink(fname) + except FileNotFoundError: + # Don't clean up non-existent files + pass + render_nrpe_check_config(checkctxt) + process_local_monitors() + process_user_monitors() + + +def process_user_monitors(): + """Collect the user defined local monitors from config.""" + if hookenv.config("monitors"): + monitors = yaml.safe_load(hookenv.config("monitors")) + else: + return + try: + local_user_checks = monitors["monitors"]["local"].keys() + except KeyError as e: + hookenv.log("no local monitors found in monitors config: {}".format(e)) + return + for checktype in local_user_checks: + for check in monitors["monitors"]["local"][checktype].keys(): + check_def = nrpe_helpers.NRPECheckCtxt( + checktype, monitors["monitors"]["local"][checktype][check], "user" + ) + render_nrpe_check_config(check_def) + + +def process_local_monitors(): + """Get all the monitor dicts and write out and local checks.""" + monitor_dicts = nrpe_helpers.MonitorsRelation().get_monitor_dicts() + for monitor_src in monitor_dicts.keys(): + monitor_dict = monitor_dicts[monitor_src] + if not (monitor_dict and "local" in monitor_dict["monitors"]): + continue + monitors = monitor_dict["monitors"]["local"] + for checktype in monitors: + for check in monitors[checktype]: + render_nrpe_check_config( + nrpe_helpers.NRPECheckCtxt( + checktype, + monitors[checktype][check], + monitor_src, + ) + ) + + +def update_nrpe_external_master_relation(service_name): + """Update nrpe external master relation. + + Send updated nagios_hostname to charms attached + to nrpe_external_master relation. + """ + principal_relation = nrpe_helpers.PrincipalRelation() + for rid in hookenv.relation_ids("nrpe-external-master"): + hookenv.relation_set( + relation_id=rid, relation_settings=principal_relation.provide_data() + ) + + +def update_monitor_relation(service_name): + """Send updated monitor yaml to charms attached to monitor relation.""" + monitor_relation = nrpe_helpers.MonitorsRelation() + for rid in hookenv.relation_ids("monitors"): + hookenv.relation_set( + relation_id=rid, relation_settings=monitor_relation.provide_data() + ) + + +def has_consumer(): + """Check for the monitor relation or external monitor config.""" + return hookenv.config("nagios_master") not in ["None", "", None] or bool( + hookenv.relation_ids("monitors") + ) + + +class TolerantPortManagerCallback(PortManagerCallback): + """Manage unit ports. + + Specialization of the PortManagerCallback. It will open or close + ports as its superclass, but will not raise an error on conflicts + for opening ports + + For context, see: + https://bugs.launchpad.net/juju/+bug/1750079 and + https://github.com/juju/charm-helpers/pull/152 + """ + + def __call__(self, manager, service_name, event_name): + """Open unit ports.""" + service = manager.get_service(service_name) + new_ports = service.get("ports", []) + port_file = os.path.join(hookenv.charm_dir(), ".{}.ports".format(service_name)) + if os.path.exists(port_file): + with open(port_file) as fp: + old_ports = fp.read().split(",") + for old_port in old_ports: + if bool(old_port) and not self.ports_contains(old_port, new_ports): + hookenv.close_port(old_port) + with open(port_file, "w") as fp: + fp.write(",".join(str(port) for port in new_ports)) + for port in new_ports: + # A port is either a number or 'ICMP' + protocol = "TCP" + if str(port).upper() == "ICMP": + protocol = "ICMP" + if event_name == "start": + try: + hookenv.open_port(port, protocol) + except subprocess.CalledProcessError as err: + if err.returncode == 1: + hookenv.log( + "open_port returns: {}, ignoring".format(err), + level=hookenv.INFO, + ) + else: + raise + elif event_name == "stop": + hookenv.close_port(port, protocol) + + +maybe_open_ports = TolerantPortManagerCallback() + + +class ExportManagerCallback(ManagerCallback): + """Defer lookup of nagios_hostname. + + This class exists in order to defer lookup of nagios_hostname() + until the template is ready to be rendered. This should reduce the + incidence of incorrectly-rendered hostnames in /var/lib/nagios/exports. + See charmhelpers.core.services.base.ManagerCallback and + charmhelpers.core.services.helpers.TemplateCallback for more background. + """ + + def __call__(self, manager, service_name, event_name): + """Render export_host.cfg.""" + nag_hostname = nrpe_helpers.PrincipalRelation().nagios_hostname() + target = "/var/lib/nagios/export/host__{}.cfg".format(nag_hostname) + renderer = helpers.render_template( + source="export_host.cfg.tmpl", + target=target, + perms=0o644, + ) + renderer(manager, service_name, event_name) + + +create_host_export_fragment = ExportManagerCallback() diff --git a/nrpe/hooks/services.py b/nrpe/hooks/services.py new file mode 100644 index 0000000..b99014f --- /dev/null +++ b/nrpe/hooks/services.py @@ -0,0 +1,94 @@ +"""Nrpe service definifition.""" + +import os + +from charmhelpers.core import hookenv +from charmhelpers.core.hookenv import status_set +from charmhelpers.core.services import helpers +from charmhelpers.core.services.base import ServiceManager + +import nrpe_helpers + +import nrpe_utils + + +def get_revision(): + """Get charm revision str.""" + revision = "" + if os.path.exists("version"): + with open("version") as f: + line = f.readline().strip() + # We only want the first 8 characters, that's enough to tell + # which version of the charm we're using. + if len(line) > 8: + revision = " (source version/commit {}...)".format(line[:8]) + else: + revision = " (source version/commit {})".format(line) + return revision + + +def manage(): + """Manage nrpe service.""" + status_set("maintenance", "starting") + config = hookenv.config() + manager = ServiceManager( + [ + { + "service": "nrpe-install", + "data_ready": [ + nrpe_utils.install_packages, + nrpe_utils.install_charm_files, + ], + "start": [], + "stop": [], + }, + { + "service": "nrpe-config", + "required_data": [ + config, + nrpe_helpers.MonitorsRelation(), + nrpe_helpers.PrincipalRelation(), + nrpe_helpers.NagiosInfo(), + ], + "data_ready": [ + nrpe_utils.update_nrpe_external_master_relation, + nrpe_utils.update_monitor_relation, + nrpe_utils.create_host_export_fragment, + nrpe_utils.render_nrped_files, + helpers.render_template( + source="nrpe.tmpl", target="/etc/nagios/nrpe.cfg" + ), + ], + "provided_data": [nrpe_helpers.PrincipalRelation()], + "ports": [hookenv.config("server_port"), "ICMP"], + "start": [nrpe_utils.maybe_open_ports, nrpe_utils.restart_nrpe], + "stop": [], + }, + { + "service": "nrpe-rsync", + "required_data": [ + config, + nrpe_helpers.PrincipalRelation(), + nrpe_helpers.RsyncEnabled(), + nrpe_helpers.NagiosInfo(), + ], + "data_ready": [ + nrpe_utils.remove_host_export_fragments, + helpers.render_template( + source="rsync-juju.d.tmpl", + target="/etc/rsync-juju.d/010-nrpe-external-master.conf", + ), + nrpe_utils.create_host_export_fragment, + ], + "start": [nrpe_utils.restart_rsync], + "stop": [], + }, + ] + ) + manager.manage() + if not nrpe_utils.has_consumer(): + status_set("blocked", "Nagios server not configured or related") + elif nrpe_helpers.has_netlinks_error(): + status_set("blocked", "Netlinks parsing encountered failure; see logs") + else: + status_set("active", "Ready{}".format(get_revision())) diff --git a/nrpe/hooks/start b/nrpe/hooks/start new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/start @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/stop b/nrpe/hooks/stop new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/stop @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/hooks/update-status b/nrpe/hooks/update-status new file mode 100755 index 0000000..7cdefd9 --- /dev/null +++ b/nrpe/hooks/update-status @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Nrpe update-status hook""" + +import os +import subprocess +from charmhelpers.core.hookenv import status_set +from services import get_revision + +SERVICE = "nagios-nrpe-server" + + +def update_status(): + """Update Nrpe Juju status.""" + retcode = subprocess.call(["systemctl", "is-active", "--quiet", SERVICE]) + if retcode == 0: + status_set("active", "Ready{}".format(get_revision())) + else: + status_set("blocked", "{} service inactive.".format(SERVICE)) + + +if __name__ == '__main__': + update_status() + diff --git a/nrpe/hooks/upgrade-charm b/nrpe/hooks/upgrade-charm new file mode 120000 index 0000000..f73cfd2 --- /dev/null +++ b/nrpe/hooks/upgrade-charm @@ -0,0 +1 @@ +nrpe_hooks.py \ No newline at end of file diff --git a/nrpe/icon.svg b/nrpe/icon.svg new file mode 100644 index 0000000..56d8416 --- /dev/null +++ b/nrpe/icon.svg @@ -0,0 +1,212 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + diff --git a/nrpe/metadata.yaml b/nrpe/metadata.yaml new file mode 100644 index 0000000..e3b05f5 --- /dev/null +++ b/nrpe/metadata.yaml @@ -0,0 +1,32 @@ +name: nrpe +format: 2 +summary: Nagios Remote Plugin Executor Server +maintainer: LMA Charmers +subordinate: true +description: | + Nagios is a host/service/network monitoring and management system. The + purpose of this addon is to allow you to execute Nagios plugins on a + remote host in as transparent a manner as possible. This program runs + as a background process on the remote host and processes command + execution requests from the check_nrpe plugin on the Nagios host. +tags: + - misc +provides: + nrpe: + interface: nrpe + monitors: + interface: monitors +requires: + nrpe-external-master: + interface: nrpe-external-master + scope: container + general-info: + interface: juju-info + scope: container + local-monitors: + interface: local-monitors + scope: container +series: + - bionic + - focal + - xenial diff --git a/nrpe/mod/charmhelpers/.bzrignore b/nrpe/mod/charmhelpers/.bzrignore new file mode 100644 index 0000000..398f08f --- /dev/null +++ b/nrpe/mod/charmhelpers/.bzrignore @@ -0,0 +1,17 @@ +*.pyc +__pycache__/ +dist/ +build/ +MANIFEST +charmhelpers.egg-info/ +charmhelpers/version.py +.coverage +.env/ +coverage.xml +docs/_build +.idea +.project +.pydevproject +.settings +.venv +.venv3 diff --git a/nrpe/mod/charmhelpers/.github/workflows/build.yml b/nrpe/mod/charmhelpers/.github/workflows/build.yml new file mode 100644 index 0000000..d07309d --- /dev/null +++ b/nrpe/mod/charmhelpers/.github/workflows/build.yml @@ -0,0 +1,46 @@ +name: charm-helpers CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-18.04 + strategy: + matrix: + include: + - python-version: 2.7 + env: pep8,py27 + - python-version: 3.4 + env: pep8,py34 + - python-version: 3.5 + env: pep8,py35 + - python-version: 3.6 + env: pep8,py36 + - python-version: 3.7 + env: pep8,py37 + - python-version: 3.8 + env: pep8,py38 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install juju + run: | + sudo snap install juju --classic + - name: Install packages + run: | + sudo apt -qq update + sudo apt install --yes libapt-pkg-dev # For python-apt wheel build + sudo apt install --yes bzr + - name: Install tox + run: pip install tox + - name: Test + run: tox -c tox.ini -e ${{ matrix.env }} diff --git a/nrpe/mod/charmhelpers/.gitignore b/nrpe/mod/charmhelpers/.gitignore new file mode 100644 index 0000000..9b13d46 --- /dev/null +++ b/nrpe/mod/charmhelpers/.gitignore @@ -0,0 +1,125 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv* +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +*.pyc +__pycache__/ +dist/ +build/ +MANIFEST +charmhelpers.egg-info/ +charmhelpers/version.py +.coverage +.env/ +coverage.xml +docs/_build +.idea +.project +.pydevproject +.settings +.venv +.venv3 +.bzr +.unit-state.db + +AUTHORS +ChangeLog diff --git a/nrpe/mod/charmhelpers/HACKING.md b/nrpe/mod/charmhelpers/HACKING.md new file mode 100644 index 0000000..d9d5996 --- /dev/null +++ b/nrpe/mod/charmhelpers/HACKING.md @@ -0,0 +1,102 @@ +# Hacking on charmhelpers + +## Run testsuite (tox method) + +CAUTION: the charm-helpers library has some unit tests which do unsavory things +such as making real, unmocked calls out to sudo foo, juju binaries, and perhaps +other things. This is not ideal for a number of reasons. One of those reasons +is that it pollutes the test runner (your) system. + +The current recommendation for testing locally is to do so in a fresh Xenial +(16.04) lxc container. 16.04 is selected for consistency with what is available +in the Travis CI test gates. As of this writing, 18.04 is not available there. + +The fresh Xenial lxc system container will need to have the following packages +installed in order to satisfy test runner dependencies: + + sudo apt install git bzr tox libapt-pkg-dev python-dev python3-dev build-essential juju -y + +The tests can be executed as follows: + + tox -e pep8 + tox -e py3 + tox -e py2 + +See also: .travis.yaml for what is happening in the test gate. + +## Run testsuite (legacy Makefile method) + + make test + +Run `make` without arguments for more options. + +## Test it in a charm + +Use following instructions to build a charm that uses your own development branch of +charmhelpers. + +Step 1: Make sure your version of charmhelpers is recognised as the latest version by +by appending `dev0` to the version number in the `VERSION` file. + +Step 2: Create an override file `override-wheelhouse.txt` that points to your own +charmhelpers branch. *The format of this file is the same as pip's +[`requirements.txt`](https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format) +file. + + # Override charmhelpers by the version found in folder + -e /path/to/charmhelpers + # Or point it to a github repo with + -e git+https://github.com//charm-helpers#egg=charmhelpers + +Step 3: Build the charm specifying the override file. *You might need to install the +candidate channel of the charm snap* + + charm build -w wheelhouse-overrides.txt + +Now when you deploy your charm, it will use your own branch of charmhelpers. + +*Note: If you want to verify this or change the charmhelpers code on a built +charm, get the path of the installed charmhelpers by running following command.* + + python3 -c "import charmhelpers; print(charmhelpers.__file__)" + + +# Hacking on Docs + +Install html doc dependencies: + +```bash +sudo apt-get install python-flake8 python-shelltoolbox python-tempita \ +python-nose python-mock python-testtools python-jinja2 python-coverage \ +python-git python-netifaces python-netaddr python-pip zip +``` + +To build the html documentation: + +```bash +make docs +``` + +To browse the html documentation locally: + +```bash +make docs +cd docs/_build/html +python -m SimpleHTTPServer 8765 +# point web browser to http://localhost:8765 +``` + +To build and upload package and doc updates to PyPI: + +```bash +make release +# note: if the package version already exists on PyPI +# this command will upload doc updates only +``` + +# PyPI Package and Docs + +The published package and docs currently live at: + + https://pypi.python.org/pypi/charmhelpers + http://pythonhosted.org/charmhelpers/ diff --git a/nrpe/mod/charmhelpers/LICENSE b/nrpe/mod/charmhelpers/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/nrpe/mod/charmhelpers/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/nrpe/mod/charmhelpers/MANIFEST.in b/nrpe/mod/charmhelpers/MANIFEST.in new file mode 100644 index 0000000..72bbf0a --- /dev/null +++ b/nrpe/mod/charmhelpers/MANIFEST.in @@ -0,0 +1,7 @@ +include *.txt +include Makefile +include VERSION +include MANIFEST.in +include scripts/* +include README.rst +recursive-include debian * diff --git a/nrpe/mod/charmhelpers/Makefile b/nrpe/mod/charmhelpers/Makefile new file mode 100644 index 0000000..8771f31 --- /dev/null +++ b/nrpe/mod/charmhelpers/Makefile @@ -0,0 +1,89 @@ +PROJECT=charmhelpers +PYTHON := /usr/bin/env python +SUITE=unstable +TESTS=tests/ + +all: + @echo "make source - Create source package" + @echo "make sdeb - Create debian source package" + @echo "make deb - Create debian package" + @echo "make clean" + @echo "make userinstall - Install locally" + @echo "make docs - Build html documentation" + @echo "make release - Build and upload package and docs to PyPI" + @echo "make test" + +sdeb: source + scripts/build source + +deb: source + scripts/build + +source: setup.py + scripts/update-revno + python setup.py sdist + +clean: + -python setup.py clean + rm -rf build/ MANIFEST + find . -name '*.pyc' -delete + find . -name '__pycache__' -delete + rm -rf dist/* + rm -rf .venv + rm -rf .venv3 + (which dh_clean && dh_clean) || true + +userinstall: + scripts/update-revno + python setup.py install --user + + +.venv: + dpkg-query -W -f='$${status}' gcc python-dev python-virtualenv 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python-dev python-virtualenv + virtualenv .venv --system-site-packages + .venv/bin/pip install -U pip + .venv/bin/pip install -I -r test-requirements.txt + .venv/bin/pip install bzr + +.venv3: + dpkg-query -W -f='$${status}' gcc python3-dev python-virtualenv python3-apt 2>/dev/null | grep --invert-match "not-installed" || sudo apt-get install -y python3-dev python-virtualenv python3-apt + virtualenv .venv3 --python=python3 --system-site-packages + .venv3/bin/pip install -U pip + .venv3/bin/pip install -I -r test-requirements.txt + +# Note we don't even attempt to run tests if lint isn't passing. +test: lint test2 test3 + @echo OK + +test2: + @echo Starting Py2 tests... + .venv/bin/nosetests -s --nologcapture tests/ + +test3: + @echo Starting Py3 tests... + .venv3/bin/nosetests -s --nologcapture tests/ + +ftest: lint + @echo Starting fast tests... + .venv/bin/nosetests --attr '!slow' --nologcapture tests/ + .venv3/bin/nosetests --attr '!slow' --nologcapture tests/ + +lint: .venv .venv3 + @echo Checking for Python syntax... + @.venv/bin/flake8 --ignore=E402,E501,W504 $(PROJECT) $(TESTS) tools/ \ + && echo Py2 OK + @.venv3/bin/flake8 --ignore=E402,E501,W504 $(PROJECT) $(TESTS) tools/ \ + && echo Py3 OK + +docs: + - [ -z "`dpkg -l | grep python-sphinx`" ] && sudo apt-get install python-sphinx -y + - [ -z "`dpkg -l | grep python-pip`" ] && sudo apt-get install python-pip -y + - [ -z "`pip list | grep -i sphinx-pypi-upload`" ] && sudo pip install sphinx-pypi-upload + - [ -z "`pip list | grep -i sphinx_rtd_theme`" ] && sudo pip install sphinx_rtd_theme + cd docs && make html && cd - +.PHONY: docs + +release: docs + $(PYTHON) setup.py sdist upload upload_sphinx + +build: test lint docs diff --git a/nrpe/mod/charmhelpers/README.rst b/nrpe/mod/charmhelpers/README.rst new file mode 100644 index 0000000..35c168c --- /dev/null +++ b/nrpe/mod/charmhelpers/README.rst @@ -0,0 +1,52 @@ +CharmHelpers |badge| +-------------------- + +.. |badge| image:: https://github.com/juju/charm-helpers/actions/workflows/build.yml/badge.svg?branch=master + :target: https://github.com/juju/charm-helpers/actions/workflows/build.yml + +Overview +======== + +CharmHelpers provides an opinionated set of tools for building Juju charms. + +The full documentation is available online at: https://charm-helpers.readthedocs.io/ + +Common Usage Examples +===================== + +* interaction with charm-specific Juju unit agents via hook tools; +* processing of events and execution of decorated functions based on event names; +* handling of persistent storage between independent charm invocations; +* rendering of configuration file templates; +* modification of system configuration files; +* installation of packages; +* retrieval of machine-specific details; +* implementation of application-specific code reused in similar charms. + +Why Python? +=========== + +* Python is an extremely popular, easy to learn, and powerful language which is also common in automation tools; +* An interpreted language helps with charm portability across different CPU architectures; +* Doesn't require debugging symbols (just use pdb in-place); +* An author or a user is able to make debugging changes without recompiling a charm. + +Dev/Test +======== + +See the HACKING.md file for information about testing and development. + +License +======= + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/nrpe/mod/charmhelpers/bin/README b/nrpe/mod/charmhelpers/bin/README new file mode 100644 index 0000000..df4a504 --- /dev/null +++ b/nrpe/mod/charmhelpers/bin/README @@ -0,0 +1,4 @@ +This directory contains executables for accessing charmhelpers functionality + + +Please see charmhelpers.cli for the recommended way to add scripts. diff --git a/nrpe/mod/charmhelpers/bin/chlp b/nrpe/mod/charmhelpers/bin/chlp new file mode 100755 index 0000000..0c1c38d --- /dev/null +++ b/nrpe/mod/charmhelpers/bin/chlp @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from charmhelpers.cli import cmdline +from charmhelpers.cli.commands import * + + +if __name__ == '__main__': + cmdline.run() diff --git a/nrpe/mod/charmhelpers/bin/contrib/charmsupport/charmsupport b/nrpe/mod/charmhelpers/bin/contrib/charmsupport/charmsupport new file mode 100755 index 0000000..7a28beb --- /dev/null +++ b/nrpe/mod/charmhelpers/bin/contrib/charmsupport/charmsupport @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import argparse +from charmhelpers.contrib.charmsupport import execd + + +def run_execd(args): + execd.execd_run(args.module, args.dir, die_on_error=True) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Perform common charm tasks') + subparsers = parser.add_subparsers(help='Commands') + + execd_parser = subparsers.add_parser('execd', + help='Execute a directory of commands') + execd_parser.add_argument('--module', default='charm-pre-install', + help='module to run (default: charm-pre-install)') + execd_parser.add_argument('--dir', + help="Override the exec.d directory path") + execd_parser.set_defaults(func=run_execd) + + return parser.parse_args() + + +def main(): + arguments = parse_args() + arguments.func(arguments) + +if __name__ == '__main__': + exit(main()) diff --git a/nrpe/mod/charmhelpers/bin/contrib/saltstack/salt-call b/nrpe/mod/charmhelpers/bin/contrib/saltstack/salt-call new file mode 100755 index 0000000..5b8a8f3 --- /dev/null +++ b/nrpe/mod/charmhelpers/bin/contrib/saltstack/salt-call @@ -0,0 +1,11 @@ +#!/usr/bin/env python +''' +Directly call a salt command in the modules, does not require a running salt +minion to run. +''' + +from salt.scripts import salt_call + + +if __name__ == '__main__': + salt_call() diff --git a/nrpe/mod/charmhelpers/charmhelpers/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/__init__.py new file mode 100644 index 0000000..1f57ed2 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/__init__.py @@ -0,0 +1,99 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Bootstrap charm-helpers, installing its dependencies if necessary using +# only standard libraries. +from __future__ import print_function +from __future__ import absolute_import + +import functools +import inspect +import subprocess +import sys + +try: + import six # NOQA:F401 +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) + import six # NOQA:F401 + +try: + import yaml # NOQA:F401 +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + import yaml # NOQA:F401 + + +# Holds a list of mapping of mangled function names that have been deprecated +# using the @deprecate decorator below. This is so that the warning is only +# printed once for each usage of the function. +__deprecated_functions = {} + + +def deprecate(warning, date=None, log=None): + """Add a deprecation warning the first time the function is used. + + The date which is a string in semi-ISO8660 format indicates the year-month + that the function is officially going to be removed. + + usage: + + @deprecate('use core/fetch/add_source() instead', '2017-04') + def contributed_add_source_thing(...): + ... + + And it then prints to the log ONCE that the function is deprecated. + The reason for passing the logging function (log) is so that hookenv.log + can be used for a charm if needed. + + :param warning: String to indicate what is to be used instead. + :param date: Optional string in YYYY-MM format to indicate when the + function will definitely (probably) be removed. + :param log: The log function to call in order to log. If None, logs to + stdout + """ + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + try: + module = inspect.getmodule(f) + file = inspect.getsourcefile(f) + lines = inspect.getsourcelines(f) + f_name = "{}-{}-{}..{}-{}".format( + module.__name__, file, lines[0], lines[-1], f.__name__) + except (IOError, TypeError): + # assume it was local, so just use the name of the function + f_name = f.__name__ + if f_name not in __deprecated_functions: + __deprecated_functions[f_name] = True + s = "DEPRECATION WARNING: Function {} is being removed".format( + f.__name__) + if date: + s = "{} on/around {}".format(s, date) + if warning: + s = "{} : {}".format(s, warning) + if log: + log(s) + else: + print(s) + return f(*args, **kwargs) + return wrapped_f + return wrap diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/README.rst b/nrpe/mod/charmhelpers/charmhelpers/cli/README.rst new file mode 100644 index 0000000..4c211ee --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/README.rst @@ -0,0 +1,57 @@ +========== +Commandant +========== + +----------------------------------------------------- +Automatic command-line interfaces to Python functions +----------------------------------------------------- + +One of the benefits of ``libvirt`` is the uniformity of the interface: the C API (as well as the bindings in other languages) is a set of functions that accept parameters that are nearly identical to the command-line arguments. If you run ``virsh``, you get an interactive command prompt that supports all of the same commands that your shell scripts use as ``virsh`` subcommands. + +Command execution and stdio manipulation is the greatest common factor across all development systems in the POSIX environment. By exposing your functions as commands that manipulate streams of text, you can make life easier for all the Ruby and Erlang and Go programmers in your life. + +Goals +===== + +* Single decorator to expose a function as a command. + * now two decorators - one "automatic" and one that allows authors to manipulate the arguments for fine-grained control.(MW) +* Automatic analysis of function signature through ``inspect.getargspec()`` on python 2 or ``inspect.getfullargspec()`` on python 3 +* Command argument parser built automatically with ``argparse`` +* Interactive interpreter loop object made with ``Cmd`` +* Options to output structured return value data via ``pprint``, ``yaml`` or ``json`` dumps. + +Other Important Features that need writing +------------------------------------------ + +* Help and Usage documentation can be automatically generated, but it will be important to let users override this behaviour +* The decorator should allow specifying further parameters to the parser's add_argument() calls, to specify types or to make arguments behave as boolean flags, etc. + - Filename arguments are important, as good practice is for functions to accept file objects as parameters. + - choices arguments help to limit bad input before the function is called +* Some automatic behaviour could make for better defaults, once the user can override them. + - We could automatically detect arguments that default to False or True, and automatically support --no-foo for foo=True. + - We could automatically support hyphens as alternates for underscores + - Arguments defaulting to sequence types could support the ``append`` action. + + +----------------------------------------------------- +Implementing subcommands +----------------------------------------------------- + +(WIP) + +So as to avoid dependencies on the cli module, subcommands should be defined separately from their implementations. The recommmendation would be to place definitions into separate modules near the implementations which they expose. + +Some examples:: + + from charmhelpers.cli import CommandLine + from charmhelpers.payload import execd + from charmhelpers.foo import bar + + cli = CommandLine() + + cli.subcommand(execd.execd_run) + + @cli.subcommand_builder("bar", help="Bar baz qux") + def barcmd_builder(subparser): + subparser.add_argument('argument1', help="yackety") + return bar diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/cli/__init__.py new file mode 100644 index 0000000..74ea729 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/__init__.py @@ -0,0 +1,196 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import argparse +import sys + +import six +from six.moves import zip + +import charmhelpers.core.unitdata + + +class OutputFormatter(object): + def __init__(self, outfile=sys.stdout): + self.formats = ( + "raw", + "json", + "py", + "yaml", + "csv", + "tab", + ) + self.outfile = outfile + + def add_arguments(self, argument_parser): + formatgroup = argument_parser.add_mutually_exclusive_group() + choices = self.supported_formats + formatgroup.add_argument("--format", metavar='FMT', + help="Select output format for returned data, " + "where FMT is one of: {}".format(choices), + choices=choices, default='raw') + for fmt in self.formats: + fmtfunc = getattr(self, fmt) + formatgroup.add_argument("-{}".format(fmt[0]), + "--{}".format(fmt), action='store_const', + const=fmt, dest='format', + help=fmtfunc.__doc__) + + @property + def supported_formats(self): + return self.formats + + def raw(self, output): + """Output data as raw string (default)""" + if isinstance(output, (list, tuple)): + output = '\n'.join(map(str, output)) + self.outfile.write(str(output)) + + def py(self, output): + """Output data as a nicely-formatted python data structure""" + import pprint + pprint.pprint(output, stream=self.outfile) + + def json(self, output): + """Output data in JSON format""" + import json + json.dump(output, self.outfile) + + def yaml(self, output): + """Output data in YAML format""" + import yaml + yaml.safe_dump(output, self.outfile) + + def csv(self, output): + """Output data as excel-compatible CSV""" + import csv + csvwriter = csv.writer(self.outfile) + csvwriter.writerows(output) + + def tab(self, output): + """Output data in excel-compatible tab-delimited format""" + import csv + csvwriter = csv.writer(self.outfile, dialect=csv.excel_tab) + csvwriter.writerows(output) + + def format_output(self, output, fmt='raw'): + fmtfunc = getattr(self, fmt) + fmtfunc(output) + + +class CommandLine(object): + argument_parser = None + subparsers = None + formatter = None + exit_code = 0 + + def __init__(self): + if not self.argument_parser: + self.argument_parser = argparse.ArgumentParser(description='Perform common charm tasks') + if not self.formatter: + self.formatter = OutputFormatter() + self.formatter.add_arguments(self.argument_parser) + if not self.subparsers: + self.subparsers = self.argument_parser.add_subparsers(help='Commands') + + def subcommand(self, command_name=None): + """ + Decorate a function as a subcommand. Use its arguments as the + command-line arguments""" + def wrapper(decorated): + cmd_name = command_name or decorated.__name__ + subparser = self.subparsers.add_parser(cmd_name, + description=decorated.__doc__) + for args, kwargs in describe_arguments(decorated): + subparser.add_argument(*args, **kwargs) + subparser.set_defaults(func=decorated) + return decorated + return wrapper + + def test_command(self, decorated): + """ + Subcommand is a boolean test function, so bool return values should be + converted to a 0/1 exit code. + """ + decorated._cli_test_command = True + return decorated + + def no_output(self, decorated): + """ + Subcommand is not expected to return a value, so don't print a spurious None. + """ + decorated._cli_no_output = True + return decorated + + def subcommand_builder(self, command_name, description=None): + """ + Decorate a function that builds a subcommand. Builders should accept a + single argument (the subparser instance) and return the function to be + run as the command.""" + def wrapper(decorated): + subparser = self.subparsers.add_parser(command_name) + func = decorated(subparser) + subparser.set_defaults(func=func) + subparser.description = description or func.__doc__ + return wrapper + + def run(self): + "Run cli, processing arguments and executing subcommands." + arguments = self.argument_parser.parse_args() + if six.PY2: + argspec = inspect.getargspec(arguments.func) + else: + argspec = inspect.getfullargspec(arguments.func) + vargs = [] + for arg in argspec.args: + vargs.append(getattr(arguments, arg)) + if argspec.varargs: + vargs.extend(getattr(arguments, argspec.varargs)) + output = arguments.func(*vargs) + if getattr(arguments.func, '_cli_test_command', False): + self.exit_code = 0 if output else 1 + output = '' + if getattr(arguments.func, '_cli_no_output', False): + output = '' + self.formatter.format_output(output, arguments.format) + if charmhelpers.core.unitdata._KV: + charmhelpers.core.unitdata._KV.flush() + + +cmdline = CommandLine() + + +def describe_arguments(func): + """ + Analyze a function's signature and return a data structure suitable for + passing in as arguments to an argparse parser's add_argument() method.""" + + if six.PY2: + argspec = inspect.getargspec(func) + else: + argspec = inspect.getfullargspec(func) + # we should probably raise an exception somewhere if func includes **kwargs + if argspec.defaults: + positional_args = argspec.args[:-len(argspec.defaults)] + keyword_names = argspec.args[-len(argspec.defaults):] + for arg, default in zip(keyword_names, argspec.defaults): + yield ('--{}'.format(arg),), {'default': default} + else: + positional_args = argspec.args + + for arg in positional_args: + yield (arg,), {} + if argspec.varargs: + yield (argspec.varargs,), {'nargs': '*'} diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/benchmark.py b/nrpe/mod/charmhelpers/charmhelpers/cli/benchmark.py new file mode 100644 index 0000000..303af14 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/benchmark.py @@ -0,0 +1,34 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import cmdline +from charmhelpers.contrib.benchmark import Benchmark + + +@cmdline.subcommand(command_name='benchmark-start') +def start(): + Benchmark.start() + + +@cmdline.subcommand(command_name='benchmark-finish') +def finish(): + Benchmark.finish() + + +@cmdline.subcommand_builder('benchmark-composite', description="Set the benchmark composite score") +def service(subparser): + subparser.add_argument("value", help="The composite score.") + subparser.add_argument("units", help="The units the composite score represents, i.e., 'reads/sec'.") + subparser.add_argument("direction", help="'asc' if a lower score is better, 'desc' if a higher score is better.") + return Benchmark.set_composite_score diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/commands.py b/nrpe/mod/charmhelpers/charmhelpers/cli/commands.py new file mode 100644 index 0000000..b931056 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/commands.py @@ -0,0 +1,30 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module loads sub-modules into the python runtime so they can be +discovered via the inspect module. In order to prevent flake8 from (rightfully) +telling us these are unused modules, throw a ' # noqa' at the end of each import +so that the warning is suppressed. +""" + +from . import CommandLine # noqa + +""" +Import the sub-modules which have decorated subcommands to register with chlp. +""" +from . import host # noqa +from . import benchmark # noqa +from . import unitdata # noqa +from . import hookenv # noqa diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/hookenv.py b/nrpe/mod/charmhelpers/charmhelpers/cli/hookenv.py new file mode 100644 index 0000000..bd72f44 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/hookenv.py @@ -0,0 +1,21 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import cmdline +from charmhelpers.core import hookenv + + +cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped) +cmdline.subcommand('service-name')(hookenv.service_name) +cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped) diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/host.py b/nrpe/mod/charmhelpers/charmhelpers/cli/host.py new file mode 100644 index 0000000..4039684 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/host.py @@ -0,0 +1,29 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import cmdline +from charmhelpers.core import host + + +@cmdline.subcommand() +def mounts(): + "List mounts" + return host.mounts() + + +@cmdline.subcommand_builder('service', description="Control system services") +def service(subparser): + subparser.add_argument("action", help="The action to perform (start, stop, etc...)") + subparser.add_argument("service_name", help="Name of the service to control") + return host.service diff --git a/nrpe/mod/charmhelpers/charmhelpers/cli/unitdata.py b/nrpe/mod/charmhelpers/charmhelpers/cli/unitdata.py new file mode 100644 index 0000000..acce846 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/cli/unitdata.py @@ -0,0 +1,46 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import cmdline +from charmhelpers.core import unitdata + + +@cmdline.subcommand_builder('unitdata', description="Store and retrieve data") +def unitdata_cmd(subparser): + nested = subparser.add_subparsers() + + get_cmd = nested.add_parser('get', help='Retrieve data') + get_cmd.add_argument('key', help='Key to retrieve the value of') + get_cmd.set_defaults(action='get', value=None) + + getrange_cmd = nested.add_parser('getrange', help='Retrieve multiple data') + getrange_cmd.add_argument('key', metavar='prefix', + help='Prefix of the keys to retrieve') + getrange_cmd.set_defaults(action='getrange', value=None) + + set_cmd = nested.add_parser('set', help='Store data') + set_cmd.add_argument('key', help='Key to set') + set_cmd.add_argument('value', help='Value to store') + set_cmd.set_defaults(action='set') + + def _unitdata_cmd(action, key, value): + if action == 'get': + return unitdata.kv().get(key) + elif action == 'getrange': + return unitdata.kv().getrange(key) + elif action == 'set': + unitdata.kv().set(key, value) + unitdata.kv().flush() + return '' + return _unitdata_cmd diff --git a/nrpe/mod/charmhelpers/charmhelpers/context.py b/nrpe/mod/charmhelpers/charmhelpers/context.py new file mode 100644 index 0000000..0186474 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/context.py @@ -0,0 +1,205 @@ +# Copyright 2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +A Pythonic API to interact with the charm hook environment. + +:author: Stuart Bishop +''' + +import six + +from charmhelpers.core import hookenv + +from collections import OrderedDict +if six.PY3: + from collections import UserDict # pragma: nocover +else: + from UserDict import IterableUserDict as UserDict # pragma: nocover + + +class Relations(OrderedDict): + '''Mapping relation name -> relation id -> Relation. + + >>> rels = Relations() + >>> rels['sprog']['sprog:12']['client/6']['widget'] + 'remote widget' + >>> rels['sprog']['sprog:12'].local['widget'] = 'local widget' + >>> rels['sprog']['sprog:12'].local['widget'] + 'local widget' + >>> rels.peer.local['widget'] + 'local widget on the peer relation' + ''' + def __init__(self): + super(Relations, self).__init__() + for relname in sorted(hookenv.relation_types()): + self[relname] = OrderedDict() + relids = hookenv.relation_ids(relname) + relids.sort(key=lambda x: int(x.split(':', 1)[-1])) + for relid in relids: + self[relname][relid] = Relation(relid) + + @property + def peer(self): + peer_relid = hookenv.peer_relation_id() + for rels in self.values(): + if peer_relid in rels: + return rels[peer_relid] + + +class Relation(OrderedDict): + '''Mapping of unit -> remote RelationInfo for a relation. + + This is an OrderedDict mapping, ordered numerically by + by unit number. + + Also provides access to the local RelationInfo, and peer RelationInfo + instances by the 'local' and 'peers' attributes. + + >>> r = Relation('sprog:12') + >>> r.keys() + ['client/9', 'client/10'] # Ordered numerically + >>> r['client/10']['widget'] # A remote RelationInfo setting + 'remote widget' + >>> r.local['widget'] # The local RelationInfo setting + 'local widget' + ''' + relid = None # The relation id. + relname = None # The relation name (also known as relation type). + service = None # The remote service name, if known. + local = None # The local end's RelationInfo. + peers = None # Map of peer -> RelationInfo. None if no peer relation. + + def __init__(self, relid): + remote_units = hookenv.related_units(relid) + remote_units.sort(key=lambda u: int(u.split('/', 1)[-1])) + super(Relation, self).__init__((unit, RelationInfo(relid, unit)) + for unit in remote_units) + + self.relname = relid.split(':', 1)[0] + self.relid = relid + self.local = RelationInfo(relid, hookenv.local_unit()) + + for relinfo in self.values(): + self.service = relinfo.service + break + + # If we have peers, and they have joined both the provided peer + # relation and this relation, we can peek at their data too. + # This is useful for creating consensus without leadership. + peer_relid = hookenv.peer_relation_id() + if peer_relid and peer_relid != relid: + peers = hookenv.related_units(peer_relid) + if peers: + peers.sort(key=lambda u: int(u.split('/', 1)[-1])) + self.peers = OrderedDict((peer, RelationInfo(relid, peer)) + for peer in peers) + else: + self.peers = OrderedDict() + else: + self.peers = None + + def __str__(self): + return '{} ({})'.format(self.relid, self.service) + + +class RelationInfo(UserDict): + '''The bag of data at an end of a relation. + + Every unit participating in a relation has a single bag of + data associated with that relation. This is that bag. + + The bag of data for the local unit may be updated. Remote data + is immutable and will remain static for the duration of the hook. + + Changes made to the local units relation data only become visible + to other units after the hook completes successfully. If the hook + does not complete successfully, the changes are rolled back. + + Unlike standard Python mappings, setting an item to None is the + same as deleting it. + + >>> relinfo = RelationInfo('db:12') # Default is the local unit. + >>> relinfo['user'] = 'fred' + >>> relinfo['user'] + 'fred' + >>> relinfo['user'] = None + >>> 'fred' in relinfo + False + + This class wraps hookenv.relation_get and hookenv.relation_set. + All caching is left up to these two methods to avoid synchronization + issues. Data is only loaded on demand. + ''' + relid = None # The relation id. + relname = None # The relation name (also know as the relation type). + unit = None # The unit id. + number = None # The unit number (integer). + service = None # The service name. + + def __init__(self, relid, unit): + self.relname = relid.split(':', 1)[0] + self.relid = relid + self.unit = unit + self.service, num = self.unit.split('/', 1) + self.number = int(num) + + def __str__(self): + return '{} ({})'.format(self.relid, self.unit) + + @property + def data(self): + return hookenv.relation_get(rid=self.relid, unit=self.unit) + + def __setitem__(self, key, value): + if self.unit != hookenv.local_unit(): + raise TypeError('Attempting to set {} on remote unit {}' + ''.format(key, self.unit)) + if value is not None and not isinstance(value, six.string_types): + # We don't do implicit casting. This would cause simple + # types like integers to be read back as strings in subsequent + # hooks, and mutable types would require a lot of wrapping + # to ensure relation-set gets called when they are mutated. + raise ValueError('Only string values allowed') + hookenv.relation_set(self.relid, {key: value}) + + def __delitem__(self, key): + # Deleting a key and setting it to null is the same thing in + # Juju relations. + self[key] = None + + +class Leader(UserDict): + def __init__(self): + pass # Don't call superclass initializer, as it will nuke self.data + + @property + def data(self): + return hookenv.leader_get() + + def __setitem__(self, key, value): + if not hookenv.is_leader(): + raise TypeError('Not the leader. Cannot change leader settings.') + if value is not None and not isinstance(value, six.string_types): + # We don't do implicit casting. This would cause simple + # types like integers to be read back as strings in subsequent + # hooks, and mutable types would require a lot of wrapping + # to ensure leader-set gets called when they are mutated. + raise ValueError('Only string values allowed') + hookenv.leader_set({key: value}) + + def __delitem__(self, key): + # Deleting a key and setting it to null is the same thing in + # Juju leadership settings. + self[key] = None diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/ansible/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/ansible/__init__.py new file mode 100644 index 0000000..8a56972 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/ansible/__init__.py @@ -0,0 +1,306 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +""" +The ansible package enables you to easily use the configuration management +tool `Ansible`_ to setup and configure your charm. All of your charm +configuration options and relation-data are available as regular Ansible +variables which can be used in your playbooks and templates. + +.. _Ansible: https://www.ansible.com/ + +Usage +===== + +Here is an example directory structure for a charm to get you started:: + + charm-ansible-example/ + |-- ansible + | |-- playbook.yaml + | `-- templates + | `-- example.j2 + |-- config.yaml + |-- copyright + |-- icon.svg + |-- layer.yaml + |-- metadata.yaml + |-- reactive + | `-- example.py + |-- README.md + +Running a playbook called ``playbook.yaml`` when the ``install`` hook is run +can be as simple as:: + + from charmhelpers.contrib import ansible + from charms.reactive import hook + + @hook('install') + def install(): + ansible.install_ansible_support() + ansible.apply_playbook('ansible/playbook.yaml') + +Here is an example playbook that uses the ``template`` module to template the +file ``example.j2`` to the charm host and then uses the ``debug`` module to +print out all the host and Juju variables that you can use in your playbooks. +Note that you must target ``localhost`` as the playbook is run locally on the +charm host:: + + --- + - hosts: localhost + tasks: + - name: Template a file + template: + src: templates/example.j2 + dest: /tmp/example.j2 + + - name: Print all variables available to Ansible + debug: + var: vars + +Read more online about `playbooks`_ and standard Ansible `modules`_. + +.. _playbooks: https://docs.ansible.com/ansible/latest/user_guide/playbooks.html +.. _modules: https://docs.ansible.com/ansible/latest/user_guide/modules.html + +A further feature of the Ansible hooks is to provide a light weight "action" +scripting tool. This is a decorator that you apply to a function, and that +function can now receive cli args, and can pass extra args to the playbook:: + + @hooks.action() + def some_action(amount, force="False"): + "Usage: some-action AMOUNT [force=True]" # <-- shown on error + # process the arguments + # do some calls + # return extra-vars to be passed to ansible-playbook + return { + 'amount': int(amount), + 'type': force, + } + +You can now create a symlink to hooks.py that can be invoked like a hook, but +with cli params:: + + # link actions/some-action to hooks/hooks.py + + actions/some-action amount=10 force=true + +Install Ansible via pip +======================= + +If you want to install a specific version of Ansible via pip instead of +``install_ansible_support`` which uses APT, consider using the layer options +of `layer-basic`_ to install Ansible in a virtualenv:: + + options: + basic: + python_packages: ['ansible==2.9.0'] + include_system_packages: true + use_venv: true + +.. _layer-basic: https://charmsreactive.readthedocs.io/en/latest/layer-basic.html#layer-configuration + +""" +import os +import json +import stat +import subprocess +import functools + +import charmhelpers.contrib.templating.contexts +import charmhelpers.core.host +import charmhelpers.core.hookenv +import charmhelpers.fetch + + +charm_dir = os.environ.get('CHARM_DIR', '') +ansible_hosts_path = '/etc/ansible/hosts' +# Ansible will automatically include any vars in the following +# file in its inventory when run locally. +ansible_vars_path = '/etc/ansible/host_vars/localhost' + + +def install_ansible_support(from_ppa=True, ppa_location='ppa:ansible/ansible'): + """Installs Ansible via APT. + + By default this installs Ansible from the `PPA`_ linked from + the Ansible `website`_ or from a PPA set in ``ppa_location``. + + .. _PPA: https://launchpad.net/~ansible/+archive/ubuntu/ansible + .. _website: http://docs.ansible.com/intro_installation.html#latest-releases-via-apt-ubuntu + + If ``from_ppa`` is ``False``, then Ansible will be installed from + Ubuntu's Universe repositories. + """ + if from_ppa: + charmhelpers.fetch.add_source(ppa_location) + charmhelpers.fetch.apt_update(fatal=True) + charmhelpers.fetch.apt_install('ansible') + with open(ansible_hosts_path, 'w+') as hosts_file: + hosts_file.write('localhost ansible_connection=local ansible_remote_tmp=/root/.ansible/tmp') + + +def apply_playbook(playbook, tags=None, extra_vars=None): + """Run a playbook. + + This helper runs a playbook with juju state variables as context, + therefore variables set in application config can be used directly. + List of tags (--tags) and dictionary with extra_vars (--extra-vars) + can be passed as additional parameters. + + Read more about playbook `_variables`_ online. + + .. _variables: https://docs.ansible.com/ansible/latest/user_guide/playbooks_variables.html + + Example:: + + # Run ansible/playbook.yaml with tag install and pass extra + # variables var_a and var_b + apply_playbook( + playbook='ansible/playbook.yaml', + tags=['install'], + extra_vars={'var_a': 'val_a', 'var_b': 'val_b'} + ) + + # Run ansible/playbook.yaml with tag config and extra variable nested, + # which is passed as json and can be used as dictionary in playbook + apply_playbook( + playbook='ansible/playbook.yaml', + tags=['config'], + extra_vars={'nested': {'a': 'value1', 'b': 'value2'}} + ) + + # Custom config file can be passed within extra_vars + apply_playbook( + playbook='ansible/playbook.yaml', + extra_vars="@some_file.json" + ) + + """ + tags = tags or [] + tags = ",".join(tags) + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + ansible_vars_path, namespace_separator='__', + allow_hyphens_in_keys=False, mode=(stat.S_IRUSR | stat.S_IWUSR)) + + # we want ansible's log output to be unbuffered + env = os.environ.copy() + proxy_settings = charmhelpers.core.hookenv.env_proxy_settings() + if proxy_settings: + env.update(proxy_settings) + env['PYTHONUNBUFFERED'] = "1" + call = [ + 'ansible-playbook', + '-c', + 'local', + playbook, + ] + if tags: + call.extend(['--tags', '{}'.format(tags)]) + if extra_vars: + call.extend(['--extra-vars', json.dumps(extra_vars)]) + subprocess.check_call(call, env=env) + + +class AnsibleHooks(charmhelpers.core.hookenv.Hooks): + """Run a playbook with the hook-name as the tag. + + This helper builds on the standard hookenv.Hooks helper, + but additionally runs the playbook with the hook-name specified + using --tags (ie. running all the tasks tagged with the hook-name). + + Example:: + + hooks = AnsibleHooks(playbook_path='ansible/my_machine_state.yaml') + + # All the tasks within my_machine_state.yaml tagged with 'install' + # will be run automatically after do_custom_work() + @hooks.hook() + def install(): + do_custom_work() + + # For most of your hooks, you won't need to do anything other + # than run the tagged tasks for the hook: + @hooks.hook('config-changed', 'start', 'stop') + def just_use_playbook(): + pass + + # As a convenience, you can avoid the above noop function by specifying + # the hooks which are handled by ansible-only and they'll be registered + # for you: + # hooks = AnsibleHooks( + # 'ansible/my_machine_state.yaml', + # default_hooks=['config-changed', 'start', 'stop']) + + if __name__ == "__main__": + # execute a hook based on the name the program is called by + hooks.execute(sys.argv) + """ + + def __init__(self, playbook_path, default_hooks=None): + """Register any hooks handled by ansible.""" + super(AnsibleHooks, self).__init__() + + self._actions = {} + self.playbook_path = playbook_path + + default_hooks = default_hooks or [] + + def noop(*args, **kwargs): + pass + + for hook in default_hooks: + self.register(hook, noop) + + def register_action(self, name, function): + """Register a hook""" + self._actions[name] = function + + def execute(self, args): + """Execute the hook followed by the playbook using the hook as tag.""" + hook_name = os.path.basename(args[0]) + extra_vars = None + if hook_name in self._actions: + extra_vars = self._actions[hook_name](args[1:]) + else: + super(AnsibleHooks, self).execute(args) + + charmhelpers.contrib.ansible.apply_playbook( + self.playbook_path, tags=[hook_name], extra_vars=extra_vars) + + def action(self, *action_names): + """Decorator, registering them as actions""" + def action_wrapper(decorated): + + @functools.wraps(decorated) + def wrapper(argv): + kwargs = dict(arg.split('=') for arg in argv) + try: + return decorated(**kwargs) + except TypeError as e: + if decorated.__doc__: + e.args += (decorated.__doc__,) + raise + + self.register_action(decorated.__name__, wrapper) + if '_' in decorated.__name__: + self.register_action( + decorated.__name__.replace('_', '-'), wrapper) + + return wrapper + + return action_wrapper diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/benchmark/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/benchmark/__init__.py new file mode 100644 index 0000000..c35f7fe --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/benchmark/__init__.py @@ -0,0 +1,124 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import time +import os +from distutils.spawn import find_executable + +from charmhelpers.core.hookenv import ( + in_relation_hook, + relation_ids, + relation_set, + relation_get, +) + + +def action_set(key, val): + if find_executable('action-set'): + action_cmd = ['action-set'] + + if isinstance(val, dict): + for k, v in iter(val.items()): + action_set('%s.%s' % (key, k), v) + return True + + action_cmd.append('%s=%s' % (key, val)) + subprocess.check_call(action_cmd) + return True + return False + + +class Benchmark(): + """ + Helper class for the `benchmark` interface. + + :param list actions: Define the actions that are also benchmarks + + From inside the benchmark-relation-changed hook, you would + Benchmark(['memory', 'cpu', 'disk', 'smoke', 'custom']) + + Examples: + + siege = Benchmark(['siege']) + siege.start() + [... run siege ...] + # The higher the score, the better the benchmark + siege.set_composite_score(16.70, 'trans/sec', 'desc') + siege.finish() + + + """ + + BENCHMARK_CONF = '/etc/benchmark.conf' # Replaced in testing + + required_keys = [ + 'hostname', + 'port', + 'graphite_port', + 'graphite_endpoint', + 'api_port' + ] + + def __init__(self, benchmarks=None): + if in_relation_hook(): + if benchmarks is not None: + for rid in sorted(relation_ids('benchmark')): + relation_set(relation_id=rid, relation_settings={ + 'benchmarks': ",".join(benchmarks) + }) + + # Check the relation data + config = {} + for key in self.required_keys: + val = relation_get(key) + if val is not None: + config[key] = val + else: + # We don't have all of the required keys + config = {} + break + + if len(config): + with open(self.BENCHMARK_CONF, 'w') as f: + for key, val in iter(config.items()): + f.write("%s=%s\n" % (key, val)) + + @staticmethod + def start(): + action_set('meta.start', time.strftime('%Y-%m-%dT%H:%M:%SZ')) + + """ + If the collectd charm is also installed, tell it to send a snapshot + of the current profile data. + """ + COLLECT_PROFILE_DATA = '/usr/local/bin/collect-profile-data' + if os.path.exists(COLLECT_PROFILE_DATA): + subprocess.check_output([COLLECT_PROFILE_DATA]) + + @staticmethod + def finish(): + action_set('meta.stop', time.strftime('%Y-%m-%dT%H:%M:%SZ')) + + @staticmethod + def set_composite_score(value, units, direction='asc'): + """ + Set the composite score for a benchmark run. This is a single number + representative of the benchmark results. This could be the most + important metric, or an amalgamation of metric scores. + """ + return action_set( + "meta.composite", + {'value': value, 'units': units, 'direction': direction} + ) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/IMPORT b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/IMPORT new file mode 100644 index 0000000..d41cb04 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/IMPORT @@ -0,0 +1,4 @@ +Source lp:charm-tools/trunk + +charm-tools/helpers/python/charmhelpers/__init__.py -> charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py +charm-tools/helpers/python/charmhelpers/tests/test_charmhelpers.py -> charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py new file mode 100644 index 0000000..ed63e81 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmhelpers/__init__.py @@ -0,0 +1,203 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +warnings.warn("contrib.charmhelpers is deprecated", DeprecationWarning) # noqa + +import operator +import tempfile +import time +import yaml +import subprocess + +import six +if six.PY3: + from urllib.request import urlopen + from urllib.error import (HTTPError, URLError) +else: + from urllib2 import (urlopen, HTTPError, URLError) + +"""Helper functions for writing Juju charms in Python.""" + +__metaclass__ = type +__all__ = [ + # 'get_config', # core.hookenv.config() + # 'log', # core.hookenv.log() + # 'log_entry', # core.hookenv.log() + # 'log_exit', # core.hookenv.log() + # 'relation_get', # core.hookenv.relation_get() + # 'relation_set', # core.hookenv.relation_set() + # 'relation_ids', # core.hookenv.relation_ids() + # 'relation_list', # core.hookenv.relation_units() + # 'config_get', # core.hookenv.config() + # 'unit_get', # core.hookenv.unit_get() + # 'open_port', # core.hookenv.open_port() + # 'close_port', # core.hookenv.close_port() + # 'service_control', # core.host.service() + 'unit_info', # client-side, NOT IMPLEMENTED + 'wait_for_machine', # client-side, NOT IMPLEMENTED + 'wait_for_page_contents', # client-side, NOT IMPLEMENTED + 'wait_for_relation', # client-side, NOT IMPLEMENTED + 'wait_for_unit', # client-side, NOT IMPLEMENTED +] + + +SLEEP_AMOUNT = 0.1 + + +# We create a juju_status Command here because it makes testing much, +# much easier. +def juju_status(): + subprocess.check_call(['juju', 'status']) + +# re-implemented as charmhelpers.fetch.configure_sources() +# def configure_source(update=False): +# source = config_get('source') +# if ((source.startswith('ppa:') or +# source.startswith('cloud:') or +# source.startswith('http:'))): +# run('add-apt-repository', source) +# if source.startswith("http:"): +# run('apt-key', 'import', config_get('key')) +# if update: +# run('apt-get', 'update') + + +# DEPRECATED: client-side only +def make_charm_config_file(charm_config): + charm_config_file = tempfile.NamedTemporaryFile(mode='w+') + charm_config_file.write(yaml.dump(charm_config)) + charm_config_file.flush() + # The NamedTemporaryFile instance is returned instead of just the name + # because we want to take advantage of garbage collection-triggered + # deletion of the temp file when it goes out of scope in the caller. + return charm_config_file + + +# DEPRECATED: client-side only +def unit_info(service_name, item_name, data=None, unit=None): + if data is None: + data = yaml.safe_load(juju_status()) + service = data['services'].get(service_name) + if service is None: + # XXX 2012-02-08 gmb: + # This allows us to cope with the race condition that we + # have between deploying a service and having it come up in + # `juju status`. We could probably do with cleaning it up so + # that it fails a bit more noisily after a while. + return '' + units = service['units'] + if unit is not None: + item = units[unit][item_name] + else: + # It might seem odd to sort the units here, but we do it to + # ensure that when no unit is specified, the first unit for the + # service (or at least the one with the lowest number) is the + # one whose data gets returned. + sorted_unit_names = sorted(units.keys()) + item = units[sorted_unit_names[0]][item_name] + return item + + +# DEPRECATED: client-side only +def get_machine_data(): + return yaml.safe_load(juju_status())['machines'] + + +# DEPRECATED: client-side only +def wait_for_machine(num_machines=1, timeout=300): + """Wait `timeout` seconds for `num_machines` machines to come up. + + This wait_for... function can be called by other wait_for functions + whose timeouts might be too short in situations where only a bare + Juju setup has been bootstrapped. + + :return: A tuple of (num_machines, time_taken). This is used for + testing. + """ + # You may think this is a hack, and you'd be right. The easiest way + # to tell what environment we're working in (LXC vs EC2) is to check + # the dns-name of the first machine. If it's localhost we're in LXC + # and we can just return here. + if get_machine_data()[0]['dns-name'] == 'localhost': + return 1, 0 + start_time = time.time() + while True: + # Drop the first machine, since it's the Zookeeper and that's + # not a machine that we need to wait for. This will only work + # for EC2 environments, which is why we return early above if + # we're in LXC. + machine_data = get_machine_data() + non_zookeeper_machines = [ + machine_data[key] for key in list(machine_data.keys())[1:]] + if len(non_zookeeper_machines) >= num_machines: + all_machines_running = True + for machine in non_zookeeper_machines: + if machine.get('instance-state') != 'running': + all_machines_running = False + break + if all_machines_running: + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for service to start') + time.sleep(SLEEP_AMOUNT) + return num_machines, time.time() - start_time + + +# DEPRECATED: client-side only +def wait_for_unit(service_name, timeout=480): + """Wait `timeout` seconds for a given service name to come up.""" + wait_for_machine(num_machines=1) + start_time = time.time() + while True: + state = unit_info(service_name, 'agent-state') + if 'error' in state or state == 'started': + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for service to start') + time.sleep(SLEEP_AMOUNT) + if state != 'started': + raise RuntimeError('unit did not start, agent-state: ' + state) + + +# DEPRECATED: client-side only +def wait_for_relation(service_name, relation_name, timeout=120): + """Wait `timeout` seconds for a given relation to come up.""" + start_time = time.time() + while True: + relation = unit_info(service_name, 'relations').get(relation_name) + if relation is not None and relation['state'] == 'up': + break + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for relation to be up') + time.sleep(SLEEP_AMOUNT) + + +# DEPRECATED: client-side only +def wait_for_page_contents(url, contents, timeout=120, validate=None): + if validate is None: + validate = operator.contains + start_time = time.time() + while True: + try: + stream = urlopen(url) + except (HTTPError, URLError): + pass + else: + page = stream.read() + if validate(page, contents): + return page + if time.time() - start_time >= timeout: + raise RuntimeError('timeout waiting for contents of ' + url) + time.sleep(SLEEP_AMOUNT) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/IMPORT b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/IMPORT new file mode 100644 index 0000000..554fddd --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/IMPORT @@ -0,0 +1,14 @@ +Source: lp:charmsupport/trunk + +charmsupport/charmsupport/execd.py -> charm-helpers/charmhelpers/contrib/charmsupport/execd.py +charmsupport/charmsupport/hookenv.py -> charm-helpers/charmhelpers/contrib/charmsupport/hookenv.py +charmsupport/charmsupport/host.py -> charm-helpers/charmhelpers/contrib/charmsupport/host.py +charmsupport/charmsupport/nrpe.py -> charm-helpers/charmhelpers/contrib/charmsupport/nrpe.py +charmsupport/charmsupport/volumes.py -> charm-helpers/charmhelpers/contrib/charmsupport/volumes.py + +charmsupport/tests/test_execd.py -> charm-helpers/tests/contrib/charmsupport/test_execd.py +charmsupport/tests/test_hookenv.py -> charm-helpers/tests/contrib/charmsupport/test_hookenv.py +charmsupport/tests/test_host.py -> charm-helpers/tests/contrib/charmsupport/test_host.py +charmsupport/tests/test_nrpe.py -> charm-helpers/tests/contrib/charmsupport/test_nrpe.py + +charmsupport/bin/charmsupport -> charm-helpers/bin/contrib/charmsupport/charmsupport diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/nrpe.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/nrpe.py new file mode 100644 index 0000000..8d1753c --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/nrpe.py @@ -0,0 +1,522 @@ +# Copyright 2012-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compatibility with the nrpe-external-master charm""" +# +# Authors: +# Matthew Wedgwood + +import glob +import grp +import os +import pwd +import re +import shlex +import shutil +import subprocess +import yaml + +from charmhelpers.core.hookenv import ( + config, + hook_name, + local_unit, + log, + relation_get, + relation_ids, + relation_set, + relations_of_type, +) + +from charmhelpers.core.host import service +from charmhelpers.core import host + +# This module adds compatibility with the nrpe-external-master and plain nrpe +# subordinate charms. To use it in your charm: +# +# 1. Update metadata.yaml +# +# provides: +# (...) +# nrpe-external-master: +# interface: nrpe-external-master +# scope: container +# +# and/or +# +# provides: +# (...) +# local-monitors: +# interface: local-monitors +# scope: container + +# +# 2. Add the following to config.yaml +# +# nagios_context: +# default: "juju" +# type: string +# description: | +# Used by the nrpe subordinate charms. +# A string that will be prepended to instance name to set the host name +# in nagios. So for instance the hostname would be something like: +# juju-myservice-0 +# If you're running multiple environments with the same services in them +# this allows you to differentiate between them. +# nagios_servicegroups: +# default: "" +# type: string +# description: | +# A comma-separated list of nagios servicegroups. +# If left empty, the nagios_context will be used as the servicegroup +# +# 3. Add custom checks (Nagios plugins) to files/nrpe-external-master +# +# 4. Update your hooks.py with something like this: +# +# from charmsupport.nrpe import NRPE +# (...) +# def update_nrpe_config(): +# nrpe_compat = NRPE() +# nrpe_compat.add_check( +# shortname = "myservice", +# description = "Check MyService", +# check_cmd = "check_http -w 2 -c 10 http://localhost" +# ) +# nrpe_compat.add_check( +# "myservice_other", +# "Check for widget failures", +# check_cmd = "/srv/myapp/scripts/widget_check" +# ) +# nrpe_compat.write() +# +# def config_changed(): +# (...) +# update_nrpe_config() +# +# def nrpe_external_master_relation_changed(): +# update_nrpe_config() +# +# def local_monitors_relation_changed(): +# update_nrpe_config() +# +# 4.a If your charm is a subordinate charm set primary=False +# +# from charmsupport.nrpe import NRPE +# (...) +# def update_nrpe_config(): +# nrpe_compat = NRPE(primary=False) +# +# 5. ln -s hooks.py nrpe-external-master-relation-changed +# ln -s hooks.py local-monitors-relation-changed + + +class CheckException(Exception): + pass + + +class Check(object): + shortname_re = '[A-Za-z0-9-_.@]+$' + service_template = (""" +#--------------------------------------------------- +# This file is Juju managed +#--------------------------------------------------- +define service {{ + use active-service + host_name {nagios_hostname} + service_description {nagios_hostname}[{shortname}] """ + """{description} + check_command check_nrpe!{command} + servicegroups {nagios_servicegroup} +{service_config_overrides} +}} +""") + + def __init__(self, shortname, description, check_cmd, max_check_attempts=None): + super(Check, self).__init__() + # XXX: could be better to calculate this from the service name + if not re.match(self.shortname_re, shortname): + raise CheckException("shortname must match {}".format( + Check.shortname_re)) + self.shortname = shortname + self.command = "check_{}".format(shortname) + # Note: a set of invalid characters is defined by the + # Nagios server config + # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()= + self.description = description + self.check_cmd = self._locate_cmd(check_cmd) + self.max_check_attempts = max_check_attempts + + def _get_check_filename(self): + return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) + + def _get_service_filename(self, hostname): + return os.path.join(NRPE.nagios_exportdir, + 'service__{}_{}.cfg'.format(hostname, self.command)) + + def _locate_cmd(self, check_cmd): + search_path = ( + '/usr/lib/nagios/plugins', + '/usr/local/lib/nagios/plugins', + ) + parts = shlex.split(check_cmd) + for path in search_path: + if os.path.exists(os.path.join(path, parts[0])): + command = os.path.join(path, parts[0]) + if len(parts) > 1: + command += " " + " ".join(parts[1:]) + return command + log('Check command not found: {}'.format(parts[0])) + return '' + + def _remove_service_files(self): + if not os.path.exists(NRPE.nagios_exportdir): + return + for f in os.listdir(NRPE.nagios_exportdir): + if f.endswith('_{}.cfg'.format(self.command)): + os.remove(os.path.join(NRPE.nagios_exportdir, f)) + + def remove(self, hostname): + nrpe_check_file = self._get_check_filename() + if os.path.exists(nrpe_check_file): + os.remove(nrpe_check_file) + self._remove_service_files() + + def write(self, nagios_context, hostname, nagios_servicegroups): + nrpe_check_file = self._get_check_filename() + with open(nrpe_check_file, 'w') as nrpe_check_config: + nrpe_check_config.write("# check {}\n".format(self.shortname)) + if nagios_servicegroups: + nrpe_check_config.write( + "# The following header was added automatically by juju\n") + nrpe_check_config.write( + "# Modifying it will affect nagios monitoring and alerting\n") + nrpe_check_config.write( + "# servicegroups: {}\n".format(nagios_servicegroups)) + nrpe_check_config.write("command[{}]={}\n".format( + self.command, self.check_cmd)) + + if not os.path.exists(NRPE.nagios_exportdir): + log('Not writing service config as {} is not accessible'.format( + NRPE.nagios_exportdir)) + else: + self.write_service_config(nagios_context, hostname, + nagios_servicegroups) + + def write_service_config(self, nagios_context, hostname, + nagios_servicegroups): + self._remove_service_files() + + if self.max_check_attempts: + service_config_overrides = ' max_check_attempts {}'.format( + self.max_check_attempts + ) # Note indentation is here rather than in the template to avoid trailing spaces + else: + service_config_overrides = '' # empty string to avoid printing 'None' + templ_vars = { + 'nagios_hostname': hostname, + 'nagios_servicegroup': nagios_servicegroups, + 'description': self.description, + 'shortname': self.shortname, + 'command': self.command, + 'service_config_overrides': service_config_overrides, + } + nrpe_service_text = Check.service_template.format(**templ_vars) + nrpe_service_file = self._get_service_filename(hostname) + with open(nrpe_service_file, 'w') as nrpe_service_config: + nrpe_service_config.write(str(nrpe_service_text)) + + def run(self): + subprocess.call(self.check_cmd) + + +class NRPE(object): + nagios_logdir = '/var/log/nagios' + nagios_exportdir = '/var/lib/nagios/export' + nrpe_confdir = '/etc/nagios/nrpe.d' + homedir = '/var/lib/nagios' # home dir provided by nagios-nrpe-server + + def __init__(self, hostname=None, primary=True): + super(NRPE, self).__init__() + self.config = config() + self.primary = primary + self.nagios_context = self.config['nagios_context'] + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: + self.nagios_servicegroups = self.config['nagios_servicegroups'] + else: + self.nagios_servicegroups = self.nagios_context + self.unit_name = local_unit().replace('/', '-') + if hostname: + self.hostname = hostname + else: + nagios_hostname = get_nagios_hostname() + if nagios_hostname: + self.hostname = nagios_hostname + else: + self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) + self.checks = [] + # Iff in an nrpe-external-master relation hook, set primary status + relation = relation_ids('nrpe-external-master') + if relation: + log("Setting charm primary status {}".format(primary)) + for rid in relation: + relation_set(relation_id=rid, relation_settings={'primary': self.primary}) + self.remove_check_queue = set() + + @classmethod + def does_nrpe_conf_dir_exist(cls): + """Return True if th nrpe_confdif directory exists.""" + return os.path.isdir(cls.nrpe_confdir) + + def add_check(self, *args, **kwargs): + shortname = None + if kwargs.get('shortname') is None: + if len(args) > 0: + shortname = args[0] + else: + shortname = kwargs['shortname'] + + self.checks.append(Check(*args, **kwargs)) + try: + self.remove_check_queue.remove(shortname) + except KeyError: + pass + + def remove_check(self, *args, **kwargs): + if kwargs.get('shortname') is None: + raise ValueError('shortname of check must be specified') + + # Use sensible defaults if they're not specified - these are not + # actually used during removal, but they're required for constructing + # the Check object; check_disk is chosen because it's part of the + # nagios-plugins-basic package. + if kwargs.get('check_cmd') is None: + kwargs['check_cmd'] = 'check_disk' + if kwargs.get('description') is None: + kwargs['description'] = '' + + check = Check(*args, **kwargs) + check.remove(self.hostname) + self.remove_check_queue.add(kwargs['shortname']) + + def write(self): + try: + nagios_uid = pwd.getpwnam('nagios').pw_uid + nagios_gid = grp.getgrnam('nagios').gr_gid + except Exception: + log("Nagios user not set up, nrpe checks not updated") + return + + if not os.path.exists(NRPE.nagios_logdir): + os.mkdir(NRPE.nagios_logdir) + os.chown(NRPE.nagios_logdir, nagios_uid, nagios_gid) + + nrpe_monitors = {} + monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}} + + # check that the charm can write to the conf dir. If not, then nagios + # probably isn't installed, and we can defer. + if not self.does_nrpe_conf_dir_exist(): + return + + for nrpecheck in self.checks: + nrpecheck.write(self.nagios_context, self.hostname, + self.nagios_servicegroups) + nrpe_monitors[nrpecheck.shortname] = { + "command": nrpecheck.command, + } + # If we were passed max_check_attempts, add that to the relation data + if nrpecheck.max_check_attempts is not None: + nrpe_monitors[nrpecheck.shortname]['max_check_attempts'] = nrpecheck.max_check_attempts + + # update-status hooks are configured to firing every 5 minutes by + # default. When nagios-nrpe-server is restarted, the nagios server + # reports checks failing causing unnecessary alerts. Let's not restart + # on update-status hooks. + if not hook_name() == 'update-status': + service('restart', 'nagios-nrpe-server') + + monitor_ids = relation_ids("local-monitors") + \ + relation_ids("nrpe-external-master") + for rid in monitor_ids: + reldata = relation_get(unit=local_unit(), rid=rid) + if 'monitors' in reldata: + # update the existing set of monitors with the new data + old_monitors = yaml.safe_load(reldata['monitors']) + old_nrpe_monitors = old_monitors['monitors']['remote']['nrpe'] + # remove keys that are in the remove_check_queue + old_nrpe_monitors = {k: v for k, v in old_nrpe_monitors.items() + if k not in self.remove_check_queue} + # update/add nrpe_monitors + old_nrpe_monitors.update(nrpe_monitors) + old_monitors['monitors']['remote']['nrpe'] = old_nrpe_monitors + # write back to the relation + relation_set(relation_id=rid, monitors=yaml.dump(old_monitors)) + else: + # write a brand new set of monitors, as no existing ones. + relation_set(relation_id=rid, monitors=yaml.dump(monitors)) + + self.remove_check_queue.clear() + + +def get_nagios_hostcontext(relation_name='nrpe-external-master'): + """ + Query relation with nrpe subordinate, return the nagios_host_context + + :param str relation_name: Name of relation nrpe sub joined to + """ + for rel in relations_of_type(relation_name): + if 'nagios_host_context' in rel: + return rel['nagios_host_context'] + + +def get_nagios_hostname(relation_name='nrpe-external-master'): + """ + Query relation with nrpe subordinate, return the nagios_hostname + + :param str relation_name: Name of relation nrpe sub joined to + """ + for rel in relations_of_type(relation_name): + if 'nagios_hostname' in rel: + return rel['nagios_hostname'] + + +def get_nagios_unit_name(relation_name='nrpe-external-master'): + """ + Return the nagios unit name prepended with host_context if needed + + :param str relation_name: Name of relation nrpe sub joined to + """ + host_context = get_nagios_hostcontext(relation_name) + if host_context: + unit = "%s:%s" % (host_context, local_unit()) + else: + unit = local_unit() + return unit + + +def add_init_service_checks(nrpe, services, unit_name, immediate_check=True): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param list services: List of services to check + :param str unit_name: Unit name to use in check description + :param bool immediate_check: For sysv init, run the service check immediately + """ + for svc in services: + # Don't add a check for these services from neutron-gateway + if svc in ['ext-port', 'os-charm-phy-nic-mtu']: + next + + upstart_init = '/etc/init/%s.conf' % svc + sysv_init = '/etc/init.d/%s' % svc + + if host.init_is_systemd(service_name=svc): + nrpe.add_check( + shortname=svc, + description='process check {%s}' % unit_name, + check_cmd='check_systemd.py %s' % svc + ) + elif os.path.exists(upstart_init): + nrpe.add_check( + shortname=svc, + description='process check {%s}' % unit_name, + check_cmd='check_upstart_job %s' % svc + ) + elif os.path.exists(sysv_init): + cronpath = '/etc/cron.d/nagios-service-check-%s' % svc + checkpath = '%s/service-check-%s.txt' % (nrpe.homedir, svc) + croncmd = ( + '/usr/local/lib/nagios/plugins/check_exit_status.pl ' + '-e -s /etc/init.d/%s status' % svc + ) + cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath) + f = open(cronpath, 'w') + f.write(cron_file) + f.close() + nrpe.add_check( + shortname=svc, + description='service check {%s}' % unit_name, + check_cmd='check_status_file.py -f %s' % checkpath, + ) + # if /var/lib/nagios doesn't exist open(checkpath, 'w') will fail + # (LP: #1670223). + if immediate_check and os.path.isdir(nrpe.homedir): + f = open(checkpath, 'w') + subprocess.call( + croncmd.split(), + stdout=f, + stderr=subprocess.STDOUT + ) + f.close() + os.chmod(checkpath, 0o644) + + +def copy_nrpe_checks(nrpe_files_dir=None): + """ + Copy the nrpe checks into place + + """ + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' + if nrpe_files_dir is None: + # determine if "charmhelpers" is in CHARMDIR or CHARMDIR/hooks + for segment in ['.', 'hooks']: + nrpe_files_dir = os.path.abspath(os.path.join( + os.getenv('CHARM_DIR'), + segment, + 'charmhelpers', + 'contrib', + 'openstack', + 'files')) + if os.path.isdir(nrpe_files_dir): + break + else: + raise RuntimeError("Couldn't find charmhelpers directory") + if not os.path.exists(NAGIOS_PLUGINS): + os.makedirs(NAGIOS_PLUGINS) + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): + if os.path.isfile(fname): + shutil.copy2(fname, + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) + + +def add_haproxy_checks(nrpe, unit_name): + """ + Add checks for each service in list + + :param NRPE nrpe: NRPE object to add check to + :param str unit_name: Unit name to use in check description + """ + nrpe.add_check( + shortname='haproxy_servers', + description='Check HAProxy {%s}' % unit_name, + check_cmd='check_haproxy.sh') + nrpe.add_check( + shortname='haproxy_queue', + description='Check HAProxy queue depth {%s}' % unit_name, + check_cmd='check_haproxy_queue_depth.sh') + + +def remove_deprecated_check(nrpe, deprecated_services): + """ + Remove checks for deprecated services in list + + :param nrpe: NRPE object to remove check from + :type nrpe: NRPE + :param deprecated_services: List of deprecated services that are removed + :type deprecated_services: list + """ + for dep_svc in deprecated_services: + log('Deprecated service: {}'.format(dep_svc)) + nrpe.remove_check(shortname=dep_svc) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/volumes.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/volumes.py new file mode 100644 index 0000000..f7c6fbd --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/charmsupport/volumes.py @@ -0,0 +1,173 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Functions for managing volumes in juju units. One volume is supported per unit. +Subordinates may have their own storage, provided it is on its own partition. + +Configuration stanzas:: + + volume-ephemeral: + type: boolean + default: true + description: > + If false, a volume is mounted as specified in "volume-map" + If true, ephemeral storage will be used, meaning that log data + will only exist as long as the machine. YOU HAVE BEEN WARNED. + volume-map: + type: string + default: {} + description: > + YAML map of units to device names, e.g: + "{ rsyslog/0: /dev/vdb, rsyslog/1: /dev/vdb }" + Service units will raise a configure-error if volume-ephemeral + is 'true' and no volume-map value is set. Use 'juju set' to set a + value and 'juju resolved' to complete configuration. + +Usage:: + + from charmsupport.volumes import configure_volume, VolumeConfigurationError + from charmsupport.hookenv import log, ERROR + def post_mount_hook(): + stop_service('myservice') + def post_mount_hook(): + start_service('myservice') + + if __name__ == '__main__': + try: + configure_volume(before_change=pre_mount_hook, + after_change=post_mount_hook) + except VolumeConfigurationError: + log('Storage could not be configured', ERROR) + +''' + +# XXX: Known limitations +# - fstab is neither consulted nor updated + +import os +from charmhelpers.core import hookenv +from charmhelpers.core import host +import yaml + + +MOUNT_BASE = '/srv/juju/volumes' + + +class VolumeConfigurationError(Exception): + '''Volume configuration data is missing or invalid''' + pass + + +def get_config(): + '''Gather and sanity-check volume configuration data''' + volume_config = {} + config = hookenv.config() + + errors = False + + if config.get('volume-ephemeral') in (True, 'True', 'true', 'Yes', 'yes'): + volume_config['ephemeral'] = True + else: + volume_config['ephemeral'] = False + + try: + volume_map = yaml.safe_load(config.get('volume-map', '{}')) + except yaml.YAMLError as e: + hookenv.log("Error parsing YAML volume-map: {}".format(e), + hookenv.ERROR) + errors = True + if volume_map is None: + # probably an empty string + volume_map = {} + elif not isinstance(volume_map, dict): + hookenv.log("Volume-map should be a dictionary, not {}".format( + type(volume_map))) + errors = True + + volume_config['device'] = volume_map.get(os.environ['JUJU_UNIT_NAME']) + if volume_config['device'] and volume_config['ephemeral']: + # asked for ephemeral storage but also defined a volume ID + hookenv.log('A volume is defined for this unit, but ephemeral ' + 'storage was requested', hookenv.ERROR) + errors = True + elif not volume_config['device'] and not volume_config['ephemeral']: + # asked for permanent storage but did not define volume ID + hookenv.log('Ephemeral storage was requested, but there is no volume ' + 'defined for this unit.', hookenv.ERROR) + errors = True + + unit_mount_name = hookenv.local_unit().replace('/', '-') + volume_config['mountpoint'] = os.path.join(MOUNT_BASE, unit_mount_name) + + if errors: + return None + return volume_config + + +def mount_volume(config): + if os.path.exists(config['mountpoint']): + if not os.path.isdir(config['mountpoint']): + hookenv.log('Not a directory: {}'.format(config['mountpoint'])) + raise VolumeConfigurationError() + else: + host.mkdir(config['mountpoint']) + if os.path.ismount(config['mountpoint']): + unmount_volume(config) + if not host.mount(config['device'], config['mountpoint'], persist=True): + raise VolumeConfigurationError() + + +def unmount_volume(config): + if os.path.ismount(config['mountpoint']): + if not host.umount(config['mountpoint'], persist=True): + raise VolumeConfigurationError() + + +def managed_mounts(): + '''List of all mounted managed volumes''' + return filter(lambda mount: mount[0].startswith(MOUNT_BASE), host.mounts()) + + +def configure_volume(before_change=lambda: None, after_change=lambda: None): + '''Set up storage (or don't) according to the charm's volume configuration. + Returns the mount point or "ephemeral". before_change and after_change + are optional functions to be called if the volume configuration changes. + ''' + + config = get_config() + if not config: + hookenv.log('Failed to read volume configuration', hookenv.CRITICAL) + raise VolumeConfigurationError() + + if config['ephemeral']: + if os.path.ismount(config['mountpoint']): + before_change() + unmount_volume(config) + after_change() + return 'ephemeral' + else: + # persistent storage + if os.path.ismount(config['mountpoint']): + mounts = dict(managed_mounts()) + if mounts.get(config['mountpoint']) != config['device']: + before_change() + unmount_volume(config) + mount_volume(config) + after_change() + else: + before_change() + mount_volume(config) + after_change() + return config['mountpoint'] diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/database/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/database/__init__.py new file mode 100644 index 0000000..64fac9d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/database/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/database/mysql.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/database/mysql.py new file mode 100644 index 0000000..ca79924 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/database/mysql.py @@ -0,0 +1,840 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper for working with a MySQL database""" +import collections +import copy +import json +import re +import sys +import platform +import os +import glob +import six + +# from string import upper + +from charmhelpers.core.host import ( + CompareHostReleases, + lsb_release, + mkdir, + pwgen, + write_file +) +from charmhelpers.core.hookenv import ( + config as config_get, + relation_get, + related_units, + unit_get, + log, + DEBUG, + ERROR, + INFO, + WARNING, + leader_get, + leader_set, + is_leader, +) +from charmhelpers.fetch import ( + apt_install, + apt_update, + filter_installed_packages, +) +from charmhelpers.contrib.network.ip import get_host_ip + +try: + import MySQLdb +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install(filter_installed_packages(['python-mysqldb']), fatal=True) + else: + apt_install(filter_installed_packages(['python3-mysqldb']), fatal=True) + import MySQLdb + + +class MySQLSetPasswordError(Exception): + pass + + +class MySQLHelper(object): + + def __init__(self, rpasswdf_template, upasswdf_template, host='localhost', + migrate_passwd_to_leader_storage=True, + delete_ondisk_passwd_file=True, user="root", password=None, + port=None, connect_timeout=None): + self.user = user + self.host = host + self.password = password + self.port = port + # default timeout of 30 seconds. + self.connect_timeout = connect_timeout or 30 + + # Password file path templates + self.root_passwd_file_template = rpasswdf_template + self.user_passwd_file_template = upasswdf_template + + self.migrate_passwd_to_leader_storage = migrate_passwd_to_leader_storage + # If we migrate we have the option to delete local copy of root passwd + self.delete_ondisk_passwd_file = delete_ondisk_passwd_file + self.connection = None + + def connect(self, user='root', password=None, host=None, port=None, + connect_timeout=None): + _connection_info = { + "user": user or self.user, + "passwd": password or self.password, + "host": host or self.host + } + # set the connection timeout; for mysql8 it can hang forever, so some + # timeout is required. + timeout = connect_timeout or self.connect_timeout + if timeout: + _connection_info["connect_timeout"] = timeout + # port cannot be None but we also do not want to specify it unless it + # has been explicit set. + port = port or self.port + if port is not None: + _connection_info["port"] = port + + log("Opening db connection for %s@%s" % (user, host), level=DEBUG) + try: + self.connection = MySQLdb.connect(**_connection_info) + except Exception as e: + log("Failed to connect to database due to '{}'".format(str(e)), + level=ERROR) + raise + + def database_exists(self, db_name): + cursor = self.connection.cursor() + try: + cursor.execute("SHOW DATABASES") + databases = [i[0] for i in cursor.fetchall()] + finally: + cursor.close() + + return db_name in databases + + def create_database(self, db_name): + cursor = self.connection.cursor() + try: + cursor.execute("CREATE DATABASE `{}` CHARACTER SET UTF8" + .format(db_name)) + finally: + cursor.close() + + def grant_exists(self, db_name, db_user, remote_ip): + cursor = self.connection.cursor() + priv_string = "GRANT ALL PRIVILEGES ON `{}`.* " \ + "TO '{}'@'{}'".format(db_name, db_user, remote_ip) + try: + cursor.execute("SHOW GRANTS for '{}'@'{}'".format(db_user, + remote_ip)) + grants = [i[0] for i in cursor.fetchall()] + except MySQLdb.OperationalError: + return False + finally: + cursor.close() + + # TODO: review for different grants + return priv_string in grants + + def create_grant(self, db_name, db_user, remote_ip, password): + cursor = self.connection.cursor() + try: + # TODO: review for different grants + cursor.execute("GRANT ALL PRIVILEGES ON `{}`.* TO '{}'@'{}' " + "IDENTIFIED BY '{}'".format(db_name, + db_user, + remote_ip, + password)) + finally: + cursor.close() + + def create_admin_grant(self, db_user, remote_ip, password): + cursor = self.connection.cursor() + try: + cursor.execute("GRANT ALL PRIVILEGES ON *.* TO '{}'@'{}' " + "IDENTIFIED BY '{}'".format(db_user, + remote_ip, + password)) + finally: + cursor.close() + + def cleanup_grant(self, db_user, remote_ip): + cursor = self.connection.cursor() + try: + cursor.execute("DROP FROM mysql.user WHERE user='{}' " + "AND HOST='{}'".format(db_user, + remote_ip)) + finally: + cursor.close() + + def flush_priviledges(self): + cursor = self.connection.cursor() + try: + cursor.execute("FLUSH PRIVILEGES") + finally: + cursor.close() + + def execute(self, sql): + """Execute arbitrary SQL against the database.""" + cursor = self.connection.cursor() + try: + cursor.execute(sql) + finally: + cursor.close() + + def select(self, sql): + """ + Execute arbitrary SQL select query against the database + and return the results. + + :param sql: SQL select query to execute + :type sql: string + :returns: SQL select query result + :rtype: list of lists + :raises: MySQLdb.Error + """ + cursor = self.connection.cursor() + try: + cursor.execute(sql) + results = [list(i) for i in cursor.fetchall()] + finally: + cursor.close() + return results + + def migrate_passwords_to_leader_storage(self, excludes=None): + """Migrate any passwords storage on disk to leader storage.""" + if not is_leader(): + log("Skipping password migration as not the lead unit", + level=DEBUG) + return + dirname = os.path.dirname(self.root_passwd_file_template) + path = os.path.join(dirname, '*.passwd') + for f in glob.glob(path): + if excludes and f in excludes: + log("Excluding %s from leader storage migration" % (f), + level=DEBUG) + continue + + key = os.path.basename(f) + with open(f, 'r') as passwd: + _value = passwd.read().strip() + + try: + leader_set(settings={key: _value}) + + if self.delete_ondisk_passwd_file: + os.unlink(f) + except ValueError: + # NOTE cluster relation not yet ready - skip for now + pass + + def get_mysql_password_on_disk(self, username=None, password=None): + """Retrieve, generate or store a mysql password for the provided + username on disk.""" + if username: + template = self.user_passwd_file_template + passwd_file = template.format(username) + else: + passwd_file = self.root_passwd_file_template + + _password = None + if os.path.exists(passwd_file): + log("Using existing password file '%s'" % passwd_file, level=DEBUG) + with open(passwd_file, 'r') as passwd: + _password = passwd.read().strip() + else: + log("Generating new password file '%s'" % passwd_file, level=DEBUG) + if not os.path.isdir(os.path.dirname(passwd_file)): + # NOTE: need to ensure this is not mysql root dir (which needs + # to be mysql readable) + mkdir(os.path.dirname(passwd_file), owner='root', group='root', + perms=0o770) + # Force permissions - for some reason the chmod in makedirs + # fails + os.chmod(os.path.dirname(passwd_file), 0o770) + + _password = password or pwgen(length=32) + write_file(passwd_file, _password, owner='root', group='root', + perms=0o660) + + return _password + + def passwd_keys(self, username): + """Generator to return keys used to store passwords in peer store. + + NOTE: we support both legacy and new format to support mysql + charm prior to refactor. This is necessary to avoid LP 1451890. + """ + keys = [] + if username == 'mysql': + log("Bad username '%s'" % (username), level=WARNING) + + if username: + # IMPORTANT: *newer* format must be returned first + keys.append('mysql-%s.passwd' % (username)) + keys.append('%s.passwd' % (username)) + else: + keys.append('mysql.passwd') + + for key in keys: + yield key + + def get_mysql_password(self, username=None, password=None): + """Retrieve, generate or store a mysql password for the provided + username using peer relation cluster.""" + excludes = [] + + # First check peer relation. + try: + for key in self.passwd_keys(username): + _password = leader_get(key) + if _password: + break + + # If root password available don't update peer relation from local + if _password and not username: + excludes.append(self.root_passwd_file_template) + + except ValueError: + # cluster relation is not yet started; use on-disk + _password = None + + # If none available, generate new one + if not _password: + _password = self.get_mysql_password_on_disk(username, password) + + # Put on wire if required + if self.migrate_passwd_to_leader_storage: + self.migrate_passwords_to_leader_storage(excludes=excludes) + + return _password + + def get_mysql_root_password(self, password=None): + """Retrieve or generate mysql root password for service units.""" + return self.get_mysql_password(username=None, password=password) + + def set_mysql_password(self, username, password, current_password=None): + """Update a mysql password for the provided username changing the + leader settings + + To update root's password pass `None` in the username + + :param username: Username to change password of + :type username: str + :param password: New password for user. + :type password: str + :param current_password: Existing password for user. + :type current_password: str + """ + + if username is None: + username = 'root' + + # get root password via leader-get, it may be that in the past (when + # changes to root-password were not supported) the user changed the + # password, so leader-get is more reliable source than + # config.previous('root-password'). + rel_username = None if username == 'root' else username + if not current_password: + current_password = self.get_mysql_password(rel_username) + + # password that needs to be set + new_passwd = password + + # update password for all users (e.g. root@localhost, root@::1, etc) + try: + self.connect(user=username, password=current_password) + cursor = self.connection.cursor() + except MySQLdb.OperationalError as ex: + raise MySQLSetPasswordError(('Cannot connect using password in ' + 'leader settings (%s)') % ex, ex) + + try: + # NOTE(freyes): Due to skip-name-resolve root@$HOSTNAME account + # fails when using SET PASSWORD so using UPDATE against the + # mysql.user table is needed, but changes to this table are not + # replicated across the cluster, so this update needs to run in + # all the nodes. More info at + # http://galeracluster.com/documentation-webpages/userchanges.html + release = CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) + if release < 'bionic': + SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET password = " + "PASSWORD( %s ) WHERE user = %s;") + else: + # PXC 5.7 (introduced in Bionic) uses authentication_string + SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET " + "authentication_string = " + "PASSWORD( %s ) WHERE user = %s;") + cursor.execute(SQL_UPDATE_PASSWD, (new_passwd, username)) + cursor.execute('FLUSH PRIVILEGES;') + self.connection.commit() + except MySQLdb.OperationalError as ex: + raise MySQLSetPasswordError('Cannot update password: %s' % str(ex), + ex) + finally: + cursor.close() + + # check the password was changed + try: + self.connect(user=username, password=new_passwd) + self.execute('select 1;') + except MySQLdb.OperationalError as ex: + raise MySQLSetPasswordError(('Cannot connect using new password: ' + '%s') % str(ex), ex) + + if not is_leader(): + log('Only the leader can set a new password in the relation', + level=DEBUG) + return + + for key in self.passwd_keys(rel_username): + _password = leader_get(key) + if _password: + log('Updating password for %s (%s)' % (key, rel_username), + level=DEBUG) + leader_set(settings={key: new_passwd}) + + def set_mysql_root_password(self, password, current_password=None): + """Update mysql root password changing the leader settings + + :param password: New password for user. + :type password: str + :param current_password: Existing password for user. + :type current_password: str + """ + self.set_mysql_password( + 'root', + password, + current_password=current_password) + + def normalize_address(self, hostname): + """Ensure that address returned is an IP address (i.e. not fqdn)""" + if config_get('prefer-ipv6'): + # TODO: add support for ipv6 dns + return hostname + + if hostname != unit_get('private-address'): + return get_host_ip(hostname, fallback=hostname) + + # Otherwise assume localhost + return '127.0.0.1' + + def get_allowed_units(self, database, username, relation_id=None, prefix=None): + """Get list of units with access grants for database with username. + + This is typically used to provide shared-db relations with a list of + which units have been granted access to the given database. + """ + if not self.connection: + self.connect(password=self.get_mysql_root_password()) + allowed_units = set() + if not prefix: + prefix = database + for unit in related_units(relation_id): + settings = relation_get(rid=relation_id, unit=unit) + # First check for setting with prefix, then without + for attr in ["%s_hostname" % (prefix), 'hostname']: + hosts = settings.get(attr, None) + if hosts: + break + + if hosts: + # hostname can be json-encoded list of hostnames + try: + hosts = json.loads(hosts) + except ValueError: + hosts = [hosts] + else: + hosts = [settings['private-address']] + + if hosts: + for host in hosts: + host = self.normalize_address(host) + if self.grant_exists(database, username, host): + log("Grant exists for host '%s' on db '%s'" % + (host, database), level=DEBUG) + if unit not in allowed_units: + allowed_units.add(unit) + else: + log("Grant does NOT exist for host '%s' on db '%s'" % + (host, database), level=DEBUG) + else: + log("No hosts found for grant check", level=INFO) + + return allowed_units + + def configure_db(self, hostname, database, username, admin=False): + """Configure access to database for username from hostname.""" + if not self.connection: + self.connect(password=self.get_mysql_root_password()) + if not self.database_exists(database): + self.create_database(database) + + remote_ip = self.normalize_address(hostname) + password = self.get_mysql_password(username) + if not self.grant_exists(database, username, remote_ip): + if not admin: + self.create_grant(database, username, remote_ip, password) + else: + self.create_admin_grant(username, remote_ip, password) + self.flush_priviledges() + + return password + + +# `_singleton_config_helper` stores the instance of the helper class that is +# being used during a hook invocation. +_singleton_config_helper = None + + +def get_mysql_config_helper(): + global _singleton_config_helper + if _singleton_config_helper is None: + _singleton_config_helper = MySQLConfigHelper() + return _singleton_config_helper + + +class MySQLConfigHelper(object): + """Base configuration helper for MySQL.""" + + # Going for the biggest page size to avoid wasted bytes. + # InnoDB page size is 16MB + + DEFAULT_PAGE_SIZE = 16 * 1024 * 1024 + DEFAULT_INNODB_BUFFER_FACTOR = 0.50 + DEFAULT_INNODB_BUFFER_SIZE_MAX = 512 * 1024 * 1024 + + # Validation and lookups for InnoDB configuration + INNODB_VALID_BUFFERING_VALUES = [ + 'none', + 'inserts', + 'deletes', + 'changes', + 'purges', + 'all' + ] + INNODB_FLUSH_CONFIG_VALUES = { + 'fast': 2, + 'safest': 1, + 'unsafe': 0, + } + + def human_to_bytes(self, human): + """Convert human readable configuration options to bytes.""" + num_re = re.compile('^[0-9]+$') + if num_re.match(human): + return human + + factors = { + 'K': 1024, + 'M': 1048576, + 'G': 1073741824, + 'T': 1099511627776 + } + modifier = human[-1] + if modifier in factors: + return int(human[:-1]) * factors[modifier] + + if modifier == '%': + total_ram = self.human_to_bytes(self.get_mem_total()) + if self.is_32bit_system() and total_ram > self.sys_mem_limit(): + total_ram = self.sys_mem_limit() + factor = int(human[:-1]) * 0.01 + pctram = total_ram * factor + return int(pctram - (pctram % self.DEFAULT_PAGE_SIZE)) + + raise ValueError("Can only convert K,M,G, or T") + + def is_32bit_system(self): + """Determine whether system is 32 or 64 bit.""" + try: + return sys.maxsize < 2 ** 32 + except OverflowError: + return False + + def sys_mem_limit(self): + """Determine the default memory limit for the current service unit.""" + if platform.machine() in ['armv7l']: + _mem_limit = self.human_to_bytes('2700M') # experimentally determined + else: + # Limit for x86 based 32bit systems + _mem_limit = self.human_to_bytes('4G') + + return _mem_limit + + def get_mem_total(self): + """Calculate the total memory in the current service unit.""" + with open('/proc/meminfo') as meminfo_file: + for line in meminfo_file: + key, mem = line.split(':', 2) + if key == 'MemTotal': + mtot, modifier = mem.strip().split(' ') + return '%s%s' % (mtot, modifier[0].upper()) + + def get_innodb_flush_log_at_trx_commit(self): + """Get value for innodb_flush_log_at_trx_commit. + + Use the innodb-flush-log-at-trx-commit or the tunning-level setting + translated by INNODB_FLUSH_CONFIG_VALUES to get the + innodb_flush_log_at_trx_commit value. + + :returns: Numeric value for innodb_flush_log_at_trx_commit + :rtype: Union[None, int] + """ + _iflatc = config_get('innodb-flush-log-at-trx-commit') + _tuning_level = config_get('tuning-level') + if _iflatc: + return _iflatc + elif _tuning_level: + return self.INNODB_FLUSH_CONFIG_VALUES.get(_tuning_level, 1) + + def get_innodb_change_buffering(self): + """Get value for innodb_change_buffering. + + Use the innodb-change-buffering validated against + INNODB_VALID_BUFFERING_VALUES to get the innodb_change_buffering value. + + :returns: String value for innodb_change_buffering. + :rtype: Union[None, str] + """ + _icb = config_get('innodb-change-buffering') + if _icb and _icb in self.INNODB_VALID_BUFFERING_VALUES: + return _icb + + def get_innodb_buffer_pool_size(self): + """Get value for innodb_buffer_pool_size. + + Return the number value of innodb-buffer-pool-size or dataset-size. If + neither is set, calculate a sane default based on total memory. + + :returns: Numeric value for innodb_buffer_pool_size. + :rtype: int + """ + total_memory = self.human_to_bytes(self.get_mem_total()) + + dataset_bytes = config_get('dataset-size') + innodb_buffer_pool_size = config_get('innodb-buffer-pool-size') + + if innodb_buffer_pool_size: + innodb_buffer_pool_size = self.human_to_bytes( + innodb_buffer_pool_size) + elif dataset_bytes: + log("Option 'dataset-size' has been deprecated, please use" + "innodb_buffer_pool_size option instead", level="WARN") + innodb_buffer_pool_size = self.human_to_bytes( + dataset_bytes) + else: + # NOTE(jamespage): pick the smallest of 50% of RAM or 512MB + # to ensure that deployments in containers + # without constraints don't try to consume + # silly amounts of memory. + innodb_buffer_pool_size = min( + int(total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR), + self.DEFAULT_INNODB_BUFFER_SIZE_MAX + ) + + if innodb_buffer_pool_size > total_memory: + log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format( + innodb_buffer_pool_size, + total_memory), level='WARN') + + return innodb_buffer_pool_size + + +class PerconaClusterHelper(MySQLConfigHelper): + """Percona-cluster specific configuration helper.""" + + def parse_config(self): + """Parse charm configuration and calculate values for config files.""" + config = config_get() + mysql_config = {} + if 'max-connections' in config: + mysql_config['max_connections'] = config['max-connections'] + + if 'wait-timeout' in config: + mysql_config['wait_timeout'] = config['wait-timeout'] + + if self.get_innodb_flush_log_at_trx_commit() is not None: + mysql_config['innodb_flush_log_at_trx_commit'] = \ + self.get_innodb_flush_log_at_trx_commit() + + if self.get_innodb_change_buffering() is not None: + mysql_config['innodb_change_buffering'] = config['innodb-change-buffering'] + + if 'innodb-io-capacity' in config: + mysql_config['innodb_io_capacity'] = config['innodb-io-capacity'] + + # Set a sane default key_buffer size + mysql_config['key_buffer'] = self.human_to_bytes('32M') + mysql_config['innodb_buffer_pool_size'] = self.get_innodb_buffer_pool_size() + return mysql_config + + +class MySQL8Helper(MySQLHelper): + + def grant_exists(self, db_name, db_user, remote_ip): + cursor = self.connection.cursor() + priv_string = ("GRANT ALL PRIVILEGES ON {}.* " + "TO {}@{}".format(db_name, db_user, remote_ip)) + try: + cursor.execute("SHOW GRANTS FOR '{}'@'{}'".format(db_user, + remote_ip)) + grants = [i[0] for i in cursor.fetchall()] + except MySQLdb.OperationalError: + return False + finally: + cursor.close() + + # Different versions of MySQL use ' or `. Ignore these in the check. + return priv_string in [ + i.replace("'", "").replace("`", "") for i in grants] + + def create_grant(self, db_name, db_user, remote_ip, password): + if self.grant_exists(db_name, db_user, remote_ip): + return + + # Make sure the user exists + # MySQL8 must create the user before the grant + self.create_user(db_user, remote_ip, password) + + cursor = self.connection.cursor() + try: + cursor.execute("GRANT ALL PRIVILEGES ON `{}`.* TO '{}'@'{}'" + .format(db_name, db_user, remote_ip)) + finally: + cursor.close() + + def create_user(self, db_user, remote_ip, password): + + SQL_USER_CREATE = ( + "CREATE USER '{db_user}'@'{remote_ip}' " + "IDENTIFIED BY '{password}'") + + cursor = self.connection.cursor() + try: + cursor.execute(SQL_USER_CREATE.format( + db_user=db_user, + remote_ip=remote_ip, + password=password) + ) + except MySQLdb._exceptions.OperationalError: + log("DB user {} already exists.".format(db_user), + "WARNING") + finally: + cursor.close() + + def create_router_grant(self, db_user, remote_ip, password): + + # Make sure the user exists + # MySQL8 must create the user before the grant + self.create_user(db_user, remote_ip, password) + + # Mysql-Router specific grants + cursor = self.connection.cursor() + try: + cursor.execute("GRANT CREATE USER ON *.* TO '{}'@'{}' WITH GRANT " + "OPTION".format(db_user, remote_ip)) + cursor.execute("GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON " + "mysql_innodb_cluster_metadata.* TO '{}'@'{}'" + .format(db_user, remote_ip)) + cursor.execute("GRANT SELECT ON mysql.user TO '{}'@'{}'" + .format(db_user, remote_ip)) + cursor.execute("GRANT SELECT ON " + "performance_schema.replication_group_members " + "TO '{}'@'{}'".format(db_user, remote_ip)) + cursor.execute("GRANT SELECT ON " + "performance_schema.replication_group_member_stats " + "TO '{}'@'{}'".format(db_user, remote_ip)) + cursor.execute("GRANT SELECT ON " + "performance_schema.global_variables " + "TO '{}'@'{}'".format(db_user, remote_ip)) + finally: + cursor.close() + + def configure_router(self, hostname, username): + + if self.connection is None: + self.connect(password=self.get_mysql_root_password()) + + remote_ip = self.normalize_address(hostname) + password = self.get_mysql_password(username) + self.create_user(username, remote_ip, password) + self.create_router_grant(username, remote_ip, password) + + return password + + +def get_prefix(requested, keys=None): + """Return existing prefix or None. + + :param requested: Request string. i.e. novacell0_username + :type requested: str + :param keys: Keys to determine prefix. Defaults set in function. + :type keys: List of str keys + :returns: String prefix i.e. novacell0 + :rtype: Union[None, str] + """ + if keys is None: + # Shared-DB default keys + keys = ["_database", "_username", "_hostname"] + for key in keys: + if requested.endswith(key): + return requested[:-len(key)] + + +def get_db_data(relation_data, unprefixed): + """Organize database requests into a collections.OrderedDict + + :param relation_data: shared-db relation data + :type relation_data: dict + :param unprefixed: Prefix to use for requests without a prefix. This should + be unique for each side of the relation to avoid + conflicts. + :type unprefixed: str + :returns: Order dict of databases and users + :rtype: collections.OrderedDict + """ + # Deep copy to avoid unintentionally changing relation data + settings = copy.deepcopy(relation_data) + databases = collections.OrderedDict() + + # Clear non-db related elements + if "egress-subnets" in settings.keys(): + settings.pop("egress-subnets") + if "ingress-address" in settings.keys(): + settings.pop("ingress-address") + if "private-address" in settings.keys(): + settings.pop("private-address") + + singleset = {"database", "username", "hostname"} + if singleset.issubset(settings): + settings["{}_{}".format(unprefixed, "hostname")] = ( + settings["hostname"]) + settings.pop("hostname") + settings["{}_{}".format(unprefixed, "database")] = ( + settings["database"]) + settings.pop("database") + settings["{}_{}".format(unprefixed, "username")] = ( + settings["username"]) + settings.pop("username") + + for k, v in settings.items(): + db = k.split("_")[0] + x = "_".join(k.split("_")[1:]) + if db not in databases: + databases[db] = collections.OrderedDict() + databases[db][x] = v + + return databases diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/apache.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/apache.py new file mode 100644 index 0000000..a54702b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/apache.py @@ -0,0 +1,90 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copyright 2012 Canonical Ltd. +# +# This file is sourced from lp:openstack-charm-helpers +# +# Authors: +# James Page +# Adam Gandelman +# + +import os + +from charmhelpers.core import host +from charmhelpers.core.hookenv import ( + config as config_get, + relation_get, + relation_ids, + related_units as relation_list, + log, + INFO, +) + +# This file contains the CA cert from the charms ssl_ca configuration +# option, in future the file name should be updated reflect that. +CONFIG_CA_CERT_FILE = 'keystone_juju_ca_cert' + + +def get_cert(cn=None): + # TODO: deal with multiple https endpoints via charm config + cert = config_get('ssl_cert') + key = config_get('ssl_key') + if not (cert and key): + log("Inspecting identity-service relations for SSL certificate.", + level=INFO) + cert = key = None + if cn: + ssl_cert_attr = 'ssl_cert_{}'.format(cn) + ssl_key_attr = 'ssl_key_{}'.format(cn) + else: + ssl_cert_attr = 'ssl_cert' + ssl_key_attr = 'ssl_key' + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + if not cert: + cert = relation_get(ssl_cert_attr, + rid=r_id, unit=unit) + if not key: + key = relation_get(ssl_key_attr, + rid=r_id, unit=unit) + return (cert, key) + + +def get_ca_cert(): + ca_cert = config_get('ssl_ca') + if ca_cert is None: + log("Inspecting identity-service relations for CA SSL certificate.", + level=INFO) + for r_id in (relation_ids('identity-service') + + relation_ids('identity-credentials')): + for unit in relation_list(r_id): + if ca_cert is None: + ca_cert = relation_get('ca_cert', + rid=r_id, unit=unit) + return ca_cert + + +def retrieve_ca_cert(cert_file): + cert = None + if os.path.isfile(cert_file): + with open(cert_file, 'rb') as crt: + cert = crt.read() + return cert + + +def install_ca_cert(ca_cert): + host.install_ca_cert(ca_cert, CONFIG_CA_CERT_FILE) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/cluster.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/cluster.py new file mode 100644 index 0000000..f0b629a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hahelpers/cluster.py @@ -0,0 +1,451 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copyright 2012 Canonical Ltd. +# +# Authors: +# James Page +# Adam Gandelman +# + +""" +Helpers for clustering and determining "cluster leadership" and other +clustering-related helpers. +""" + +import functools +import subprocess +import os +import time + +from socket import gethostname as get_unit_hostname + +import six + +from charmhelpers.core.hookenv import ( + log, + relation_ids, + related_units as relation_list, + relation_get, + config as config_get, + INFO, + DEBUG, + WARNING, + unit_get, + is_leader as juju_is_leader, + status_set, +) +from charmhelpers.core.host import ( + modulo_distribution, +) +from charmhelpers.core.decorators import ( + retry_on_exception, +) +from charmhelpers.core.strutils import ( + bool_from_string, +) + +DC_RESOURCE_NAME = 'DC' + + +class HAIncompleteConfig(Exception): + pass + + +class HAIncorrectConfig(Exception): + pass + + +class CRMResourceNotFound(Exception): + pass + + +class CRMDCNotFound(Exception): + pass + + +def is_elected_leader(resource): + """ + Returns True if the charm executing this is the elected cluster leader. + + It relies on two mechanisms to determine leadership: + 1. If juju is sufficiently new and leadership election is supported, + the is_leader command will be used. + 2. If the charm is part of a corosync cluster, call corosync to + determine leadership. + 3. If the charm is not part of a corosync cluster, the leader is + determined as being "the alive unit with the lowest unit number". In + other words, the oldest surviving unit. + """ + try: + return juju_is_leader() + except NotImplementedError: + log('Juju leadership election feature not enabled' + ', using fallback support', + level=WARNING) + + if is_clustered(): + if not is_crm_leader(resource): + log('Deferring action to CRM leader.', level=INFO) + return False + else: + peers = peer_units() + if peers and not oldest_peer(peers): + log('Deferring action to oldest service unit.', level=INFO) + return False + return True + + +def is_clustered(): + for r_id in (relation_ids('ha') or []): + for unit in (relation_list(r_id) or []): + clustered = relation_get('clustered', + rid=r_id, + unit=unit) + if clustered: + return True + return False + + +def is_crm_dc(): + """ + Determine leadership by querying the pacemaker Designated Controller + """ + cmd = ['crm', 'status'] + try: + status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + if not isinstance(status, six.text_type): + status = six.text_type(status, "utf-8") + except subprocess.CalledProcessError as ex: + raise CRMDCNotFound(str(ex)) + + current_dc = '' + for line in status.split('\n'): + if line.startswith('Current DC'): + # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum + current_dc = line.split(':')[1].split()[0] + if current_dc == get_unit_hostname(): + return True + elif current_dc == 'NONE': + raise CRMDCNotFound('Current DC: NONE') + + return False + + +@retry_on_exception(5, base_delay=2, + exc_type=(CRMResourceNotFound, CRMDCNotFound)) +def is_crm_leader(resource, retry=False): + """ + Returns True if the charm calling this is the elected corosync leader, + as returned by calling the external "crm" command. + + We allow this operation to be retried to avoid the possibility of getting a + false negative. See LP #1396246 for more info. + """ + if resource == DC_RESOURCE_NAME: + return is_crm_dc() + cmd = ['crm', 'resource', 'show', resource] + try: + status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + if not isinstance(status, six.text_type): + status = six.text_type(status, "utf-8") + except subprocess.CalledProcessError: + status = None + + if status and get_unit_hostname() in status: + return True + + if status and "resource %s is NOT running" % (resource) in status: + raise CRMResourceNotFound("CRM resource %s not found" % (resource)) + + return False + + +def is_leader(resource): + log("is_leader is deprecated. Please consider using is_crm_leader " + "instead.", level=WARNING) + return is_crm_leader(resource) + + +def peer_units(peer_relation="cluster"): + peers = [] + for r_id in (relation_ids(peer_relation) or []): + for unit in (relation_list(r_id) or []): + peers.append(unit) + return peers + + +def peer_ips(peer_relation='cluster', addr_key='private-address'): + '''Return a dict of peers and their private-address''' + peers = {} + for r_id in relation_ids(peer_relation): + for unit in relation_list(r_id): + peers[unit] = relation_get(addr_key, rid=r_id, unit=unit) + return peers + + +def oldest_peer(peers): + """Determines who the oldest peer is by comparing unit numbers.""" + local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1]) + for peer in peers: + remote_unit_no = int(peer.split('/')[1]) + if remote_unit_no < local_unit_no: + return False + return True + + +def eligible_leader(resource): + log("eligible_leader is deprecated. Please consider using " + "is_elected_leader instead.", level=WARNING) + return is_elected_leader(resource) + + +def https(): + ''' + Determines whether enough data has been provided in configuration + or relation data to configure HTTPS + . + returns: boolean + ''' + use_https = config_get('use-https') + if use_https and bool_from_string(use_https): + return True + if config_get('ssl_cert') and config_get('ssl_key'): + return True + for r_id in relation_ids('certificates'): + for unit in relation_list(r_id): + ca = relation_get('ca', rid=r_id, unit=unit) + if ca: + return True + for r_id in relation_ids('identity-service'): + for unit in relation_list(r_id): + # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN + rel_state = [ + relation_get('https_keystone', rid=r_id, unit=unit), + relation_get('ca_cert', rid=r_id, unit=unit), + ] + # NOTE: works around (LP: #1203241) + if (None not in rel_state) and ('' not in rel_state): + return True + return False + + +def determine_api_port(public_port, singlenode_mode=False): + ''' + Determine correct API server listening port based on + existence of HTTPS reverse proxy and/or haproxy. + + public_port: int: standard public port for given service + + singlenode_mode: boolean: Shuffle ports when only a single unit is present + + returns: int: the correct listening port for the API service + ''' + i = 0 + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): + i += 1 + if https(): + i += 1 + return public_port - (i * 10) + + +def determine_apache_port(public_port, singlenode_mode=False): + ''' + Description: Determine correct apache listening port based on public IP + + state of the cluster. + + public_port: int: standard public port for given service + + singlenode_mode: boolean: Shuffle ports when only a single unit is present + + returns: int: the correct listening port for the HAProxy service + ''' + i = 0 + if singlenode_mode: + i += 1 + elif len(peer_units()) > 0 or is_clustered(): + i += 1 + return public_port - (i * 10) + + +determine_apache_port_single = functools.partial( + determine_apache_port, singlenode_mode=True) + + +def get_hacluster_config(exclude_keys=None): + ''' + Obtains all relevant configuration from charm configuration required + for initiating a relation to hacluster: + + ha-bindiface, ha-mcastport, vip, os-internal-hostname, + os-admin-hostname, os-public-hostname, os-access-hostname + + param: exclude_keys: list of setting key(s) to be excluded. + returns: dict: A dict containing settings keyed by setting name. + raises: HAIncompleteConfig if settings are missing or incorrect. + ''' + settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname', + 'os-admin-hostname', 'os-public-hostname', 'os-access-hostname'] + conf = {} + for setting in settings: + if exclude_keys and setting in exclude_keys: + continue + + conf[setting] = config_get(setting) + + if not valid_hacluster_config(): + raise HAIncorrectConfig('Insufficient or incorrect config data to ' + 'configure hacluster.') + return conf + + +def valid_hacluster_config(): + ''' + Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname + must be set. + + Note: ha-bindiface and ha-macastport both have defaults and will always + be set. We only care that either vip or dns-ha is set. + + :returns: boolean: valid config returns true. + raises: HAIncompatibileConfig if settings conflict. + raises: HAIncompleteConfig if settings are missing. + ''' + vip = config_get('vip') + dns = config_get('dns-ha') + if not(bool(vip) ^ bool(dns)): + msg = ('HA: Either vip or dns-ha must be set but not both in order to ' + 'use high availability') + status_set('blocked', msg) + raise HAIncorrectConfig(msg) + + # If dns-ha then one of os-*-hostname must be set + if dns: + dns_settings = ['os-internal-hostname', 'os-admin-hostname', + 'os-public-hostname', 'os-access-hostname'] + # At this point it is unknown if one or all of the possible + # network spaces are in HA. Validate at least one is set which is + # the minimum required. + for setting in dns_settings: + if config_get(setting): + log('DNS HA: At least one hostname is set {}: {}' + ''.format(setting, config_get(setting)), + level=DEBUG) + return True + + msg = ('DNS HA: At least one os-*-hostname(s) must be set to use ' + 'DNS HA') + status_set('blocked', msg) + raise HAIncompleteConfig(msg) + + log('VIP HA: VIP is set {}'.format(vip), level=DEBUG) + return True + + +def canonical_url(configs, vip_setting='vip'): + ''' + Returns the correct HTTP URL to this host given the state of HTTPS + configuration and hacluster. + + :configs : OSTemplateRenderer: A config tempating object to inspect for + a complete https context. + + :vip_setting: str: Setting in charm config that specifies + VIP address. + ''' + scheme = 'http' + if 'https' in configs.complete_contexts(): + scheme = 'https' + if is_clustered(): + addr = config_get(vip_setting) + else: + addr = unit_get('private-address') + return '%s://%s' % (scheme, addr) + + +def distributed_wait(modulo=None, wait=None, operation_name='operation'): + ''' Distribute operations by waiting based on modulo_distribution + + If modulo and or wait are not set, check config_get for those values. + If config values are not set, default to modulo=3 and wait=30. + + :param modulo: int The modulo number creates the group distribution + :param wait: int The constant time wait value + :param operation_name: string Operation name for status message + i.e. 'restart' + :side effect: Calls config_get() + :side effect: Calls log() + :side effect: Calls status_set() + :side effect: Calls time.sleep() + ''' + if modulo is None: + modulo = config_get('modulo-nodes') or 3 + if wait is None: + wait = config_get('known-wait') or 30 + if juju_is_leader(): + # The leader should never wait + calculated_wait = 0 + else: + # non_zero_wait=True guarantees the non-leader who gets modulo 0 + # will still wait + calculated_wait = modulo_distribution(modulo=modulo, wait=wait, + non_zero_wait=True) + msg = "Waiting {} seconds for {} ...".format(calculated_wait, + operation_name) + log(msg, DEBUG) + status_set('maintenance', msg) + time.sleep(calculated_wait) + + +def get_managed_services_and_ports(services, external_ports, + external_services=None, + port_conv_f=determine_apache_port_single): + """Get the services and ports managed by this charm. + + Return only the services and corresponding ports that are managed by this + charm. This excludes haproxy when there is a relation with hacluster. This + is because this charm passes responsibility for stopping and starting + haproxy to hacluster. + + Similarly, if a relation with hacluster exists then the ports returned by + this method correspond to those managed by the apache server rather than + haproxy. + + :param services: List of services. + :type services: List[str] + :param external_ports: List of ports managed by external services. + :type external_ports: List[int] + :param external_services: List of services to be removed if ha relation is + present. + :type external_services: List[str] + :param port_conv_f: Function to apply to ports to calculate the ports + managed by services controlled by this charm. + :type port_convert_func: f() + :returns: A tuple containing a list of services first followed by a list of + ports. + :rtype: Tuple[List[str], List[int]] + """ + if external_services is None: + external_services = ['haproxy'] + if relation_ids('ha'): + for svc in external_services: + try: + services.remove(svc) + except ValueError: + pass + external_ports = [port_conv_f(p) for p in external_ports] + return services, external_ports diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/README.hardening.md b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/README.hardening.md new file mode 100644 index 0000000..91280c0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/README.hardening.md @@ -0,0 +1,38 @@ +# Juju charm-helpers hardening library + +## Description + +This library provides multiple implementations of system and application +hardening that conform to the standards of http://hardening.io/. + +Current implementations include: + + * OS + * SSH + * MySQL + * Apache + +## Requirements + +* Juju Charms + +## Usage + +1. Synchronise this library into your charm and add the harden() decorator + (from contrib.hardening.harden) to any functions or methods you want to use + to trigger hardening of your application/system. + +2. Add a config option called 'harden' to your charm config.yaml and set it to + a space-delimited list of hardening modules you want to run e.g. "os ssh" + +3. Override any config defaults (contrib.hardening.defaults) by adding a file + called hardening.yaml to your charm root containing the name(s) of the + modules whose settings you want override at root level and then any settings + with overrides e.g. + + os: + general: + desktop_enable: True + +4. Now just run your charm as usual and hardening will be applied each time the + hook runs. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/__init__.py new file mode 100644 index 0000000..30a3e94 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/__init__.py new file mode 100644 index 0000000..58bebd8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path + +TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/__init__.py new file mode 100644 index 0000000..3bc2ebd --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.contrib.hardening.apache.checks import config + + +def run_apache_checks(): + log("Starting Apache hardening checks.", level=DEBUG) + checks = config.get_audits() + for check in checks: + log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) + check.ensure_compliance() + + log("Apache hardening checks complete.", level=DEBUG) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/config.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/config.py new file mode 100644 index 0000000..341da9e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/checks/config.py @@ -0,0 +1,104 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import six +import subprocess + + +from charmhelpers.core.hookenv import ( + log, + INFO, +) +from charmhelpers.contrib.hardening.audits.file import ( + FilePermissionAudit, + DirectoryPermissionAudit, + NoReadWriteForOther, + TemplatedFile, + DeletedFile +) +from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit +from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get Apache hardening config audits. + + :returns: dictionary of audits + """ + if subprocess.call(['which', 'apache2'], stdout=subprocess.PIPE) != 0: + log("Apache server does not appear to be installed on this node - " + "skipping apache hardening", level=INFO) + return [] + + context = ApacheConfContext() + settings = utils.get_settings('apache') + audits = [ + FilePermissionAudit(paths=os.path.join( + settings['common']['apache_dir'], 'apache2.conf'), + user='root', group='root', mode=0o0640), + + TemplatedFile(os.path.join(settings['common']['apache_dir'], + 'mods-available/alias.conf'), + context, + TEMPLATES_DIR, + mode=0o0640, + user='root', + service_actions=[{'service': 'apache2', + 'actions': ['restart']}]), + + TemplatedFile(os.path.join(settings['common']['apache_dir'], + 'conf-enabled/99-hardening.conf'), + context, + TEMPLATES_DIR, + mode=0o0640, + user='root', + service_actions=[{'service': 'apache2', + 'actions': ['restart']}]), + + DirectoryPermissionAudit(settings['common']['apache_dir'], + user='root', + group='root', + mode=0o0750), + + DisabledModuleAudit(settings['hardening']['modules_to_disable']), + + NoReadWriteForOther(settings['common']['apache_dir']), + + DeletedFile(['/var/www/html/index.html']) + ] + + return audits + + +class ApacheConfContext(object): + """Defines the set of key/value pairs to set in a apache config file. + + This context, when called, will return a dictionary containing the + key/value pairs of setting to specify in the + /etc/apache/conf-enabled/hardening.conf file. + """ + def __call__(self): + settings = utils.get_settings('apache') + ctxt = settings['hardening'] + + out = subprocess.check_output(['apache2', '-v']) + if six.PY3: + out = out.decode('utf-8') + ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+', + out).group(1) + ctxt['apache_icondir'] = '/usr/share/apache2/icons/' + return ctxt diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf new file mode 100644 index 0000000..22b6804 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/99-hardening.conf @@ -0,0 +1,32 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### + + + + # http://httpd.apache.org/docs/2.4/upgrading.html + {% if apache_version > '2.2' -%} + Require all granted + {% else -%} + Order Allow,Deny + Deny from all + {% endif %} + + + + + Options -Indexes -FollowSymLinks + AllowOverride None + + + + Options -Indexes -FollowSymLinks + AllowOverride None + + +TraceEnable {{ traceenable }} +ServerTokens {{ servertokens }} + +SSLHonorCipherOrder {{ honor_cipher_order }} +SSLCipherSuite {{ cipher_suite }} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/alias.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/alias.conf new file mode 100644 index 0000000..e46a58a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/apache/templates/alias.conf @@ -0,0 +1,31 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### + + # + # Aliases: Add here as many aliases as you need (with no limit). The format is + # Alias fakename realname + # + # Note that if you include a trailing / on fakename then the server will + # require it to be present in the URL. So "/icons" isn't aliased in this + # example, only "/icons/". If the fakename is slash-terminated, then the + # realname must also be slash terminated, and if the fakename omits the + # trailing slash, the realname must also omit it. + # + # We include the /icons/ alias for FancyIndexed directory listings. If + # you do not use FancyIndexing, you may comment this out. + # + Alias /icons/ "{{ apache_icondir }}/" + + + Options -Indexes -MultiViews -FollowSymLinks + AllowOverride None +{% if apache_version == '2.4' -%} + Require all granted +{% else -%} + Order allow,deny + Allow from all +{% endif %} + + diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/__init__.py new file mode 100644 index 0000000..6dd5b05 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/__init__.py @@ -0,0 +1,54 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class BaseAudit(object): # NO-QA + """Base class for hardening checks. + + The lifecycle of a hardening check is to first check to see if the system + is in compliance for the specified check. If it is not in compliance, the + check method will return a value which will be supplied to the. + """ + def __init__(self, *args, **kwargs): + self.unless = kwargs.get('unless', None) + super(BaseAudit, self).__init__() + + def ensure_compliance(self): + """Checks to see if the current hardening check is in compliance or + not. + + If the check that is performed is not in compliance, then an exception + should be raised. + """ + pass + + def _take_action(self): + """Determines whether to perform the action or not. + + Checks whether or not an action should be taken. This is determined by + the truthy value for the unless parameter. If unless is a callback + method, it will be invoked with no parameters in order to determine + whether or not the action should be taken. Otherwise, the truthy value + of the unless attribute will determine if the action should be + performed. + """ + # Do the action if there isn't an unless override. + if self.unless is None: + return True + + # Invoke the callback if there is one. + if hasattr(self.unless, '__call__'): + return not self.unless() + + return not self.unless diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apache.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apache.py new file mode 100644 index 0000000..c153762 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apache.py @@ -0,0 +1,105 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + +import six + +from charmhelpers.core.hookenv import ( + log, + INFO, + ERROR, +) + +from charmhelpers.contrib.hardening.audits import BaseAudit + + +class DisabledModuleAudit(BaseAudit): + """Audits Apache2 modules. + + Determines if the apache2 modules are enabled. If the modules are enabled + then they are removed in the ensure_compliance. + """ + def __init__(self, modules): + if modules is None: + self.modules = [] + elif isinstance(modules, six.string_types): + self.modules = [modules] + else: + self.modules = modules + + def ensure_compliance(self): + """Ensures that the modules are not loaded.""" + if not self.modules: + return + + try: + loaded_modules = self._get_loaded_modules() + non_compliant_modules = [] + for module in self.modules: + if module in loaded_modules: + log("Module '%s' is enabled but should not be." % + (module), level=INFO) + non_compliant_modules.append(module) + + if len(non_compliant_modules) == 0: + return + + for module in non_compliant_modules: + self._disable_module(module) + self._restart_apache() + except subprocess.CalledProcessError as e: + log('Error occurred auditing apache module compliance. ' + 'This may have been already reported. ' + 'Output is: %s' % e.output, level=ERROR) + + @staticmethod + def _get_loaded_modules(): + """Returns the modules which are enabled in Apache.""" + output = subprocess.check_output(['apache2ctl', '-M']) + if six.PY3: + output = output.decode('utf-8') + modules = [] + for line in output.splitlines(): + # Each line of the enabled module output looks like: + # module_name (static|shared) + # Plus a header line at the top of the output which is stripped + # out by the regex. + matcher = re.search(r'^ (\S*)_module (\S*)', line) + if matcher: + modules.append(matcher.group(1)) + return modules + + @staticmethod + def _disable_module(module): + """Disables the specified module in Apache.""" + try: + subprocess.check_call(['a2dismod', module]) + except subprocess.CalledProcessError as e: + # Note: catch error here to allow the attempt of disabling + # multiple modules in one go rather than failing after the + # first module fails. + log('Error occurred disabling module %s. ' + 'Output is: %s' % (module, e.output), level=ERROR) + + @staticmethod + def _restart_apache(): + """Restarts the apache process""" + subprocess.check_output(['service', 'apache2', 'restart']) + + @staticmethod + def is_ssl_enabled(): + """Check if SSL module is enabled or not""" + return 'ssl' in DisabledModuleAudit._get_loaded_modules() diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apt.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apt.py new file mode 100644 index 0000000..cad7bf7 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/apt.py @@ -0,0 +1,104 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import # required for external apt import +from six import string_types + +from charmhelpers.fetch import ( + apt_cache, + apt_purge +) +from charmhelpers.core.hookenv import ( + log, + DEBUG, + WARNING, +) +from charmhelpers.contrib.hardening.audits import BaseAudit +from charmhelpers.fetch import ubuntu_apt_pkg as apt_pkg + + +class AptConfig(BaseAudit): + + def __init__(self, config, **kwargs): + self.config = config + + def verify_config(self): + apt_pkg.init() + for cfg in self.config: + value = apt_pkg.config.get(cfg['key'], cfg.get('default', '')) + if value and value != cfg['expected']: + log("APT config '%s' has unexpected value '%s' " + "(expected='%s')" % + (cfg['key'], value, cfg['expected']), level=WARNING) + + def ensure_compliance(self): + self.verify_config() + + +class RestrictedPackages(BaseAudit): + """Class used to audit restricted packages on the system.""" + + def __init__(self, pkgs, **kwargs): + super(RestrictedPackages, self).__init__(**kwargs) + if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'): + self.pkgs = pkgs.split() + else: + self.pkgs = pkgs + + def ensure_compliance(self): + cache = apt_cache() + + for p in self.pkgs: + if p not in cache: + continue + + pkg = cache[p] + if not self.is_virtual_package(pkg): + if not pkg.current_ver: + log("Package '%s' is not installed." % pkg.name, + level=DEBUG) + continue + else: + log("Restricted package '%s' is installed" % pkg.name, + level=WARNING) + self.delete_package(cache, pkg) + else: + log("Checking restricted virtual package '%s' provides" % + pkg.name, level=DEBUG) + self.delete_package(cache, pkg) + + def delete_package(self, cache, pkg): + """Deletes the package from the system. + + Deletes the package form the system, properly handling virtual + packages. + + :param cache: the apt cache + :param pkg: the package to remove + """ + if self.is_virtual_package(pkg): + log("Package '%s' appears to be virtual - purging provides" % + pkg.name, level=DEBUG) + for _p in pkg.provides_list: + self.delete_package(cache, _p[2].parent_pkg) + elif not pkg.current_ver: + log("Package '%s' not installed" % pkg.name, level=DEBUG) + return + else: + log("Purging package '%s'" % pkg.name, level=DEBUG) + apt_purge(pkg.name) + + def is_virtual_package(self, pkg): + return (pkg.get('has_provides', False) and + not pkg.get('has_versions', False)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/file.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/file.py new file mode 100644 index 0000000..257c635 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/audits/file.py @@ -0,0 +1,550 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grp +import os +import pwd +import re + +from subprocess import ( + CalledProcessError, + check_output, + check_call, +) +from traceback import format_exc +from six import string_types +from stat import ( + S_ISGID, + S_ISUID +) + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + INFO, + WARNING, + ERROR, +) +from charmhelpers.core import unitdata +from charmhelpers.core.host import file_hash +from charmhelpers.contrib.hardening.audits import BaseAudit +from charmhelpers.contrib.hardening.templating import ( + get_template_path, + render_and_write, +) +from charmhelpers.contrib.hardening import utils + + +class BaseFileAudit(BaseAudit): + """Base class for file audits. + + Provides api stubs for compliance check flow that must be used by any class + that implemented this one. + """ + + def __init__(self, paths, always_comply=False, *args, **kwargs): + """ + :param paths: string path of list of paths of files we want to apply + compliance checks are criteria to. + :param always_comply: if true compliance criteria is always applied + else compliance is skipped for non-existent + paths. + """ + super(BaseFileAudit, self).__init__(*args, **kwargs) + self.always_comply = always_comply + if isinstance(paths, string_types) or not hasattr(paths, '__iter__'): + self.paths = [paths] + else: + self.paths = paths + + def ensure_compliance(self): + """Ensure that the all registered files comply to registered criteria. + """ + for p in self.paths: + if os.path.exists(p): + if self.is_compliant(p): + continue + + log('File %s is not in compliance.' % p, level=INFO) + else: + if not self.always_comply: + log("Non-existent path '%s' - skipping compliance check" + % (p), level=INFO) + continue + + if self._take_action(): + log("Applying compliance criteria to '%s'" % (p), level=INFO) + self.comply(p) + + def is_compliant(self, path): + """Audits the path to see if it is compliance. + + :param path: the path to the file that should be checked. + """ + raise NotImplementedError + + def comply(self, path): + """Enforces the compliance of a path. + + :param path: the path to the file that should be enforced. + """ + raise NotImplementedError + + @classmethod + def _get_stat(cls, path): + """Returns the Posix st_stat information for the specified file path. + + :param path: the path to get the st_stat information for. + :returns: an st_stat object for the path or None if the path doesn't + exist. + """ + return os.stat(path) + + +class FilePermissionAudit(BaseFileAudit): + """Implements an audit for file permissions and ownership for a user. + + This class implements functionality that ensures that a specific user/group + will own the file(s) specified and that the permissions specified are + applied properly to the file. + """ + def __init__(self, paths, user, group=None, mode=0o600, **kwargs): + self.user = user + self.group = group + self.mode = mode + super(FilePermissionAudit, self).__init__(paths, user, group, mode, + **kwargs) + + @property + def user(self): + return self._user + + @user.setter + def user(self, name): + try: + user = pwd.getpwnam(name) + except KeyError: + log('Unknown user %s' % name, level=ERROR) + user = None + self._user = user + + @property + def group(self): + return self._group + + @group.setter + def group(self, name): + try: + group = None + if name: + group = grp.getgrnam(name) + else: + group = grp.getgrgid(self.user.pw_gid) + except KeyError: + log('Unknown group %s' % name, level=ERROR) + self._group = group + + def is_compliant(self, path): + """Checks if the path is in compliance. + + Used to determine if the path specified meets the necessary + requirements to be in compliance with the check itself. + + :param path: the file path to check + :returns: True if the path is compliant, False otherwise. + """ + stat = self._get_stat(path) + user = self.user + group = self.group + + compliant = True + if stat.st_uid != user.pw_uid or stat.st_gid != group.gr_gid: + log('File %s is not owned by %s:%s.' % (path, user.pw_name, + group.gr_name), + level=INFO) + compliant = False + + # POSIX refers to the st_mode bits as corresponding to both the + # file type and file permission bits, where the least significant 12 + # bits (o7777) are the suid (11), sgid (10), sticky bits (9), and the + # file permission bits (8-0) + perms = stat.st_mode & 0o7777 + if perms != self.mode: + log('File %s has incorrect permissions, currently set to %s' % + (path, oct(stat.st_mode & 0o7777)), level=INFO) + compliant = False + + return compliant + + def comply(self, path): + """Issues a chown and chmod to the file paths specified.""" + utils.ensure_permissions(path, self.user.pw_name, self.group.gr_name, + self.mode) + + +class DirectoryPermissionAudit(FilePermissionAudit): + """Performs a permission check for the specified directory path.""" + + def __init__(self, paths, user, group=None, mode=0o600, + recursive=True, **kwargs): + super(DirectoryPermissionAudit, self).__init__(paths, user, group, + mode, **kwargs) + self.recursive = recursive + + def is_compliant(self, path): + """Checks if the directory is compliant. + + Used to determine if the path specified and all of its children + directories are in compliance with the check itself. + + :param path: the directory path to check + :returns: True if the directory tree is compliant, otherwise False. + """ + if not os.path.isdir(path): + log('Path specified %s is not a directory.' % path, level=ERROR) + raise ValueError("%s is not a directory." % path) + + if not self.recursive: + return super(DirectoryPermissionAudit, self).is_compliant(path) + + compliant = True + for root, dirs, _ in os.walk(path): + if len(dirs) > 0: + continue + + if not super(DirectoryPermissionAudit, self).is_compliant(root): + compliant = False + continue + + return compliant + + def comply(self, path): + for root, dirs, _ in os.walk(path): + if len(dirs) > 0: + super(DirectoryPermissionAudit, self).comply(root) + + +class ReadOnly(BaseFileAudit): + """Audits that files and folders are read only.""" + def __init__(self, paths, *args, **kwargs): + super(ReadOnly, self).__init__(paths=paths, *args, **kwargs) + + def is_compliant(self, path): + try: + output = check_output(['find', path, '-perm', '-go+w', + '-type', 'f']).strip() + + # The find above will find any files which have permission sets + # which allow too broad of write access. As such, the path is + # compliant if there is no output. + if output: + return False + + return True + except CalledProcessError as e: + log('Error occurred checking finding writable files for %s. ' + 'Error information is: command %s failed with returncode ' + '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, + format_exc(e)), level=ERROR) + return False + + def comply(self, path): + try: + check_output(['chmod', 'go-w', '-R', path]) + except CalledProcessError as e: + log('Error occurred removing writeable permissions for %s. ' + 'Error information is: command %s failed with returncode ' + '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, + format_exc(e)), level=ERROR) + + +class NoReadWriteForOther(BaseFileAudit): + """Ensures that the files found under the base path are readable or + writable by anyone other than the owner or the group. + """ + def __init__(self, paths): + super(NoReadWriteForOther, self).__init__(paths) + + def is_compliant(self, path): + try: + cmd = ['find', path, '-perm', '-o+r', '-type', 'f', '-o', + '-perm', '-o+w', '-type', 'f'] + output = check_output(cmd).strip() + + # The find above here will find any files which have read or + # write permissions for other, meaning there is too broad of access + # to read/write the file. As such, the path is compliant if there's + # no output. + if output: + return False + + return True + except CalledProcessError as e: + log('Error occurred while finding files which are readable or ' + 'writable to the world in %s. ' + 'Command output is: %s.' % (path, e.output), level=ERROR) + + def comply(self, path): + try: + check_output(['chmod', '-R', 'o-rw', path]) + except CalledProcessError as e: + log('Error occurred attempting to change modes of files under ' + 'path %s. Output of command is: %s' % (path, e.output)) + + +class NoSUIDSGIDAudit(BaseFileAudit): + """Audits that specified files do not have SUID/SGID bits set.""" + def __init__(self, paths, *args, **kwargs): + super(NoSUIDSGIDAudit, self).__init__(paths=paths, *args, **kwargs) + + def is_compliant(self, path): + stat = self._get_stat(path) + if (stat.st_mode & (S_ISGID | S_ISUID)) != 0: + return False + + return True + + def comply(self, path): + try: + log('Removing suid/sgid from %s.' % path, level=DEBUG) + check_output(['chmod', '-s', path]) + except CalledProcessError as e: + log('Error occurred removing suid/sgid from %s.' + 'Error information is: command %s failed with returncode ' + '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output, + format_exc(e)), level=ERROR) + + +class TemplatedFile(BaseFileAudit): + """The TemplatedFileAudit audits the contents of a templated file. + + This audit renders a file from a template, sets the appropriate file + permissions, then generates a hashsum with which to check the content + changed. + """ + def __init__(self, path, context, template_dir, mode, user='root', + group='root', service_actions=None, **kwargs): + self.context = context + self.user = user + self.group = group + self.mode = mode + self.template_dir = template_dir + self.service_actions = service_actions + super(TemplatedFile, self).__init__(paths=path, always_comply=True, + **kwargs) + + def is_compliant(self, path): + """Determines if the templated file is compliant. + + A templated file is only compliant if it has not changed (as + determined by its sha256 hashsum) AND its file permissions are set + appropriately. + + :param path: the path to check compliance. + """ + same_templates = self.templates_match(path) + same_content = self.contents_match(path) + same_permissions = self.permissions_match(path) + + if same_content and same_permissions and same_templates: + return True + + return False + + def run_service_actions(self): + """Run any actions on services requested.""" + if not self.service_actions: + return + + for svc_action in self.service_actions: + name = svc_action['service'] + actions = svc_action['actions'] + log("Running service '%s' actions '%s'" % (name, actions), + level=DEBUG) + for action in actions: + cmd = ['service', name, action] + try: + check_call(cmd) + except CalledProcessError as exc: + log("Service name='%s' action='%s' failed - %s" % + (name, action, exc), level=WARNING) + + def comply(self, path): + """Ensures the contents and the permissions of the file. + + :param path: the path to correct + """ + dirname = os.path.dirname(path) + if not os.path.exists(dirname): + os.makedirs(dirname) + + self.pre_write() + render_and_write(self.template_dir, path, self.context()) + utils.ensure_permissions(path, self.user, self.group, self.mode) + self.run_service_actions() + self.save_checksum(path) + self.post_write() + + def pre_write(self): + """Invoked prior to writing the template.""" + pass + + def post_write(self): + """Invoked after writing the template.""" + pass + + def templates_match(self, path): + """Determines if the template files are the same. + + The template file equality is determined by the hashsum of the + template files themselves. If there is no hashsum, then the content + cannot be sure to be the same so treat it as if they changed. + Otherwise, return whether or not the hashsums are the same. + + :param path: the path to check + :returns: boolean + """ + template_path = get_template_path(self.template_dir, path) + key = 'hardening:template:%s' % template_path + template_checksum = file_hash(template_path) + kv = unitdata.kv() + stored_tmplt_checksum = kv.get(key) + if not stored_tmplt_checksum: + kv.set(key, template_checksum) + kv.flush() + log('Saved template checksum for %s.' % template_path, + level=DEBUG) + # Since we don't have a template checksum, then assume it doesn't + # match and return that the template is different. + return False + elif stored_tmplt_checksum != template_checksum: + kv.set(key, template_checksum) + kv.flush() + log('Updated template checksum for %s.' % template_path, + level=DEBUG) + return False + + # Here the template hasn't changed based upon the calculated + # checksum of the template and what was previously stored. + return True + + def contents_match(self, path): + """Determines if the file content is the same. + + This is determined by comparing hashsum of the file contents and + the saved hashsum. If there is no hashsum, then the content cannot + be sure to be the same so treat them as if they are not the same. + Otherwise, return True if the hashsums are the same, False if they + are not the same. + + :param path: the file to check. + """ + checksum = file_hash(path) + + kv = unitdata.kv() + stored_checksum = kv.get('hardening:%s' % path) + if not stored_checksum: + # If the checksum hasn't been generated, return False to ensure + # the file is written and the checksum stored. + log('Checksum for %s has not been calculated.' % path, level=DEBUG) + return False + elif stored_checksum != checksum: + log('Checksum mismatch for %s.' % path, level=DEBUG) + return False + + return True + + def permissions_match(self, path): + """Determines if the file owner and permissions match. + + :param path: the path to check. + """ + audit = FilePermissionAudit(path, self.user, self.group, self.mode) + return audit.is_compliant(path) + + def save_checksum(self, path): + """Calculates and saves the checksum for the path specified. + + :param path: the path of the file to save the checksum. + """ + checksum = file_hash(path) + kv = unitdata.kv() + kv.set('hardening:%s' % path, checksum) + kv.flush() + + +class DeletedFile(BaseFileAudit): + """Audit to ensure that a file is deleted.""" + def __init__(self, paths): + super(DeletedFile, self).__init__(paths) + + def is_compliant(self, path): + return not os.path.exists(path) + + def comply(self, path): + os.remove(path) + + +class FileContentAudit(BaseFileAudit): + """Audit the contents of a file.""" + def __init__(self, paths, cases, **kwargs): + # Cases we expect to pass + self.pass_cases = cases.get('pass', []) + # Cases we expect to fail + self.fail_cases = cases.get('fail', []) + super(FileContentAudit, self).__init__(paths, **kwargs) + + def is_compliant(self, path): + """ + Given a set of content matching cases i.e. tuple(regex, bool) where + bool value denotes whether or not regex is expected to match, check that + all cases match as expected with the contents of the file. Cases can be + expected to pass of fail. + + :param path: Path of file to check. + :returns: Boolean value representing whether or not all cases are + found to be compliant. + """ + log("Auditing contents of file '%s'" % (path), level=DEBUG) + with open(path, 'r') as fd: + contents = fd.read() + + matches = 0 + for pattern in self.pass_cases: + key = re.compile(pattern, flags=re.MULTILINE) + results = re.search(key, contents) + if results: + matches += 1 + else: + log("Pattern '%s' was expected to pass but instead it failed" + % (pattern), level=WARNING) + + for pattern in self.fail_cases: + key = re.compile(pattern, flags=re.MULTILINE) + results = re.search(key, contents) + if not results: + matches += 1 + else: + log("Pattern '%s' was expected to fail but instead it passed" + % (pattern), level=WARNING) + + total = len(self.pass_cases) + len(self.fail_cases) + log("Checked %s cases and %s passed" % (total, matches), level=DEBUG) + return matches == total + + def comply(self, *args, **kwargs): + """NOOP since we just issue warnings. This is to avoid the + NotImplememtedError. + """ + log("Not applying any compliance criteria, only checks.", level=INFO) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml new file mode 100644 index 0000000..0f940d4 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml @@ -0,0 +1,16 @@ +# NOTE: this file contains the default configuration for the 'apache' hardening +# code. If you want to override any settings you must add them to a file +# called hardening.yaml in the root directory of your charm using the +# name 'apache' as the root key followed by any of the following with new +# values. + +common: + apache_dir: '/etc/apache2' + +hardening: + traceenable: 'off' + allowed_http_methods: "GET POST" + modules_to_disable: [ cgi, cgid ] + servertokens: 'Prod' + honor_cipher_order: 'on' + cipher_suite: 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml.schema b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml.schema new file mode 100644 index 0000000..c112137 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/apache.yaml.schema @@ -0,0 +1,12 @@ +# NOTE: this schema must contain all valid keys from it's associated defaults +# file. It is used to validate user-provided overrides. +common: + apache_dir: + traceenable: + +hardening: + allowed_http_methods: + modules_to_disable: + servertokens: + honor_cipher_order: + cipher_suite: diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml new file mode 100644 index 0000000..682d22b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml @@ -0,0 +1,38 @@ +# NOTE: this file contains the default configuration for the 'mysql' hardening +# code. If you want to override any settings you must add them to a file +# called hardening.yaml in the root directory of your charm using the +# name 'mysql' as the root key followed by any of the following with new +# values. + +hardening: + mysql-conf: /etc/mysql/my.cnf + hardening-conf: /etc/mysql/conf.d/hardening.cnf + +security: + # @see http://www.symantec.com/connect/articles/securing-mysql-step-step + # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_chroot + chroot: None + + # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_safe-user-create + safe-user-create: 1 + + # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-auth + secure-auth: 1 + + # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_symbolic-links + skip-symbolic-links: 1 + + # @see http://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_skip-show-database + skip-show-database: True + + # @see http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_local_infile + local-infile: 0 + + # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_allow-suspicious-udfs + allow-suspicious-udfs: 0 + + # @see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_automatic_sp_privileges + automatic-sp-privileges: 0 + + # @see https://dev.mysql.com/doc/refman/5.7/en/server-options.html#option_mysqld_secure-file-priv + secure-file-priv: /tmp diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema new file mode 100644 index 0000000..2edf325 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/mysql.yaml.schema @@ -0,0 +1,15 @@ +# NOTE: this schema must contain all valid keys from it's associated defaults +# file. It is used to validate user-provided overrides. +hardening: + mysql-conf: + hardening-conf: +security: + chroot: + safe-user-create: + secure-auth: + skip-symbolic-links: + skip-show-database: + local-infile: + allow-suspicious-udfs: + automatic-sp-privileges: + secure-file-priv: diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml new file mode 100644 index 0000000..9a8627b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml @@ -0,0 +1,68 @@ +# NOTE: this file contains the default configuration for the 'os' hardening +# code. If you want to override any settings you must add them to a file +# called hardening.yaml in the root directory of your charm using the +# name 'os' as the root key followed by any of the following with new +# values. + +general: + desktop_enable: False # (type:boolean) + +environment: + extra_user_paths: [] + umask: 027 + root_path: / + +auth: + pw_max_age: 60 + # discourage password cycling + pw_min_age: 7 + retries: 5 + lockout_time: 600 + timeout: 60 + allow_homeless: False # (type:boolean) + pam_passwdqc_enable: True # (type:boolean) + pam_passwdqc_options: 'min=disabled,disabled,16,12,8' + root_ttys: + console + tty1 + tty2 + tty3 + tty4 + tty5 + tty6 + uid_min: 1000 + gid_min: 1000 + sys_uid_min: 100 + sys_uid_max: 999 + sys_gid_min: 100 + sys_gid_max: 999 + chfn_restrict: + +security: + users_allow: [] + suid_sgid_enforce: True # (type:boolean) + # user-defined blacklist and whitelist + suid_sgid_blacklist: [] + suid_sgid_whitelist: [] + # if this is True, remove any suid/sgid bits from files that were not in the whitelist + suid_sgid_dry_run_on_unknown: False # (type:boolean) + suid_sgid_remove_from_unknown: False # (type:boolean) + # remove packages with known issues + packages_clean: True # (type:boolean) + packages_list: + xinetd + inetd + ypserv + telnet-server + rsh-server + rsync + kernel_enable_module_loading: True # (type:boolean) + kernel_enable_core_dump: False # (type:boolean) + ssh_tmout: 300 + +sysctl: + kernel_secure_sysrq: 244 # 4 + 16 + 32 + 64 + 128 + kernel_enable_sysrq: False # (type:boolean) + forwarding: False # (type:boolean) + ipv6_enable: False # (type:boolean) + arp_restricted: True # (type:boolean) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml.schema b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml.schema new file mode 100644 index 0000000..cc3b9c2 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/os.yaml.schema @@ -0,0 +1,43 @@ +# NOTE: this schema must contain all valid keys from it's associated defaults +# file. It is used to validate user-provided overrides. +general: + desktop_enable: +environment: + extra_user_paths: + umask: + root_path: +auth: + pw_max_age: + pw_min_age: + retries: + lockout_time: + timeout: + allow_homeless: + pam_passwdqc_enable: + pam_passwdqc_options: + root_ttys: + uid_min: + gid_min: + sys_uid_min: + sys_uid_max: + sys_gid_min: + sys_gid_max: + chfn_restrict: +security: + users_allow: + suid_sgid_enforce: + suid_sgid_blacklist: + suid_sgid_whitelist: + suid_sgid_dry_run_on_unknown: + suid_sgid_remove_from_unknown: + packages_clean: + packages_list: + kernel_enable_module_loading: + kernel_enable_core_dump: + ssh_tmout: +sysctl: + kernel_secure_sysrq: + kernel_enable_sysrq: + forwarding: + ipv6_enable: + arp_restricted: diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml new file mode 100644 index 0000000..cd529bc --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml @@ -0,0 +1,49 @@ +# NOTE: this file contains the default configuration for the 'ssh' hardening +# code. If you want to override any settings you must add them to a file +# called hardening.yaml in the root directory of your charm using the +# name 'ssh' as the root key followed by any of the following with new +# values. + +common: + service_name: 'ssh' + network_ipv6_enable: False # (type:boolean) + ports: [22] + remote_hosts: [] + +client: + package: 'openssh-client' + cbc_required: False # (type:boolean) + weak_hmac: False # (type:boolean) + weak_kex: False # (type:boolean) + roaming: False + password_authentication: 'no' + +server: + host_key_files: ['/etc/ssh/ssh_host_rsa_key', '/etc/ssh/ssh_host_dsa_key', + '/etc/ssh/ssh_host_ecdsa_key'] + cbc_required: False # (type:boolean) + weak_hmac: False # (type:boolean) + weak_kex: False # (type:boolean) + allow_root_with_key: False # (type:boolean) + allow_tcp_forwarding: 'no' + allow_agent_forwarding: 'no' + allow_x11_forwarding: 'no' + use_privilege_separation: 'sandbox' + listen_to: ['0.0.0.0'] + use_pam: 'no' + package: 'openssh-server' + password_authentication: 'no' + alive_interval: '600' + alive_count: '3' + sftp_enable: False # (type:boolean) + sftp_group: 'sftponly' + sftp_chroot: '/home/%u' + deny_users: [] + allow_users: [] + deny_groups: [] + allow_groups: [] + print_motd: 'no' + print_last_log: 'no' + use_dns: 'no' + max_auth_tries: 2 + max_sessions: 10 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema new file mode 100644 index 0000000..d05e054 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/defaults/ssh.yaml.schema @@ -0,0 +1,42 @@ +# NOTE: this schema must contain all valid keys from it's associated defaults +# file. It is used to validate user-provided overrides. +common: + service_name: + network_ipv6_enable: + ports: + remote_hosts: +client: + package: + cbc_required: + weak_hmac: + weak_kex: + roaming: + password_authentication: +server: + host_key_files: + cbc_required: + weak_hmac: + weak_kex: + allow_root_with_key: + allow_tcp_forwarding: + allow_agent_forwarding: + allow_x11_forwarding: + use_privilege_separation: + listen_to: + use_pam: + package: + password_authentication: + alive_interval: + alive_count: + sftp_enable: + sftp_group: + sftp_chroot: + deny_users: + allow_users: + deny_groups: + allow_groups: + print_motd: + print_last_log: + use_dns: + max_auth_tries: + max_sessions: diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/harden.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/harden.py new file mode 100644 index 0000000..63f21b9 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/harden.py @@ -0,0 +1,96 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from collections import OrderedDict + +from charmhelpers.core.hookenv import ( + config, + log, + DEBUG, + WARNING, +) +from charmhelpers.contrib.hardening.host.checks import run_os_checks +from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks +from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks +from charmhelpers.contrib.hardening.apache.checks import run_apache_checks + +_DISABLE_HARDENING_FOR_UNIT_TEST = False + + +def harden(overrides=None): + """Hardening decorator. + + This is the main entry point for running the hardening stack. In order to + run modules of the stack you must add this decorator to charm hook(s) and + ensure that your charm config.yaml contains the 'harden' option set to + one or more of the supported modules. Setting these will cause the + corresponding hardening code to be run when the hook fires. + + This decorator can and should be applied to more than one hook or function + such that hardening modules are called multiple times. This is because + subsequent calls will perform auditing checks that will report any changes + to resources hardened by the first run (and possibly perform compliance + actions as a result of any detected infractions). + + :param overrides: Optional list of stack modules used to override those + provided with 'harden' config. + :returns: Returns value returned by decorated function once executed. + """ + if overrides is None: + overrides = [] + + def _harden_inner1(f): + # As this has to be py2.7 compat, we can't use nonlocal. Use a trick + # to capture the dictionary that can then be updated. + _logged = {'done': False} + + def _harden_inner2(*args, **kwargs): + # knock out hardening via a config var; normally it won't get + # disabled. + if _DISABLE_HARDENING_FOR_UNIT_TEST: + return f(*args, **kwargs) + if not _logged['done']: + log("Hardening function '%s'" % (f.__name__), level=DEBUG) + _logged['done'] = True + RUN_CATALOG = OrderedDict([('os', run_os_checks), + ('ssh', run_ssh_checks), + ('mysql', run_mysql_checks), + ('apache', run_apache_checks)]) + + enabled = overrides[:] or (config("harden") or "").split() + if enabled: + modules_to_run = [] + # modules will always be performed in the following order + for module, func in six.iteritems(RUN_CATALOG): + if module in enabled: + enabled.remove(module) + modules_to_run.append(func) + + if enabled: + log("Unknown hardening modules '%s' - ignoring" % + (', '.join(enabled)), level=WARNING) + + for hardener in modules_to_run: + log("Executing hardening module '%s'" % + (hardener.__name__), level=DEBUG) + hardener() + else: + log("No hardening applied to '%s'" % (f.__name__), level=DEBUG) + + return f(*args, **kwargs) + return _harden_inner2 + + return _harden_inner1 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/__init__.py new file mode 100644 index 0000000..58bebd8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path + +TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/__init__.py new file mode 100644 index 0000000..0e7e409 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.contrib.hardening.host.checks import ( + apt, + limits, + login, + minimize_access, + pam, + profile, + securetty, + suid_sgid, + sysctl +) + + +def run_os_checks(): + log("Starting OS hardening checks.", level=DEBUG) + checks = apt.get_audits() + checks.extend(limits.get_audits()) + checks.extend(login.get_audits()) + checks.extend(minimize_access.get_audits()) + checks.extend(pam.get_audits()) + checks.extend(profile.get_audits()) + checks.extend(securetty.get_audits()) + checks.extend(suid_sgid.get_audits()) + checks.extend(sysctl.get_audits()) + + for check in checks: + log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) + check.ensure_compliance() + + log("OS hardening checks complete.", level=DEBUG) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/apt.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/apt.py new file mode 100644 index 0000000..7ce41b0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/apt.py @@ -0,0 +1,37 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.contrib.hardening.utils import get_settings +from charmhelpers.contrib.hardening.audits.apt import ( + AptConfig, + RestrictedPackages, +) + + +def get_audits(): + """Get OS hardening apt audits. + + :returns: dictionary of audits + """ + audits = [AptConfig([{'key': 'APT::Get::AllowUnauthenticated', + 'expected': 'false'}])] + + settings = get_settings('os') + clean_packages = settings['security']['packages_clean'] + if clean_packages: + security_packages = settings['security']['packages_list'] + if security_packages: + audits.append(RestrictedPackages(security_packages)) + + return audits diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/limits.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/limits.py new file mode 100644 index 0000000..e94f5eb --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/limits.py @@ -0,0 +1,53 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.contrib.hardening.audits.file import ( + DirectoryPermissionAudit, + TemplatedFile, +) +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get OS hardening security limits audits. + + :returns: dictionary of audits + """ + audits = [] + settings = utils.get_settings('os') + + # Ensure that the /etc/security/limits.d directory is only writable + # by the root user, but others can execute and read. + audits.append(DirectoryPermissionAudit('/etc/security/limits.d', + user='root', group='root', + mode=0o755)) + + # If core dumps are not enabled, then don't allow core dumps to be + # created as they may contain sensitive information. + if not settings['security']['kernel_enable_core_dump']: + audits.append(TemplatedFile('/etc/security/limits.d/10.hardcore.conf', + SecurityLimitsContext(), + template_dir=TEMPLATES_DIR, + user='root', group='root', mode=0o0440)) + return audits + + +class SecurityLimitsContext(object): + + def __call__(self): + settings = utils.get_settings('os') + ctxt = {'disable_core_dump': + not settings['security']['kernel_enable_core_dump']} + return ctxt diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/login.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/login.py new file mode 100644 index 0000000..fe2bc6e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/login.py @@ -0,0 +1,65 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from six import string_types + +from charmhelpers.contrib.hardening.audits.file import TemplatedFile +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get OS hardening login.defs audits. + + :returns: dictionary of audits + """ + audits = [TemplatedFile('/etc/login.defs', LoginContext(), + template_dir=TEMPLATES_DIR, + user='root', group='root', mode=0o0444)] + return audits + + +class LoginContext(object): + + def __call__(self): + settings = utils.get_settings('os') + + # Octal numbers in yaml end up being turned into decimal, + # so check if the umask is entered as a string (e.g. '027') + # or as an octal umask as we know it (e.g. 002). If its not + # a string assume it to be octal and turn it into an octal + # string. + umask = settings['environment']['umask'] + if not isinstance(umask, string_types): + umask = '%s' % oct(umask) + + ctxt = { + 'additional_user_paths': + settings['environment']['extra_user_paths'], + 'umask': umask, + 'pwd_max_age': settings['auth']['pw_max_age'], + 'pwd_min_age': settings['auth']['pw_min_age'], + 'uid_min': settings['auth']['uid_min'], + 'sys_uid_min': settings['auth']['sys_uid_min'], + 'sys_uid_max': settings['auth']['sys_uid_max'], + 'gid_min': settings['auth']['gid_min'], + 'sys_gid_min': settings['auth']['sys_gid_min'], + 'sys_gid_max': settings['auth']['sys_gid_max'], + 'login_retries': settings['auth']['retries'], + 'login_timeout': settings['auth']['timeout'], + 'chfn_restrict': settings['auth']['chfn_restrict'], + 'allow_login_without_home': settings['auth']['allow_homeless'] + } + + return ctxt diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/minimize_access.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/minimize_access.py new file mode 100644 index 0000000..6e64be0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/minimize_access.py @@ -0,0 +1,50 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.contrib.hardening.audits.file import ( + FilePermissionAudit, + ReadOnly, +) +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get OS hardening access audits. + + :returns: dictionary of audits + """ + audits = [] + settings = utils.get_settings('os') + + # Remove write permissions from $PATH folders for all regular users. + # This prevents changing system-wide commands from normal users. + path_folders = {'/usr/local/sbin', + '/usr/local/bin', + '/usr/sbin', + '/usr/bin', + '/bin'} + extra_user_paths = settings['environment']['extra_user_paths'] + path_folders.update(extra_user_paths) + audits.append(ReadOnly(path_folders)) + + # Only allow the root user to have access to the shadow file. + audits.append(FilePermissionAudit('/etc/shadow', 'root', 'root', 0o0600)) + + if 'change_user' not in settings['security']['users_allow']: + # su should only be accessible to user and group root, unless it is + # expressly defined to allow users to change to root via the + # security_users_allow config option. + audits.append(FilePermissionAudit('/bin/su', 'root', 'root', 0o750)) + + return audits diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/pam.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/pam.py new file mode 100644 index 0000000..9b38d5f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/pam.py @@ -0,0 +1,132 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from subprocess import ( + check_output, + CalledProcessError, +) + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + ERROR, +) +from charmhelpers.fetch import ( + apt_install, + apt_purge, + apt_update, +) +from charmhelpers.contrib.hardening.audits.file import ( + TemplatedFile, + DeletedFile, +) +from charmhelpers.contrib.hardening import utils +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR + + +def get_audits(): + """Get OS hardening PAM authentication audits. + + :returns: dictionary of audits + """ + audits = [] + + settings = utils.get_settings('os') + + if settings['auth']['pam_passwdqc_enable']: + audits.append(PasswdqcPAM('/etc/passwdqc.conf')) + + if settings['auth']['retries']: + audits.append(Tally2PAM('/usr/share/pam-configs/tally2')) + else: + audits.append(DeletedFile('/usr/share/pam-configs/tally2')) + + return audits + + +class PasswdqcPAMContext(object): + + def __call__(self): + ctxt = {} + settings = utils.get_settings('os') + + ctxt['auth_pam_passwdqc_options'] = \ + settings['auth']['pam_passwdqc_options'] + + return ctxt + + +class PasswdqcPAM(TemplatedFile): + """The PAM Audit verifies the linux PAM settings.""" + def __init__(self, path): + super(PasswdqcPAM, self).__init__(path=path, + template_dir=TEMPLATES_DIR, + context=PasswdqcPAMContext(), + user='root', + group='root', + mode=0o0640) + + def pre_write(self): + # Always remove? + for pkg in ['libpam-ccreds', 'libpam-cracklib']: + log("Purging package '%s'" % pkg, level=DEBUG), + apt_purge(pkg) + + apt_update(fatal=True) + for pkg in ['libpam-passwdqc']: + log("Installing package '%s'" % pkg, level=DEBUG), + apt_install(pkg) + + def post_write(self): + """Updates the PAM configuration after the file has been written""" + try: + check_output(['pam-auth-update', '--package']) + except CalledProcessError as e: + log('Error calling pam-auth-update: %s' % e, level=ERROR) + + +class Tally2PAMContext(object): + + def __call__(self): + ctxt = {} + settings = utils.get_settings('os') + + ctxt['auth_lockout_time'] = settings['auth']['lockout_time'] + ctxt['auth_retries'] = settings['auth']['retries'] + + return ctxt + + +class Tally2PAM(TemplatedFile): + """The PAM Audit verifies the linux PAM settings.""" + def __init__(self, path): + super(Tally2PAM, self).__init__(path=path, + template_dir=TEMPLATES_DIR, + context=Tally2PAMContext(), + user='root', + group='root', + mode=0o0640) + + def pre_write(self): + # Always remove? + apt_purge('libpam-ccreds') + apt_update(fatal=True) + apt_install('libpam-modules') + + def post_write(self): + """Updates the PAM configuration after the file has been written""" + try: + check_output(['pam-auth-update', '--package']) + except CalledProcessError as e: + log('Error calling pam-auth-update: %s' % e, level=ERROR) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/profile.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/profile.py new file mode 100644 index 0000000..2727428 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/profile.py @@ -0,0 +1,49 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.contrib.hardening.audits.file import TemplatedFile +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get OS hardening profile audits. + + :returns: dictionary of audits + """ + audits = [] + + settings = utils.get_settings('os') + # If core dumps are not enabled, then don't allow core dumps to be + # created as they may contain sensitive information. + if not settings['security']['kernel_enable_core_dump']: + audits.append(TemplatedFile('/etc/profile.d/pinerolo_profile.sh', + ProfileContext(), + template_dir=TEMPLATES_DIR, + mode=0o0755, user='root', group='root')) + if settings['security']['ssh_tmout']: + audits.append(TemplatedFile('/etc/profile.d/99-hardening.sh', + ProfileContext(), + template_dir=TEMPLATES_DIR, + mode=0o0644, user='root', group='root')) + return audits + + +class ProfileContext(object): + + def __call__(self): + settings = utils.get_settings('os') + ctxt = {'ssh_tmout': + settings['security']['ssh_tmout']} + return ctxt diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/securetty.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/securetty.py new file mode 100644 index 0000000..34cd021 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/securetty.py @@ -0,0 +1,37 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.contrib.hardening.audits.file import TemplatedFile +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get OS hardening Secure TTY audits. + + :returns: dictionary of audits + """ + audits = [] + audits.append(TemplatedFile('/etc/securetty', SecureTTYContext(), + template_dir=TEMPLATES_DIR, + mode=0o0400, user='root', group='root')) + return audits + + +class SecureTTYContext(object): + + def __call__(self): + settings = utils.get_settings('os') + ctxt = {'ttys': settings['auth']['root_ttys']} + return ctxt diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/suid_sgid.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/suid_sgid.py new file mode 100644 index 0000000..bcbe3fd --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/suid_sgid.py @@ -0,0 +1,129 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess + +from charmhelpers.core.hookenv import ( + log, + INFO, +) +from charmhelpers.contrib.hardening.audits.file import NoSUIDSGIDAudit +from charmhelpers.contrib.hardening import utils + + +BLACKLIST = ['/usr/bin/rcp', '/usr/bin/rlogin', '/usr/bin/rsh', + '/usr/libexec/openssh/ssh-keysign', + '/usr/lib/openssh/ssh-keysign', + '/sbin/netreport', + '/usr/sbin/usernetctl', + '/usr/sbin/userisdnctl', + '/usr/sbin/pppd', + '/usr/bin/lockfile', + '/usr/bin/mail-lock', + '/usr/bin/mail-unlock', + '/usr/bin/mail-touchlock', + '/usr/bin/dotlockfile', + '/usr/bin/arping', + '/usr/sbin/uuidd', + '/usr/bin/mtr', + '/usr/lib/evolution/camel-lock-helper-1.2', + '/usr/lib/pt_chown', + '/usr/lib/eject/dmcrypt-get-device', + '/usr/lib/mc/cons.saver'] + +WHITELIST = ['/bin/mount', '/bin/ping', '/bin/su', '/bin/umount', + '/sbin/pam_timestamp_check', '/sbin/unix_chkpwd', '/usr/bin/at', + '/usr/bin/gpasswd', '/usr/bin/locate', '/usr/bin/newgrp', + '/usr/bin/passwd', '/usr/bin/ssh-agent', + '/usr/libexec/utempter/utempter', '/usr/sbin/lockdev', + '/usr/sbin/sendmail.sendmail', '/usr/bin/expiry', + '/bin/ping6', '/usr/bin/traceroute6.iputils', + '/sbin/mount.nfs', '/sbin/umount.nfs', + '/sbin/mount.nfs4', '/sbin/umount.nfs4', + '/usr/bin/crontab', + '/usr/bin/wall', '/usr/bin/write', + '/usr/bin/screen', + '/usr/bin/mlocate', + '/usr/bin/chage', '/usr/bin/chfn', '/usr/bin/chsh', + '/bin/fusermount', + '/usr/bin/pkexec', + '/usr/bin/sudo', '/usr/bin/sudoedit', + '/usr/sbin/postdrop', '/usr/sbin/postqueue', + '/usr/sbin/suexec', + '/usr/lib/squid/ncsa_auth', '/usr/lib/squid/pam_auth', + '/usr/kerberos/bin/ksu', + '/usr/sbin/ccreds_validate', + '/usr/bin/Xorg', + '/usr/bin/X', + '/usr/lib/dbus-1.0/dbus-daemon-launch-helper', + '/usr/lib/vte/gnome-pty-helper', + '/usr/lib/libvte9/gnome-pty-helper', + '/usr/lib/libvte-2.90-9/gnome-pty-helper'] + + +def get_audits(): + """Get OS hardening suid/sgid audits. + + :returns: dictionary of audits + """ + checks = [] + settings = utils.get_settings('os') + if not settings['security']['suid_sgid_enforce']: + log("Skipping suid/sgid hardening", level=INFO) + return checks + + # Build the blacklist and whitelist of files for suid/sgid checks. + # There are a total of 4 lists: + # 1. the system blacklist + # 2. the system whitelist + # 3. the user blacklist + # 4. the user whitelist + # + # The blacklist is the set of paths which should NOT have the suid/sgid bit + # set and the whitelist is the set of paths which MAY have the suid/sgid + # bit setl. The user whitelist/blacklist effectively override the system + # whitelist/blacklist. + u_b = settings['security']['suid_sgid_blacklist'] + u_w = settings['security']['suid_sgid_whitelist'] + + blacklist = set(BLACKLIST) - set(u_w + u_b) + whitelist = set(WHITELIST) - set(u_b + u_w) + + checks.append(NoSUIDSGIDAudit(blacklist)) + + dry_run = settings['security']['suid_sgid_dry_run_on_unknown'] + + if settings['security']['suid_sgid_remove_from_unknown'] or dry_run: + # If the policy is a dry_run (e.g. complain only) or remove unknown + # suid/sgid bits then find all of the paths which have the suid/sgid + # bit set and then remove the whitelisted paths. + root_path = settings['environment']['root_path'] + unknown_paths = find_paths_with_suid_sgid(root_path) - set(whitelist) + checks.append(NoSUIDSGIDAudit(unknown_paths, unless=dry_run)) + + return checks + + +def find_paths_with_suid_sgid(root_path): + """Finds all paths/files which have an suid/sgid bit enabled. + + Starting with the root_path, this will recursively find all paths which + have an suid or sgid bit set. + """ + cmd = ['find', root_path, '-perm', '-4000', '-o', '-perm', '-2000', + '-type', 'f', '!', '-path', '/proc/*', '-print'] + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = p.communicate() + return set(out.split('\n')) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/sysctl.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/sysctl.py new file mode 100644 index 0000000..f1ea581 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/checks/sysctl.py @@ -0,0 +1,209 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import platform +import re +import six +import subprocess + +from charmhelpers.core.hookenv import ( + log, + INFO, + WARNING, +) +from charmhelpers.contrib.hardening import utils +from charmhelpers.contrib.hardening.audits.file import ( + FilePermissionAudit, + TemplatedFile, +) +from charmhelpers.contrib.hardening.host import TEMPLATES_DIR + + +SYSCTL_DEFAULTS = """net.ipv4.ip_forward=%(net_ipv4_ip_forward)s +net.ipv6.conf.all.forwarding=%(net_ipv6_conf_all_forwarding)s +net.ipv4.conf.all.rp_filter=1 +net.ipv4.conf.default.rp_filter=1 +net.ipv4.icmp_echo_ignore_broadcasts=1 +net.ipv4.icmp_ignore_bogus_error_responses=1 +net.ipv4.icmp_ratelimit=100 +net.ipv4.icmp_ratemask=88089 +net.ipv6.conf.all.disable_ipv6=%(net_ipv6_conf_all_disable_ipv6)s +net.ipv4.tcp_timestamps=%(net_ipv4_tcp_timestamps)s +net.ipv4.conf.all.arp_ignore=%(net_ipv4_conf_all_arp_ignore)s +net.ipv4.conf.all.arp_announce=%(net_ipv4_conf_all_arp_announce)s +net.ipv4.tcp_rfc1337=1 +net.ipv4.tcp_syncookies=1 +net.ipv4.conf.all.shared_media=1 +net.ipv4.conf.default.shared_media=1 +net.ipv4.conf.all.accept_source_route=0 +net.ipv4.conf.default.accept_source_route=0 +net.ipv4.conf.all.accept_redirects=0 +net.ipv4.conf.default.accept_redirects=0 +net.ipv6.conf.all.accept_redirects=0 +net.ipv6.conf.default.accept_redirects=0 +net.ipv4.conf.all.secure_redirects=0 +net.ipv4.conf.default.secure_redirects=0 +net.ipv4.conf.all.send_redirects=0 +net.ipv4.conf.default.send_redirects=0 +net.ipv4.conf.all.log_martians=0 +net.ipv6.conf.default.router_solicitations=0 +net.ipv6.conf.default.accept_ra_rtr_pref=0 +net.ipv6.conf.default.accept_ra_pinfo=0 +net.ipv6.conf.default.accept_ra_defrtr=0 +net.ipv6.conf.default.autoconf=0 +net.ipv6.conf.default.dad_transmits=0 +net.ipv6.conf.default.max_addresses=1 +net.ipv6.conf.all.accept_ra=0 +net.ipv6.conf.default.accept_ra=0 +kernel.modules_disabled=%(kernel_modules_disabled)s +kernel.sysrq=%(kernel_sysrq)s +fs.suid_dumpable=%(fs_suid_dumpable)s +kernel.randomize_va_space=2 +""" + + +def get_audits(): + """Get OS hardening sysctl audits. + + :returns: dictionary of audits + """ + audits = [] + settings = utils.get_settings('os') + + # Apply the sysctl settings which are configured to be applied. + audits.append(SysctlConf()) + # Make sure that only root has access to the sysctl.conf file, and + # that it is read-only. + audits.append(FilePermissionAudit('/etc/sysctl.conf', + user='root', + group='root', mode=0o0440)) + # If module loading is not enabled, then ensure that the modules + # file has the appropriate permissions and rebuild the initramfs + if not settings['security']['kernel_enable_module_loading']: + audits.append(ModulesTemplate()) + + return audits + + +class ModulesContext(object): + + def __call__(self): + settings = utils.get_settings('os') + with open('/proc/cpuinfo', 'r') as fd: + cpuinfo = fd.readlines() + + for line in cpuinfo: + match = re.search(r"^vendor_id\s+:\s+(.+)", line) + if match: + vendor = match.group(1) + + if vendor == "GenuineIntel": + vendor = "intel" + elif vendor == "AuthenticAMD": + vendor = "amd" + + ctxt = {'arch': platform.processor(), + 'cpuVendor': vendor, + 'desktop_enable': settings['general']['desktop_enable']} + + return ctxt + + +class ModulesTemplate(object): + + def __init__(self): + super(ModulesTemplate, self).__init__('/etc/initramfs-tools/modules', + ModulesContext(), + templates_dir=TEMPLATES_DIR, + user='root', group='root', + mode=0o0440) + + def post_write(self): + subprocess.check_call(['update-initramfs', '-u']) + + +class SysCtlHardeningContext(object): + def __call__(self): + settings = utils.get_settings('os') + ctxt = {'sysctl': {}} + + log("Applying sysctl settings", level=INFO) + extras = {'net_ipv4_ip_forward': 0, + 'net_ipv6_conf_all_forwarding': 0, + 'net_ipv6_conf_all_disable_ipv6': 1, + 'net_ipv4_tcp_timestamps': 0, + 'net_ipv4_conf_all_arp_ignore': 0, + 'net_ipv4_conf_all_arp_announce': 0, + 'kernel_sysrq': 0, + 'fs_suid_dumpable': 0, + 'kernel_modules_disabled': 1} + + if settings['sysctl']['ipv6_enable']: + extras['net_ipv6_conf_all_disable_ipv6'] = 0 + + if settings['sysctl']['forwarding']: + extras['net_ipv4_ip_forward'] = 1 + extras['net_ipv6_conf_all_forwarding'] = 1 + + if settings['sysctl']['arp_restricted']: + extras['net_ipv4_conf_all_arp_ignore'] = 1 + extras['net_ipv4_conf_all_arp_announce'] = 2 + + if settings['security']['kernel_enable_module_loading']: + extras['kernel_modules_disabled'] = 0 + + if settings['sysctl']['kernel_enable_sysrq']: + sysrq_val = settings['sysctl']['kernel_secure_sysrq'] + extras['kernel_sysrq'] = sysrq_val + + if settings['security']['kernel_enable_core_dump']: + extras['fs_suid_dumpable'] = 1 + + settings.update(extras) + for d in (SYSCTL_DEFAULTS % settings).split(): + d = d.strip().partition('=') + key = d[0].strip() + path = os.path.join('/proc/sys', key.replace('.', '/')) + if not os.path.exists(path): + log("Skipping '%s' since '%s' does not exist" % (key, path), + level=WARNING) + continue + + ctxt['sysctl'][key] = d[2] or None + + # Translate for python3 + return {'sysctl_settings': + [(k, v) for k, v in six.iteritems(ctxt['sysctl'])]} + + +class SysctlConf(TemplatedFile): + """An audit check for sysctl settings.""" + def __init__(self): + self.conffile = '/etc/sysctl.d/99-juju-hardening.conf' + super(SysctlConf, self).__init__(self.conffile, + SysCtlHardeningContext(), + template_dir=TEMPLATES_DIR, + user='root', group='root', + mode=0o0440) + + def post_write(self): + try: + subprocess.check_call(['sysctl', '-p', self.conffile]) + except subprocess.CalledProcessError as e: + # NOTE: on some systems if sysctl cannot apply all settings it + # will return non-zero as well. + log("sysctl command returned an error (maybe some " + "keys could not be set) - %s" % (e), + level=WARNING) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf new file mode 100644 index 0000000..0014191 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/10.hardcore.conf @@ -0,0 +1,8 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +{% if disable_core_dump -%} +# Prevent core dumps for all users. These are usually only needed by developers and may contain sensitive information. +* hard core 0 +{% endif %} \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-hardening.sh b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-hardening.sh new file mode 100644 index 0000000..616cef4 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-hardening.sh @@ -0,0 +1,5 @@ +TMOUT={{ tmout }} +readonly TMOUT +export TMOUT + +readonly HISTFILE diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf new file mode 100644 index 0000000..101f1e1 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/99-juju-hardening.conf @@ -0,0 +1,7 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +{% for key, value in sysctl_settings -%} +{{ key }}={{ value }} +{% endfor -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/login.defs b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/login.defs new file mode 100644 index 0000000..7d10763 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/login.defs @@ -0,0 +1,349 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# +# /etc/login.defs - Configuration control definitions for the login package. +# +# Three items must be defined: MAIL_DIR, ENV_SUPATH, and ENV_PATH. +# If unspecified, some arbitrary (and possibly incorrect) value will +# be assumed. All other items are optional - if not specified then +# the described action or option will be inhibited. +# +# Comment lines (lines beginning with "#") and blank lines are ignored. +# +# Modified for Linux. --marekm + +# REQUIRED for useradd/userdel/usermod +# Directory where mailboxes reside, _or_ name of file, relative to the +# home directory. If you _do_ define MAIL_DIR and MAIL_FILE, +# MAIL_DIR takes precedence. +# +# Essentially: +# - MAIL_DIR defines the location of users mail spool files +# (for mbox use) by appending the username to MAIL_DIR as defined +# below. +# - MAIL_FILE defines the location of the users mail spool files as the +# fully-qualified filename obtained by prepending the user home +# directory before $MAIL_FILE +# +# NOTE: This is no more used for setting up users MAIL environment variable +# which is, starting from shadow 4.0.12-1 in Debian, entirely the +# job of the pam_mail PAM modules +# See default PAM configuration files provided for +# login, su, etc. +# +# This is a temporary situation: setting these variables will soon +# move to /etc/default/useradd and the variables will then be +# no more supported +MAIL_DIR /var/mail +#MAIL_FILE .mail + +# +# Enable logging and display of /var/log/faillog login failure info. +# This option conflicts with the pam_tally PAM module. +# +FAILLOG_ENAB yes + +# +# Enable display of unknown usernames when login failures are recorded. +# +# WARNING: Unknown usernames may become world readable. +# See #290803 and #298773 for details about how this could become a security +# concern +LOG_UNKFAIL_ENAB no + +# +# Enable logging of successful logins +# +LOG_OK_LOGINS yes + +# +# Enable "syslog" logging of su activity - in addition to sulog file logging. +# SYSLOG_SG_ENAB does the same for newgrp and sg. +# +SYSLOG_SU_ENAB yes +SYSLOG_SG_ENAB yes + +# +# If defined, all su activity is logged to this file. +# +#SULOG_FILE /var/log/sulog + +# +# If defined, file which maps tty line to TERM environment parameter. +# Each line of the file is in a format something like "vt100 tty01". +# +#TTYTYPE_FILE /etc/ttytype + +# +# If defined, login failures will be logged here in a utmp format +# last, when invoked as lastb, will read /var/log/btmp, so... +# +FTMP_FILE /var/log/btmp + +# +# If defined, the command name to display when running "su -". For +# example, if this is defined as "su" then a "ps" will display the +# command is "-su". If not defined, then "ps" would display the +# name of the shell actually being run, e.g. something like "-sh". +# +SU_NAME su + +# +# If defined, file which inhibits all the usual chatter during the login +# sequence. If a full pathname, then hushed mode will be enabled if the +# user's name or shell are found in the file. If not a full pathname, then +# hushed mode will be enabled if the file exists in the user's home directory. +# +HUSHLOGIN_FILE .hushlogin +#HUSHLOGIN_FILE /etc/hushlogins + +# +# *REQUIRED* The default PATH settings, for superuser and normal users. +# +# (they are minimal, add the rest in the shell startup files) +ENV_SUPATH PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV_PATH PATH=/usr/local/bin:/usr/bin:/bin{% if additional_user_paths %}{{ additional_user_paths }}{% endif %} + +# +# Terminal permissions +# +# TTYGROUP Login tty will be assigned this group ownership. +# TTYPERM Login tty will be set to this permission. +# +# If you have a "write" program which is "setgid" to a special group +# which owns the terminals, define TTYGROUP to the group number and +# TTYPERM to 0620. Otherwise leave TTYGROUP commented out and assign +# TTYPERM to either 622 or 600. +# +# In Debian /usr/bin/bsd-write or similar programs are setgid tty +# However, the default and recommended value for TTYPERM is still 0600 +# to not allow anyone to write to anyone else console or terminal + +# Users can still allow other people to write them by issuing +# the "mesg y" command. + +TTYGROUP tty +TTYPERM 0600 + +# +# Login configuration initializations: +# +# ERASECHAR Terminal ERASE character ('\010' = backspace). +# KILLCHAR Terminal KILL character ('\025' = CTRL/U). +# UMASK Default "umask" value. +# +# The ERASECHAR and KILLCHAR are used only on System V machines. +# +# UMASK is the default umask value for pam_umask and is used by +# useradd and newusers to set the mode of the new home directories. +# 022 is the "historical" value in Debian for UMASK +# 027, or even 077, could be considered better for privacy +# There is no One True Answer here : each sysadmin must make up his/her +# mind. +# +# If USERGROUPS_ENAB is set to "yes", that will modify this UMASK default value +# for private user groups, i. e. the uid is the same as gid, and username is +# the same as the primary group name: for these, the user permissions will be +# used as group permissions, e. g. 022 will become 002. +# +# Prefix these values with "0" to get octal, "0x" to get hexadecimal. +# +ERASECHAR 0177 +KILLCHAR 025 +UMASK {{ umask }} + +# Enable setting of the umask group bits to be the same as owner bits (examples: `022` -> `002`, `077` -> `007`) for non-root users, if the uid is the same as gid, and username is the same as the primary group name. +# If set to yes, userdel will remove the user´s group if it contains no more members, and useradd will create by default a group with the name of the user. +USERGROUPS_ENAB yes + +# +# Password aging controls: +# +# PASS_MAX_DAYS Maximum number of days a password may be used. +# PASS_MIN_DAYS Minimum number of days allowed between password changes. +# PASS_WARN_AGE Number of days warning given before a password expires. +# +PASS_MAX_DAYS {{ pwd_max_age }} +PASS_MIN_DAYS {{ pwd_min_age }} +PASS_WARN_AGE 7 + +# +# Min/max values for automatic uid selection in useradd +# +UID_MIN {{ uid_min }} +UID_MAX 60000 +# System accounts +SYS_UID_MIN {{ sys_uid_min }} +SYS_UID_MAX {{ sys_uid_max }} + +# Min/max values for automatic gid selection in groupadd +GID_MIN {{ gid_min }} +GID_MAX 60000 +# System accounts +SYS_GID_MIN {{ sys_gid_min }} +SYS_GID_MAX {{ sys_gid_max }} + +# +# Max number of login retries if password is bad. This will most likely be +# overridden by PAM, since the default pam_unix module has it's own built +# in of 3 retries. However, this is a safe fallback in case you are using +# an authentication module that does not enforce PAM_MAXTRIES. +# +LOGIN_RETRIES {{ login_retries }} + +# +# Max time in seconds for login +# +LOGIN_TIMEOUT {{ login_timeout }} + +# +# Which fields may be changed by regular users using chfn - use +# any combination of letters "frwh" (full name, room number, work +# phone, home phone). If not defined, no changes are allowed. +# For backward compatibility, "yes" = "rwh" and "no" = "frwh". +# +{% if chfn_restrict %} +CHFN_RESTRICT {{ chfn_restrict }} +{% endif %} + +# +# Should login be allowed if we can't cd to the home directory? +# Default in no. +# +DEFAULT_HOME {% if allow_login_without_home %} yes {% else %} no {% endif %} + +# +# If defined, this command is run when removing a user. +# It should remove any at/cron/print jobs etc. owned by +# the user to be removed (passed as the first argument). +# +#USERDEL_CMD /usr/sbin/userdel_local + +# +# Enable setting of the umask group bits to be the same as owner bits +# (examples: 022 -> 002, 077 -> 007) for non-root users, if the uid is +# the same as gid, and username is the same as the primary group name. +# +# If set to yes, userdel will remove the user´s group if it contains no +# more members, and useradd will create by default a group with the name +# of the user. +# +USERGROUPS_ENAB yes + +# +# Instead of the real user shell, the program specified by this parameter +# will be launched, although its visible name (argv[0]) will be the shell's. +# The program may do whatever it wants (logging, additional authentication, +# banner, ...) before running the actual shell. +# +# FAKE_SHELL /bin/fakeshell + +# +# If defined, either full pathname of a file containing device names or +# a ":" delimited list of device names. Root logins will be allowed only +# upon these devices. +# +# This variable is used by login and su. +# +#CONSOLE /etc/consoles +#CONSOLE console:tty01:tty02:tty03:tty04 + +# +# List of groups to add to the user's supplementary group set +# when logging in on the console (as determined by the CONSOLE +# setting). Default is none. +# +# Use with caution - it is possible for users to gain permanent +# access to these groups, even when not logged in on the console. +# How to do it is left as an exercise for the reader... +# +# This variable is used by login and su. +# +#CONSOLE_GROUPS floppy:audio:cdrom + +# +# If set to "yes", new passwords will be encrypted using the MD5-based +# algorithm compatible with the one used by recent releases of FreeBSD. +# It supports passwords of unlimited length and longer salt strings. +# Set to "no" if you need to copy encrypted passwords to other systems +# which don't understand the new algorithm. Default is "no". +# +# This variable is deprecated. You should use ENCRYPT_METHOD. +# +MD5_CRYPT_ENAB no + +# +# If set to MD5 , MD5-based algorithm will be used for encrypting password +# If set to SHA256, SHA256-based algorithm will be used for encrypting password +# If set to SHA512, SHA512-based algorithm will be used for encrypting password +# If set to DES, DES-based algorithm will be used for encrypting password (default) +# Overrides the MD5_CRYPT_ENAB option +# +# Note: It is recommended to use a value consistent with +# the PAM modules configuration. +# +ENCRYPT_METHOD SHA512 + +# +# Only used if ENCRYPT_METHOD is set to SHA256 or SHA512. +# +# Define the number of SHA rounds. +# With a lot of rounds, it is more difficult to brute forcing the password. +# But note also that it more CPU resources will be needed to authenticate +# users. +# +# If not specified, the libc will choose the default number of rounds (5000). +# The values must be inside the 1000-999999999 range. +# If only one of the MIN or MAX values is set, then this value will be used. +# If MIN > MAX, the highest value will be used. +# +# SHA_CRYPT_MIN_ROUNDS 5000 +# SHA_CRYPT_MAX_ROUNDS 5000 + +################# OBSOLETED BY PAM ############## +# # +# These options are now handled by PAM. Please # +# edit the appropriate file in /etc/pam.d/ to # +# enable the equivelants of them. +# +############### + +#MOTD_FILE +#DIALUPS_CHECK_ENAB +#LASTLOG_ENAB +#MAIL_CHECK_ENAB +#OBSCURE_CHECKS_ENAB +#PORTTIME_CHECKS_ENAB +#SU_WHEEL_ONLY +#CRACKLIB_DICTPATH +#PASS_CHANGE_TRIES +#PASS_ALWAYS_WARN +#ENVIRON_FILE +#NOLOGINS_FILE +#ISSUE_FILE +#PASS_MIN_LEN +#PASS_MAX_LEN +#ULIMIT +#ENV_HZ +#CHFN_AUTH +#CHSH_AUTH +#FAIL_DELAY + +################# OBSOLETED ####################### +# # +# These options are no more handled by shadow. # +# # +# Shadow utilities will display a warning if they # +# still appear. # +# # +################################################### + +# CLOSE_SESSIONS +# LOGIN_STRING +# NO_PASSWORD_CONSOLE +# QMAIL_DIR + + + diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/modules b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/modules new file mode 100644 index 0000000..ef0354e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/modules @@ -0,0 +1,117 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# /etc/modules: kernel modules to load at boot time. +# +# This file contains the names of kernel modules that should be loaded +# at boot time, one per line. Lines beginning with "#" are ignored. +# Parameters can be specified after the module name. + +# Arch +# ---- +# +# Modules for certains builds, contains support modules and some CPU-specific optimizations. + +{% if arch == "x86_64" -%} +# Optimize for x86_64 cryptographic features +twofish-x86_64-3way +twofish-x86_64 +aes-x86_64 +salsa20-x86_64 +blowfish-x86_64 +{% endif -%} + +{% if cpuVendor == "intel" -%} +# Intel-specific optimizations +ghash-clmulni-intel +aesni-intel +kvm-intel +{% endif -%} + +{% if cpuVendor == "amd" -%} +# AMD-specific optimizations +kvm-amd +{% endif -%} + +kvm + + +# Crypto +# ------ + +# Some core modules which comprise strong cryptography. +blowfish_common +blowfish_generic +ctr +cts +lrw +lzo +rmd160 +rmd256 +rmd320 +serpent +sha512_generic +twofish_common +twofish_generic +xts +zlib + + +# Drivers +# ------- + +# Basics +lp +rtc +loop + +# Filesystems +ext2 +btrfs + +{% if desktop_enable -%} +# Desktop +psmouse +snd +snd_ac97_codec +snd_intel8x0 +snd_page_alloc +snd_pcm +snd_timer +soundcore +usbhid +{% endif -%} + +# Lib +# --- +xz + + +# Net +# --- + +# All packets needed for netfilter rules (ie iptables, ebtables). +ip_tables +x_tables +iptable_filter +iptable_nat + +# Targets +ipt_LOG +ipt_REJECT + +# Modules +xt_connlimit +xt_tcpudp +xt_recent +xt_limit +xt_conntrack +nf_conntrack +nf_conntrack_ipv4 +nf_defrag_ipv4 +xt_state +nf_nat + +# Addons +xt_pknock \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/passwdqc.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/passwdqc.conf new file mode 100644 index 0000000..f98d14e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/passwdqc.conf @@ -0,0 +1,11 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +Name: passwdqc password strength enforcement +Default: yes +Priority: 1024 +Conflicts: cracklib +Password-Type: Primary +Password: + requisite pam_passwdqc.so {{ auth_pam_passwdqc_options }} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh new file mode 100644 index 0000000..fd2de79 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/pinerolo_profile.sh @@ -0,0 +1,8 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# Disable core dumps via soft limits for all users. Compliance to this setting +# is voluntary and can be modified by users up to a hard limit. This setting is +# a sane default. +ulimit -S -c 0 > /dev/null 2>&1 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/securetty b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/securetty new file mode 100644 index 0000000..15b18d4 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/securetty @@ -0,0 +1,11 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# A list of TTYs, from which root can log in +# see `man securetty` for reference +{% if ttys -%} +{% for tty in ttys -%} +{{ tty }} +{% endfor -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/tally2 b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/tally2 new file mode 100644 index 0000000..d962029 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/host/templates/tally2 @@ -0,0 +1,14 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +Name: tally2 lockout after failed attempts enforcement +Default: yes +Priority: 1024 +Conflicts: cracklib +Auth-Type: Primary +Auth-Initial: + required pam_tally2.so deny={{ auth_retries }} onerr=fail unlock_time={{ auth_lockout_time }} +Account-Type: Primary +Account-Initial: + required pam_tally2.so diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/__init__.py new file mode 100644 index 0000000..58bebd8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path + +TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/__init__.py new file mode 100644 index 0000000..1990d85 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.contrib.hardening.mysql.checks import config + + +def run_mysql_checks(): + log("Starting MySQL hardening checks.", level=DEBUG) + checks = config.get_audits() + for check in checks: + log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) + check.ensure_compliance() + + log("MySQL hardening checks complete.", level=DEBUG) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/config.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/config.py new file mode 100644 index 0000000..a79f33b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/checks/config.py @@ -0,0 +1,87 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import subprocess + +from charmhelpers.core.hookenv import ( + log, + WARNING, +) +from charmhelpers.contrib.hardening.audits.file import ( + FilePermissionAudit, + DirectoryPermissionAudit, + TemplatedFile, +) +from charmhelpers.contrib.hardening.mysql import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get MySQL hardening config audits. + + :returns: dictionary of audits + """ + if subprocess.call(['which', 'mysql'], stdout=subprocess.PIPE) != 0: + log("MySQL does not appear to be installed on this node - " + "skipping mysql hardening", level=WARNING) + return [] + + settings = utils.get_settings('mysql') + hardening_settings = settings['hardening'] + my_cnf = hardening_settings['mysql-conf'] + + audits = [ + FilePermissionAudit(paths=[my_cnf], user='root', + group='root', mode=0o0600), + + TemplatedFile(hardening_settings['hardening-conf'], + MySQLConfContext(), + TEMPLATES_DIR, + mode=0o0750, + user='mysql', + group='root', + service_actions=[{'service': 'mysql', + 'actions': ['restart']}]), + + # MySQL and Percona charms do not allow configuration of the + # data directory, so use the default. + DirectoryPermissionAudit('/var/lib/mysql', + user='mysql', + group='mysql', + recursive=False, + mode=0o755), + + DirectoryPermissionAudit('/etc/mysql', + user='root', + group='root', + recursive=False, + mode=0o700), + ] + + return audits + + +class MySQLConfContext(object): + """Defines the set of key/value pairs to set in a mysql config file. + + This context, when called, will return a dictionary containing the + key/value pairs of setting to specify in the + /etc/mysql/conf.d/hardening.cnf file. + """ + def __call__(self): + settings = utils.get_settings('mysql') + # Translate for python3 + return {'mysql_settings': + [(k, v) for k, v in six.iteritems(settings['security'])]} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf new file mode 100644 index 0000000..8242586 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/mysql/templates/hardening.cnf @@ -0,0 +1,12 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +[mysqld] +{% for setting, value in mysql_settings -%} +{% if value == 'True' -%} +{{ setting }} +{% elif value != 'None' and value != None -%} +{{ setting }} = {{ value }} +{% endif -%} +{% endfor -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/__init__.py new file mode 100644 index 0000000..58bebd8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from os import path + +TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates') diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/__init__.py new file mode 100644 index 0000000..edaf484 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.contrib.hardening.ssh.checks import config + + +def run_ssh_checks(): + log("Starting SSH hardening checks.", level=DEBUG) + checks = config.get_audits() + for check in checks: + log("Running '%s' check" % (check.__class__.__name__), level=DEBUG) + check.ensure_compliance() + + log("SSH hardening checks complete.", level=DEBUG) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/config.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/config.py new file mode 100644 index 0000000..41bed2d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/checks/config.py @@ -0,0 +1,435 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + get_iface_addr, + is_ip, +) +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.fetch import ( + apt_install, + apt_update, +) +from charmhelpers.core.host import ( + lsb_release, + CompareHostReleases, +) +from charmhelpers.contrib.hardening.audits.file import ( + TemplatedFile, + FileContentAudit, +) +from charmhelpers.contrib.hardening.ssh import TEMPLATES_DIR +from charmhelpers.contrib.hardening import utils + + +def get_audits(): + """Get SSH hardening config audits. + + :returns: dictionary of audits + """ + audits = [SSHConfig(), SSHDConfig(), SSHConfigFileContentAudit(), + SSHDConfigFileContentAudit()] + return audits + + +class SSHConfigContext(object): + + type = 'client' + + def get_macs(self, allow_weak_mac): + if allow_weak_mac: + weak_macs = 'weak' + else: + weak_macs = 'default' + + default = 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160' + macs = {'default': default, + 'weak': default + ',hmac-sha1'} + + default = ('hmac-sha2-512-etm@openssh.com,' + 'hmac-sha2-256-etm@openssh.com,' + 'hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,' + 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160') + macs_66 = {'default': default, + 'weak': default + ',hmac-sha1'} + + # Use newer ciphers on Ubuntu Trusty and above + _release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(_release) >= 'trusty': + log("Detected Ubuntu 14.04 or newer, using new macs", level=DEBUG) + macs = macs_66 + + return macs[weak_macs] + + def get_kexs(self, allow_weak_kex): + if allow_weak_kex: + weak_kex = 'weak' + else: + weak_kex = 'default' + + default = 'diffie-hellman-group-exchange-sha256' + weak = (default + ',diffie-hellman-group14-sha1,' + 'diffie-hellman-group-exchange-sha1,' + 'diffie-hellman-group1-sha1') + kex = {'default': default, + 'weak': weak} + + default = ('curve25519-sha256@libssh.org,' + 'diffie-hellman-group-exchange-sha256') + weak = (default + ',diffie-hellman-group14-sha1,' + 'diffie-hellman-group-exchange-sha1,' + 'diffie-hellman-group1-sha1') + kex_66 = {'default': default, + 'weak': weak} + + # Use newer kex on Ubuntu Trusty and above + _release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(_release) >= 'trusty': + log('Detected Ubuntu 14.04 or newer, using new key exchange ' + 'algorithms', level=DEBUG) + kex = kex_66 + + return kex[weak_kex] + + def get_ciphers(self, cbc_required): + if cbc_required: + weak_ciphers = 'weak' + else: + weak_ciphers = 'default' + + default = 'aes256-ctr,aes192-ctr,aes128-ctr' + cipher = {'default': default, + 'weak': default + 'aes256-cbc,aes192-cbc,aes128-cbc'} + + default = ('chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,' + 'aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr') + ciphers_66 = {'default': default, + 'weak': default + ',aes256-cbc,aes192-cbc,aes128-cbc'} + + # Use newer ciphers on ubuntu Trusty and above + _release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(_release) >= 'trusty': + log('Detected Ubuntu 14.04 or newer, using new ciphers', + level=DEBUG) + cipher = ciphers_66 + + return cipher[weak_ciphers] + + def get_listening(self, listen=['0.0.0.0']): + """Returns a list of addresses SSH can list on + + Turns input into a sensible list of IPs SSH can listen on. Input + must be a python list of interface names, IPs and/or CIDRs. + + :param listen: list of IPs, CIDRs, interface names + + :returns: list of IPs available on the host + """ + if listen == ['0.0.0.0']: + return listen + + value = [] + for network in listen: + try: + ip = get_address_in_network(network=network, fatal=True) + except ValueError: + if is_ip(network): + ip = network + else: + try: + ip = get_iface_addr(iface=network, fatal=False)[0] + except IndexError: + continue + value.append(ip) + if value == []: + return ['0.0.0.0'] + return value + + def __call__(self): + settings = utils.get_settings('ssh') + if settings['common']['network_ipv6_enable']: + addr_family = 'any' + else: + addr_family = 'inet' + + ctxt = { + 'addr_family': addr_family, + 'remote_hosts': settings['common']['remote_hosts'], + 'password_auth_allowed': + settings['client']['password_authentication'], + 'ports': settings['common']['ports'], + 'ciphers': self.get_ciphers(settings['client']['cbc_required']), + 'macs': self.get_macs(settings['client']['weak_hmac']), + 'kexs': self.get_kexs(settings['client']['weak_kex']), + 'roaming': settings['client']['roaming'], + } + return ctxt + + +class SSHConfig(TemplatedFile): + def __init__(self): + path = '/etc/ssh/ssh_config' + super(SSHConfig, self).__init__(path=path, + template_dir=TEMPLATES_DIR, + context=SSHConfigContext(), + user='root', + group='root', + mode=0o0644) + + def pre_write(self): + settings = utils.get_settings('ssh') + apt_update(fatal=True) + apt_install(settings['client']['package']) + if not os.path.exists('/etc/ssh'): + os.makedir('/etc/ssh') + # NOTE: don't recurse + utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, + maxdepth=0) + + def post_write(self): + # NOTE: don't recurse + utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, + maxdepth=0) + + +class SSHDConfigContext(SSHConfigContext): + + type = 'server' + + def __call__(self): + settings = utils.get_settings('ssh') + if settings['common']['network_ipv6_enable']: + addr_family = 'any' + else: + addr_family = 'inet' + + ctxt = { + 'ssh_ip': self.get_listening(settings['server']['listen_to']), + 'password_auth_allowed': + settings['server']['password_authentication'], + 'ports': settings['common']['ports'], + 'addr_family': addr_family, + 'ciphers': self.get_ciphers(settings['server']['cbc_required']), + 'macs': self.get_macs(settings['server']['weak_hmac']), + 'kexs': self.get_kexs(settings['server']['weak_kex']), + 'host_key_files': settings['server']['host_key_files'], + 'allow_root_with_key': settings['server']['allow_root_with_key'], + 'password_authentication': + settings['server']['password_authentication'], + 'use_priv_sep': settings['server']['use_privilege_separation'], + 'use_pam': settings['server']['use_pam'], + 'allow_x11_forwarding': settings['server']['allow_x11_forwarding'], + 'print_motd': settings['server']['print_motd'], + 'print_last_log': settings['server']['print_last_log'], + 'client_alive_interval': + settings['server']['alive_interval'], + 'client_alive_count': settings['server']['alive_count'], + 'allow_tcp_forwarding': settings['server']['allow_tcp_forwarding'], + 'allow_agent_forwarding': + settings['server']['allow_agent_forwarding'], + 'deny_users': settings['server']['deny_users'], + 'allow_users': settings['server']['allow_users'], + 'deny_groups': settings['server']['deny_groups'], + 'allow_groups': settings['server']['allow_groups'], + 'use_dns': settings['server']['use_dns'], + 'sftp_enable': settings['server']['sftp_enable'], + 'sftp_group': settings['server']['sftp_group'], + 'sftp_chroot': settings['server']['sftp_chroot'], + 'max_auth_tries': settings['server']['max_auth_tries'], + 'max_sessions': settings['server']['max_sessions'], + } + return ctxt + + +class SSHDConfig(TemplatedFile): + def __init__(self): + path = '/etc/ssh/sshd_config' + super(SSHDConfig, self).__init__(path=path, + template_dir=TEMPLATES_DIR, + context=SSHDConfigContext(), + user='root', + group='root', + mode=0o0600, + service_actions=[{'service': 'ssh', + 'actions': + ['restart']}]) + + def pre_write(self): + settings = utils.get_settings('ssh') + apt_update(fatal=True) + apt_install(settings['server']['package']) + if not os.path.exists('/etc/ssh'): + os.makedir('/etc/ssh') + # NOTE: don't recurse + utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, + maxdepth=0) + + def post_write(self): + # NOTE: don't recurse + utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755, + maxdepth=0) + + +class SSHConfigFileContentAudit(FileContentAudit): + def __init__(self): + self.path = '/etc/ssh/ssh_config' + super(SSHConfigFileContentAudit, self).__init__(self.path, {}) + + def is_compliant(self, *args, **kwargs): + self.pass_cases = [] + self.fail_cases = [] + settings = utils.get_settings('ssh') + + _release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(_release) >= 'trusty': + if not settings['server']['weak_hmac']: + self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') + else: + self.pass_cases.append(r'^MACs.+,hmac-sha1$') + + if settings['server']['weak_kex']: + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + else: + self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa + self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa + + if settings['server']['cbc_required']: + self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa + self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$') + self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + if not settings['client']['weak_hmac']: + self.fail_cases.append(r'^MACs.+,hmac-sha1$') + else: + self.pass_cases.append(r'^MACs.+,hmac-sha1$') + + if settings['client']['weak_kex']: + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + else: + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + + if settings['client']['cbc_required']: + self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + + if settings['client']['roaming']: + self.pass_cases.append(r'^UseRoaming yes$') + else: + self.fail_cases.append(r'^UseRoaming yes$') + + return super(SSHConfigFileContentAudit, self).is_compliant(*args, + **kwargs) + + +class SSHDConfigFileContentAudit(FileContentAudit): + def __init__(self): + self.path = '/etc/ssh/sshd_config' + super(SSHDConfigFileContentAudit, self).__init__(self.path, {}) + + def is_compliant(self, *args, **kwargs): + self.pass_cases = [] + self.fail_cases = [] + settings = utils.get_settings('ssh') + + _release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(_release) >= 'trusty': + if not settings['server']['weak_hmac']: + self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') + else: + self.pass_cases.append(r'^MACs.+,hmac-sha1$') + + if settings['server']['weak_kex']: + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + else: + self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa + self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa + + if settings['server']['cbc_required']: + self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa + self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$') + self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + if not settings['server']['weak_hmac']: + self.pass_cases.append(r'^MACs.+,hmac-ripemd160$') + else: + self.pass_cases.append(r'^MACs.+,hmac-sha1$') + + if settings['server']['weak_kex']: + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + else: + self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa + self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa + + if settings['server']['cbc_required']: + self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + else: + self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?') + self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?') + + if settings['server']['sftp_enable']: + self.pass_cases.append(r'^Subsystem\ssftp') + else: + self.fail_cases.append(r'^Subsystem\ssftp') + + return super(SSHDConfigFileContentAudit, self).is_compliant(*args, + **kwargs) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/ssh_config b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/ssh_config new file mode 100644 index 0000000..9742d8e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/ssh_config @@ -0,0 +1,70 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# This is the ssh client system-wide configuration file. See +# ssh_config(5) for more information. This file provides defaults for +# users, and the values can be changed in per-user configuration files +# or on the command line. + +# Configuration data is parsed as follows: +# 1. command line options +# 2. user-specific file +# 3. system-wide file +# Any configuration value is only changed the first time it is set. +# Thus, host-specific definitions should be at the beginning of the +# configuration file, and defaults at the end. + +# Site-wide defaults for some commonly used options. For a comprehensive +# list of available options, their meanings and defaults, please see the +# ssh_config(5) man page. + +# Restrict the following configuration to be limited to this Host. +{% if remote_hosts -%} +Host {{ ' '.join(remote_hosts) }} +{% endif %} +ForwardAgent no +ForwardX11 no +ForwardX11Trusted yes +RhostsRSAAuthentication no +RSAAuthentication yes +PasswordAuthentication {{ password_auth_allowed }} +HostbasedAuthentication no +GSSAPIAuthentication no +GSSAPIDelegateCredentials no +GSSAPIKeyExchange no +GSSAPITrustDNS no +BatchMode no +CheckHostIP yes +AddressFamily {{ addr_family }} +ConnectTimeout 0 +StrictHostKeyChecking ask +IdentityFile ~/.ssh/identity +IdentityFile ~/.ssh/id_rsa +IdentityFile ~/.ssh/id_dsa +# The port at the destination should be defined +{% for port in ports -%} +Port {{ port }} +{% endfor %} +Protocol 2 +Cipher 3des +{% if ciphers -%} +Ciphers {{ ciphers }} +{%- endif %} +{% if macs -%} +MACs {{ macs }} +{%- endif %} +{% if kexs -%} +KexAlgorithms {{ kexs }} +{%- endif %} +EscapeChar ~ +Tunnel no +TunnelDevice any:any +PermitLocalCommand no +VisualHostKey no +RekeyLimit 1G 1h +SendEnv LANG LC_* +HashKnownHosts yes +{% if roaming -%} +UseRoaming {{ roaming }} +{% endif %} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/sshd_config b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/sshd_config new file mode 100644 index 0000000..5f87298 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/ssh/templates/sshd_config @@ -0,0 +1,159 @@ +############################################################################### +# WARNING: This configuration file is maintained by Juju. Local changes may +# be overwritten. +############################################################################### +# Package generated configuration file +# See the sshd_config(5) manpage for details + +# What ports, IPs and protocols we listen for +{% for port in ports -%} +Port {{ port }} +{% endfor -%} +AddressFamily {{ addr_family }} +# Use these options to restrict which interfaces/protocols sshd will bind to +{% if ssh_ip -%} +{% for ip in ssh_ip -%} +ListenAddress {{ ip }} +{% endfor %} +{%- else -%} +ListenAddress :: +ListenAddress 0.0.0.0 +{% endif -%} +Protocol 2 +{% if ciphers -%} +Ciphers {{ ciphers }} +{% endif -%} +{% if macs -%} +MACs {{ macs }} +{% endif -%} +{% if kexs -%} +KexAlgorithms {{ kexs }} +{% endif -%} +# HostKeys for protocol version 2 +{% for keyfile in host_key_files -%} +HostKey {{ keyfile }} +{% endfor -%} + +# Privilege Separation is turned on for security +{% if use_priv_sep -%} +UsePrivilegeSeparation {{ use_priv_sep }} +{% endif -%} + +# Lifetime and size of ephemeral version 1 server key +KeyRegenerationInterval 3600 +ServerKeyBits 1024 + +# Logging +SyslogFacility AUTH +LogLevel VERBOSE + +# Authentication: +LoginGraceTime 30s +{% if allow_root_with_key -%} +PermitRootLogin without-password +{% else -%} +PermitRootLogin no +{% endif %} +PermitTunnel no +PermitUserEnvironment no +StrictModes yes + +RSAAuthentication yes +PubkeyAuthentication yes +AuthorizedKeysFile %h/.ssh/authorized_keys + +# Don't read the user's ~/.rhosts and ~/.shosts files +IgnoreRhosts yes +# For this to work you will also need host keys in /etc/ssh_known_hosts +RhostsRSAAuthentication no +# similar for protocol version 2 +HostbasedAuthentication no +# Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication +IgnoreUserKnownHosts yes + +# To enable empty passwords, change to yes (NOT RECOMMENDED) +PermitEmptyPasswords no + +# Change to yes to enable challenge-response passwords (beware issues with +# some PAM modules and threads) +ChallengeResponseAuthentication no + +# Change to no to disable tunnelled clear text passwords +PasswordAuthentication {{ password_authentication }} + +# Kerberos options +KerberosAuthentication no +KerberosGetAFSToken no +KerberosOrLocalPasswd no +KerberosTicketCleanup yes + +# GSSAPI options +GSSAPIAuthentication no +GSSAPICleanupCredentials yes + +X11Forwarding {{ allow_x11_forwarding }} +X11DisplayOffset 10 +X11UseLocalhost yes +GatewayPorts no +PrintMotd {{ print_motd }} +PrintLastLog {{ print_last_log }} +TCPKeepAlive no +UseLogin no + +ClientAliveInterval {{ client_alive_interval }} +ClientAliveCountMax {{ client_alive_count }} +AllowTcpForwarding {{ allow_tcp_forwarding }} +AllowAgentForwarding {{ allow_agent_forwarding }} + +MaxStartups 10:30:100 +#Banner /etc/issue.net + +# Allow client to pass locale environment variables +AcceptEnv LANG LC_* + +# Set this to 'yes' to enable PAM authentication, account processing, +# and session processing. If this is enabled, PAM authentication will +# be allowed through the ChallengeResponseAuthentication and +# PasswordAuthentication. Depending on your PAM configuration, +# PAM authentication via ChallengeResponseAuthentication may bypass +# the setting of "PermitRootLogin without-password". +# If you just want the PAM account and session checks to run without +# PAM authentication, then enable this but set PasswordAuthentication +# and ChallengeResponseAuthentication to 'no'. +UsePAM {{ use_pam }} + +{% if deny_users -%} +DenyUsers {{ deny_users }} +{% endif -%} +{% if allow_users -%} +AllowUsers {{ allow_users }} +{% endif -%} +{% if deny_groups -%} +DenyGroups {{ deny_groups }} +{% endif -%} +{% if allow_groups -%} +AllowGroups allow_groups +{% endif -%} +UseDNS {{ use_dns }} +MaxAuthTries {{ max_auth_tries }} +MaxSessions {{ max_sessions }} + +{% if sftp_enable -%} +# Configuration, in case SFTP is used +## override default of no subsystems +## Subsystem sftp /opt/app/openssh5/libexec/sftp-server +Subsystem sftp internal-sftp -l VERBOSE + +## These lines must appear at the *end* of sshd_config +Match Group {{ sftp_group }} +ForceCommand internal-sftp -l VERBOSE +ChrootDirectory {{ sftp_chroot }} +{% else -%} +# Configuration, in case SFTP is used +## override default of no subsystems +## Subsystem sftp /opt/app/openssh5/libexec/sftp-server +## These lines must appear at the *end* of sshd_config +Match Group sftponly +ForceCommand internal-sftp -l VERBOSE +ChrootDirectory /sftpchroot/home/%u +{% endif %} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/templating.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/templating.py new file mode 100644 index 0000000..5b6765f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/templating.py @@ -0,0 +1,73 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import six + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + WARNING, +) + +try: + from jinja2 import FileSystemLoader, Environment +except ImportError: + from charmhelpers.fetch import apt_install + from charmhelpers.fetch import apt_update + apt_update(fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) + from jinja2 import FileSystemLoader, Environment + + +# NOTE: function separated from main rendering code to facilitate easier +# mocking in unit tests. +def write(path, data): + with open(path, 'wb') as out: + out.write(data) + + +def get_template_path(template_dir, path): + """Returns the template file which would be used to render the path. + + The path to the template file is returned. + :param template_dir: the directory the templates are located in + :param path: the file path to be written to. + :returns: path to the template file + """ + return os.path.join(template_dir, os.path.basename(path)) + + +def render_and_write(template_dir, path, context): + """Renders the specified template into the file. + + :param template_dir: the directory to load the template from + :param path: the path to write the templated contents to + :param context: the parameters to pass to the rendering engine + """ + env = Environment(loader=FileSystemLoader(template_dir)) + template_file = os.path.basename(path) + template = env.get_template(template_file) + log('Rendering from template: %s' % template.name, level=DEBUG) + rendered_content = template.render(context) + if not rendered_content: + log("Render returned None - skipping '%s'" % path, + level=WARNING) + return + + write(path, rendered_content.encode('utf-8').strip()) + log('Wrote template %s' % path, level=DEBUG) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/utils.py new file mode 100644 index 0000000..56afa4b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/hardening/utils.py @@ -0,0 +1,155 @@ +# Copyright 2016-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import grp +import os +import pwd +import six +import yaml + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + INFO, + WARNING, + ERROR, +) + + +# Global settings cache. Since each hook fire entails a fresh module import it +# is safe to hold this in memory and not risk missing config changes (since +# they will result in a new hook fire and thus re-import). +__SETTINGS__ = {} + + +def _get_defaults(modules): + """Load the default config for the provided modules. + + :param modules: stack modules config defaults to lookup. + :returns: modules default config dictionary. + """ + default = os.path.join(os.path.dirname(__file__), + 'defaults/%s.yaml' % (modules)) + return yaml.safe_load(open(default)) + + +def _get_schema(modules): + """Load the config schema for the provided modules. + + NOTE: this schema is intended to have 1-1 relationship with they keys in + the default config and is used a means to verify valid overrides provided + by the user. + + :param modules: stack modules config schema to lookup. + :returns: modules default schema dictionary. + """ + schema = os.path.join(os.path.dirname(__file__), + 'defaults/%s.yaml.schema' % (modules)) + return yaml.safe_load(open(schema)) + + +def _get_user_provided_overrides(modules): + """Load user-provided config overrides. + + :param modules: stack modules to lookup in user overrides yaml file. + :returns: overrides dictionary. + """ + overrides = os.path.join(os.environ['JUJU_CHARM_DIR'], + 'hardening.yaml') + if os.path.exists(overrides): + log("Found user-provided config overrides file '%s'" % + (overrides), level=DEBUG) + settings = yaml.safe_load(open(overrides)) + if settings and settings.get(modules): + log("Applying '%s' overrides" % (modules), level=DEBUG) + return settings.get(modules) + + log("No overrides found for '%s'" % (modules), level=DEBUG) + else: + log("No hardening config overrides file '%s' found in charm " + "root dir" % (overrides), level=DEBUG) + + return {} + + +def _apply_overrides(settings, overrides, schema): + """Get overrides config overlaid onto modules defaults. + + :param modules: require stack modules config. + :returns: dictionary of modules config with user overrides applied. + """ + if overrides: + for k, v in six.iteritems(overrides): + if k in schema: + if schema[k] is None: + settings[k] = v + elif type(schema[k]) is dict: + settings[k] = _apply_overrides(settings[k], overrides[k], + schema[k]) + else: + raise Exception("Unexpected type found in schema '%s'" % + type(schema[k]), level=ERROR) + else: + log("Unknown override key '%s' - ignoring" % (k), level=INFO) + + return settings + + +def get_settings(modules): + global __SETTINGS__ + if modules in __SETTINGS__: + return __SETTINGS__[modules] + + schema = _get_schema(modules) + settings = _get_defaults(modules) + overrides = _get_user_provided_overrides(modules) + __SETTINGS__[modules] = _apply_overrides(settings, overrides, schema) + return __SETTINGS__[modules] + + +def ensure_permissions(path, user, group, permissions, maxdepth=-1): + """Ensure permissions for path. + + If path is a file, apply to file and return. If path is a directory, + apply recursively (if required) to directory contents and return. + + :param user: user name + :param group: group name + :param permissions: octal permissions + :param maxdepth: maximum recursion depth. A negative maxdepth allows + infinite recursion and maxdepth=0 means no recursion. + :returns: None + """ + if not os.path.exists(path): + log("File '%s' does not exist - cannot set permissions" % (path), + level=WARNING) + return + + _user = pwd.getpwnam(user) + os.chown(path, _user.pw_uid, grp.getgrnam(group).gr_gid) + os.chmod(path, permissions) + + if maxdepth == 0: + log("Max recursion depth reached - skipping further recursion", + level=DEBUG) + return + elif maxdepth > 0: + maxdepth -= 1 + + if os.path.isdir(path): + contents = glob.glob("%s/*" % (path)) + for c in contents: + ensure_permissions(c, user=user, group=group, + permissions=permissions, maxdepth=maxdepth) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/infiniband.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/infiniband.py new file mode 100644 index 0000000..0edb231 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/mellanox/infiniband.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Jorge Niedbalski " + +import six + +from charmhelpers.fetch import ( + apt_install, + apt_update, +) + +from charmhelpers.core.hookenv import ( + log, + INFO, +) + +try: + from netifaces import interfaces as network_interfaces +except ImportError: + if six.PY2: + apt_install('python-netifaces') + else: + apt_install('python3-netifaces') + from netifaces import interfaces as network_interfaces + +import os +import re +import subprocess + +from charmhelpers.core.kernel import modprobe + +REQUIRED_MODULES = ( + "mlx4_ib", + "mlx4_en", + "mlx4_core", + "ib_ipath", + "ib_mthca", + "ib_srpt", + "ib_srp", + "ib_ucm", + "ib_isert", + "ib_iser", + "ib_ipoib", + "ib_cm", + "ib_uverbs" + "ib_umad", + "ib_sa", + "ib_mad", + "ib_core", + "ib_addr", + "rdma_ucm", +) + +REQUIRED_PACKAGES = ( + "ibutils", + "infiniband-diags", + "ibverbs-utils", +) + +IPOIB_DRIVERS = ( + "ib_ipoib", +) + +ABI_VERSION_FILE = "/sys/class/infiniband_mad/abi_version" + + +class DeviceInfo(object): + pass + + +def install_packages(): + apt_update() + apt_install(REQUIRED_PACKAGES, fatal=True) + + +def load_modules(): + for module in REQUIRED_MODULES: + modprobe(module, persist=True) + + +def is_enabled(): + """Check if infiniband is loaded on the system""" + return os.path.exists(ABI_VERSION_FILE) + + +def stat(): + """Return full output of ibstat""" + return subprocess.check_output(["ibstat"]) + + +def devices(): + """Returns a list of IB enabled devices""" + return subprocess.check_output(['ibstat', '-l']).splitlines() + + +def device_info(device): + """Returns a DeviceInfo object with the current device settings""" + + status = subprocess.check_output([ + 'ibstat', device, '-s']).splitlines() + + regexes = { + "CA type: (.*)": "device_type", + "Number of ports: (.*)": "num_ports", + "Firmware version: (.*)": "fw_ver", + "Hardware version: (.*)": "hw_ver", + "Node GUID: (.*)": "node_guid", + "System image GUID: (.*)": "sys_guid", + } + + device = DeviceInfo() + + for line in status: + for expression, key in regexes.items(): + matches = re.search(expression, line) + if matches: + setattr(device, key, matches.group(1)) + + return device + + +def ipoib_interfaces(): + """Return a list of IPOIB capable ethernet interfaces""" + interfaces = [] + + for interface in network_interfaces(): + try: + driver = re.search('^driver: (.+)$', subprocess.check_output([ + 'ethtool', '-i', + interface]), re.M).group(1) + + if driver in IPOIB_DRIVERS: + interfaces.append(interface) + except Exception: + log("Skipping interface %s" % interface, level=INFO) + continue + + return interfaces diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ip.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ip.py new file mode 100644 index 0000000..b356d64 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ip.py @@ -0,0 +1,603 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import re +import subprocess +import six +import socket + +from functools import partial + +from charmhelpers.fetch import apt_install, apt_update +from charmhelpers.core.hookenv import ( + config, + log, + network_get_primary_address, + unit_get, + WARNING, + NoNetworkBinding, +) + +from charmhelpers.core.host import ( + lsb_release, + CompareHostReleases, +) + +try: + import netifaces +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) + import netifaces + +try: + import netaddr +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install('python-netaddr', fatal=True) + else: + apt_install('python3-netaddr', fatal=True) + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def no_ip_found_error_out(network): + errmsg = ("No IP address found in network(s): %s" % network) + raise ValueError(errmsg) + + +def _get_ipv6_network_from_address(address): + """Get an netaddr.IPNetwork for the given IPv6 address + :param address: a dict as returned by netifaces.ifaddresses + :returns netaddr.IPNetwork: None if the address is a link local or loopback + address + """ + if address['addr'].startswith('fe80') or address['addr'] == "::1": + return None + + prefix = address['netmask'].split("/") + if len(prefix) > 1: + netmask = prefix[1] + else: + netmask = address['netmask'] + return netaddr.IPNetwork("%s/%s" % (address['addr'], + netmask)) + + +def get_address_in_network(network, fallback=None, fatal=False): + """Get an IPv4 or IPv6 address within the network from the host. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. Supports multiple networks as a space-delimited list. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ + if network is None: + if fallback is not None: + return fallback + + if fatal: + no_ip_found_error_out(network) + else: + return None + + networks = network.split() or [network] + for network in networks: + _validate_cidr(network) + network = netaddr.IPNetwork(network) + for iface in netifaces.interfaces(): + try: + addresses = netifaces.ifaddresses(iface) + except ValueError: + # If an instance was deleted between + # netifaces.interfaces() run and now, its interfaces are gone + continue + if network.version == 4 and netifaces.AF_INET in addresses: + for addr in addresses[netifaces.AF_INET]: + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) + + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + cidr = _get_ipv6_network_from_address(addr) + if cidr and cidr in network: + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + no_ip_found_error_out(network) + + return None + + +def is_ipv6(address): + """Determine whether provided address is IPv6 or not.""" + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + + return address.version == 6 + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + + if address in network: + return True + else: + return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + network = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + cidr = network.cidr + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + network = _get_ipv6_network_from_address(addr) + if not network: + continue + + cidr = network.cidr + if address in cidr: + if key == 'iface': + return iface + elif key == 'netmask' and cidr: + return str(cidr).split('/')[1] + else: + return addr[key] + return None + + +get_iface_for_address = partial(_get_for_address, key='iface') + + +get_netmask_for_address = partial(_get_for_address, key='netmask') + + +def resolve_network_cidr(ip_address): + ''' + Resolves the full address cidr of an ip_address based on + configured network interfaces + ''' + netmask = get_netmask_for_address(ip_address) + return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr) + + +def format_ipv6_addr(address): + """If address is IPv6, wrap it in '[]' otherwise return None. + + This is required by most configuration files when specifying IPv6 + addresses. + """ + if is_ipv6(address): + return "[%s]" % address + + return None + + +def is_ipv6_disabled(): + try: + result = subprocess.check_output( + ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], + stderr=subprocess.STDOUT, + universal_newlines=True) + except subprocess.CalledProcessError: + return True + + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + + +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, + fatal=True, exc_list=None): + """Return the assigned IP address for a given interface, if any. + + :param iface: network interface on which address(es) are expected to + be found. + :param inet_type: inet address family + :param inc_aliases: include alias interfaces in search + :param fatal: if True, raise exception if address not found + :param exc_list: list of addresses to ignore + :return: list of ip addresses + """ + # Extract nic if passed /dev/ethX + if '/' in iface: + iface = iface.split('/')[-1] + + if not exc_list: + exc_list = [] + + try: + inet_num = getattr(netifaces, inet_type) + except AttributeError: + raise Exception("Unknown inet type '%s'" % str(inet_type)) + + interfaces = netifaces.interfaces() + if inc_aliases: + ifaces = [] + for _iface in interfaces: + if iface == _iface or _iface.split(':')[0] == iface: + ifaces.append(_iface) + + if fatal and not ifaces: + raise Exception("Invalid interface '%s'" % iface) + + ifaces.sort() + else: + if iface not in interfaces: + if fatal: + raise Exception("Interface '%s' not found " % (iface)) + else: + return [] + + else: + ifaces = [iface] + + addresses = [] + for netiface in ifaces: + net_info = netifaces.ifaddresses(netiface) + if inet_num in net_info: + for entry in net_info[inet_num]: + if 'addr' in entry and entry['addr'] not in exc_list: + addresses.append(entry['addr']) + + if fatal and not addresses: + raise Exception("Interface '%s' doesn't have any %s addresses." % + (iface, inet_type)) + + return sorted(addresses) + + +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') + + +def get_iface_from_addr(addr): + """Work out on which interface the provided address is configured.""" + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + for inet_type in addresses: + for _addr in addresses[inet_type]: + _addr = _addr['addr'] + # link local + ll_key = re.compile("(.+)%.*") + raw = re.match(ll_key, _addr) + if raw: + _addr = raw.group(1) + + if _addr == addr: + log("Address '%s' is configured on iface '%s'" % + (addr, iface)) + return iface + + msg = "Unable to infer net iface on which '%s' is configured" % (addr) + raise Exception(msg) + + +def sniff_iface(f): + """Ensure decorated function is called with a value for iface. + + If no iface provided, inject net iface inferred from unit private address. + """ + def iface_sniffer(*args, **kwargs): + if not kwargs.get('iface', None): + kwargs['iface'] = get_iface_from_addr(unit_get('private-address')) + + return f(*args, **kwargs) + + return iface_sniffer + + +@sniff_iface +def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, + dynamic_only=True): + """Get assigned IPv6 address for a given interface. + + Returns list of addresses found. If no address found, returns empty list. + + If iface is None, we infer the current primary interface by doing a reverse + lookup on the unit private-address. + + We currently only support scope global IPv6 addresses i.e. non-temporary + addresses. If no global IPv6 address is found, return the first one found + in the ipv6 address list. + + :param iface: network interface on which ipv6 address(es) are expected to + be found. + :param inc_aliases: include alias interfaces in search + :param fatal: if True, raise exception if address not found + :param exc_list: list of addresses to ignore + :param dynamic_only: only recognise dynamic addresses + :return: list of ipv6 addresses + """ + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', + inc_aliases=inc_aliases, fatal=fatal, + exc_list=exc_list) + + if addresses: + global_addrs = [] + for addr in addresses: + key_scope_link_local = re.compile("^fe80::..(.+)%(.+)") + m = re.match(key_scope_link_local, addr) + if m: + eui_64_mac = m.group(1) + iface = m.group(2) + else: + global_addrs.append(addr) + + if global_addrs: + # Make sure any found global addresses are not temporary + cmd = ['ip', 'addr', 'show', iface] + out = subprocess.check_output( + cmd).decode('UTF-8', errors='replace') + if dynamic_only: + key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*") + else: + key = re.compile("inet6 (.+)/[0-9]+ scope global.*") + + addrs = [] + for line in out.split('\n'): + line = line.strip() + m = re.match(key, line) + if m and 'temporary' not in line: + # Return the first valid address we find + for addr in global_addrs: + if m.group(1) == addr: + if not dynamic_only or \ + m.group(1).endswith(eui_64_mac): + addrs.append(addr) + + if addrs: + return addrs + + if fatal: + raise Exception("Interface '%s' does not have a scope global " + "non-temporary ipv6 address." % iface) + + return [] + + +def get_bridges(vnic_dir='/sys/devices/virtual/net'): + """Return a list of bridges on the system.""" + b_regex = "%s/*/bridge" % vnic_dir + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)] + + +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): + """Return a list of nics comprising a given bridge on the system.""" + brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_regex)] + + +def is_bridge_member(nic): + """Check if a given nic is a member of a bridge.""" + for bridge in get_bridges(): + if nic in get_bridge_nics(bridge): + return True + + return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4/IPv6 address + address = netaddr.IPAddress(address) + return True + except (netaddr.AddrFormatError, ValueError): + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + if six.PY2: + apt_install('python-dnspython', fatal=True) + else: + apt_install('python3-dnspython', fatal=True) + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + try: + answers = dns.resolver.query(address, rtype) + except dns.resolver.NXDOMAIN: + return None + + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except Exception: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + if six.PY2: + apt_install("python-dnspython", fatal=True) + else: + apt_install("python3-dnspython", fatal=True) + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + + if not result: + try: + result = socket.gethostbyaddr(address)[0] + except Exception: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] + + +def port_has_listener(address, port): + """ + Returns True if the address:port is open and being listened to, + else False. + + @param address: an IP address or hostname + @param port: integer port + + Note calls 'zc' via a subprocess shell + """ + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not(bool(result)) + + +def assert_charm_supports_ipv6(): + """Check whether we are able to support charms ipv6.""" + release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(release) < "trusty": + raise Exception("IPv6 is not supported in the charms for Ubuntu " + "versions less than Trusty 14.04") + + +def get_relation_ip(interface, cidr_network=None): + """Return this unit's IP for the given interface. + + Allow for an arbitrary interface to use with network-get to select an IP. + Handle all address selection options including passed cidr network and + IPv6. + + Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8') + + @param interface: string name of the relation. + @param cidr_network: string CIDR Network to select an address from. + @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. + @returns IPv6 or IPv4 address + """ + # Select the interface address first + # For possible use as a fallback below with get_address_in_network + try: + # Get the interface specific IP + address = network_get_primary_address(interface) + except NotImplementedError: + # If network-get is not available + address = get_host_ip(unit_get('private-address')) + except NoNetworkBinding: + log("No network binding for {}".format(interface), WARNING) + address = get_host_ip(unit_get('private-address')) + + if config('prefer-ipv6'): + # Currently IPv6 has priority, eventually we want IPv6 to just be + # another network space. + assert_charm_supports_ipv6() + return get_ipv6_addr()[0] + elif cidr_network: + # If a specific CIDR network is passed get the address from that + # network. + return get_address_in_network(cidr_network, address) + + # Return the interface address + return address diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/__init__.py new file mode 100644 index 0000000..d9f004a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/__init__.py @@ -0,0 +1,693 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Helpers for interacting with OpenvSwitch ''' +import collections +import hashlib +import os +import re +import six +import subprocess + +from charmhelpers import deprecate +from charmhelpers.contrib.network.ovs import ovsdb as ch_ovsdb +from charmhelpers.fetch import apt_install + +from charmhelpers.core.hookenv import ( + log, WARNING, INFO, DEBUG, charm_name +) +from charmhelpers.core.host import ( + CompareHostReleases, + lsb_release, + service +) + + +BRIDGE_TEMPLATE = """\ +# This veth pair is required when neutron data-port is mapped to an existing linux bridge. lp:1635067 + +auto {linuxbridge_port} +iface {linuxbridge_port} inet manual + pre-up ip link add name {linuxbridge_port} type veth peer name {ovsbridge_port} + pre-up ip link set {ovsbridge_port} master {bridge} + pre-up ip link set {ovsbridge_port} up + up ip link set {linuxbridge_port} up + down ip link del {linuxbridge_port} +""" + +MAX_KERNEL_INTERFACE_NAME_LEN = 15 + + +def get_bridges(): + """Return list of the bridges on the default openvswitch + + :returns: List of bridge names + :rtype: List[str] + :raises: subprocess.CalledProcessError if ovs-vsctl fails + """ + cmd = ["ovs-vsctl", "list-br"] + lines = subprocess.check_output(cmd).decode('utf-8').split("\n") + maybe_bridges = [l.strip() for l in lines] + return [b for b in maybe_bridges if b] + + +def get_bridge_ports(name): + """Return a list the ports on a named bridge + + :param name: the name of the bridge to list + :type name: str + :returns: List of ports on the named bridge + :rtype: List[str] + :raises: subprocess.CalledProcessError if the ovs-vsctl command fails. If + the named bridge doesn't exist, then the exception will be raised. + """ + cmd = ["ovs-vsctl", "--", "list-ports", name] + lines = subprocess.check_output(cmd).decode('utf-8').split("\n") + maybe_ports = [l.strip() for l in lines] + return [p for p in maybe_ports if p] + + +def get_bridges_and_ports_map(): + """Return dictionary of bridge to ports for the default openvswitch + + :returns: a mapping of bridge name to a list of ports. + :rtype: Dict[str, List[str]] + :raises: subprocess.CalledProcessError if any of the underlying ovs-vsctl + command fail. + """ + return {b: get_bridge_ports(b) for b in get_bridges()} + + +def _dict_to_vsctl_set(data, table, entity): + """Helper that takes dictionary and provides ``ovs-vsctl set`` commands + + :param data: Additional data to attach to interface + The keys in the data dictionary map directly to column names in the + OpenvSwitch table specified as defined in DB-SCHEMA [0] referenced in + RFC 7047 [1] + + There are some established conventions for keys in the external-ids + column of various tables, consult the OVS Integration Guide [2] for + more details. + + NOTE(fnordahl): Technically the ``external-ids`` column is called + ``external_ids`` (with an underscore) and we rely on ``ovs-vsctl``'s + behaviour of transforming dashes to underscores for us [3] so we can + have a more pleasant data structure. + + 0: http://www.openvswitch.org/ovs-vswitchd.conf.db.5.pdf + 1: https://tools.ietf.org/html/rfc7047 + 2: http://docs.openvswitch.org/en/latest/topics/integration/ + 3: https://github.com/openvswitch/ovs/blob/ + 20dac08fdcce4b7fda1d07add3b346aa9751cfbc/ + lib/db-ctl-base.c#L189-L215 + :type data: Optional[Dict[str,Union[str,Dict[str,str]]]] + :param table: Name of table to operate on + :type table: str + :param entity: Name of entity to operate on + :type entity: str + :returns: '--' separated ``ovs-vsctl set`` commands + :rtype: Iterator[Tuple[str, str, str, str, str]] + """ + for (k, v) in data.items(): + if isinstance(v, dict): + entries = { + '{}:{}'.format(k, dk): dv for (dk, dv) in v.items()} + else: + entries = {k: v} + for (colk, colv) in entries.items(): + yield ('--', 'set', table, entity, '{}={}'.format(colk, colv)) + + +def add_bridge(name, datapath_type=None, brdata=None, exclusive=False): + """Add the named bridge to openvswitch and set/update bridge data for it + + :param name: Name of bridge to create + :type name: str + :param datapath_type: Add datapath_type to bridge (DEPRECATED, use brdata) + :type datapath_type: Optional[str] + :param brdata: Additional data to attach to bridge + The keys in the brdata dictionary map directly to column names in the + OpenvSwitch bridge table as defined in DB-SCHEMA [0] referenced in + RFC 7047 [1] + + There are some established conventions for keys in the external-ids + column of various tables, consult the OVS Integration Guide [2] for + more details. + + NOTE(fnordahl): Technically the ``external-ids`` column is called + ``external_ids`` (with an underscore) and we rely on ``ovs-vsctl``'s + behaviour of transforming dashes to underscores for us [3] so we can + have a more pleasant data structure. + + 0: http://www.openvswitch.org/ovs-vswitchd.conf.db.5.pdf + 1: https://tools.ietf.org/html/rfc7047 + 2: http://docs.openvswitch.org/en/latest/topics/integration/ + 3: https://github.com/openvswitch/ovs/blob/ + 20dac08fdcce4b7fda1d07add3b346aa9751cfbc/ + lib/db-ctl-base.c#L189-L215 + :type brdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + :param exclusive: If True, raise exception if bridge exists + :type exclusive: bool + :raises: subprocess.CalledProcessError + """ + log('Creating bridge {}'.format(name)) + cmd = ['ovs-vsctl', '--'] + if not exclusive: + cmd.append('--may-exist') + cmd.extend(('add-br', name)) + if brdata: + for setcmd in _dict_to_vsctl_set(brdata, 'bridge', name): + cmd.extend(setcmd) + if datapath_type is not None: + log('DEPRECATION WARNING: add_bridge called with datapath_type, ' + 'please use the brdata keyword argument instead.') + cmd += ['--', 'set', 'bridge', name, + 'datapath_type={}'.format(datapath_type)] + subprocess.check_call(cmd) + + +def del_bridge(name): + """Delete the named bridge from openvswitch + + :param name: Name of bridge to remove + :type name: str + :raises: subprocess.CalledProcessError + """ + log('Deleting bridge {}'.format(name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name]) + + +def add_bridge_port(name, port, promisc=False, ifdata=None, exclusive=False, + linkup=True, portdata=None): + """Add port to bridge and optionally set/update interface data for it + + :param name: Name of bridge to attach port to + :type name: str + :param port: Name of port as represented in netdev + :type port: str + :param promisc: Whether to set promiscuous mode on interface + True=on, False=off, None leave untouched + :type promisc: Optional[bool] + :param ifdata: Additional data to attach to interface + The keys in the ifdata dictionary map directly to column names in the + OpenvSwitch Interface table as defined in DB-SCHEMA [0] referenced in + RFC 7047 [1] + + There are some established conventions for keys in the external-ids + column of various tables, consult the OVS Integration Guide [2] for + more details. + + NOTE(fnordahl): Technically the ``external-ids`` column is called + ``external_ids`` (with an underscore) and we rely on ``ovs-vsctl``'s + behaviour of transforming dashes to underscores for us [3] so we can + have a more pleasant data structure. + + 0: http://www.openvswitch.org/ovs-vswitchd.conf.db.5.pdf + 1: https://tools.ietf.org/html/rfc7047 + 2: http://docs.openvswitch.org/en/latest/topics/integration/ + 3: https://github.com/openvswitch/ovs/blob/ + 20dac08fdcce4b7fda1d07add3b346aa9751cfbc/ + lib/db-ctl-base.c#L189-L215 + :type ifdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + :param exclusive: If True, raise exception if port exists + :type exclusive: bool + :param linkup: Bring link up + :type linkup: bool + :param portdata: Additional data to attach to port. Similar to ifdata. + :type portdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + :raises: subprocess.CalledProcessError + """ + cmd = ['ovs-vsctl', '--'] + if not exclusive: + cmd.append('--may-exist') + cmd.extend(('add-port', name, port)) + for ovs_table, data in (('Interface', ifdata), ('Port', portdata)): + if data: + for setcmd in _dict_to_vsctl_set(data, ovs_table, port): + cmd.extend(setcmd) + + log('Adding port {} to bridge {}'.format(port, name)) + subprocess.check_call(cmd) + if linkup: + # This is mostly a workaround for CI environments, in the real world + # the bare metal provider would most likely have configured and brought + # up the link for us. + subprocess.check_call(["ip", "link", "set", port, "up"]) + if promisc: + subprocess.check_call(["ip", "link", "set", port, "promisc", "on"]) + elif promisc is False: + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) + + +def del_bridge_port(name, port, linkdown=True): + """Delete a port from the named openvswitch bridge + + :param name: Name of bridge to remove port from + :type name: str + :param port: Name of port to remove + :type port: str + :param linkdown: Whether to set link down on interface (default: True) + :type linkdown: bool + :raises: subprocess.CalledProcessError + """ + log('Deleting port {} from bridge {}'.format(port, name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port", + name, port]) + if linkdown: + subprocess.check_call(["ip", "link", "set", port, "down"]) + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) + + +def add_bridge_bond(bridge, port, interfaces, portdata=None, ifdatamap=None, + exclusive=False): + """Add bonded port in bridge from interfaces. + + :param bridge: Name of bridge to add bonded port to + :type bridge: str + :param port: Name of created port + :type port: str + :param interfaces: Underlying interfaces that make up the bonded port + :type interfaces: Iterator[str] + :param portdata: Additional data to attach to the created bond port + See _dict_to_vsctl_set() for detailed description. + Example: + { + 'bond-mode': 'balance-tcp', + 'lacp': 'active', + 'other-config': { + 'lacp-time': 'fast', + }, + } + :type portdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + :param ifdatamap: Map of data to attach to created bond interfaces + See _dict_to_vsctl_set() for detailed description. + Example: + { + 'eth0': { + 'type': 'dpdk', + 'mtu-request': '9000', + 'options': { + 'dpdk-devargs': '0000:01:00.0', + }, + }, + } + :type ifdatamap: Optional[Dict[str,Dict[str,Union[str,Dict[str,str]]]]] + :param exclusive: If True, raise exception if port exists + :type exclusive: bool + :raises: subprocess.CalledProcessError + """ + cmd = ['ovs-vsctl', '--'] + if not exclusive: + cmd.append('--may-exist') + cmd.extend(('add-bond', bridge, port)) + cmd.extend(interfaces) + if portdata: + for setcmd in _dict_to_vsctl_set(portdata, 'port', port): + cmd.extend(setcmd) + if ifdatamap: + for ifname, ifdata in ifdatamap.items(): + for setcmd in _dict_to_vsctl_set(ifdata, 'Interface', ifname): + cmd.extend(setcmd) + subprocess.check_call(cmd) + + +@deprecate('see lp:1877594', '2021-01', log=log) +def add_ovsbridge_linuxbridge(name, bridge, ifdata=None, portdata=None): + """Add linux bridge to the named openvswitch bridge + + :param name: Name of ovs bridge to be added to Linux bridge + :type name: str + :param bridge: Name of Linux bridge to be added to ovs bridge + :type name: str + :param ifdata: Additional data to attach to interface + The keys in the ifdata dictionary map directly to column names in the + OpenvSwitch Interface table as defined in DB-SCHEMA [0] referenced in + RFC 7047 [1] + + There are some established conventions for keys in the external-ids + column of various tables, consult the OVS Integration Guide [2] for + more details. + + NOTE(fnordahl): Technically the ``external-ids`` column is called + ``external_ids`` (with an underscore) and we rely on ``ovs-vsctl``'s + behaviour of transforming dashes to underscores for us [3] so we can + have a more pleasant data structure. + + 0: http://www.openvswitch.org/ovs-vswitchd.conf.db.5.pdf + 1: https://tools.ietf.org/html/rfc7047 + 2: http://docs.openvswitch.org/en/latest/topics/integration/ + 3: https://github.com/openvswitch/ovs/blob/ + 20dac08fdcce4b7fda1d07add3b346aa9751cfbc/ + lib/db-ctl-base.c#L189-L215 + :type ifdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + :param portdata: Additional data to attach to port. Similar to ifdata. + :type portdata: Optional[Dict[str,Union[str,Dict[str,str]]]] + + WARNINGS: + * The `ifup` command (NetworkManager) must be available on the system for + this to work. Before bionic this was shipped by default. On bionic and + newer you need to install the package `ifupdown`. This might however cause + issues when deploying to LXD, see lp:1877594, which is why this function + isn't supported anymore. + * On focal and newer this function won't even try to run `ifup` and raise + directly. + """ + try: + import netifaces + except ImportError: + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) + import netifaces + + # NOTE(jamespage): + # Older code supported addition of a linuxbridge directly + # to an OVS bridge; ensure we don't break uses on upgrade + existing_ovs_bridge = port_to_br(bridge) + if existing_ovs_bridge is not None: + log('Linuxbridge {} is already directly in use' + ' by OVS bridge {}'.format(bridge, existing_ovs_bridge), + level=INFO) + return + + # NOTE(jamespage): + # preserve existing naming because interfaces may already exist. + ovsbridge_port = "veth-" + name + linuxbridge_port = "veth-" + bridge + if (len(ovsbridge_port) > MAX_KERNEL_INTERFACE_NAME_LEN or + len(linuxbridge_port) > MAX_KERNEL_INTERFACE_NAME_LEN): + # NOTE(jamespage): + # use parts of hashed bridgename (openstack style) when + # a bridge name exceeds 15 chars + hashed_bridge = hashlib.sha256(bridge.encode('UTF-8')).hexdigest() + base = '{}-{}'.format(hashed_bridge[:8], hashed_bridge[-2:]) + ovsbridge_port = "cvo{}".format(base) + linuxbridge_port = "cvb{}".format(base) + + network_interface_already_exists = False + interfaces = netifaces.interfaces() + for interface in interfaces: + if interface == ovsbridge_port or interface == linuxbridge_port: + log('Interface {} already exists'.format(interface), level=INFO) + network_interface_already_exists = True + break + + log('Adding linuxbridge {} to ovsbridge {}'.format(bridge, name), + level=INFO) + + if not network_interface_already_exists: + setup_eni() # will raise on focal+ + + with open('/etc/network/interfaces.d/{}.cfg'.format( + linuxbridge_port), 'w') as config: + config.write(BRIDGE_TEMPLATE.format( + linuxbridge_port=linuxbridge_port, + ovsbridge_port=ovsbridge_port, bridge=bridge)) + + try: + # NOTE(lourot): 'ifup ' can't be replaced by + # 'ip link set up' as the latter won't parse + # /etc/network/interfaces* + subprocess.check_call(['ifup', linuxbridge_port]) + except FileNotFoundError: + # NOTE(lourot): on bionic and newer, 'ifup' isn't installed by + # default. It has been replaced by netplan.io but we can't use it + # yet because of lp:1876730. For the time being, charms using this + # have to install 'ifupdown' on bionic and newer. This will however + # cause issues when deploying to LXD, see lp:1877594. + raise RuntimeError('ifup: command not found. Did this charm forget ' + 'to install ifupdown?') + + add_bridge_port(name, linuxbridge_port, ifdata=ifdata, exclusive=False, + portdata=portdata) + + +def is_linuxbridge_interface(port): + ''' Check if the interface is a linuxbridge bridge + :param port: Name of an interface to check whether it is a Linux bridge + :returns: True if port is a Linux bridge''' + + if os.path.exists('/sys/class/net/' + port + '/bridge'): + log('Interface {} is a Linux bridge'.format(port), level=DEBUG) + return True + else: + log('Interface {} is not a Linux bridge'.format(port), level=DEBUG) + return False + + +def set_manager(manager): + ''' Set the controller for the local openvswitch ''' + log('Setting manager for local ovs to {}'.format(manager)) + subprocess.check_call(['ovs-vsctl', 'set-manager', + 'ssl:{}'.format(manager)]) + + +def set_Open_vSwitch_column_value(column_value): + """ + Calls ovs-vsctl and sets the 'column_value' in the Open_vSwitch table. + + :param column_value: + See http://www.openvswitch.org//ovs-vswitchd.conf.db.5.pdf for + details of the relevant values. + :type str + :raises CalledProcessException: possibly ovsdb-server is not running + """ + log('Setting {} in the Open_vSwitch table'.format(column_value)) + subprocess.check_call(['ovs-vsctl', 'set', 'Open_vSwitch', '.', column_value]) + + +CERT_PATH = '/etc/openvswitch/ovsclient-cert.pem' + + +def get_certificate(): + ''' Read openvswitch certificate from disk ''' + if os.path.exists(CERT_PATH): + log('Reading ovs certificate from {}'.format(CERT_PATH)) + with open(CERT_PATH, 'r') as cert: + full_cert = cert.read() + begin_marker = "-----BEGIN CERTIFICATE-----" + end_marker = "-----END CERTIFICATE-----" + begin_index = full_cert.find(begin_marker) + end_index = full_cert.rfind(end_marker) + if end_index == -1 or begin_index == -1: + raise RuntimeError("Certificate does not contain valid begin" + " and end markers.") + full_cert = full_cert[begin_index:(end_index + len(end_marker))] + return full_cert + else: + log('Certificate not found', level=WARNING) + return None + + +@deprecate('see lp:1877594', '2021-01', log=log) +def setup_eni(): + """Makes sure /etc/network/interfaces.d/ exists and will be parsed. + + When setting up interfaces, Juju removes from + /etc/network/interfaces the line sourcing interfaces.d/ + + WARNING: Not supported on focal and newer anymore. Will raise. + """ + release = CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) + if release >= 'focal': + raise RuntimeError("NetworkManager isn't supported anymore") + + if not os.path.exists('/etc/network/interfaces.d'): + os.makedirs('/etc/network/interfaces.d', mode=0o755) + with open('/etc/network/interfaces', 'r') as eni: + for line in eni: + if re.search(r'^\s*source\s+/etc/network/interfaces.d/\*\s*$', + line): + return + with open('/etc/network/interfaces', 'a') as eni: + eni.write('\nsource /etc/network/interfaces.d/*') + + +@deprecate('use setup_eni() instead', '2021-01', log=log) +def check_for_eni_source(): + setup_eni() + + +def full_restart(): + ''' Full restart and reload of openvswitch ''' + if os.path.exists('/etc/init/openvswitch-force-reload-kmod.conf'): + service('start', 'openvswitch-force-reload-kmod') + else: + service('force-reload-kmod', 'openvswitch-switch') + + +def enable_ipfix(bridge, target, + cache_active_timeout=60, + cache_max_flows=128, + sampling=64): + '''Enable IPFIX on bridge to target. + :param bridge: Bridge to monitor + :param target: IPFIX remote endpoint + :param cache_active_timeout: The maximum period in seconds for + which an IPFIX flow record is cached + and aggregated before being sent + :param cache_max_flows: The maximum number of IPFIX flow records + that can be cached at a time + :param sampling: The rate at which packets should be sampled and + sent to each target collector + ''' + cmd = [ + 'ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--', + '--id=@i', 'create', 'IPFIX', + 'targets="{}"'.format(target), + 'sampling={}'.format(sampling), + 'cache_active_timeout={}'.format(cache_active_timeout), + 'cache_max_flows={}'.format(cache_max_flows), + ] + log('Enabling IPfix on {}.'.format(bridge)) + subprocess.check_call(cmd) + + +def disable_ipfix(bridge): + '''Diable IPFIX on target bridge. + :param bridge: Bridge to modify + ''' + cmd = ['ovs-vsctl', 'clear', 'Bridge', bridge, 'ipfix'] + subprocess.check_call(cmd) + + +def port_to_br(port): + '''Determine the bridge that contains a port + :param port: Name of port to check for + :returns str: OVS bridge containing port or None if not found + ''' + try: + return subprocess.check_output( + ['ovs-vsctl', 'port-to-br', port] + ).decode('UTF-8').strip() + except subprocess.CalledProcessError: + return None + + +def ovs_appctl(target, args): + """Run `ovs-appctl` for target with args and return output. + + :param target: Name of daemon to contact. Unless target begins with '/', + `ovs-appctl` looks for a pidfile and will build the path to + a /var/run/openvswitch/target.pid.ctl for you. + :type target: str + :param args: Command and arguments to pass to `ovs-appctl` + :type args: Tuple[str, ...] + :returns: Output from command + :rtype: str + :raises: subprocess.CalledProcessError + """ + cmd = ['ovs-appctl', '-t', target] + cmd.extend(args) + return subprocess.check_output(cmd, universal_newlines=True) + + +def uuid_for_port(port_name): + """Get UUID of named port. + + :param port_name: Name of port. + :type port_name: str + :returns: Port UUID. + :rtype: Optional[uuid.UUID] + """ + for port in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').port.find('name={}'.format(port_name)): + return port['_uuid'] + + +def bridge_for_port(port_uuid): + """Find which bridge a port is on. + + :param port_uuid: UUID of port. + :type port_uuid: uuid.UUID + :returns: Name of bridge or None. + :rtype: Optional[str] + """ + for bridge in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').bridge: + # If there is a single port on a bridge the ports property will not be + # a list. ref: juju/charm-helpers#510 + if (isinstance(bridge['ports'], list) and + port_uuid in bridge['ports'] or + port_uuid == bridge['ports']): + return bridge['name'] + + +PatchPort = collections.namedtuple('PatchPort', ('bridge', 'port')) +Patch = collections.namedtuple('Patch', ('this_end', 'other_end')) + + +def patch_ports_on_bridge(bridge): + """Find patch ports on a bridge. + + :param bridge: Name of bridge + :type bridge: str + :returns: Iterator with bridge and port name for both ends of a patch. + :rtype: Iterator[Patch[PatchPort[str,str],PatchPort[str,str]]] + :raises: ValueError + """ + # On any given vSwitch there will be a small number of patch ports, so we + # start by iterating over ports with type `patch` then look up which bridge + # they belong to and act on any ports that match the criteria. + for interface in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').interface.find('type=patch'): + for port in ch_ovsdb.SimpleOVSDB( + 'ovs-vsctl').port.find('name={}'.format(interface['name'])): + if bridge_for_port(port['_uuid']) == bridge: + this_end = PatchPort(bridge, port['name']) + other_end = PatchPort(bridge_for_port( + uuid_for_port( + interface['options']['peer'])), + interface['options']['peer']) + yield(Patch(this_end, other_end)) + # We expect one result and it is ok if it turns out to be a port + # for a different bridge. However we need a break here to satisfy + # the for/else check which is in place to detect interface referring + # to non-existent port. + break + else: + raise ValueError('Port for interface named "{}" does unexpectedly ' + 'not exist.'.format(interface['name'])) + else: + # Allow our caller to handle no patch ports found gracefully, in + # reference to PEP479 just doing a return will provide a empty iterator + # and not None. + return + + +def generate_external_ids(external_id_value=None): + """Generate external-ids dictionary that can be used to mark OVS bridges + and ports as managed by the charm. + + :param external_id_value: Value of the external-ids entry. + Note: 'managed' will be used if not specified. + :type external_id_value: Optional[str] + :returns: Dict with a single external-ids entry. + { + 'external-ids': { + charm-``charm_name``: ``external_id_value`` + } + } + :rtype: Dict[str, Dict[str]] + """ + external_id_key = "charm-{}".format(charm_name()) + external_id_value = ('managed' if external_id_value is None + else external_id_value) + return { + 'external-ids': { + external_id_key: external_id_value + } + } diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovn.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovn.py new file mode 100644 index 0000000..2075f11 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovn.py @@ -0,0 +1,233 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import subprocess +import uuid + +from . import utils + + +OVN_RUNDIR = '/var/run/ovn' +OVN_SYSCONFDIR = '/etc/ovn' + + +def ovn_appctl(target, args, rundir=None, use_ovs_appctl=False): + """Run ovn/ovs-appctl for target with args and return output. + + :param target: Name of daemon to contact. Unless target begins with '/', + `ovn-appctl` looks for a pidfile and will build the path to + a /var/run/ovn/target.pid.ctl for you. + :type target: str + :param args: Command and arguments to pass to `ovn-appctl` + :type args: Tuple[str, ...] + :param rundir: Override path to sockets + :type rundir: Optional[str] + :param use_ovs_appctl: The ``ovn-appctl`` command appeared in OVN 20.03, + set this to True to use ``ovs-appctl`` instead. + :type use_ovs_appctl: bool + :returns: Output from command + :rtype: str + :raises: subprocess.CalledProcessError + """ + # NOTE(fnordahl): The ovsdb-server processes for the OVN databases use a + # non-standard naming scheme for their daemon control socket and we need + # to pass the full path to the socket. + if target in ('ovnnb_db', 'ovnsb_db',): + target = os.path.join(rundir or OVN_RUNDIR, target + '.ctl') + + if use_ovs_appctl: + tool = 'ovs-appctl' + else: + tool = 'ovn-appctl' + + return utils._run(tool, '-t', target, *args) + + +class OVNClusterStatus(object): + + def __init__(self, name, cluster_id, server_id, address, status, role, + term, leader, vote, election_timer, log, + entries_not_yet_committed, entries_not_yet_applied, + connections, servers): + """Initialize and populate OVNClusterStatus object. + + Use class initializer so we can define types in a compatible manner. + + :param name: Name of schema used for database + :type name: str + :param cluster_id: UUID of cluster + :type cluster_id: uuid.UUID + :param server_id: UUID of server + :type server_id: uuid.UUID + :param address: OVSDB connection method + :type address: str + :param status: Status text + :type status: str + :param role: Role of server + :type role: str + :param term: Election term + :type term: int + :param leader: Short form UUID of leader + :type leader: str + :param vote: Vote + :type vote: str + :param election_timer: Current value of election timer + :type election_timer: int + :param log: Log + :type log: str + :param entries_not_yet_committed: Entries not yet committed + :type entries_not_yet_committed: int + :param entries_not_yet_applied: Entries not yet applied + :type entries_not_yet_applied: int + :param connections: Connections + :type connections: str + :param servers: Servers in the cluster + [('0ea6', 'ssl:192.0.2.42:6643')] + :type servers: List[Tuple[str,str]] + """ + self.name = name + self.cluster_id = cluster_id + self.server_id = server_id + self.address = address + self.status = status + self.role = role + self.term = term + self.leader = leader + self.vote = vote + self.election_timer = election_timer + self.log = log + self.entries_not_yet_committed = entries_not_yet_committed + self.entries_not_yet_applied = entries_not_yet_applied + self.connections = connections + self.servers = servers + + def __eq__(self, other): + return ( + self.name == other.name and + self.cluster_id == other.cluster_id and + self.server_id == other.server_id and + self.address == other.address and + self.status == other.status and + self.role == other.role and + self.term == other.term and + self.leader == other.leader and + self.vote == other.vote and + self.election_timer == other.election_timer and + self.log == other.log and + self.entries_not_yet_committed == other.entries_not_yet_committed and + self.entries_not_yet_applied == other.entries_not_yet_applied and + self.connections == other.connections and + self.servers == other.servers) + + @property + def is_cluster_leader(self): + """Retrieve status information from clustered OVSDB. + + :returns: Whether target is cluster leader + :rtype: bool + """ + return self.leader == 'self' + + +def cluster_status(target, schema=None, use_ovs_appctl=False, rundir=None): + """Retrieve status information from clustered OVSDB. + + :param target: Usually one of 'ovsdb-server', 'ovnnb_db', 'ovnsb_db', can + also be full path to control socket. + :type target: str + :param schema: Database schema name, deduced from target if not provided + :type schema: Optional[str] + :param use_ovs_appctl: The ``ovn-appctl`` command appeared in OVN 20.03, + set this to True to use ``ovs-appctl`` instead. + :type use_ovs_appctl: bool + :param rundir: Override path to sockets + :type rundir: Optional[str] + :returns: cluster status data object + :rtype: OVNClusterStatus + :raises: subprocess.CalledProcessError, KeyError, RuntimeError + """ + schema_map = { + 'ovnnb_db': 'OVN_Northbound', + 'ovnsb_db': 'OVN_Southbound', + } + if schema and schema not in schema_map.keys(): + raise RuntimeError('Unknown schema provided: "{}"'.format(schema)) + + status = {} + k = '' + for line in ovn_appctl(target, + ('cluster/status', schema or schema_map[target]), + rundir=rundir, + use_ovs_appctl=use_ovs_appctl).splitlines(): + if k and line.startswith(' '): + # there is no key which means this is a instance of a multi-line/ + # multi-value item, populate the List which is already stored under + # the key. + if k == 'servers': + status[k].append( + tuple(line.replace(')', '').lstrip().split()[0:4:3])) + else: + status[k].append(line.lstrip()) + elif ':' in line: + # this is a line with a key + k, v = line.split(':', 1) + k = k.lower() + k = k.replace(' ', '_') + if v: + # this is a line with both key and value + if k in ('cluster_id', 'server_id',): + v = v.replace('(', '') + v = v.replace(')', '') + status[k] = tuple(v.split()) + else: + status[k] = v.lstrip() + else: + # this is a line with only key which means a multi-line/ + # multi-value item. Store key as List which will be + # populated on subsequent iterations. + status[k] = [] + return OVNClusterStatus( + status['name'], + uuid.UUID(status['cluster_id'][1]), + uuid.UUID(status['server_id'][1]), + status['address'], + status['status'], + status['role'], + int(status['term']), + status['leader'], + status['vote'], + int(status['election_timer']), + status['log'], + int(status['entries_not_yet_committed']), + int(status['entries_not_yet_applied']), + status['connections'], + status['servers']) + + +def is_northd_active(): + """Query `ovn-northd` for active status. + + Note that the active status information for ovn-northd is available for + OVN 20.03 and onward. + + :returns: True if local `ovn-northd` instance is active, False otherwise + :rtype: bool + """ + try: + for line in ovn_appctl('ovn-northd', ('status',)).splitlines(): + if line.startswith('Status:') and 'active' in line: + return True + except subprocess.CalledProcessError: + pass + return False diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovsdb.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovsdb.py new file mode 100644 index 0000000..2f1e53d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/ovsdb.py @@ -0,0 +1,246 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import uuid + +from . import utils + + +class SimpleOVSDB(object): + """Simple interface to OVSDB through the use of command line tools. + + OVS and OVN is managed through a set of databases. These databases have + similar command line tools to manage them. We make use of the similarity + to provide a generic class that can be used to manage them. + + The OpenvSwitch project does provide a Python API, but on the surface it + appears to be a bit too involved for our simple use case. + + Examples: + sbdb = SimpleOVSDB('ovn-sbctl') + for chs in sbdb.chassis: + print(chs) + + ovsdb = SimpleOVSDB('ovs-vsctl') + for br in ovsdb.bridge: + if br['name'] == 'br-test': + ovsdb.bridge.set(br['uuid'], 'external_ids:charm', 'managed') + + WARNING: If a list type field only have one item `ovs-vsctl` will present + it as a single item. Since we do not know the schema we have no way of + knowing what fields should be de-serialized as lists so the caller has + to be careful of checking the type of values returned from this library. + """ + + # For validation we keep a complete map of currently known good tool and + # table combinations. This requires maintenance down the line whenever + # upstream adds things that downstream wants, and the cost of maintaining + # that will most likely be lower then the cost of finding the needle in + # the haystack whenever downstream code misspells something. + _tool_table_map = { + 'ovs-vsctl': ( + 'autoattach', + 'bridge', + 'ct_timeout_policy', + 'ct_zone', + 'controller', + 'datapath', + 'flow_sample_collector_set', + 'flow_table', + 'ipfix', + 'interface', + 'manager', + 'mirror', + 'netflow', + 'open_vswitch', + 'port', + 'qos', + 'queue', + 'ssl', + 'sflow', + ), + 'ovn-nbctl': ( + 'acl', + 'address_set', + 'connection', + 'dhcp_options', + 'dns', + 'forwarding_group', + 'gateway_chassis', + 'ha_chassis', + 'ha_chassis_group', + 'load_balancer', + 'load_balancer_health_check', + 'logical_router', + 'logical_router_policy', + 'logical_router_port', + 'logical_router_static_route', + 'logical_switch', + 'logical_switch_port', + 'meter', + 'meter_band', + 'nat', + 'nb_global', + 'port_group', + 'qos', + 'ssl', + ), + 'ovn-sbctl': ( + 'address_set', + 'chassis', + 'connection', + 'controller_event', + 'dhcp_options', + 'dhcpv6_options', + 'dns', + 'datapath_binding', + 'encap', + 'gateway_chassis', + 'ha_chassis', + 'ha_chassis_group', + 'igmp_group', + 'ip_multicast', + 'logical_flow', + 'mac_binding', + 'meter', + 'meter_band', + 'multicast_group', + 'port_binding', + 'port_group', + 'rbac_permission', + 'rbac_role', + 'sb_global', + 'ssl', + 'service_monitor', + ), + } + + def __init__(self, tool): + """SimpleOVSDB constructor. + + :param tool: Which tool with database commands to operate on. + Usually one of `ovs-vsctl`, `ovn-nbctl`, `ovn-sbctl` + :type tool: str + """ + if tool not in self._tool_table_map: + raise RuntimeError( + 'tool must be one of "{}"'.format(self._tool_table_map.keys())) + self._tool = tool + + def __getattr__(self, table): + if table not in self._tool_table_map[self._tool]: + raise AttributeError( + 'table "{}" not known for use with "{}"' + .format(table, self._tool)) + return self.Table(self._tool, table) + + class Table(object): + """Methods to interact with contents of OVSDB tables. + + NOTE: At the time of this writing ``find`` is the only command + line argument to OVSDB manipulating tools that actually supports + JSON output. + """ + + def __init__(self, tool, table): + """SimpleOVSDBTable constructor. + + :param table: Which table to operate on + :type table: str + """ + self._tool = tool + self._table = table + + def _deserialize_ovsdb(self, data): + """Deserialize OVSDB RFC7047 section 5.1 data. + + :param data: Multidimensional list where first row contains RFC7047 + type information + :type data: List[str,any] + :returns: Deserialized data. + :rtype: any + """ + # When using json formatted output to OVS commands Internal OVSDB + # notation may occur that require further deserializing. + # Reference: https://tools.ietf.org/html/rfc7047#section-5.1 + ovs_type_cb_map = { + 'uuid': uuid.UUID, + # NOTE: OVSDB sets have overloaded type + # see special handling below + 'set': list, + 'map': dict, + } + assert len(data) > 1, ('Invalid data provided, expecting list ' + 'with at least two elements.') + if data[0] == 'set': + # special handling for set + # + # it is either a list of strings or a list of typed lists. + # taste first element to see which it is + for el in data[1]: + # NOTE: We lock this handling down to the `uuid` type as + # that is the only one we have a practical example of. + # We could potentially just handle this generally based on + # the types listed in `ovs_type_cb_map` but let's open for + # that as soon as we have a concrete example to validate on + if isinstance( + el, list) and len(el) and el[0] == 'uuid': + decoded_set = [] + for el in data[1]: + decoded_set.append(self._deserialize_ovsdb(el)) + return(decoded_set) + # fall back to normal processing below + break + + # Use map to deserialize data with fallback to `str` + f = ovs_type_cb_map.get(data[0], str) + return f(data[1]) + + def _find_tbl(self, condition=None): + """Run and parse output of OVSDB `find` command. + + :param condition: An optional RFC 7047 5.1 match condition + :type condition: Optional[str] + :returns: Dictionary with data + :rtype: Dict[str, any] + """ + cmd = [self._tool, '-f', 'json', 'find', self._table] + if condition: + cmd.append(condition) + output = utils._run(*cmd) + data = json.loads(output) + for row in data['data']: + values = [] + for col in row: + if isinstance(col, list) and len(col) > 1: + values.append(self._deserialize_ovsdb(col)) + else: + values.append(col) + yield dict(zip(data['headings'], values)) + + def __iter__(self): + return self._find_tbl() + + def clear(self, rec, col): + utils._run(self._tool, 'clear', self._table, rec, col) + + def find(self, condition): + return self._find_tbl(condition=condition) + + def remove(self, rec, col, value): + utils._run(self._tool, 'remove', self._table, rec, col, value) + + def set(self, rec, col, value): + utils._run(self._tool, 'set', self._table, rec, + '{}={}'.format(col, value)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/utils.py new file mode 100644 index 0000000..53c9b4d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ovs/utils.py @@ -0,0 +1,26 @@ +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import subprocess + + +def _run(*args): + """Run a process, check result, capture decoded output from STDOUT. + + :param args: Command and arguments to run + :type args: Tuple[str, ...] + :returns: Information about the completed process + :rtype: str + :raises subprocess.CalledProcessError + """ + return subprocess.check_output(args, universal_newlines=True) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ufw.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ufw.py new file mode 100644 index 0000000..b9bf7c9 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/network/ufw.py @@ -0,0 +1,386 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains helpers to add and remove ufw rules. + +Examples: + +- open SSH port for subnet 10.0.3.0/24: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.grant_access(src='10.0.3.0/24', dst='any', port='22', proto='tcp') + +- open service by name as defined in /etc/services: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('ssh', 'open') + +- close service by port number: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('4949', 'close') # munin +""" +import os +import re +import subprocess + +from charmhelpers.core import hookenv +from charmhelpers.core.kernel import modprobe, is_module_loaded + +__author__ = "Felipe Reyes " + + +class UFWError(Exception): + pass + + +class UFWIPv6Error(UFWError): + pass + + +def is_enabled(): + """ + Check if `ufw` is enabled + + :returns: True if ufw is enabled + """ + output = subprocess.check_output(['ufw', 'status'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall(r'^Status: active\n', output, re.M) + + return len(m) >= 1 + + +def is_ipv6_ok(soft_fail=False): + """ + Check if IPv6 support is present and ip6tables functional + + :param soft_fail: If set to True and IPv6 support is broken, then reports + that the host doesn't have IPv6 support, otherwise a + UFWIPv6Error exception is raised. + :returns: True if IPv6 is working, False otherwise + """ + + # do we have IPv6 in the machine? + if os.path.isdir('/proc/sys/net/ipv6'): + # is ip6tables kernel module loaded? + if not is_module_loaded('ip6_tables'): + # ip6tables support isn't complete, let's try to load it + try: + modprobe('ip6_tables') + # great, we can load the module + return True + except subprocess.CalledProcessError as ex: + hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, + level="WARN") + # we are in a world where ip6tables isn't working + if soft_fail: + # so we inform that the machine doesn't have IPv6 + return False + else: + raise UFWIPv6Error("IPv6 firewall support broken") + else: + # the module is present :) + return True + + else: + # the system doesn't have IPv6 + return False + + +def disable_ipv6(): + """ + Disable ufw IPv6 support in /etc/default/ufw + """ + exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + if exit_code == 0: + hookenv.log('IPv6 support in ufw disabled', level='INFO') + else: + hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") + raise UFWError("Couldn't disable IPv6 support in ufw") + + +def enable(soft_fail=False): + """ + Enable ufw + + :param soft_fail: If set to True silently disables IPv6 support in ufw, + otherwise a UFWIPv6Error exception is raised when IP6 + support is broken. + :returns: True if ufw is successfully enabled + """ + if is_enabled(): + return True + + if not is_ipv6_ok(soft_fail): + disable_ipv6() + + output = subprocess.check_output(['ufw', 'enable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall('^Firewall is active and enabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be enabled", level='WARN') + return False + else: + hookenv.log("ufw enabled", level='INFO') + return True + + +def reload(): + """ + Reload ufw + + :returns: True if ufw is successfully enabled + """ + output = subprocess.check_output(['ufw', 'reload'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall('^Firewall reloaded\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be reloaded", level='WARN') + return False + else: + hookenv.log("ufw reloaded", level='INFO') + return True + + +def disable(): + """ + Disable ufw + + :returns: True if ufw is successfully disabled + """ + if not is_enabled(): + return True + + output = subprocess.check_output(['ufw', 'disable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall(r'^Firewall stopped and disabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be disabled", level='WARN') + return False + else: + hookenv.log("ufw disabled", level='INFO') + return True + + +def default_policy(policy='deny', direction='incoming'): + """ + Changes the default policy for traffic `direction` + + :param policy: allow, deny or reject + :param direction: traffic direction, possible values: incoming, outgoing, + routed + """ + if policy not in ['allow', 'deny', 'reject']: + raise UFWError(('Unknown policy %s, valid values: ' + 'allow, deny, reject') % policy) + + if direction not in ['incoming', 'outgoing', 'routed']: + raise UFWError(('Unknown direction %s, valid values: ' + 'incoming, outgoing, routed') % direction) + + output = subprocess.check_output(['ufw', 'default', policy, direction], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + hookenv.log(output, level='DEBUG') + + m = re.findall("^Default %s policy changed to '%s'\n" % (direction, + policy), + output, re.M) + if len(m) == 0: + hookenv.log("ufw couldn't change the default policy to %s for %s" + % (policy, direction), level='WARN') + return False + else: + hookenv.log("ufw default policy for %s changed to %s" + % (direction, policy), level='INFO') + return True + + +def modify_access(src, dst='any', port=None, proto=None, action='allow', + index=None, prepend=False, comment=None): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :type src: Optional[str] + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :type dst: Optional[str] + :param port: destiny port + :type port: Optional[int] + :param proto: protocol (tcp or udp) + :type proto: Optional[str] + :param action: `allow` or `delete` + :type action: str + :param index: if different from None the rule is inserted at the given + `index`. + :type index: Optional[int] + :param prepend: Whether to insert the rule before all other rules matching + the rule's IP type. + :type prepend: bool + :param comment: Create the rule with a comment + :type comment: Optional[str] + """ + if not is_enabled(): + hookenv.log('ufw is disabled, skipping modify_access()', level='WARN') + return + + if action == 'delete': + if index is not None: + cmd = ['ufw', '--force', 'delete', str(index)] + else: + cmd = ['ufw', 'delete', 'allow'] + elif index is not None: + cmd = ['ufw', 'insert', str(index), action] + elif prepend: + cmd = ['ufw', 'prepend', action] + else: + cmd = ['ufw', action] + + if src is not None: + cmd += ['from', src] + + if dst is not None: + cmd += ['to', dst] + + if port is not None: + cmd += ['port', str(port)] + + if proto is not None: + cmd += ['proto', proto] + + if comment: + cmd.extend(['comment', comment]) + + hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + hookenv.log(stdout, level='INFO') + + if p.returncode != 0: + hookenv.log(stderr, level='ERROR') + hookenv.log('Error running: {}, exit code: {}'.format(' '.join(cmd), + p.returncode), + level='ERROR') + + +def grant_access(src, dst='any', port=None, proto=None, index=None): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + :param index: if different from None the rule is inserted at the given + `index`. + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='allow', + index=index) + + +def revoke_access(src, dst='any', port=None, proto=None): + """ + Revoke access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='delete') + + +def service(name, action): + """ + Open/close access to a service + + :param name: could be a service name defined in `/etc/services` or a port + number. + :param action: `open` or `close` + """ + if action == 'open': + subprocess.check_output(['ufw', 'allow', str(name)], + universal_newlines=True) + elif action == 'close': + subprocess.check_output(['ufw', 'delete', 'allow', str(name)], + universal_newlines=True) + else: + raise UFWError(("'{}' not supported, use 'allow' " + "or 'delete'").format(action)) + + +def status(): + """Retrieve firewall rules as represented by UFW. + + :returns: Tuples with rule number and data + (1, {'to': '', 'action':, 'from':, '', ipv6: True, 'comment': ''}) + :rtype: Iterator[Tuple[int, Dict[str, Union[bool, str]]]] + """ + cp = subprocess.check_output(('ufw', 'status', 'numbered',), + stderr=subprocess.STDOUT, + universal_newlines=True) + for line in cp.splitlines(): + if not line.startswith('['): + continue + ipv6 = True if '(v6)' in line else False + line = line.replace('(v6)', '') + line = line.replace('[', '') + line = line.replace(']', '') + line = line.replace('Anywhere', 'any') + row = line.split() + yield (int(row[0]), { + 'to': row[1], + 'action': ' '.join(row[2:4]).lower(), + 'from': row[4], + 'ipv6': ipv6, + 'comment': row[6] if len(row) > 5 and row[5] == '#' else '', + }) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/alternatives.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/alternatives.py new file mode 100644 index 0000000..547de09 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/alternatives.py @@ -0,0 +1,44 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Helper for managing alternatives for file conflict resolution ''' + +import subprocess +import shutil +import os + + +def install_alternative(name, target, source, priority=50): + ''' Install alternative configuration ''' + if (os.path.exists(target) and not os.path.islink(target)): + # Move existing file/directory away before installing + shutil.move(target, '{}.bak'.format(target)) + cmd = [ + 'update-alternatives', '--force', '--install', + target, name, source, str(priority) + ] + subprocess.check_call(cmd) + + +def remove_alternative(name, source): + """Remove an installed alternative configuration file + + :param name: string name of the alternative to remove + :param source: string full path to alternative to remove + """ + cmd = [ + 'update-alternatives', '--remove', + name, source + ] + subprocess.check_call(cmd) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/__init__.py new file mode 100644 index 0000000..7f7e5f7 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/__init__.py @@ -0,0 +1,212 @@ +# Copyright 2019 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenStack Security Audit code""" + +import collections +from enum import Enum +import traceback + +from charmhelpers.core.host import cmp_pkgrevno +import charmhelpers.contrib.openstack.utils as openstack_utils +import charmhelpers.core.hookenv as hookenv + + +class AuditType(Enum): + OpenStackSecurityGuide = 1 + + +_audits = {} + +Audit = collections.namedtuple('Audit', 'func filters') + + +def audit(*args): + """Decorator to register an audit. + + These are used to generate audits that can be run on a + deployed system that matches the given configuration + + :param args: List of functions to filter tests against + :type args: List[Callable[Dict]] + """ + def wrapper(f): + test_name = f.__name__ + if _audits.get(test_name): + raise RuntimeError( + "Test name '{}' used more than once" + .format(test_name)) + non_callables = [fn for fn in args if not callable(fn)] + if non_callables: + raise RuntimeError( + "Configuration includes non-callable filters: {}" + .format(non_callables)) + _audits[test_name] = Audit(func=f, filters=args) + return f + return wrapper + + +def is_audit_type(*args): + """This audit is included in the specified kinds of audits. + + :param *args: List of AuditTypes to include this audit in + :type args: List[AuditType] + :rtype: Callable[Dict] + """ + def _is_audit_type(audit_options): + if audit_options.get('audit_type') in args: + return True + else: + return False + return _is_audit_type + + +def since_package(pkg, pkg_version): + """This audit should be run after the specified package version (incl). + + :param pkg: Package name to compare + :type pkg: str + :param release: The package version + :type release: str + :rtype: Callable[Dict] + """ + def _since_package(audit_options=None): + return cmp_pkgrevno(pkg, pkg_version) >= 0 + + return _since_package + + +def before_package(pkg, pkg_version): + """This audit should be run before the specified package version (excl). + + :param pkg: Package name to compare + :type pkg: str + :param release: The package version + :type release: str + :rtype: Callable[Dict] + """ + def _before_package(audit_options=None): + return not since_package(pkg, pkg_version)() + + return _before_package + + +def since_openstack_release(pkg, release): + """This audit should run after the specified OpenStack version (incl). + + :param pkg: Package name to compare + :type pkg: str + :param release: The OpenStack release codename + :type release: str + :rtype: Callable[Dict] + """ + def _since_openstack_release(audit_options=None): + _release = openstack_utils.get_os_codename_package(pkg) + return openstack_utils.CompareOpenStackReleases(_release) >= release + + return _since_openstack_release + + +def before_openstack_release(pkg, release): + """This audit should run before the specified OpenStack version (excl). + + :param pkg: Package name to compare + :type pkg: str + :param release: The OpenStack release codename + :type release: str + :rtype: Callable[Dict] + """ + def _before_openstack_release(audit_options=None): + return not since_openstack_release(pkg, release)() + + return _before_openstack_release + + +def it_has_config(config_key): + """This audit should be run based on specified config keys. + + :param config_key: Config key to look for + :type config_key: str + :rtype: Callable[Dict] + """ + def _it_has_config(audit_options): + return audit_options.get(config_key) is not None + + return _it_has_config + + +def run(audit_options): + """Run the configured audits with the specified audit_options. + + :param audit_options: Configuration for the audit + :type audit_options: Config + + :rtype: Dict[str, str] + """ + errors = {} + results = {} + for name, audit in sorted(_audits.items()): + result_name = name.replace('_', '-') + if result_name in audit_options.get('excludes', []): + print( + "Skipping {} because it is" + "excluded in audit config" + .format(result_name)) + continue + if all(p(audit_options) for p in audit.filters): + try: + audit.func(audit_options) + print("{}: PASS".format(name)) + results[result_name] = { + 'success': True, + } + except AssertionError as e: + print("{}: FAIL ({})".format(name, e)) + results[result_name] = { + 'success': False, + 'message': e, + } + except Exception as e: + print("{}: ERROR ({})".format(name, e)) + errors[name] = e + results[result_name] = { + 'success': False, + 'message': e, + } + for name, error in errors.items(): + print("=" * 20) + print("Error in {}: ".format(name)) + traceback.print_tb(error.__traceback__) + print() + return results + + +def action_parse_results(result): + """Parse the result of `run` in the context of an action. + + :param result: The result of running the security-checklist + action on a unit + :type result: Dict[str, Dict[str, str]] + :rtype: int + """ + passed = True + for test, result in result.items(): + if result['success']: + hookenv.action_set({test: 'PASS'}) + else: + hookenv.action_set({test: 'FAIL - {}'.format(result['message'])}) + passed = False + if not passed: + hookenv.action_fail("One or more tests failed") + return 0 if passed else 1 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/openstack_security_guide.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/openstack_security_guide.py new file mode 100644 index 0000000..79740ed --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/audits/openstack_security_guide.py @@ -0,0 +1,270 @@ +# Copyright 2019 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import configparser +import glob +import os.path +import subprocess + +from charmhelpers.contrib.openstack.audits import ( + audit, + AuditType, + # filters + is_audit_type, + it_has_config, +) + +from charmhelpers.core.hookenv import ( + cached, +) + +""" +The Security Guide suggests a specific list of files inside the +config directory for the service having 640 specifically, but +by ensuring the containing directory is 750, only the owner can +write, and only the group can read files within the directory. + +By restricting access to the containing directory, we can more +effectively ensure that there is no accidental leakage if a new +file is added to the service without being added to the security +guide, and to this check. +""" +FILE_ASSERTIONS = { + 'barbican': { + '/etc/barbican': {'group': 'barbican', 'mode': '750'}, + }, + 'ceph-mon': { + '/var/lib/charm/ceph-mon/ceph.conf': + {'owner': 'root', 'group': 'root', 'mode': '644'}, + '/etc/ceph/ceph.client.admin.keyring': + {'owner': 'ceph', 'group': 'ceph'}, + '/etc/ceph/rbdmap': {'mode': '644'}, + '/var/lib/ceph': {'owner': 'ceph', 'group': 'ceph', 'mode': '750'}, + '/var/lib/ceph/bootstrap-*/ceph.keyring': + {'owner': 'ceph', 'group': 'ceph', 'mode': '600'} + }, + 'ceph-osd': { + '/var/lib/charm/ceph-osd/ceph.conf': + {'owner': 'ceph', 'group': 'ceph', 'mode': '644'}, + '/var/lib/ceph': {'owner': 'ceph', 'group': 'ceph', 'mode': '750'}, + '/var/lib/ceph/*': {'owner': 'ceph', 'group': 'ceph', 'mode': '755'}, + '/var/lib/ceph/bootstrap-*/ceph.keyring': + {'owner': 'ceph', 'group': 'ceph', 'mode': '600'}, + '/var/lib/ceph/radosgw': + {'owner': 'ceph', 'group': 'ceph', 'mode': '755'}, + }, + 'cinder': { + '/etc/cinder': {'group': 'cinder', 'mode': '750'}, + }, + 'glance': { + '/etc/glance': {'group': 'glance', 'mode': '750'}, + }, + 'keystone': { + '/etc/keystone': + {'owner': 'keystone', 'group': 'keystone', 'mode': '750'}, + }, + 'manilla': { + '/etc/manila': {'group': 'manilla', 'mode': '750'}, + }, + 'neutron-gateway': { + '/etc/neutron': {'group': 'neutron', 'mode': '750'}, + }, + 'neutron-api': { + '/etc/neutron/': {'group': 'neutron', 'mode': '750'}, + }, + 'nova-cloud-controller': { + '/etc/nova': {'group': 'nova', 'mode': '750'}, + }, + 'nova-compute': { + '/etc/nova/': {'group': 'nova', 'mode': '750'}, + }, + 'openstack-dashboard': { + # From security guide + '/etc/openstack-dashboard/local_settings.py': + {'group': 'horizon', 'mode': '640'}, + }, +} + +Ownership = collections.namedtuple('Ownership', 'owner group mode') + + +@cached +def _stat(file): + """ + Get the Ownership information from a file. + + :param file: The path to a file to stat + :type file: str + :returns: owner, group, and mode of the specified file + :rtype: Ownership + :raises subprocess.CalledProcessError: If the underlying stat fails + """ + out = subprocess.check_output( + ['stat', '-c', '%U %G %a', file]).decode('utf-8') + return Ownership(*out.strip().split(' ')) + + +@cached +def _config_ini(path): + """ + Parse an ini file + + :param path: The path to a file to parse + :type file: str + :returns: Configuration contained in path + :rtype: Dict + """ + # When strict is enabled, duplicate options are not allowed in the + # parsed INI; however, Oslo allows duplicate values. This change + # causes us to ignore the duplicate values which is acceptable as + # long as we don't validate any multi-value options + conf = configparser.ConfigParser(strict=False) + conf.read(path) + return dict(conf) + + +def _validate_file_ownership(owner, group, file_name, optional=False): + """ + Validate that a specified file is owned by `owner:group`. + + :param owner: Name of the owner + :type owner: str + :param group: Name of the group + :type group: str + :param file_name: Path to the file to verify + :type file_name: str + :param optional: Is this file optional, + ie: Should this test fail when it's missing + :type optional: bool + """ + try: + ownership = _stat(file_name) + except subprocess.CalledProcessError as e: + print("Error reading file: {}".format(e)) + if not optional: + assert False, "Specified file does not exist: {}".format(file_name) + assert owner == ownership.owner, \ + "{} has an incorrect owner: {} should be {}".format( + file_name, ownership.owner, owner) + assert group == ownership.group, \ + "{} has an incorrect group: {} should be {}".format( + file_name, ownership.group, group) + print("Validate ownership of {}: PASS".format(file_name)) + + +def _validate_file_mode(mode, file_name, optional=False): + """ + Validate that a specified file has the specified permissions. + + :param mode: file mode that is desires + :type owner: str + :param file_name: Path to the file to verify + :type file_name: str + :param optional: Is this file optional, + ie: Should this test fail when it's missing + :type optional: bool + """ + try: + ownership = _stat(file_name) + except subprocess.CalledProcessError as e: + print("Error reading file: {}".format(e)) + if not optional: + assert False, "Specified file does not exist: {}".format(file_name) + assert mode == ownership.mode, \ + "{} has an incorrect mode: {} should be {}".format( + file_name, ownership.mode, mode) + print("Validate mode of {}: PASS".format(file_name)) + + +@cached +def _config_section(config, section): + """Read the configuration file and return a section.""" + path = os.path.join(config.get('config_path'), config.get('config_file')) + conf = _config_ini(path) + return conf.get(section) + + +@audit(is_audit_type(AuditType.OpenStackSecurityGuide), + it_has_config('files')) +def validate_file_ownership(config): + """Verify that configuration files are owned by the correct user/group.""" + files = config.get('files', {}) + for file_name, options in files.items(): + for key in options.keys(): + if key not in ["owner", "group", "mode"]: + raise RuntimeError( + "Invalid ownership configuration: {}".format(key)) + owner = options.get('owner', config.get('owner', 'root')) + group = options.get('group', config.get('group', 'root')) + optional = options.get('optional', config.get('optional', False)) + if '*' in file_name: + for file in glob.glob(file_name): + if file not in files.keys(): + if os.path.isfile(file): + _validate_file_ownership(owner, group, file, optional) + else: + if os.path.isfile(file_name): + _validate_file_ownership(owner, group, file_name, optional) + + +@audit(is_audit_type(AuditType.OpenStackSecurityGuide), + it_has_config('files')) +def validate_file_permissions(config): + """Verify that permissions on configuration files are secure enough.""" + files = config.get('files', {}) + for file_name, options in files.items(): + for key in options.keys(): + if key not in ["owner", "group", "mode"]: + raise RuntimeError( + "Invalid ownership configuration: {}".format(key)) + mode = options.get('mode', config.get('permissions', '600')) + optional = options.get('optional', config.get('optional', False)) + if '*' in file_name: + for file in glob.glob(file_name): + if file not in files.keys(): + if os.path.isfile(file): + _validate_file_mode(mode, file, optional) + else: + if os.path.isfile(file_name): + _validate_file_mode(mode, file_name, optional) + + +@audit(is_audit_type(AuditType.OpenStackSecurityGuide)) +def validate_uses_keystone(audit_options): + """Validate that the service uses Keystone for authentication.""" + section = _config_section(audit_options, 'api') or _config_section(audit_options, 'DEFAULT') + assert section is not None, "Missing section 'api / DEFAULT'" + assert section.get('auth_strategy') == "keystone", \ + "Application is not using Keystone" + + +@audit(is_audit_type(AuditType.OpenStackSecurityGuide)) +def validate_uses_tls_for_keystone(audit_options): + """Verify that TLS is used to communicate with Keystone.""" + section = _config_section(audit_options, 'keystone_authtoken') + assert section is not None, "Missing section 'keystone_authtoken'" + assert not section.get('insecure') and \ + "https://" in section.get("auth_uri"), \ + "TLS is not used for Keystone" + + +@audit(is_audit_type(AuditType.OpenStackSecurityGuide)) +def validate_uses_tls_for_glance(audit_options): + """Verify that TLS is used to communicate with Glance.""" + section = _config_section(audit_options, 'glance') + assert section is not None, "Missing section 'glance'" + assert not section.get('insecure') and \ + "https://" in section.get("api_servers"), \ + "TLS is not used for Glance" diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/cert_utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/cert_utils.py new file mode 100644 index 0000000..5c961c5 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/cert_utils.py @@ -0,0 +1,443 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common python helper functions used for OpenStack charm certificates. + +import os +import json +from base64 import b64decode + +from charmhelpers.contrib.network.ip import ( + get_hostname, + resolve_network_cidr, +) +from charmhelpers.core.hookenv import ( + local_unit, + network_get_primary_address, + config, + related_units, + relation_get, + relation_ids, + remote_service_name, + NoNetworkBinding, + log, + WARNING, + INFO, +) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + get_vip_in_network, + ADDRESS_MAP, + get_default_api_bindings, + local_address, +) +from charmhelpers.contrib.network.ip import ( + get_relation_ip, +) + +from charmhelpers.core.host import ( + ca_cert_absolute_path, + install_ca_cert, + mkdir, + write_file, +) + +from charmhelpers.contrib.hahelpers.apache import ( + CONFIG_CA_CERT_FILE, +) + + +class CertRequest(object): + + """Create a request for certificates to be generated + """ + + def __init__(self, json_encode=True): + self.entries = [] + self.hostname_entry = None + self.json_encode = json_encode + + def add_entry(self, net_type, cn, addresses): + """Add a request to the batch + + :param net_type: str network space name request is for + :param cn: str Canonical Name for certificate + :param addresses: [] List of addresses to be used as SANs + """ + self.entries.append({ + 'cn': cn, + 'addresses': addresses}) + + def add_hostname_cn(self): + """Add a request for the hostname of the machine""" + ip = local_address(unit_get_fallback='private-address') + addresses = [ip] + # If a vip is being used without os-hostname config or + # network spaces then we need to ensure the local units + # cert has the appropriate vip in the SAN list + vip = get_vip_in_network(resolve_network_cidr(ip)) + if vip: + addresses.append(vip) + self.hostname_entry = { + 'cn': get_hostname(ip), + 'addresses': addresses} + + def add_hostname_cn_ip(self, addresses): + """Add an address to the SAN list for the hostname request + + :param addr: [] List of address to be added + """ + for addr in addresses: + if addr not in self.hostname_entry['addresses']: + self.hostname_entry['addresses'].append(addr) + + def get_request(self): + """Generate request from the batched up entries + + """ + if self.hostname_entry: + self.entries.append(self.hostname_entry) + request = {} + for entry in self.entries: + sans = sorted(list(set(entry['addresses']))) + request[entry['cn']] = {'sans': sans} + if self.json_encode: + req = {'cert_requests': json.dumps(request, sort_keys=True)} + else: + req = {'cert_requests': request} + req['unit_name'] = local_unit().replace('/', '_') + return req + + +def get_certificate_request(json_encode=True, bindings=None): + """Generate a certificate requests based on the network configuration + + :param json_encode: Encode request in JSON or not. Used for setting + directly on a relation. + :type json_encode: boolean + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + :returns: CertRequest request as dictionary or JSON string. + :rtype: Union[dict, json] + """ + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + req = CertRequest(json_encode=json_encode) + req.add_hostname_cn() + # Add os-hostname entries + _sans = get_certificate_sans(bindings=bindings) + + # Handle specific hostnames per binding + for binding in bindings: + try: + hostname_override = config(ADDRESS_MAP[binding]['override']) + except KeyError: + hostname_override = None + try: + try: + net_addr = resolve_address(endpoint_type=binding) + except KeyError: + net_addr = None + ip = network_get_primary_address(binding) + addresses = [net_addr, ip] + vip = get_vip_in_network(resolve_network_cidr(ip)) + if vip: + addresses.append(vip) + + # Clear any Nones or duplicates + addresses = list(set([i for i in addresses if i])) + # Add hostname certificate request + if hostname_override: + req.add_entry( + binding, + hostname_override, + addresses) + # Remove hostname specific addresses from _sans + for addr in addresses: + try: + _sans.remove(addr) + except (ValueError, KeyError): + pass + + except NoNetworkBinding: + log("Skipping request for certificate for ip in {} space, no " + "local address found".format(binding), WARNING) + # Guarantee all SANs are covered + # These are network addresses with no corresponding hostname. + # Add the ips to the hostname cert to allow for this. + req.add_hostname_cn_ip(_sans) + return req.get_request() + + +def get_certificate_sans(bindings=None): + """Get all possible IP addresses for certificate SANs. + + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + :returns: List of binding string names + :rtype: List[str] + """ + _sans = [local_address(unit_get_fallback='private-address')] + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + + for binding in bindings: + # Check for config override + try: + net_config = config(ADDRESS_MAP[binding]['config']) + except KeyError: + # There is no configuration network for this binding name + net_config = None + # Using resolve_address is likely redundant. Keeping it here in + # case there is an edge case it handles. + try: + net_addr = resolve_address(endpoint_type=binding) + except KeyError: + net_addr = None + ip = get_relation_ip(binding, cidr_network=net_config) + _sans = _sans + [net_addr, ip] + vip = get_vip_in_network(resolve_network_cidr(ip)) + if vip: + _sans.append(vip) + # Clear any Nones and duplicates + return list(set([i for i in _sans if i])) + + +def create_ip_cert_links(ssl_dir, custom_hostname_link=None, bindings=None): + """Create symlinks for SAN records + + :param ssl_dir: str Directory to create symlinks in + :param custom_hostname_link: str Additional link to be created + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + """ + + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + + # This includes the hostname cert and any specific bindng certs: + # admin, internal, public + req = get_certificate_request(json_encode=False, bindings=bindings)["cert_requests"] + # Specific certs + for cert_req in req.keys(): + requested_cert = os.path.join( + ssl_dir, + 'cert_{}'.format(cert_req)) + requested_key = os.path.join( + ssl_dir, + 'key_{}'.format(cert_req)) + for addr in req[cert_req]['sans']: + cert = os.path.join(ssl_dir, 'cert_{}'.format(addr)) + key = os.path.join(ssl_dir, 'key_{}'.format(addr)) + if os.path.isfile(requested_cert) and not os.path.isfile(cert): + os.symlink(requested_cert, cert) + os.symlink(requested_key, key) + + # Handle custom hostnames + hostname = get_hostname(local_address(unit_get_fallback='private-address')) + hostname_cert = os.path.join( + ssl_dir, + 'cert_{}'.format(hostname)) + hostname_key = os.path.join( + ssl_dir, + 'key_{}'.format(hostname)) + if custom_hostname_link: + custom_cert = os.path.join( + ssl_dir, + 'cert_{}'.format(custom_hostname_link)) + custom_key = os.path.join( + ssl_dir, + 'key_{}'.format(custom_hostname_link)) + if os.path.isfile(hostname_cert) and not os.path.isfile(custom_cert): + os.symlink(hostname_cert, custom_cert) + os.symlink(hostname_key, custom_key) + + +def install_certs(ssl_dir, certs, chain=None, user='root', group='root'): + """Install the certs passed into the ssl dir and append the chain if + provided. + + :param ssl_dir: str Directory to create symlinks in + :param certs: {} {'cn': {'cert': 'CERT', 'key': 'KEY'}} + :param chain: str Chain to be appended to certs + :param user: (Optional) Owner of certificate files. Defaults to 'root' + :type user: str + :param group: (Optional) Group of certificate files. Defaults to 'root' + :type group: str + """ + for cn, bundle in certs.items(): + cert_filename = 'cert_{}'.format(cn) + key_filename = 'key_{}'.format(cn) + cert_data = bundle['cert'] + if chain: + # Append chain file so that clients that trust the root CA will + # trust certs signed by an intermediate in the chain + cert_data = cert_data + os.linesep + chain + write_file( + path=os.path.join(ssl_dir, cert_filename), owner=user, group=group, + content=cert_data, perms=0o640) + write_file( + path=os.path.join(ssl_dir, key_filename), owner=user, group=group, + content=bundle['key'], perms=0o640) + + +def get_cert_relation_ca_name(cert_relation_id=None): + """Determine CA certificate name as provided by relation. + + The filename on disk depends on the name chosen for the application on the + providing end of the certificates relation. + + :param cert_relation_id: (Optional) Relation id providing the certs + :type cert_relation_id: str + :returns: CA certificate filename without path nor extension + :rtype: str + """ + if cert_relation_id is None: + try: + cert_relation_id = relation_ids('certificates')[0] + except IndexError: + return '' + return '{}_juju_ca_cert'.format( + remote_service_name(relid=cert_relation_id)) + + +def _manage_ca_certs(ca, cert_relation_id): + """Manage CA certs. + + :param ca: CA Certificate from certificate relation. + :type ca: str + :param cert_relation_id: Relation id providing the certs + :type cert_relation_id: str + """ + config_ssl_ca = config('ssl_ca') + config_cert_file = ca_cert_absolute_path(CONFIG_CA_CERT_FILE) + if config_ssl_ca: + log("Installing CA certificate from charm ssl_ca config to {}".format( + config_cert_file), INFO) + install_ca_cert( + b64decode(config_ssl_ca).rstrip(), + name=CONFIG_CA_CERT_FILE) + elif os.path.exists(config_cert_file): + log("Removing CA certificate {}".format(config_cert_file), INFO) + os.remove(config_cert_file) + log("Installing CA certificate from certificate relation", INFO) + install_ca_cert( + ca.encode(), + name=get_cert_relation_ca_name(cert_relation_id)) + + +def process_certificates(service_name, relation_id, unit, + custom_hostname_link=None, user='root', group='root', + bindings=None): + """Process the certificates supplied down the relation + + :param service_name: str Name of service the certificates are for. + :param relation_id: str Relation id providing the certs + :param unit: str Unit providing the certs + :param custom_hostname_link: str Name of custom link to create + :param user: (Optional) Owner of certificate files. Defaults to 'root' + :type user: str + :param group: (Optional) Group of certificate files. Defaults to 'root' + :type group: str + :param bindings: List of bindings to check in addition to default api + bindings. + :type bindings: list of strings + :returns: True if certificates processed for local unit or False + :rtype: bool + """ + if bindings: + # Add default API bindings to bindings list + bindings = list(bindings + get_default_api_bindings()) + else: + # Use default API bindings + bindings = get_default_api_bindings() + + data = relation_get(rid=relation_id, unit=unit) + ssl_dir = os.path.join('/etc/apache2/ssl/', service_name) + mkdir(path=ssl_dir) + name = local_unit().replace('/', '_') + certs = data.get('{}.processed_requests'.format(name)) + chain = data.get('chain') + ca = data.get('ca') + if certs: + certs = json.loads(certs) + _manage_ca_certs(ca, relation_id) + install_certs(ssl_dir, certs, chain, user=user, group=group) + create_ip_cert_links( + ssl_dir, + custom_hostname_link=custom_hostname_link, + bindings=bindings) + return True + return False + + +def get_requests_for_local_unit(relation_name=None): + """Extract any certificates data targeted at this unit down relation_name. + + :param relation_name: str Name of relation to check for data. + :returns: List of bundles of certificates. + :rtype: List of dicts + """ + local_name = local_unit().replace('/', '_') + raw_certs_key = '{}.processed_requests'.format(local_name) + relation_name = relation_name or 'certificates' + bundles = [] + for rid in relation_ids(relation_name): + for unit in related_units(rid): + data = relation_get(rid=rid, unit=unit) + if data.get(raw_certs_key): + bundles.append({ + 'ca': data['ca'], + 'chain': data.get('chain'), + 'certs': json.loads(data[raw_certs_key])}) + return bundles + + +def get_bundle_for_cn(cn, relation_name=None): + """Extract certificates for the given cn. + + :param cn: str Canonical Name on certificate. + :param relation_name: str Relation to check for certificates down. + :returns: Dictionary of certificate data, + :rtype: dict. + """ + entries = get_requests_for_local_unit(relation_name) + cert_bundle = {} + for entry in entries: + for _cn, bundle in entry['certs'].items(): + if _cn == cn: + cert_bundle = { + 'cert': bundle['cert'], + 'key': bundle['key'], + 'chain': entry['chain'], + 'ca': entry['ca']} + break + if cert_bundle: + break + return cert_bundle diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/context.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/context.py new file mode 100644 index 0000000..54081f0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/context.py @@ -0,0 +1,3363 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +import enum +import glob +import hashlib +import json +import math +import os +import re +import socket +import time + +from base64 import b64decode +from subprocess import ( + check_call, + check_output, + CalledProcessError) + +import six + +import charmhelpers.contrib.storage.linux.ceph as ch_ceph + +from charmhelpers.contrib.openstack.audits.openstack_security_guide import ( + _config_ini as config_ini +) + +from charmhelpers.fetch import ( + apt_install, + filter_installed_packages, +) +from charmhelpers.core.hookenv import ( + NoNetworkBinding, + config, + is_relation_made, + local_unit, + log, + relation_get, + relation_ids, + related_units, + relation_set, + unit_private_ip, + charm_name, + DEBUG, + INFO, + ERROR, + status_set, + network_get_primary_address, + WARNING, + service_name, +) + +from charmhelpers.core.sysctl import create as sysctl_create +from charmhelpers.core.strutils import bool_from_string +from charmhelpers.contrib.openstack.exceptions import OSContextError + +from charmhelpers.core.host import ( + get_bond_master, + is_phy_iface, + list_nics, + get_nic_hwaddr, + mkdir, + write_file, + pwgen, + lsb_release, + CompareHostReleases, +) +from charmhelpers.contrib.hahelpers.cluster import ( + determine_apache_port, + determine_api_port, + https, + is_clustered, +) +from charmhelpers.contrib.hahelpers.apache import ( + get_cert, + get_ca_cert, + install_ca_cert, +) +from charmhelpers.contrib.openstack.neutron import ( + neutron_plugin_attribute, + parse_data_port_mappings, +) +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + INTERNAL, + ADMIN, + PUBLIC, + ADDRESS_MAP, + local_address, +) +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + get_ipv4_addr, + get_ipv6_addr, + get_netmask_for_address, + format_ipv6_addr, + is_bridge_member, + is_ipv6_disabled, + get_relation_ip, +) +from charmhelpers.contrib.openstack.utils import ( + config_flags_parser, + get_os_codename_install_source, + enable_memcache, + CompareOpenStackReleases, + os_release, +) +from charmhelpers.core.unitdata import kv + +try: + from sriov_netplan_shim import pci +except ImportError: + # The use of the function and contexts that require the pci module is + # optional. + pass + +try: + import psutil +except ImportError: + if six.PY2: + apt_install('python-psutil', fatal=True) + else: + apt_install('python3-psutil', fatal=True) + import psutil + +CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' +ADDRESS_TYPES = ['admin', 'internal', 'public'] +HAPROXY_RUN_DIR = '/var/run/haproxy/' +DEFAULT_OSLO_MESSAGING_DRIVER = "messagingv2" + + +def ensure_packages(packages): + """Install but do not upgrade required plugin packages.""" + required = filter_installed_packages(packages) + if required: + apt_install(required, fatal=True) + + +def context_complete(ctxt): + _missing = [] + for k, v in six.iteritems(ctxt): + if v is None or v == '': + _missing.append(k) + + if _missing: + log('Missing required data: %s' % ' '.join(_missing), level=INFO) + return False + + return True + + +class OSContextGenerator(object): + """Base class for all context generators.""" + interfaces = [] + related = False + complete = False + missing_data = [] + + def __call__(self): + raise NotImplementedError + + def context_complete(self, ctxt): + """Check for missing data for the required context data. + Set self.missing_data if it exists and return False. + Set self.complete if no missing data and return True. + """ + # Fresh start + self.complete = False + self.missing_data = [] + for k, v in six.iteritems(ctxt): + if v is None or v == '': + if k not in self.missing_data: + self.missing_data.append(k) + + if self.missing_data: + self.complete = False + log('Missing required data: %s' % ' '.join(self.missing_data), + level=INFO) + else: + self.complete = True + return self.complete + + def get_related(self): + """Check if any of the context interfaces have relation ids. + Set self.related and return True if one of the interfaces + has relation ids. + """ + # Fresh start + self.related = False + try: + for interface in self.interfaces: + if relation_ids(interface): + self.related = True + return self.related + except AttributeError as e: + log("{} {}" + "".format(self, e), 'INFO') + return self.related + + +class SharedDBContext(OSContextGenerator): + interfaces = ['shared-db'] + + def __init__(self, database=None, user=None, relation_prefix=None, + ssl_dir=None, relation_id=None): + """Allows inspecting relation for settings prefixed with + relation_prefix. This is useful for parsing access for multiple + databases returned via the shared-db interface (eg, nova_password, + quantum_password) + """ + self.relation_prefix = relation_prefix + self.database = database + self.user = user + self.ssl_dir = ssl_dir + self.rel_name = self.interfaces[0] + self.relation_id = relation_id + + def __call__(self): + self.database = self.database or config('database') + self.user = self.user or config('database-user') + if None in [self.database, self.user]: + log("Could not generate shared_db context. Missing required charm " + "config options. (database name and user)", level=ERROR) + raise OSContextError + + ctxt = {} + + # NOTE(jamespage) if mysql charm provides a network upon which + # access to the database should be made, reconfigure relation + # with the service units local address and defer execution + access_network = relation_get('access-network') + if access_network is not None: + if self.relation_prefix is not None: + hostname_key = "{}_hostname".format(self.relation_prefix) + else: + hostname_key = "hostname" + access_hostname = get_address_in_network( + access_network, + local_address(unit_get_fallback='private-address')) + set_hostname = relation_get(attribute=hostname_key, + unit=local_unit()) + if set_hostname != access_hostname: + relation_set(relation_settings={hostname_key: access_hostname}) + return None # Defer any further hook execution for now.... + + password_setting = 'password' + if self.relation_prefix: + password_setting = self.relation_prefix + '_password' + + if self.relation_id: + rids = [self.relation_id] + else: + rids = relation_ids(self.interfaces[0]) + + rel = (get_os_codename_install_source(config('openstack-origin')) or + 'icehouse') + for rid in rids: + self.related = True + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + host = rdata.get('db_host') + host = format_ipv6_addr(host) or host + ctxt = { + 'database_host': host, + 'database': self.database, + 'database_user': self.user, + 'database_password': rdata.get(password_setting), + 'database_type': 'mysql+pymysql' + } + # Port is being introduced with LP Bug #1876188 + # but it not currently required and may not be set in all + # cases, particularly in classic charms. + port = rdata.get('db_port') + if port: + ctxt['database_port'] = port + if CompareOpenStackReleases(rel) < 'queens': + ctxt['database_type'] = 'mysql' + if self.context_complete(ctxt): + db_ssl(rdata, ctxt, self.ssl_dir) + return ctxt + return {} + + +class PostgresqlDBContext(OSContextGenerator): + interfaces = ['pgsql-db'] + + def __init__(self, database=None): + self.database = database + + def __call__(self): + self.database = self.database or config('database') + if self.database is None: + log('Could not generate postgresql_db context. Missing required ' + 'charm config options. (database name)', level=ERROR) + raise OSContextError + + ctxt = {} + for rid in relation_ids(self.interfaces[0]): + self.related = True + for unit in related_units(rid): + rel_host = relation_get('host', rid=rid, unit=unit) + rel_user = relation_get('user', rid=rid, unit=unit) + rel_passwd = relation_get('password', rid=rid, unit=unit) + ctxt = {'database_host': rel_host, + 'database': self.database, + 'database_user': rel_user, + 'database_password': rel_passwd, + 'database_type': 'postgresql'} + if self.context_complete(ctxt): + return ctxt + + return {} + + +def db_ssl(rdata, ctxt, ssl_dir): + if 'ssl_ca' in rdata and ssl_dir: + ca_path = os.path.join(ssl_dir, 'db-client.ca') + with open(ca_path, 'wb') as fh: + fh.write(b64decode(rdata['ssl_ca'])) + + ctxt['database_ssl_ca'] = ca_path + elif 'ssl_ca' in rdata: + log("Charm not setup for ssl support but ssl ca found", level=INFO) + return ctxt + + if 'ssl_cert' in rdata: + cert_path = os.path.join( + ssl_dir, 'db-client.cert') + if not os.path.exists(cert_path): + log("Waiting 1m for ssl client cert validity", level=INFO) + time.sleep(60) + + with open(cert_path, 'wb') as fh: + fh.write(b64decode(rdata['ssl_cert'])) + + ctxt['database_ssl_cert'] = cert_path + key_path = os.path.join(ssl_dir, 'db-client.key') + with open(key_path, 'wb') as fh: + fh.write(b64decode(rdata['ssl_key'])) + + ctxt['database_ssl_key'] = key_path + + return ctxt + + +class IdentityServiceContext(OSContextGenerator): + + def __init__(self, + service=None, + service_user=None, + rel_name='identity-service'): + self.service = service + self.service_user = service_user + self.rel_name = rel_name + self.interfaces = [self.rel_name] + + def _setup_pki_cache(self): + if self.service and self.service_user: + # This is required for pki token signing if we don't want /tmp to + # be used. + cachedir = '/var/cache/%s' % (self.service) + if not os.path.isdir(cachedir): + log("Creating service cache dir %s" % (cachedir), level=DEBUG) + mkdir(path=cachedir, owner=self.service_user, + group=self.service_user, perms=0o700) + + return cachedir + return None + + def _get_pkg_name(self, python_name='keystonemiddleware'): + """Get corresponding distro installed package for python + package name. + + :param python_name: nameof the python package + :type: string + """ + pkg_names = map(lambda x: x + python_name, ('python3-', 'python-')) + + for pkg in pkg_names: + if not filter_installed_packages((pkg,)): + return pkg + + return None + + def _get_keystone_authtoken_ctxt(self, ctxt, keystonemiddleware_os_rel): + """Build Jinja2 context for full rendering of [keystone_authtoken] + section with variable names included. Re-constructed from former + template 'section-keystone-auth-mitaka'. + + :param ctxt: Jinja2 context returned from self.__call__() + :type: dict + :param keystonemiddleware_os_rel: OpenStack release name of + keystonemiddleware package installed + """ + c = collections.OrderedDict((('auth_type', 'password'),)) + + # 'www_authenticate_uri' replaced 'auth_uri' since Stein, + # see keystonemiddleware upstream sources for more info + if CompareOpenStackReleases(keystonemiddleware_os_rel) >= 'stein': + c.update(( + ('www_authenticate_uri', "{}://{}:{}/v3".format( + ctxt.get('service_protocol', ''), + ctxt.get('service_host', ''), + ctxt.get('service_port', ''))),)) + else: + c.update(( + ('auth_uri', "{}://{}:{}/v3".format( + ctxt.get('service_protocol', ''), + ctxt.get('service_host', ''), + ctxt.get('service_port', ''))),)) + + c.update(( + ('auth_url', "{}://{}:{}/v3".format( + ctxt.get('auth_protocol', ''), + ctxt.get('auth_host', ''), + ctxt.get('auth_port', ''))), + ('project_domain_name', ctxt.get('admin_domain_name', '')), + ('user_domain_name', ctxt.get('admin_domain_name', '')), + ('project_name', ctxt.get('admin_tenant_name', '')), + ('username', ctxt.get('admin_user', '')), + ('password', ctxt.get('admin_password', '')), + ('signing_dir', ctxt.get('signing_dir', '')),)) + + return c + + def __call__(self): + log('Generating template context for ' + self.rel_name, level=DEBUG) + ctxt = {} + + keystonemiddleware_os_release = None + if self._get_pkg_name(): + keystonemiddleware_os_release = os_release(self._get_pkg_name()) + + cachedir = self._setup_pki_cache() + if cachedir: + ctxt['signing_dir'] = cachedir + + for rid in relation_ids(self.rel_name): + self.related = True + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + serv_host = rdata.get('service_host') + serv_host = format_ipv6_addr(serv_host) or serv_host + auth_host = rdata.get('auth_host') + auth_host = format_ipv6_addr(auth_host) or auth_host + int_host = rdata.get('internal_host') + int_host = format_ipv6_addr(int_host) or int_host + svc_protocol = rdata.get('service_protocol') or 'http' + auth_protocol = rdata.get('auth_protocol') or 'http' + int_protocol = rdata.get('internal_protocol') or 'http' + api_version = rdata.get('api_version') or '2.0' + ctxt.update({'service_port': rdata.get('service_port'), + 'service_host': serv_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'internal_host': int_host, + 'internal_port': rdata.get('internal_port'), + 'admin_tenant_name': rdata.get('service_tenant'), + 'admin_user': rdata.get('service_username'), + 'admin_password': rdata.get('service_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol, + 'internal_protocol': int_protocol, + 'api_version': api_version}) + + if float(api_version) > 2: + ctxt.update({ + 'admin_domain_name': rdata.get('service_domain'), + 'service_project_id': rdata.get('service_tenant_id'), + 'service_domain_id': rdata.get('service_domain_id')}) + + # we keep all veriables in ctxt for compatibility and + # add nested dictionary for keystone_authtoken generic + # templating + if keystonemiddleware_os_release: + ctxt['keystone_authtoken'] = \ + self._get_keystone_authtoken_ctxt( + ctxt, keystonemiddleware_os_release) + + if self.context_complete(ctxt): + # NOTE(jamespage) this is required for >= icehouse + # so a missing value just indicates keystone needs + # upgrading + ctxt['admin_tenant_id'] = rdata.get('service_tenant_id') + ctxt['admin_domain_id'] = rdata.get('service_domain_id') + return ctxt + + return {} + + +class IdentityCredentialsContext(IdentityServiceContext): + '''Context for identity-credentials interface type''' + + def __init__(self, + service=None, + service_user=None, + rel_name='identity-credentials'): + super(IdentityCredentialsContext, self).__init__(service, + service_user, + rel_name) + + def __call__(self): + log('Generating template context for ' + self.rel_name, level=DEBUG) + ctxt = {} + + cachedir = self._setup_pki_cache() + if cachedir: + ctxt['signing_dir'] = cachedir + + for rid in relation_ids(self.rel_name): + self.related = True + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + credentials_host = rdata.get('credentials_host') + credentials_host = ( + format_ipv6_addr(credentials_host) or credentials_host + ) + auth_host = rdata.get('auth_host') + auth_host = format_ipv6_addr(auth_host) or auth_host + svc_protocol = rdata.get('credentials_protocol') or 'http' + auth_protocol = rdata.get('auth_protocol') or 'http' + api_version = rdata.get('api_version') or '2.0' + ctxt.update({ + 'service_port': rdata.get('credentials_port'), + 'service_host': credentials_host, + 'auth_host': auth_host, + 'auth_port': rdata.get('auth_port'), + 'admin_tenant_name': rdata.get('credentials_project'), + 'admin_tenant_id': rdata.get('credentials_project_id'), + 'admin_user': rdata.get('credentials_username'), + 'admin_password': rdata.get('credentials_password'), + 'service_protocol': svc_protocol, + 'auth_protocol': auth_protocol, + 'api_version': api_version + }) + + if float(api_version) > 2: + ctxt.update({'admin_domain_name': + rdata.get('domain')}) + + if self.context_complete(ctxt): + return ctxt + + return {} + + +class NovaVendorMetadataContext(OSContextGenerator): + """Context used for configuring nova vendor metadata on nova.conf file.""" + + def __init__(self, os_release_pkg, interfaces=None): + """Initialize the NovaVendorMetadataContext object. + + :param os_release_pkg: the package name to extract the OpenStack + release codename from. + :type os_release_pkg: str + :param interfaces: list of string values to be used as the Context's + relation interfaces. + :type interfaces: List[str] + """ + self.os_release_pkg = os_release_pkg + if interfaces is not None: + self.interfaces = interfaces + + def __call__(self): + cmp_os_release = CompareOpenStackReleases( + os_release(self.os_release_pkg)) + ctxt = {'vendor_data': False} + + vdata_providers = [] + vdata = config('vendor-data') + vdata_url = config('vendor-data-url') + + if vdata: + try: + # validate the JSON. If invalid, we do not set anything here + json.loads(vdata) + except (TypeError, ValueError) as e: + log('Error decoding vendor-data. {}'.format(e), level=ERROR) + else: + ctxt['vendor_data'] = True + # Mitaka does not support DynamicJSON + # so vendordata_providers is not needed + if cmp_os_release > 'mitaka': + vdata_providers.append('StaticJSON') + + if vdata_url: + if cmp_os_release > 'mitaka': + ctxt['vendor_data_url'] = vdata_url + vdata_providers.append('DynamicJSON') + else: + log('Dynamic vendor data unsupported' + ' for {}.'.format(cmp_os_release), level=ERROR) + if vdata_providers: + ctxt['vendordata_providers'] = ','.join(vdata_providers) + + return ctxt + + +class NovaVendorMetadataJSONContext(OSContextGenerator): + """Context used for writing nova vendor metadata json file.""" + + def __init__(self, os_release_pkg): + """Initialize the NovaVendorMetadataJSONContext object. + + :param os_release_pkg: the package name to extract the OpenStack + release codename from. + :type os_release_pkg: str + """ + self.os_release_pkg = os_release_pkg + + def __call__(self): + ctxt = {'vendor_data_json': '{}'} + + vdata = config('vendor-data') + if vdata: + try: + # validate the JSON. If invalid, we return empty. + json.loads(vdata) + except (TypeError, ValueError) as e: + log('Error decoding vendor-data. {}'.format(e), level=ERROR) + else: + ctxt['vendor_data_json'] = vdata + + return ctxt + + +class AMQPContext(OSContextGenerator): + + def __init__(self, ssl_dir=None, rel_name='amqp', relation_prefix=None, + relation_id=None): + self.ssl_dir = ssl_dir + self.rel_name = rel_name + self.relation_prefix = relation_prefix + self.interfaces = [rel_name] + self.relation_id = relation_id + + def __call__(self): + log('Generating template context for amqp', level=DEBUG) + conf = config() + if self.relation_prefix: + user_setting = '%s-rabbit-user' % (self.relation_prefix) + vhost_setting = '%s-rabbit-vhost' % (self.relation_prefix) + else: + user_setting = 'rabbit-user' + vhost_setting = 'rabbit-vhost' + + try: + username = conf[user_setting] + vhost = conf[vhost_setting] + except KeyError as e: + log('Could not generate shared_db context. Missing required charm ' + 'config options: %s.' % e, level=ERROR) + raise OSContextError + + ctxt = {} + if self.relation_id: + rids = [self.relation_id] + else: + rids = relation_ids(self.rel_name) + for rid in rids: + ha_vip_only = False + self.related = True + transport_hosts = None + rabbitmq_port = '5672' + for unit in related_units(rid): + if relation_get('clustered', rid=rid, unit=unit): + ctxt['clustered'] = True + vip = relation_get('vip', rid=rid, unit=unit) + vip = format_ipv6_addr(vip) or vip + ctxt['rabbitmq_host'] = vip + transport_hosts = [vip] + else: + host = relation_get('private-address', rid=rid, unit=unit) + host = format_ipv6_addr(host) or host + ctxt['rabbitmq_host'] = host + transport_hosts = [host] + + ctxt.update({ + 'rabbitmq_user': username, + 'rabbitmq_password': relation_get('password', rid=rid, + unit=unit), + 'rabbitmq_virtual_host': vhost, + }) + + ssl_port = relation_get('ssl_port', rid=rid, unit=unit) + if ssl_port: + ctxt['rabbit_ssl_port'] = ssl_port + rabbitmq_port = ssl_port + + ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit) + if ssl_ca: + ctxt['rabbit_ssl_ca'] = ssl_ca + + if relation_get('ha_queues', rid=rid, unit=unit) is not None: + ctxt['rabbitmq_ha_queues'] = True + + ha_vip_only = relation_get('ha-vip-only', + rid=rid, unit=unit) is not None + + if self.context_complete(ctxt): + if 'rabbit_ssl_ca' in ctxt: + if not self.ssl_dir: + log("Charm not setup for ssl support but ssl ca " + "found", level=INFO) + break + + ca_path = os.path.join( + self.ssl_dir, 'rabbit-client-ca.pem') + with open(ca_path, 'wb') as fh: + fh.write(b64decode(ctxt['rabbit_ssl_ca'])) + ctxt['rabbit_ssl_ca'] = ca_path + + # Sufficient information found = break out! + break + + # Used for active/active rabbitmq >= grizzly + if (('clustered' not in ctxt or ha_vip_only) and + len(related_units(rid)) > 1): + rabbitmq_hosts = [] + for unit in related_units(rid): + host = relation_get('private-address', rid=rid, unit=unit) + if not relation_get('password', rid=rid, unit=unit): + log( + ("Skipping {} password not sent which indicates " + "unit is not ready.".format(host)), + level=DEBUG) + continue + host = format_ipv6_addr(host) or host + rabbitmq_hosts.append(host) + + rabbitmq_hosts = sorted(rabbitmq_hosts) + ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts) + transport_hosts = rabbitmq_hosts + + if transport_hosts: + transport_url_hosts = ','.join([ + "{}:{}@{}:{}".format(ctxt['rabbitmq_user'], + ctxt['rabbitmq_password'], + host_, + rabbitmq_port) + for host_ in transport_hosts]) + ctxt['transport_url'] = "rabbit://{}/{}".format( + transport_url_hosts, vhost) + + oslo_messaging_flags = conf.get('oslo-messaging-flags', None) + if oslo_messaging_flags: + ctxt['oslo_messaging_flags'] = config_flags_parser( + oslo_messaging_flags) + + oslo_messaging_driver = conf.get( + 'oslo-messaging-driver', DEFAULT_OSLO_MESSAGING_DRIVER) + if oslo_messaging_driver: + ctxt['oslo_messaging_driver'] = oslo_messaging_driver + + notification_format = conf.get('notification-format', None) + if notification_format: + ctxt['notification_format'] = notification_format + + notification_topics = conf.get('notification-topics', None) + if notification_topics: + ctxt['notification_topics'] = notification_topics + + send_notifications_to_logs = conf.get('send-notifications-to-logs', None) + if send_notifications_to_logs: + ctxt['send_notifications_to_logs'] = send_notifications_to_logs + + if not self.complete: + return {} + + return ctxt + + +class CephContext(OSContextGenerator): + """Generates context for /etc/ceph/ceph.conf templates.""" + interfaces = ['ceph'] + + def __call__(self): + if not relation_ids('ceph'): + return {} + + log('Generating template context for ceph', level=DEBUG) + mon_hosts = [] + ctxt = { + 'use_syslog': str(config('use-syslog')).lower() + } + for rid in relation_ids('ceph'): + for unit in related_units(rid): + if not ctxt.get('auth'): + ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) + if not ctxt.get('key'): + ctxt['key'] = relation_get('key', rid=rid, unit=unit) + if not ctxt.get('rbd_features'): + default_features = relation_get('rbd-features', rid=rid, unit=unit) + if default_features is not None: + ctxt['rbd_features'] = default_features + + ceph_addrs = relation_get('ceph-public-address', rid=rid, + unit=unit) + if ceph_addrs: + for addr in ceph_addrs.split(' '): + mon_hosts.append(format_ipv6_addr(addr) or addr) + else: + priv_addr = relation_get('private-address', rid=rid, + unit=unit) + mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr) + + ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts)) + + if config('pool-type') and config('pool-type') == 'erasure-coded': + base_pool_name = config('rbd-pool') or config('rbd-pool-name') + if not base_pool_name: + base_pool_name = service_name() + ctxt['rbd_default_data_pool'] = base_pool_name + + if not os.path.isdir('/etc/ceph'): + os.mkdir('/etc/ceph') + + if not self.context_complete(ctxt): + return {} + + ensure_packages(['ceph-common']) + return ctxt + + def context_complete(self, ctxt): + """Overridden here to ensure the context is actually complete. + + We set `key` and `auth` to None here, by default, to ensure + that the context will always evaluate to incomplete until the + Ceph relation has actually sent these details; otherwise, + there is a potential race condition between the relation + appearing and the first unit actually setting this data on the + relation. + + :param ctxt: The current context members + :type ctxt: Dict[str, ANY] + :returns: True if the context is complete + :rtype: bool + """ + if 'auth' not in ctxt or 'key' not in ctxt: + return False + return super(CephContext, self).context_complete(ctxt) + + +class HAProxyContext(OSContextGenerator): + """Provides half a context for the haproxy template, which describes + all peers to be included in the cluster. Each charm needs to include + its own context generator that describes the port mapping. + + :side effect: mkdir is called on HAPROXY_RUN_DIR + """ + interfaces = ['cluster'] + + def __init__(self, singlenode_mode=False, + address_types=ADDRESS_TYPES): + self.address_types = address_types + self.singlenode_mode = singlenode_mode + + def __call__(self): + if not os.path.isdir(HAPROXY_RUN_DIR): + mkdir(path=HAPROXY_RUN_DIR) + if not relation_ids('cluster') and not self.singlenode_mode: + return {} + + l_unit = local_unit().replace('/', '-') + cluster_hosts = collections.OrderedDict() + + # NOTE(jamespage): build out map of configured network endpoints + # and associated backends + for addr_type in self.address_types: + cfg_opt = 'os-{}-network'.format(addr_type) + # NOTE(thedac) For some reason the ADDRESS_MAP uses 'int' rather + # than 'internal' + if addr_type == 'internal': + _addr_map_type = INTERNAL + else: + _addr_map_type = addr_type + # Network spaces aware + laddr = get_relation_ip(ADDRESS_MAP[_addr_map_type]['binding'], + config(cfg_opt)) + if laddr: + netmask = get_netmask_for_address(laddr) + cluster_hosts[laddr] = { + 'network': "{}/{}".format(laddr, + netmask), + 'backends': collections.OrderedDict([(l_unit, + laddr)]) + } + for rid in relation_ids('cluster'): + for unit in sorted(related_units(rid)): + # API Charms will need to set {addr_type}-address with + # get_relation_ip(addr_type) + _laddr = relation_get('{}-address'.format(addr_type), + rid=rid, unit=unit) + if _laddr: + _unit = unit.replace('/', '-') + cluster_hosts[laddr]['backends'][_unit] = _laddr + + # NOTE(jamespage) add backend based on get_relation_ip - this + # will either be the only backend or the fallback if no acls + # match in the frontend + # Network spaces aware + addr = get_relation_ip('cluster') + cluster_hosts[addr] = {} + netmask = get_netmask_for_address(addr) + cluster_hosts[addr] = { + 'network': "{}/{}".format(addr, netmask), + 'backends': collections.OrderedDict([(l_unit, + addr)]) + } + for rid in relation_ids('cluster'): + for unit in sorted(related_units(rid)): + # API Charms will need to set their private-address with + # get_relation_ip('cluster') + _laddr = relation_get('private-address', + rid=rid, unit=unit) + if _laddr: + _unit = unit.replace('/', '-') + cluster_hosts[addr]['backends'][_unit] = _laddr + + ctxt = { + 'frontends': cluster_hosts, + 'default_backend': addr + } + + if config('haproxy-server-timeout'): + ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout') + + if config('haproxy-client-timeout'): + ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') + + if config('haproxy-queue-timeout'): + ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout') + + if config('haproxy-connect-timeout'): + ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout') + + if config('prefer-ipv6'): + ctxt['local_host'] = 'ip6-localhost' + ctxt['haproxy_host'] = '::' + else: + ctxt['local_host'] = '127.0.0.1' + ctxt['haproxy_host'] = '0.0.0.0' + + ctxt['ipv6_enabled'] = not is_ipv6_disabled() + + ctxt['stat_port'] = '8888' + + db = kv() + ctxt['stat_password'] = db.get('stat-password') + if not ctxt['stat_password']: + ctxt['stat_password'] = db.set('stat-password', + pwgen(32)) + db.flush() + + for frontend in cluster_hosts: + if (len(cluster_hosts[frontend]['backends']) > 1 or + self.singlenode_mode): + # Enable haproxy when we have enough peers. + log('Ensuring haproxy enabled in /etc/default/haproxy.', + level=DEBUG) + with open('/etc/default/haproxy', 'w') as out: + out.write('ENABLED=1\n') + + return ctxt + + log('HAProxy context is incomplete, this unit has no peers.', + level=INFO) + return {} + + +class ImageServiceContext(OSContextGenerator): + interfaces = ['image-service'] + + def __call__(self): + """Obtains the glance API server from the image-service relation. + Useful in nova and cinder (currently). + """ + log('Generating template context for image-service.', level=DEBUG) + rids = relation_ids('image-service') + if not rids: + return {} + + for rid in rids: + for unit in related_units(rid): + api_server = relation_get('glance-api-server', + rid=rid, unit=unit) + if api_server: + return {'glance_api_servers': api_server} + + log("ImageService context is incomplete. Missing required relation " + "data.", level=INFO) + return {} + + +class ApacheSSLContext(OSContextGenerator): + """Generates a context for an apache vhost configuration that configures + HTTPS reverse proxying for one or many endpoints. Generated context + looks something like:: + + { + 'namespace': 'cinder', + 'private_address': 'iscsi.mycinderhost.com', + 'endpoints': [(8776, 8766), (8777, 8767)] + } + + The endpoints list consists of a tuples mapping external ports + to internal ports. + """ + interfaces = ['https'] + + # charms should inherit this context and set external ports + # and service namespace accordingly. + external_ports = [] + service_namespace = None + user = group = 'root' + + def enable_modules(self): + cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers'] + check_call(cmd) + + def configure_cert(self, cn=None): + ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) + mkdir(path=ssl_dir) + cert, key = get_cert(cn) + if cert and key: + if cn: + cert_filename = 'cert_{}'.format(cn) + key_filename = 'key_{}'.format(cn) + else: + cert_filename = 'cert' + key_filename = 'key' + + write_file(path=os.path.join(ssl_dir, cert_filename), + content=b64decode(cert), owner=self.user, + group=self.group, perms=0o640) + write_file(path=os.path.join(ssl_dir, key_filename), + content=b64decode(key), owner=self.user, + group=self.group, perms=0o640) + + def configure_ca(self): + ca_cert = get_ca_cert() + if ca_cert: + install_ca_cert(b64decode(ca_cert)) + + def canonical_names(self): + """Figure out which canonical names clients will access this service. + """ + cns = [] + for r_id in relation_ids('identity-service'): + for unit in related_units(r_id): + rdata = relation_get(rid=r_id, unit=unit) + for k in rdata: + if k.startswith('ssl_key_'): + cns.append(k.lstrip('ssl_key_')) + + return sorted(list(set(cns))) + + def get_network_addresses(self): + """For each network configured, return corresponding address and + hostnamr or vip (if available). + + Returns a list of tuples of the form: + + [(address_in_net_a, hostname_in_net_a), + (address_in_net_b, hostname_in_net_b), + ...] + + or, if no hostnames(s) available: + + [(address_in_net_a, vip_in_net_a), + (address_in_net_b, vip_in_net_b), + ...] + + or, if no vip(s) available: + + [(address_in_net_a, address_in_net_a), + (address_in_net_b, address_in_net_b), + ...] + """ + addresses = [] + for net_type in [INTERNAL, ADMIN, PUBLIC]: + net_config = config(ADDRESS_MAP[net_type]['config']) + # NOTE(jamespage): Fallback must always be private address + # as this is used to bind services on the + # local unit. + fallback = local_address(unit_get_fallback="private-address") + if net_config: + addr = get_address_in_network(net_config, + fallback) + else: + try: + addr = network_get_primary_address( + ADDRESS_MAP[net_type]['binding'] + ) + except (NotImplementedError, NoNetworkBinding): + addr = fallback + + endpoint = resolve_address(net_type) + addresses.append((addr, endpoint)) + + return sorted(set(addresses)) + + def __call__(self): + if isinstance(self.external_ports, six.string_types): + self.external_ports = [self.external_ports] + + if not self.external_ports or not https(): + return {} + + use_keystone_ca = True + for rid in relation_ids('certificates'): + if related_units(rid): + use_keystone_ca = False + + if use_keystone_ca: + self.configure_ca() + + self.enable_modules() + + ctxt = {'namespace': self.service_namespace, + 'endpoints': [], + 'ext_ports': []} + + if use_keystone_ca: + cns = self.canonical_names() + if cns: + for cn in cns: + self.configure_cert(cn) + else: + # Expect cert/key provided in config (currently assumed that ca + # uses ip for cn) + for net_type in (INTERNAL, ADMIN, PUBLIC): + cn = resolve_address(endpoint_type=net_type) + self.configure_cert(cn) + + addresses = self.get_network_addresses() + for address, endpoint in addresses: + for api_port in self.external_ports: + ext_port = determine_apache_port(api_port, + singlenode_mode=True) + int_port = determine_api_port(api_port, singlenode_mode=True) + portmap = (address, endpoint, int(ext_port), int(int_port)) + ctxt['endpoints'].append(portmap) + ctxt['ext_ports'].append(int(ext_port)) + + ctxt['ext_ports'] = sorted(list(set(ctxt['ext_ports']))) + return ctxt + + +class NeutronContext(OSContextGenerator): + interfaces = [] + + @property + def plugin(self): + return None + + @property + def network_manager(self): + return None + + @property + def packages(self): + return neutron_plugin_attribute(self.plugin, 'packages', + self.network_manager) + + @property + def neutron_security_groups(self): + return None + + def _ensure_packages(self): + for pkgs in self.packages: + ensure_packages(pkgs) + + def ovs_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + ovs_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'ovs', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return ovs_ctxt + + def nuage_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nuage_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'vsp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nuage_ctxt + + def nvp_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + nvp_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'nvp', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return nvp_ctxt + + def n1kv_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + n1kv_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + n1kv_user_config_flags = config('n1kv-config-flags') + restrict_policy_profiles = config('n1kv-restrict-policy-profiles') + n1kv_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': n1kv_config, + 'vsm_ip': config('n1kv-vsm-ip'), + 'vsm_username': config('n1kv-vsm-username'), + 'vsm_password': config('n1kv-vsm-password'), + 'restrict_policy_profiles': restrict_policy_profiles} + + if n1kv_user_config_flags: + flags = config_flags_parser(n1kv_user_config_flags) + n1kv_ctxt['user_config_flags'] = flags + + return n1kv_ctxt + + def calico_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + calico_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'Calico', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + + return calico_ctxt + + def neutron_ctxt(self): + if https(): + proto = 'https' + else: + proto = 'http' + + if is_clustered(): + host = config('vip') + else: + host = local_address(unit_get_fallback='private-address') + + ctxt = {'network_manager': self.network_manager, + 'neutron_url': '%s://%s:%s' % (proto, host, '9696')} + return ctxt + + def pg_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + ovs_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'plumgrid', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': config} + return ovs_ctxt + + def midonet_ctxt(self): + driver = neutron_plugin_attribute(self.plugin, 'driver', + self.network_manager) + midonet_config = neutron_plugin_attribute(self.plugin, 'config', + self.network_manager) + mido_ctxt = {'core_plugin': driver, + 'neutron_plugin': 'midonet', + 'neutron_security_groups': self.neutron_security_groups, + 'local_ip': unit_private_ip(), + 'config': midonet_config} + + return mido_ctxt + + def __call__(self): + if self.network_manager not in ['quantum', 'neutron']: + return {} + + if not self.plugin: + return {} + + ctxt = self.neutron_ctxt() + + if self.plugin == 'ovs': + ctxt.update(self.ovs_ctxt()) + elif self.plugin in ['nvp', 'nsx']: + ctxt.update(self.nvp_ctxt()) + elif self.plugin == 'n1kv': + ctxt.update(self.n1kv_ctxt()) + elif self.plugin == 'Calico': + ctxt.update(self.calico_ctxt()) + elif self.plugin == 'vsp': + ctxt.update(self.nuage_ctxt()) + elif self.plugin == 'plumgrid': + ctxt.update(self.pg_ctxt()) + elif self.plugin == 'midonet': + ctxt.update(self.midonet_ctxt()) + + alchemy_flags = config('neutron-alchemy-flags') + if alchemy_flags: + flags = config_flags_parser(alchemy_flags) + ctxt['neutron_alchemy_flags'] = flags + + return ctxt + + +class NeutronPortContext(OSContextGenerator): + + def resolve_ports(self, ports): + """Resolve NICs not yet bound to bridge(s) + + If hwaddress provided then returns resolved hwaddress otherwise NIC. + """ + if not ports: + return None + + hwaddr_to_nic = {} + hwaddr_to_ip = {} + extant_nics = list_nics() + + for nic in extant_nics: + # Ignore virtual interfaces (bond masters will be identified from + # their slaves) + if not is_phy_iface(nic): + continue + + _nic = get_bond_master(nic) + if _nic: + log("Replacing iface '%s' with bond master '%s'" % (nic, _nic), + level=DEBUG) + nic = _nic + + hwaddr = get_nic_hwaddr(nic) + hwaddr_to_nic[hwaddr] = nic + addresses = get_ipv4_addr(nic, fatal=False) + addresses += get_ipv6_addr(iface=nic, fatal=False) + hwaddr_to_ip[hwaddr] = addresses + + resolved = [] + mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I) + for entry in ports: + if re.match(mac_regex, entry): + # NIC is in known NICs and does NOT have an IP address + if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]: + # If the nic is part of a bridge then don't use it + if is_bridge_member(hwaddr_to_nic[entry]): + continue + + # Entry is a MAC address for a valid interface that doesn't + # have an IP address assigned yet. + resolved.append(hwaddr_to_nic[entry]) + elif entry in extant_nics: + # If the passed entry is not a MAC address and the interface + # exists, assume it's a valid interface, and that the user put + # it there on purpose (we can trust it to be the real external + # network). + resolved.append(entry) + + # Ensure no duplicates + return list(set(resolved)) + + +class OSConfigFlagContext(OSContextGenerator): + """Provides support for user-defined config flags. + + Users can define a comma-seperated list of key=value pairs + in the charm configuration and apply them at any point in + any file by using a template flag. + + Sometimes users might want config flags inserted within a + specific section so this class allows users to specify the + template flag name, allowing for multiple template flags + (sections) within the same context. + + NOTE: the value of config-flags may be a comma-separated list of + key=value pairs and some Openstack config files support + comma-separated lists as values. + """ + + def __init__(self, charm_flag='config-flags', + template_flag='user_config_flags'): + """ + :param charm_flag: config flags in charm configuration. + :param template_flag: insert point for user-defined flags in template + file. + """ + super(OSConfigFlagContext, self).__init__() + self._charm_flag = charm_flag + self._template_flag = template_flag + + def __call__(self): + config_flags = config(self._charm_flag) + if not config_flags: + return {} + + return {self._template_flag: + config_flags_parser(config_flags)} + + +class LibvirtConfigFlagsContext(OSContextGenerator): + """ + This context provides support for extending + the libvirt section through user-defined flags. + """ + def __call__(self): + ctxt = {} + libvirt_flags = config('libvirt-flags') + if libvirt_flags: + ctxt['libvirt_flags'] = config_flags_parser( + libvirt_flags) + return ctxt + + +class SubordinateConfigContext(OSContextGenerator): + + """ + Responsible for inspecting relations to subordinates that + may be exporting required config via a json blob. + + The subordinate interface allows subordinates to export their + configuration requirements to the principle for multiple config + files and multiple services. Ie, a subordinate that has interfaces + to both glance and nova may export to following yaml blob as json:: + + glance: + /etc/glance/glance-api.conf: + sections: + DEFAULT: + - [key1, value1] + /etc/glance/glance-registry.conf: + MYSECTION: + - [key2, value2] + nova: + /etc/nova/nova.conf: + sections: + DEFAULT: + - [key3, value3] + + + It is then up to the principle charms to subscribe this context to + the service+config file it is interestd in. Configuration data will + be available in the template context, in glance's case, as:: + + ctxt = { + ... other context ... + 'subordinate_configuration': { + 'DEFAULT': { + 'key1': 'value1', + }, + 'MYSECTION': { + 'key2': 'value2', + }, + } + } + """ + + def __init__(self, service, config_file, interface): + """ + :param service : Service name key to query in any subordinate + data found + :param config_file : Service's config file to query sections + :param interface : Subordinate interface to inspect + """ + self.config_file = config_file + if isinstance(service, list): + self.services = service + else: + self.services = [service] + if isinstance(interface, list): + self.interfaces = interface + else: + self.interfaces = [interface] + + def __call__(self): + ctxt = {'sections': {}} + rids = [] + for interface in self.interfaces: + rids.extend(relation_ids(interface)) + for rid in rids: + for unit in related_units(rid): + sub_config = relation_get('subordinate_configuration', + rid=rid, unit=unit) + if sub_config and sub_config != '': + try: + sub_config = json.loads(sub_config) + except Exception: + log('Could not parse JSON from ' + 'subordinate_configuration setting from %s' + % rid, level=ERROR) + continue + + for service in self.services: + if service not in sub_config: + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s service' + % (rid, service), level=INFO) + continue + + sub_config = sub_config[service] + if self.config_file not in sub_config: + log('Found subordinate_configuration on %s but it ' + 'contained nothing for %s' + % (rid, self.config_file), level=INFO) + continue + + sub_config = sub_config[self.config_file] + for k, v in six.iteritems(sub_config): + if k == 'sections': + for section, config_list in six.iteritems(v): + log("adding section '%s'" % (section), + level=DEBUG) + if ctxt[k].get(section): + ctxt[k][section].extend(config_list) + else: + ctxt[k][section] = config_list + else: + ctxt[k] = v + if self.context_complete(ctxt): + log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG) + return ctxt + else: + return {} + + def context_complete(self, ctxt): + """Overridden here to ensure the context is actually complete. + + :param ctxt: The current context members + :type ctxt: Dict[str, ANY] + :returns: True if the context is complete + :rtype: bool + """ + if not ctxt.get('sections'): + return False + return super(SubordinateConfigContext, self).context_complete(ctxt) + + +class LogLevelContext(OSContextGenerator): + + def __call__(self): + ctxt = {} + ctxt['debug'] = \ + False if config('debug') is None else config('debug') + ctxt['verbose'] = \ + False if config('verbose') is None else config('verbose') + + return ctxt + + +class SyslogContext(OSContextGenerator): + + def __call__(self): + ctxt = {'use_syslog': config('use-syslog')} + return ctxt + + +class BindHostContext(OSContextGenerator): + + def __call__(self): + if config('prefer-ipv6'): + return {'bind_host': '::'} + else: + return {'bind_host': '0.0.0.0'} + + +MAX_DEFAULT_WORKERS = 4 +DEFAULT_MULTIPLIER = 2 + + +def _calculate_workers(): + ''' + Determine the number of worker processes based on the CPU + count of the unit containing the application. + + Workers will be limited to MAX_DEFAULT_WORKERS in + container environments where no worker-multipler configuration + option been set. + + @returns int: number of worker processes to use + ''' + multiplier = config('worker-multiplier') + + # distinguish an empty config and an explicit config as 0.0 + if multiplier is None: + multiplier = DEFAULT_MULTIPLIER + + count = int(_num_cpus() * multiplier) + if count <= 0: + # assign at least one worker + count = 1 + + if config('worker-multiplier') is None: + # NOTE(jamespage): Limit unconfigured worker-multiplier + # to MAX_DEFAULT_WORKERS to avoid insane + # worker configuration on large servers + # Reference: https://pad.lv/1665270 + count = min(count, MAX_DEFAULT_WORKERS) + + return count + + +def _num_cpus(): + ''' + Compatibility wrapper for calculating the number of CPU's + a unit has. + + @returns: int: number of CPU cores detected + ''' + try: + return psutil.cpu_count() + except AttributeError: + return psutil.NUM_CPUS + + +class WorkerConfigContext(OSContextGenerator): + + def __call__(self): + ctxt = {"workers": _calculate_workers()} + return ctxt + + +class WSGIWorkerConfigContext(WorkerConfigContext): + + def __init__(self, name=None, script=None, admin_script=None, + public_script=None, user=None, group=None, + process_weight=1.00, + admin_process_weight=0.25, public_process_weight=0.75): + self.service_name = name + self.user = user or name + self.group = group or name + self.script = script + self.admin_script = admin_script + self.public_script = public_script + self.process_weight = process_weight + self.admin_process_weight = admin_process_weight + self.public_process_weight = public_process_weight + + def __call__(self): + total_processes = _calculate_workers() + ctxt = { + "service_name": self.service_name, + "user": self.user, + "group": self.group, + "script": self.script, + "admin_script": self.admin_script, + "public_script": self.public_script, + "processes": int(math.ceil(self.process_weight * total_processes)), + "admin_processes": int(math.ceil(self.admin_process_weight * + total_processes)), + "public_processes": int(math.ceil(self.public_process_weight * + total_processes)), + "threads": 1, + } + return ctxt + + +class ZeroMQContext(OSContextGenerator): + interfaces = ['zeromq-configuration'] + + def __call__(self): + ctxt = {} + if is_relation_made('zeromq-configuration', 'host'): + for rid in relation_ids('zeromq-configuration'): + for unit in related_units(rid): + ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) + ctxt['zmq_host'] = relation_get('host', unit, rid) + ctxt['zmq_redis_address'] = relation_get( + 'zmq_redis_address', unit, rid) + + return ctxt + + +class NotificationDriverContext(OSContextGenerator): + + def __init__(self, zmq_relation='zeromq-configuration', + amqp_relation='amqp'): + """ + :param zmq_relation: Name of Zeromq relation to check + """ + self.zmq_relation = zmq_relation + self.amqp_relation = amqp_relation + + def __call__(self): + ctxt = {'notifications': 'False'} + if is_relation_made(self.amqp_relation): + ctxt['notifications'] = "True" + + return ctxt + + +class SysctlContext(OSContextGenerator): + """This context check if the 'sysctl' option exists on configuration + then creates a file with the loaded contents""" + def __call__(self): + sysctl_dict = config('sysctl') + if sysctl_dict: + sysctl_create(sysctl_dict, + '/etc/sysctl.d/50-{0}.conf'.format(charm_name())) + return {'sysctl': sysctl_dict} + + +class NeutronAPIContext(OSContextGenerator): + ''' + Inspects current neutron-plugin-api relation for neutron settings. Return + defaults if it is not present. + ''' + interfaces = ['neutron-plugin-api'] + + def __call__(self): + self.neutron_defaults = { + 'l2_population': { + 'rel_key': 'l2-population', + 'default': False, + }, + 'overlay_network_type': { + 'rel_key': 'overlay-network-type', + 'default': 'gre', + }, + 'neutron_security_groups': { + 'rel_key': 'neutron-security-groups', + 'default': False, + }, + 'network_device_mtu': { + 'rel_key': 'network-device-mtu', + 'default': None, + }, + 'enable_dvr': { + 'rel_key': 'enable-dvr', + 'default': False, + }, + 'enable_l3ha': { + 'rel_key': 'enable-l3ha', + 'default': False, + }, + 'dns_domain': { + 'rel_key': 'dns-domain', + 'default': None, + }, + 'polling_interval': { + 'rel_key': 'polling-interval', + 'default': 2, + }, + 'rpc_response_timeout': { + 'rel_key': 'rpc-response-timeout', + 'default': 60, + }, + 'report_interval': { + 'rel_key': 'report-interval', + 'default': 30, + }, + 'enable_qos': { + 'rel_key': 'enable-qos', + 'default': False, + }, + 'enable_nsg_logging': { + 'rel_key': 'enable-nsg-logging', + 'default': False, + }, + 'enable_nfg_logging': { + 'rel_key': 'enable-nfg-logging', + 'default': False, + }, + 'enable_port_forwarding': { + 'rel_key': 'enable-port-forwarding', + 'default': False, + }, + 'enable_fwaas': { + 'rel_key': 'enable-fwaas', + 'default': False, + }, + 'global_physnet_mtu': { + 'rel_key': 'global-physnet-mtu', + 'default': 1500, + }, + 'physical_network_mtus': { + 'rel_key': 'physical-network-mtus', + 'default': None, + }, + } + ctxt = self.get_neutron_options({}) + for rid in relation_ids('neutron-plugin-api'): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + # The l2-population key is used by the context as a way of + # checking if the api service on the other end is sending data + # in a recent format. + if 'l2-population' in rdata: + ctxt.update(self.get_neutron_options(rdata)) + + extension_drivers = [] + + if ctxt['enable_qos']: + extension_drivers.append('qos') + + if ctxt['enable_nsg_logging']: + extension_drivers.append('log') + + ctxt['extension_drivers'] = ','.join(extension_drivers) + + l3_extension_plugins = [] + + if ctxt['enable_port_forwarding']: + l3_extension_plugins.append('port_forwarding') + + if ctxt['enable_fwaas']: + l3_extension_plugins.append('fwaas_v2') + if ctxt['enable_nfg_logging']: + l3_extension_plugins.append('fwaas_v2_log') + + ctxt['l3_extension_plugins'] = l3_extension_plugins + + return ctxt + + def get_neutron_options(self, rdata): + settings = {} + for nkey in self.neutron_defaults.keys(): + defv = self.neutron_defaults[nkey]['default'] + rkey = self.neutron_defaults[nkey]['rel_key'] + if rkey in rdata.keys(): + if type(defv) is bool: + settings[nkey] = bool_from_string(rdata[rkey]) + else: + settings[nkey] = rdata[rkey] + else: + settings[nkey] = defv + return settings + + +class ExternalPortContext(NeutronPortContext): + + def __call__(self): + ctxt = {} + ports = config('ext-port') + if ports: + ports = [p.strip() for p in ports.split()] + ports = self.resolve_ports(ports) + if ports: + ctxt = {"ext_port": ports[0]} + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + if mtu: + ctxt['ext_port_mtu'] = mtu + + return ctxt + + +class DataPortContext(NeutronPortContext): + + def __call__(self): + ports = config('data-port') + if ports: + # Map of {bridge:port/mac} + portmap = parse_data_port_mappings(ports) + ports = portmap.keys() + # Resolve provided ports or mac addresses and filter out those + # already attached to a bridge. + resolved = self.resolve_ports(ports) + # Rebuild port index using resolved and filtered ports. + normalized = {get_nic_hwaddr(port): port for port in resolved + if port not in ports} + normalized.update({port: port for port in resolved + if port in ports}) + if resolved: + return {normalized[port]: bridge for port, bridge in + six.iteritems(portmap) if port in normalized.keys()} + + return None + + +class PhyNICMTUContext(DataPortContext): + + def __call__(self): + ctxt = {} + mappings = super(PhyNICMTUContext, self).__call__() + if mappings and mappings.keys(): + ports = sorted(mappings.keys()) + napi_settings = NeutronAPIContext()() + mtu = napi_settings.get('network_device_mtu') + all_ports = set() + # If any of ports is a vlan device, its underlying device must have + # mtu applied first. + for port in ports: + for lport in glob.glob("/sys/class/net/%s/lower_*" % port): + lport = os.path.basename(lport) + all_ports.add(lport.split('_')[1]) + + all_ports = list(all_ports) + all_ports.extend(ports) + if mtu: + ctxt["devs"] = '\\n'.join(all_ports) + ctxt['mtu'] = mtu + + return ctxt + + +class NetworkServiceContext(OSContextGenerator): + + def __init__(self, rel_name='quantum-network-service'): + self.rel_name = rel_name + self.interfaces = [rel_name] + + def __call__(self): + for rid in relation_ids(self.rel_name): + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + ctxt = { + 'keystone_host': rdata.get('keystone_host'), + 'service_port': rdata.get('service_port'), + 'auth_port': rdata.get('auth_port'), + 'service_tenant': rdata.get('service_tenant'), + 'service_username': rdata.get('service_username'), + 'service_password': rdata.get('service_password'), + 'quantum_host': rdata.get('quantum_host'), + 'quantum_port': rdata.get('quantum_port'), + 'quantum_url': rdata.get('quantum_url'), + 'region': rdata.get('region'), + 'service_protocol': + rdata.get('service_protocol') or 'http', + 'auth_protocol': + rdata.get('auth_protocol') or 'http', + 'api_version': + rdata.get('api_version') or '2.0', + } + if self.context_complete(ctxt): + return ctxt + return {} + + +class InternalEndpointContext(OSContextGenerator): + """Internal endpoint context. + + This context provides the endpoint type used for communication between + services e.g. between Nova and Cinder internally. Openstack uses Public + endpoints by default so this allows admins to optionally use internal + endpoints. + """ + def __call__(self): + return {'use_internal_endpoints': config('use-internal-endpoints')} + + +class VolumeAPIContext(InternalEndpointContext): + """Volume API context. + + This context provides information regarding the volume endpoint to use + when communicating between services. It determines which version of the + API is appropriate for use. + + This value will be determined in the resulting context dictionary + returned from calling the VolumeAPIContext object. Information provided + by this context is as follows: + + volume_api_version: the volume api version to use, currently + 'v2' or 'v3' + volume_catalog_info: the information to use for a cinder client + configuration that consumes API endpoints from the keystone + catalog. This is defined as the type:name:endpoint_type string. + """ + # FIXME(wolsen) This implementation is based on the provider being able + # to specify the package version to check but does not guarantee that the + # volume service api version selected is available. In practice, it is + # quite likely the volume service *is* providing the v3 volume service. + # This should be resolved when the service-discovery spec is implemented. + def __init__(self, pkg): + """ + Creates a new VolumeAPIContext for use in determining which version + of the Volume API should be used for communication. A package codename + should be supplied for determining the currently installed OpenStack + version. + + :param pkg: the package codename to use in order to determine the + component version (e.g. nova-common). See + charmhelpers.contrib.openstack.utils.PACKAGE_CODENAMES for more. + """ + super(VolumeAPIContext, self).__init__() + self._ctxt = None + if not pkg: + raise ValueError('package name must be provided in order to ' + 'determine current OpenStack version.') + self.pkg = pkg + + @property + def ctxt(self): + if self._ctxt is not None: + return self._ctxt + self._ctxt = self._determine_ctxt() + return self._ctxt + + def _determine_ctxt(self): + """Determines the Volume API endpoint information. + + Determines the appropriate version of the API that should be used + as well as the catalog_info string that would be supplied. Returns + a dict containing the volume_api_version and the volume_catalog_info. + """ + rel = os_release(self.pkg) + version = '2' + if CompareOpenStackReleases(rel) >= 'pike': + version = '3' + + service_type = 'volumev{version}'.format(version=version) + service_name = 'cinderv{version}'.format(version=version) + endpoint_type = 'publicURL' + if config('use-internal-endpoints'): + endpoint_type = 'internalURL' + catalog_info = '{type}:{name}:{endpoint}'.format( + type=service_type, name=service_name, endpoint=endpoint_type) + + return { + 'volume_api_version': version, + 'volume_catalog_info': catalog_info, + } + + def __call__(self): + return self.ctxt + + +class AppArmorContext(OSContextGenerator): + """Base class for apparmor contexts.""" + + def __init__(self, profile_name=None): + self._ctxt = None + self.aa_profile = profile_name + self.aa_utils_packages = ['apparmor-utils'] + + @property + def ctxt(self): + if self._ctxt is not None: + return self._ctxt + self._ctxt = self._determine_ctxt() + return self._ctxt + + def _determine_ctxt(self): + """ + Validate aa-profile-mode settings is disable, enforce, or complain. + + :return ctxt: Dictionary of the apparmor profile or None + """ + if config('aa-profile-mode') in ['disable', 'enforce', 'complain']: + ctxt = {'aa_profile_mode': config('aa-profile-mode'), + 'ubuntu_release': lsb_release()['DISTRIB_RELEASE']} + if self.aa_profile: + ctxt['aa_profile'] = self.aa_profile + else: + ctxt = None + return ctxt + + def __call__(self): + return self.ctxt + + def install_aa_utils(self): + """ + Install packages required for apparmor configuration. + """ + log("Installing apparmor utils.") + ensure_packages(self.aa_utils_packages) + + def manually_disable_aa_profile(self): + """ + Manually disable an apparmor profile. + + If aa-profile-mode is set to disabled (default) this is required as the + template has been written but apparmor is yet unaware of the profile + and aa-disable aa-profile fails. Without this the profile would kick + into enforce mode on the next service restart. + + """ + profile_path = '/etc/apparmor.d' + disable_path = '/etc/apparmor.d/disable' + if not os.path.lexists(os.path.join(disable_path, self.aa_profile)): + os.symlink(os.path.join(profile_path, self.aa_profile), + os.path.join(disable_path, self.aa_profile)) + + def setup_aa_profile(self): + """ + Setup an apparmor profile. + The ctxt dictionary will contain the apparmor profile mode and + the apparmor profile name. + Makes calls out to aa-disable, aa-complain, or aa-enforce to setup + the apparmor profile. + """ + self() + if not self.ctxt: + log("Not enabling apparmor Profile") + return + self.install_aa_utils() + cmd = ['aa-{}'.format(self.ctxt['aa_profile_mode'])] + cmd.append(self.ctxt['aa_profile']) + log("Setting up the apparmor profile for {} in {} mode." + "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode'])) + try: + check_call(cmd) + except CalledProcessError as e: + # If aa-profile-mode is set to disabled (default) manual + # disabling is required as the template has been written but + # apparmor is yet unaware of the profile and aa-disable aa-profile + # fails. If aa-disable learns to read profile files first this can + # be removed. + if self.ctxt['aa_profile_mode'] == 'disable': + log("Manually disabling the apparmor profile for {}." + "".format(self.ctxt['aa_profile'])) + self.manually_disable_aa_profile() + return + status_set('blocked', "Apparmor profile {} failed to be set to {}." + "".format(self.ctxt['aa_profile'], + self.ctxt['aa_profile_mode'])) + raise e + + +class MemcacheContext(OSContextGenerator): + """Memcache context + + This context provides options for configuring a local memcache client and + server for both IPv4 and IPv6 + """ + + def __init__(self, package=None): + """ + @param package: Package to examine to extrapolate OpenStack release. + Used when charms have no openstack-origin config + option (ie subordinates) + """ + self.package = package + + def __call__(self): + ctxt = {} + ctxt['use_memcache'] = enable_memcache(package=self.package) + if ctxt['use_memcache']: + # Trusty version of memcached does not support ::1 as a listen + # address so use host file entry instead + release = lsb_release()['DISTRIB_CODENAME'].lower() + if is_ipv6_disabled(): + if CompareHostReleases(release) > 'trusty': + ctxt['memcache_server'] = '127.0.0.1' + else: + ctxt['memcache_server'] = 'localhost' + ctxt['memcache_server_formatted'] = '127.0.0.1' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = '{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) + else: + if CompareHostReleases(release) > 'trusty': + ctxt['memcache_server'] = '::1' + else: + ctxt['memcache_server'] = 'ip6-localhost' + ctxt['memcache_server_formatted'] = '[::1]' + ctxt['memcache_port'] = '11211' + ctxt['memcache_url'] = 'inet6:{}:{}'.format( + ctxt['memcache_server_formatted'], + ctxt['memcache_port']) + return ctxt + + +class EnsureDirContext(OSContextGenerator): + ''' + Serves as a generic context to create a directory as a side-effect. + + Useful for software that supports drop-in files (.d) in conjunction + with config option-based templates. Examples include: + * OpenStack oslo.policy drop-in files; + * systemd drop-in config files; + * other software that supports overriding defaults with .d files + + Another use-case is when a subordinate generates a configuration for + primary to render in a separate directory. + + Some software requires a user to create a target directory to be + scanned for drop-in files with a specific format. This is why this + context is needed to do that before rendering a template. + ''' + + def __init__(self, dirname, **kwargs): + '''Used merely to ensure that a given directory exists.''' + self.dirname = dirname + self.kwargs = kwargs + + def __call__(self): + mkdir(self.dirname, **self.kwargs) + return {} + + +class VersionsContext(OSContextGenerator): + """Context to return the openstack and operating system versions. + + """ + def __init__(self, pkg='python-keystone'): + """Initialise context. + + :param pkg: Package to extrapolate openstack version from. + :type pkg: str + """ + self.pkg = pkg + + def __call__(self): + ostack = os_release(self.pkg) + osystem = lsb_release()['DISTRIB_CODENAME'].lower() + return { + 'openstack_release': ostack, + 'operating_system_release': osystem} + + +class LogrotateContext(OSContextGenerator): + """Common context generator for logrotate.""" + + def __init__(self, location, interval, count): + """ + :param location: Absolute path for the logrotate config file + :type location: str + :param interval: The interval for the rotations. Valid values are + 'daily', 'weekly', 'monthly', 'yearly' + :type interval: str + :param count: The logrotate count option configures the 'count' times + the log files are being rotated before being + :type count: int + """ + self.location = location + self.interval = interval + self.count = 'rotate {}'.format(count) + + def __call__(self): + ctxt = { + 'logrotate_logs_location': self.location, + 'logrotate_interval': self.interval, + 'logrotate_count': self.count, + } + return ctxt + + +class HostInfoContext(OSContextGenerator): + """Context to provide host information.""" + + def __init__(self, use_fqdn_hint_cb=None): + """Initialize HostInfoContext + + :param use_fqdn_hint_cb: Callback whose return value used to populate + `use_fqdn_hint` + :type use_fqdn_hint_cb: Callable[[], bool] + """ + # Store callback used to get hint for whether FQDN should be used + + # Depending on the workload a charm manages, the use of FQDN vs. + # shortname may be a deploy-time decision, i.e. behaviour can not + # change on charm upgrade or post-deployment configuration change. + + # The hint is passed on as a flag in the context to allow the decision + # to be made in the Jinja2 configuration template. + self.use_fqdn_hint_cb = use_fqdn_hint_cb + + def _get_canonical_name(self, name=None): + """Get the official FQDN of the host + + The implementation of ``socket.getfqdn()`` in the standard Python + library does not exhaust all methods of getting the official name + of a host ref Python issue https://bugs.python.org/issue5004 + + This function mimics the behaviour of a call to ``hostname -f`` to + get the official FQDN but returns an empty string if it is + unsuccessful. + + :param name: Shortname to get FQDN on + :type name: Optional[str] + :returns: The official FQDN for host or empty string ('') + :rtype: str + """ + name = name or socket.gethostname() + fqdn = '' + + if six.PY2: + exc = socket.error + else: + exc = OSError + + try: + addrs = socket.getaddrinfo( + name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME) + except exc: + pass + else: + for addr in addrs: + if addr[3]: + if '.' in addr[3]: + fqdn = addr[3] + break + return fqdn + + def __call__(self): + name = socket.gethostname() + ctxt = { + 'host_fqdn': self._get_canonical_name(name) or name, + 'host': name, + 'use_fqdn_hint': ( + self.use_fqdn_hint_cb() if self.use_fqdn_hint_cb else False) + } + return ctxt + + +def validate_ovs_use_veth(*args, **kwargs): + """Validate OVS use veth setting for dhcp agents + + The ovs_use_veth setting is considered immutable as it will break existing + deployments. Historically, we set ovs_use_veth=True in dhcp_agent.ini. It + turns out this is no longer necessary. Ideally, all new deployments would + have this set to False. + + This function validates that the config value does not conflict with + previously deployed settings in dhcp_agent.ini. + + See LP Bug#1831935 for details. + + :returns: Status state and message + :rtype: Union[(None, None), (string, string)] + """ + existing_ovs_use_veth = ( + DHCPAgentContext.get_existing_ovs_use_veth()) + config_ovs_use_veth = DHCPAgentContext.parse_ovs_use_veth() + + # Check settings are set and not None + if existing_ovs_use_veth is not None and config_ovs_use_veth is not None: + # Check for mismatch between existing config ini and juju config + if existing_ovs_use_veth != config_ovs_use_veth: + # Stop the line to avoid breakage + msg = ( + "The existing setting for dhcp_agent.ini ovs_use_veth, {}, " + "does not match the juju config setting, {}. This may lead to " + "VMs being unable to receive a DHCP IP. Either change the " + "juju config setting or dhcp agents may need to be recreated." + .format(existing_ovs_use_veth, config_ovs_use_veth)) + log(msg, ERROR) + return ( + "blocked", + "Mismatched existing and configured ovs-use-veth. See log.") + + # Everything is OK + return None, None + + +class DHCPAgentContext(OSContextGenerator): + + def __call__(self): + """Return the DHCPAGentContext. + + Return all DHCP Agent INI related configuration. + ovs unit is attached to (as a subordinate) and the 'dns_domain' from + the neutron-plugin-api relations (if one is set). + + :returns: Dictionary context + :rtype: Dict + """ + + ctxt = {} + dnsmasq_flags = config('dnsmasq-flags') + if dnsmasq_flags: + ctxt['dnsmasq_flags'] = config_flags_parser(dnsmasq_flags) + ctxt['dns_servers'] = config('dns-servers') + + neutron_api_settings = NeutronAPIContext()() + + ctxt['debug'] = config('debug') + ctxt['instance_mtu'] = config('instance-mtu') + ctxt['ovs_use_veth'] = self.get_ovs_use_veth() + + ctxt['enable_metadata_network'] = config('enable-metadata-network') + ctxt['enable_isolated_metadata'] = config('enable-isolated-metadata') + + if neutron_api_settings.get('dns_domain'): + ctxt['dns_domain'] = neutron_api_settings.get('dns_domain') + + # Override user supplied config for these plugins as these settings are + # mandatory + if config('plugin') in ['nvp', 'nsx', 'n1kv']: + ctxt['enable_metadata_network'] = True + ctxt['enable_isolated_metadata'] = True + + ctxt['append_ovs_config'] = False + cmp_release = CompareOpenStackReleases( + os_release('neutron-common', base='icehouse')) + if cmp_release >= 'queens' and config('enable-dpdk'): + ctxt['append_ovs_config'] = True + + return ctxt + + @staticmethod + def get_existing_ovs_use_veth(): + """Return existing ovs_use_veth setting from dhcp_agent.ini. + + :returns: Boolean value of existing ovs_use_veth setting or None + :rtype: Optional[Bool] + """ + DHCP_AGENT_INI = "/etc/neutron/dhcp_agent.ini" + existing_ovs_use_veth = None + # If there is a dhcp_agent.ini file read the current setting + if os.path.isfile(DHCP_AGENT_INI): + # config_ini does the right thing and returns None if the setting is + # commented. + existing_ovs_use_veth = ( + config_ini(DHCP_AGENT_INI)["DEFAULT"].get("ovs_use_veth")) + # Convert to Bool if necessary + if isinstance(existing_ovs_use_veth, six.string_types): + return bool_from_string(existing_ovs_use_veth) + return existing_ovs_use_veth + + @staticmethod + def parse_ovs_use_veth(): + """Parse the ovs-use-veth config setting. + + Parse the string config setting for ovs-use-veth and return a boolean + or None. + + bool_from_string will raise a ValueError if the string is not falsy or + truthy. + + :raises: ValueError for invalid input + :returns: Boolean value of ovs-use-veth or None + :rtype: Optional[Bool] + """ + _config = config("ovs-use-veth") + # An unset parameter returns None. Just in case we will also check for + # an empty string: "". Ironically, (the problem we are trying to avoid) + # "False" returns True and "" returns False. + if _config is None or not _config: + # Return None + return + # bool_from_string handles many variations of true and false strings + # as well as upper and lowercases including: + # ['y', 'yes', 'true', 't', 'on', 'n', 'no', 'false', 'f', 'off'] + return bool_from_string(_config) + + def get_ovs_use_veth(self): + """Return correct ovs_use_veth setting for use in dhcp_agent.ini. + + Get the right value from config or existing dhcp_agent.ini file. + Existing has precedence. Attempt to default to "False" without + disrupting existing deployments. Handle existing deployments and + upgrades safely. See LP Bug#1831935 + + :returns: Value to use for ovs_use_veth setting + :rtype: Bool + """ + _existing = self.get_existing_ovs_use_veth() + if _existing is not None: + return _existing + + _config = self.parse_ovs_use_veth() + if _config is None: + # New better default + return False + else: + return _config + + +EntityMac = collections.namedtuple('EntityMac', ['entity', 'mac']) + + +def resolve_pci_from_mapping_config(config_key): + """Resolve local PCI devices from MAC addresses in mapping config. + + Note that this function keeps record of mac->PCI address lookups + in the local unit db as the devices will disappaear from the system + once bound. + + :param config_key: Configuration option key to parse data from + :type config_key: str + :returns: PCI device address to Tuple(entity, mac) map + :rtype: collections.OrderedDict[str,Tuple[str,str]] + """ + devices = pci.PCINetDevices() + resolved_devices = collections.OrderedDict() + db = kv() + # Note that ``parse_data_port_mappings`` returns Dict regardless of input + for mac, entity in parse_data_port_mappings(config(config_key)).items(): + pcidev = devices.get_device_from_mac(mac) + if pcidev: + # NOTE: store mac->pci allocation as post binding + # it disappears from PCIDevices. + db.set(mac, pcidev.pci_address) + db.flush() + + pci_address = db.get(mac) + if pci_address: + resolved_devices[pci_address] = EntityMac(entity, mac) + + return resolved_devices + + +class DPDKDeviceContext(OSContextGenerator): + + def __init__(self, driver_key=None, bridges_key=None, bonds_key=None): + """Initialize DPDKDeviceContext. + + :param driver_key: Key to use when retrieving driver config. + :type driver_key: str + :param bridges_key: Key to use when retrieving bridge config. + :type bridges_key: str + :param bonds_key: Key to use when retrieving bonds config. + :type bonds_key: str + """ + self.driver_key = driver_key or 'dpdk-driver' + self.bridges_key = bridges_key or 'data-port' + self.bonds_key = bonds_key or 'dpdk-bond-mappings' + + def __call__(self): + """Populate context. + + :returns: context + :rtype: Dict[str,Union[str,collections.OrderedDict[str,str]]] + """ + driver = config(self.driver_key) + if driver is None: + return {} + # Resolve PCI devices for both directly used devices (_bridges) + # and devices for use in dpdk bonds (_bonds) + pci_devices = resolve_pci_from_mapping_config(self.bridges_key) + pci_devices.update(resolve_pci_from_mapping_config(self.bonds_key)) + return {'devices': pci_devices, + 'driver': driver} + + +class OVSDPDKDeviceContext(OSContextGenerator): + + def __init__(self, bridges_key=None, bonds_key=None): + """Initialize OVSDPDKDeviceContext. + + :param bridges_key: Key to use when retrieving bridge config. + :type bridges_key: str + :param bonds_key: Key to use when retrieving bonds config. + :type bonds_key: str + """ + self.bridges_key = bridges_key or 'data-port' + self.bonds_key = bonds_key or 'dpdk-bond-mappings' + + @staticmethod + def _parse_cpu_list(cpulist): + """Parses a linux cpulist for a numa node + + :returns: list of cores + :rtype: List[int] + """ + cores = [] + ranges = cpulist.split(',') + for cpu_range in ranges: + if "-" in cpu_range: + cpu_min_max = cpu_range.split('-') + cores += range(int(cpu_min_max[0]), + int(cpu_min_max[1]) + 1) + else: + cores.append(int(cpu_range)) + return cores + + def _numa_node_cores(self): + """Get map of numa node -> cpu core + + :returns: map of numa node -> cpu core + :rtype: Dict[str,List[int]] + """ + nodes = {} + node_regex = '/sys/devices/system/node/node*' + for node in glob.glob(node_regex): + index = node.lstrip('/sys/devices/system/node/node') + with open(os.path.join(node, 'cpulist')) as cpulist: + nodes[index] = self._parse_cpu_list(cpulist.read().strip()) + return nodes + + def cpu_mask(self): + """Get hex formatted CPU mask + + The mask is based on using the first config:dpdk-socket-cores + cores of each NUMA node in the unit. + :returns: hex formatted CPU mask + :rtype: str + """ + return self.cpu_masks()['dpdk_lcore_mask'] + + def cpu_masks(self): + """Get hex formatted CPU masks + + The mask is based on using the first config:dpdk-socket-cores + cores of each NUMA node in the unit, followed by the + next config:pmd-socket-cores + + :returns: Dict of hex formatted CPU masks + :rtype: Dict[str, str] + """ + num_lcores = config('dpdk-socket-cores') + pmd_cores = config('pmd-socket-cores') + lcore_mask = 0 + pmd_mask = 0 + for cores in self._numa_node_cores().values(): + for core in cores[:num_lcores]: + lcore_mask = lcore_mask | 1 << core + for core in cores[num_lcores:][:pmd_cores]: + pmd_mask = pmd_mask | 1 << core + return { + 'pmd_cpu_mask': format(pmd_mask, '#04x'), + 'dpdk_lcore_mask': format(lcore_mask, '#04x')} + + def socket_memory(self): + """Formatted list of socket memory configuration per socket. + + :returns: socket memory configuration per socket. + :rtype: str + """ + lscpu_out = check_output( + ['lscpu', '-p=socket']).decode('UTF-8').strip() + sockets = set() + for line in lscpu_out.split('\n'): + try: + sockets.add(int(line)) + except ValueError: + # lscpu output is headed by comments so ignore them. + pass + sm_size = config('dpdk-socket-memory') + mem_list = [str(sm_size) for _ in sockets] + if mem_list: + return ','.join(mem_list) + else: + return str(sm_size) + + def devices(self): + """List of PCI devices for use by DPDK + + :returns: List of PCI devices for use by DPDK + :rtype: collections.OrderedDict[str,str] + """ + pci_devices = resolve_pci_from_mapping_config(self.bridges_key) + pci_devices.update(resolve_pci_from_mapping_config(self.bonds_key)) + return pci_devices + + def _formatted_whitelist(self, flag): + """Flag formatted list of devices to whitelist + + :param flag: flag format to use + :type flag: str + :rtype: str + """ + whitelist = [] + for device in self.devices(): + whitelist.append(flag.format(device=device)) + return ' '.join(whitelist) + + def device_whitelist(self): + """Formatted list of devices to whitelist for dpdk + + using the old style '-w' flag + + :returns: devices to whitelist prefixed by '-w ' + :rtype: str + """ + return self._formatted_whitelist('-w {device}') + + def pci_whitelist(self): + """Formatted list of devices to whitelist for dpdk + + using the new style '--pci-whitelist' flag + + :returns: devices to whitelist prefixed by '--pci-whitelist ' + :rtype: str + """ + return self._formatted_whitelist('--pci-whitelist {device}') + + def __call__(self): + """Populate context. + + :returns: context + :rtype: Dict[str,Union[bool,str]] + """ + ctxt = {} + whitelist = self.device_whitelist() + if whitelist: + ctxt['dpdk_enabled'] = config('enable-dpdk') + ctxt['device_whitelist'] = self.device_whitelist() + ctxt['socket_memory'] = self.socket_memory() + ctxt['cpu_mask'] = self.cpu_mask() + return ctxt + + +class BridgePortInterfaceMap(object): + """Build a map of bridge ports and interfaces from charm configuration. + + NOTE: the handling of this detail in the charm is pre-deprecated. + + The long term goal is for network connectivity detail to be modelled in + the server provisioning layer (such as MAAS) which in turn will provide + a Netplan YAML description that will be used to drive Open vSwitch. + + Until we get to that reality the charm will need to configure this + detail based on application level configuration options. + + There is a established way of mapping interfaces to ports and bridges + in the ``neutron-openvswitch`` and ``neutron-gateway`` charms and we + will carry that forward. + + The relationship between bridge, port and interface(s). + +--------+ + | bridge | + +--------+ + | + +----------------+ + | port aka. bond | + +----------------+ + | | + +-+ +-+ + |i| |i| + |n| |n| + |t| |t| + |0| |N| + +-+ +-+ + """ + class interface_type(enum.Enum): + """Supported interface types. + + Supported interface types can be found in the ``iface_types`` column + in the ``Open_vSwitch`` table on a running system. + """ + dpdk = 'dpdk' + internal = 'internal' + system = 'system' + + def __str__(self): + """Return string representation of value. + + :returns: string representation of value. + :rtype: str + """ + return self.value + + def __init__(self, bridges_key=None, bonds_key=None, enable_dpdk_key=None, + global_mtu=None): + """Initialize map. + + :param bridges_key: Name of bridge:interface/port map config key + (default: 'data-port') + :type bridges_key: Optional[str] + :param bonds_key: Name of port-name:interface map config key + (default: 'dpdk-bond-mappings') + :type bonds_key: Optional[str] + :param enable_dpdk_key: Name of DPDK toggle config key + (default: 'enable-dpdk') + :type enable_dpdk_key: Optional[str] + :param global_mtu: Set a MTU on all interfaces at map initialization. + + The default is to have Open vSwitch get this from the underlying + interface as set up by bare metal provisioning. + + Note that you can augment the MTU on an individual interface basis + like this: + + ifdatamap = bpi.get_ifdatamap(bridge, port) + ifdatamap = { + port: { + **ifdata, + **{'mtu-request': my_individual_mtu_map[port]}, + } + for port, ifdata in ifdatamap.items() + } + :type global_mtu: Optional[int] + """ + bridges_key = bridges_key or 'data-port' + bonds_key = bonds_key or 'dpdk-bond-mappings' + enable_dpdk_key = enable_dpdk_key or 'enable-dpdk' + self._map = collections.defaultdict( + lambda: collections.defaultdict(dict)) + self._ifname_mac_map = collections.defaultdict(list) + self._mac_ifname_map = {} + self._mac_pci_address_map = {} + + # First we iterate over the list of physical interfaces visible to the + # system and update interface name to mac and mac to interface name map + for ifname in list_nics(): + if not is_phy_iface(ifname): + continue + mac = get_nic_hwaddr(ifname) + self._ifname_mac_map[ifname] = [mac] + self._mac_ifname_map[mac] = ifname + + # check if interface is part of a linux bond + _bond_name = get_bond_master(ifname) + if _bond_name and _bond_name != ifname: + log('Add linux bond "{}" to map for physical interface "{}" ' + 'with mac "{}".'.format(_bond_name, ifname, mac), + level=DEBUG) + # for bonds we want to be able to get a list of the mac + # addresses for the physical interfaces the bond is made up of. + if self._ifname_mac_map.get(_bond_name): + self._ifname_mac_map[_bond_name].append(mac) + else: + self._ifname_mac_map[_bond_name] = [mac] + + # In light of the pre-deprecation notice in the docstring of this + # class we will expose the ability to configure OVS bonds as a + # DPDK-only feature, but generally use the data structures internally. + if config(enable_dpdk_key): + # resolve PCI address of interfaces listed in the bridges and bonds + # charm configuration options. Note that for already bound + # interfaces the helper will retrieve MAC address from the unit + # KV store as the information is no longer available in sysfs. + _pci_bridge_mac = resolve_pci_from_mapping_config( + bridges_key) + _pci_bond_mac = resolve_pci_from_mapping_config( + bonds_key) + + for pci_address, bridge_mac in _pci_bridge_mac.items(): + if bridge_mac.mac in self._mac_ifname_map: + # if we already have the interface name in our map it is + # visible to the system and therefore not bound to DPDK + continue + ifname = 'dpdk-{}'.format( + hashlib.sha1( + pci_address.encode('UTF-8')).hexdigest()[:7]) + self._ifname_mac_map[ifname] = [bridge_mac.mac] + self._mac_ifname_map[bridge_mac.mac] = ifname + self._mac_pci_address_map[bridge_mac.mac] = pci_address + + for pci_address, bond_mac in _pci_bond_mac.items(): + # for bonds we want to be able to get a list of macs from + # the bond name and also get at the interface name made up + # of the hash of the PCI address + ifname = 'dpdk-{}'.format( + hashlib.sha1( + pci_address.encode('UTF-8')).hexdigest()[:7]) + self._ifname_mac_map[bond_mac.entity].append(bond_mac.mac) + self._mac_ifname_map[bond_mac.mac] = ifname + self._mac_pci_address_map[bond_mac.mac] = pci_address + + config_bridges = config(bridges_key) or '' + for bridge, ifname_or_mac in ( + pair.split(':', 1) + for pair in config_bridges.split()): + if ':' in ifname_or_mac: + try: + ifname = self.ifname_from_mac(ifname_or_mac) + except KeyError: + # The interface is destined for a different unit in the + # deployment. + continue + macs = [ifname_or_mac] + else: + ifname = ifname_or_mac + macs = self.macs_from_ifname(ifname_or_mac) + + portname = ifname + for mac in macs: + try: + pci_address = self.pci_address_from_mac(mac) + iftype = self.interface_type.dpdk + ifname = self.ifname_from_mac(mac) + except KeyError: + pci_address = None + iftype = self.interface_type.system + + self.add_interface( + bridge, portname, ifname, iftype, pci_address, global_mtu) + + if not macs: + # We have not mapped the interface and it is probably some sort + # of virtual interface. Our user have put it in the config with + # a purpose so let's carry out their wish. LP: #1884743 + log('Add unmapped interface from config: name "{}" bridge "{}"' + .format(ifname, bridge), + level=DEBUG) + self.add_interface( + bridge, ifname, ifname, self.interface_type.system, None, + global_mtu) + + def __getitem__(self, key): + """Provide a Dict-like interface, get value of item. + + :param key: Key to look up value from. + :type key: any + :returns: Value + :rtype: any + """ + return self._map.__getitem__(key) + + def __iter__(self): + """Provide a Dict-like interface, iterate over keys. + + :returns: Iterator + :rtype: Iterator[any] + """ + return self._map.__iter__() + + def __len__(self): + """Provide a Dict-like interface, measure the length of internal map. + + :returns: Length + :rtype: int + """ + return len(self._map) + + def items(self): + """Provide a Dict-like interface, iterate over items. + + :returns: Key Value pairs + :rtype: Iterator[any, any] + """ + return self._map.items() + + def keys(self): + """Provide a Dict-like interface, iterate over keys. + + :returns: Iterator + :rtype: Iterator[any] + """ + return self._map.keys() + + def ifname_from_mac(self, mac): + """ + :returns: Name of interface + :rtype: str + :raises: KeyError + """ + return (get_bond_master(self._mac_ifname_map[mac]) or + self._mac_ifname_map[mac]) + + def macs_from_ifname(self, ifname): + """ + :returns: List of hardware address (MAC) of interface + :rtype: List[str] + :raises: KeyError + """ + return self._ifname_mac_map[ifname] + + def pci_address_from_mac(self, mac): + """ + :param mac: Hardware address (MAC) of interface + :type mac: str + :returns: PCI address of device associated with mac + :rtype: str + :raises: KeyError + """ + return self._mac_pci_address_map[mac] + + def add_interface(self, bridge, port, ifname, iftype, + pci_address, mtu_request): + """Add an interface to the map. + + :param bridge: Name of bridge on which the bond will be added + :type bridge: str + :param port: Name of port which will represent the bond on bridge + :type port: str + :param ifname: Name of interface that will make up the bonded port + :type ifname: str + :param iftype: Type of interface + :type iftype: BridgeBondMap.interface_type + :param pci_address: PCI address of interface + :type pci_address: Optional[str] + :param mtu_request: MTU to request for interface + :type mtu_request: Optional[int] + """ + self._map[bridge][port][ifname] = { + 'type': str(iftype), + } + if pci_address: + self._map[bridge][port][ifname].update({ + 'pci-address': pci_address, + }) + if mtu_request is not None: + self._map[bridge][port][ifname].update({ + 'mtu-request': str(mtu_request) + }) + + def get_ifdatamap(self, bridge, port): + """Get structure suitable for charmhelpers.contrib.network.ovs helpers. + + :param bridge: Name of bridge on which the port will be added + :type bridge: str + :param port: Name of port which will represent one or more interfaces + :type port: str + """ + for _bridge, _ports in self.items(): + for _port, _interfaces in _ports.items(): + if _bridge == bridge and _port == port: + ifdatamap = {} + for name, data in _interfaces.items(): + ifdatamap.update({ + name: { + 'type': data['type'], + }, + }) + if data.get('mtu-request') is not None: + ifdatamap[name].update({ + 'mtu_request': data['mtu-request'], + }) + if data.get('pci-address'): + ifdatamap[name].update({ + 'options': { + 'dpdk-devargs': data['pci-address'], + }, + }) + return ifdatamap + + +class BondConfig(object): + """Container and helpers for bond configuration options. + + Data is put into a dictionary and a convenient config get interface is + provided. + """ + + DEFAULT_LACP_CONFIG = { + 'mode': 'balance-tcp', + 'lacp': 'active', + 'lacp-time': 'fast' + } + ALL_BONDS = 'ALL_BONDS' + + BOND_MODES = ['active-backup', 'balance-slb', 'balance-tcp'] + BOND_LACP = ['active', 'passive', 'off'] + BOND_LACP_TIME = ['fast', 'slow'] + + def __init__(self, config_key=None): + """Parse specified configuration option. + + :param config_key: Configuration key to retrieve data from + (default: ``dpdk-bond-config``) + :type config_key: Optional[str] + """ + self.config_key = config_key or 'dpdk-bond-config' + + self.lacp_config = { + self.ALL_BONDS: copy.deepcopy(self.DEFAULT_LACP_CONFIG) + } + + lacp_config = config(self.config_key) + if lacp_config: + lacp_config_map = lacp_config.split() + for entry in lacp_config_map: + bond, entry = entry.partition(':')[0:3:2] + if not bond: + bond = self.ALL_BONDS + + mode, entry = entry.partition(':')[0:3:2] + if not mode: + mode = self.DEFAULT_LACP_CONFIG['mode'] + assert mode in self.BOND_MODES, \ + "Bond mode {} is invalid".format(mode) + + lacp, entry = entry.partition(':')[0:3:2] + if not lacp: + lacp = self.DEFAULT_LACP_CONFIG['lacp'] + assert lacp in self.BOND_LACP, \ + "Bond lacp {} is invalid".format(lacp) + + lacp_time, entry = entry.partition(':')[0:3:2] + if not lacp_time: + lacp_time = self.DEFAULT_LACP_CONFIG['lacp-time'] + assert lacp_time in self.BOND_LACP_TIME, \ + "Bond lacp-time {} is invalid".format(lacp_time) + + self.lacp_config[bond] = { + 'mode': mode, + 'lacp': lacp, + 'lacp-time': lacp_time + } + + def get_bond_config(self, bond): + """Get the LACP configuration for a bond + + :param bond: the bond name + :return: a dictionary with the configuration of the bond + :rtype: Dict[str,Dict[str,str]] + """ + return self.lacp_config.get(bond, self.lacp_config[self.ALL_BONDS]) + + def get_ovs_portdata(self, bond): + """Get structure suitable for charmhelpers.contrib.network.ovs helpers. + + :param bond: the bond name + :return: a dictionary with the configuration of the bond + :rtype: Dict[str,Union[str,Dict[str,str]]] + """ + bond_config = self.get_bond_config(bond) + return { + 'bond_mode': bond_config['mode'], + 'lacp': bond_config['lacp'], + 'other_config': { + 'lacp-time': bond_config['lacp-time'], + }, + } + + +class SRIOVContext(OSContextGenerator): + """Provide context for configuring SR-IOV devices.""" + + class sriov_config_mode(enum.Enum): + """Mode in which SR-IOV is configured. + + The configuration option identified by the ``numvfs_key`` parameter + is overloaded and defines in which mode the charm should interpret + the other SR-IOV-related configuration options. + """ + auto = 'auto' + blanket = 'blanket' + explicit = 'explicit' + + PCIDeviceNumVFs = collections.namedtuple( + 'PCIDeviceNumVFs', ['device', 'numvfs']) + + def _determine_numvfs(self, device, sriov_numvfs): + """Determine number of Virtual Functions (VFs) configured for device. + + :param device: Object describing a PCI Network interface card (NIC)/ + :type device: sriov_netplan_shim.pci.PCINetDevice + :param sriov_numvfs: Number of VFs requested for blanket configuration. + :type sriov_numvfs: int + :returns: Number of VFs to configure for device + :rtype: Optional[int] + """ + + def _get_capped_numvfs(requested): + """Get a number of VFs that does not exceed individual card limits. + + Depending and make and model of NIC the number of VFs supported + vary. Requesting more VFs than a card support would be a fatal + error, cap the requested number at the total number of VFs each + individual card supports. + + :param requested: Number of VFs requested + :type requested: int + :returns: Number of VFs allowed + :rtype: int + """ + actual = min(int(requested), int(device.sriov_totalvfs)) + if actual < int(requested): + log('Requested VFs ({}) too high for device {}. Falling back ' + 'to value supported by device: {}' + .format(requested, device.interface_name, + device.sriov_totalvfs), + level=WARNING) + return actual + + if self._sriov_config_mode == self.sriov_config_mode.auto: + # auto-mode + # + # If device mapping configuration is present, return information + # on cards with mapping. + # + # If no device mapping configuration is present, return information + # for all cards. + # + # The maximum number of VFs supported by card will be used. + if (self._sriov_mapped_devices and + device.interface_name not in self._sriov_mapped_devices): + log('SR-IOV configured in auto mode: No device mapping for {}' + .format(device.interface_name), + level=DEBUG) + return + return _get_capped_numvfs(device.sriov_totalvfs) + elif self._sriov_config_mode == self.sriov_config_mode.blanket: + # blanket-mode + # + # User has specified a number of VFs that should apply to all + # cards with support for VFs. + return _get_capped_numvfs(sriov_numvfs) + elif self._sriov_config_mode == self.sriov_config_mode.explicit: + # explicit-mode + # + # User has given a list of interface names and associated number of + # VFs + if device.interface_name not in self._sriov_config_devices: + log('SR-IOV configured in explicit mode: No device:numvfs ' + 'pair for device {}, skipping.' + .format(device.interface_name), + level=DEBUG) + return + return _get_capped_numvfs( + self._sriov_config_devices[device.interface_name]) + else: + raise RuntimeError('This should not be reached') + + def __init__(self, numvfs_key=None, device_mappings_key=None): + """Initialize map from PCI devices and configuration options. + + :param numvfs_key: Config key for numvfs (default: 'sriov-numvfs') + :type numvfs_key: Optional[str] + :param device_mappings_key: Config key for device mappings + (default: 'sriov-device-mappings') + :type device_mappings_key: Optional[str] + :raises: RuntimeError + """ + numvfs_key = numvfs_key or 'sriov-numvfs' + device_mappings_key = device_mappings_key or 'sriov-device-mappings' + + devices = pci.PCINetDevices() + charm_config = config() + sriov_numvfs = charm_config.get(numvfs_key) or '' + sriov_device_mappings = charm_config.get(device_mappings_key) or '' + + # create list of devices from sriov_device_mappings config option + self._sriov_mapped_devices = [ + pair.split(':', 1)[1] + for pair in sriov_device_mappings.split() + ] + + # create map of device:numvfs from sriov_numvfs config option + self._sriov_config_devices = { + ifname: numvfs for ifname, numvfs in ( + pair.split(':', 1) for pair in sriov_numvfs.split() + if ':' in sriov_numvfs) + } + + # determine configuration mode from contents of sriov_numvfs + if sriov_numvfs == 'auto': + self._sriov_config_mode = self.sriov_config_mode.auto + elif sriov_numvfs.isdigit(): + self._sriov_config_mode = self.sriov_config_mode.blanket + elif ':' in sriov_numvfs: + self._sriov_config_mode = self.sriov_config_mode.explicit + else: + raise RuntimeError('Unable to determine mode of SR-IOV ' + 'configuration.') + + self._map = { + device.pci_address: self.PCIDeviceNumVFs( + device, self._determine_numvfs(device, sriov_numvfs)) + for device in devices.pci_devices + if device.sriov and + self._determine_numvfs(device, sriov_numvfs) is not None + } + + def __call__(self): + """Provide backward compatible SR-IOV context. + + :returns: Map interface name: min(configured, max) virtual functions. + Example: + { + 'eth0': 16, + 'eth1': 32, + 'eth2': 64, + } + :rtype: Dict[str,int] + """ + return { + pcidnvfs.device.interface_name: pcidnvfs.numvfs + for _, pcidnvfs in self._map.items() + } + + @property + def get_map(self): + """Provide map of configured SR-IOV capable PCI devices. + + :returns: Map PCI-address: (PCIDevice, min(configured, max) VFs. + Example: + { + '0000:81:00.0': self.PCIDeviceNumVFs(, 32), + '0000:81:00.1': self.PCIDeviceNumVFs(, 32), + } + :rtype: Dict[str, self.PCIDeviceNumVFs] + """ + return self._map + + +class CephBlueStoreCompressionContext(OSContextGenerator): + """Ceph BlueStore compression options.""" + + # Tuple with Tuples that map configuration option name to CephBrokerRq op + # property name + options = ( + ('bluestore-compression-algorithm', + 'compression-algorithm'), + ('bluestore-compression-mode', + 'compression-mode'), + ('bluestore-compression-required-ratio', + 'compression-required-ratio'), + ('bluestore-compression-min-blob-size', + 'compression-min-blob-size'), + ('bluestore-compression-min-blob-size-hdd', + 'compression-min-blob-size-hdd'), + ('bluestore-compression-min-blob-size-ssd', + 'compression-min-blob-size-ssd'), + ('bluestore-compression-max-blob-size', + 'compression-max-blob-size'), + ('bluestore-compression-max-blob-size-hdd', + 'compression-max-blob-size-hdd'), + ('bluestore-compression-max-blob-size-ssd', + 'compression-max-blob-size-ssd'), + ) + + def __init__(self): + """Initialize context by loading values from charm config. + + We keep two maps, one suitable for use with CephBrokerRq's and one + suitable for template generation. + """ + charm_config = config() + + # CephBrokerRq op map + self.op = {} + # Context exposed for template generation + self.ctxt = {} + for config_key, op_key in self.options: + value = charm_config.get(config_key) + self.ctxt.update({config_key.replace('-', '_'): value}) + self.op.update({op_key: value}) + + def __call__(self): + """Get context. + + :returns: Context + :rtype: Dict[str,any] + """ + return self.ctxt + + def get_op(self): + """Get values for use in CephBrokerRq op. + + :returns: Context values with CephBrokerRq op property name as key. + :rtype: Dict[str,any] + """ + return self.op + + def get_kwargs(self): + """Get values for use as keyword arguments. + + :returns: Context values with key suitable for use as kwargs to + CephBrokerRq add_op_create_*_pool methods. + :rtype: Dict[str,any] + """ + return { + k.replace('-', '_'): v + for k, v in self.op.items() + } + + def validate(self): + """Validate options. + + :raises: AssertionError + """ + # We slip in a dummy name on class instantiation to allow validation of + # the other options. It will not affect further use. + # + # NOTE: once we retire Python 3.5 we can fold this into a in-line + # dictionary comprehension in the call to the initializer. + dummy_op = {'name': 'dummy-name'} + dummy_op.update(self.op) + pool = ch_ceph.BasePool('dummy-service', op=dummy_op) + pool.validate() diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/deferred_events.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/deferred_events.py new file mode 100644 index 0000000..94eacf6 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/deferred_events.py @@ -0,0 +1,416 @@ +# Copyright 2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for managing deferred service events. + +This module is used to manage deferred service events from both charm actions +and package actions. +""" + +import datetime +import glob +import yaml +import os +import time +import uuid + +import charmhelpers.contrib.openstack.policy_rcd as policy_rcd +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as host +import charmhelpers.core.unitdata as unitdata + +import subprocess + + +# Deferred events generated from the charm are stored along side those +# generated from packaging. +DEFERRED_EVENTS_DIR = policy_rcd.POLICY_DEFERRED_EVENTS_DIR + + +class ServiceEvent(): + + def __init__(self, timestamp, service, reason, action, + policy_requestor_name=None, policy_requestor_type=None): + self.timestamp = timestamp + self.service = service + self.reason = reason + self.action = action + if policy_requestor_name: + self.policy_requestor_name = policy_requestor_name + else: + self.policy_requestor_name = hookenv.service_name() + if policy_requestor_type: + self.policy_requestor_type = policy_requestor_type + else: + self.policy_requestor_type = 'charm' + + def __eq__(self, other): + for attr in vars(self): + if getattr(self, attr) != getattr(other, attr): + return False + return True + + def matching_request(self, other): + for attr in ['service', 'action', 'reason']: + if getattr(self, attr) != getattr(other, attr): + return False + return True + + @classmethod + def from_dict(cls, data): + return cls( + data['timestamp'], + data['service'], + data['reason'], + data['action'], + data.get('policy_requestor_name'), + data.get('policy_requestor_type')) + + +def deferred_events_files(): + """Deferred event files + + Deferred event files that were generated by service_name() policy. + + :returns: Deferred event files + :rtype: List[str] + """ + return glob.glob('{}/*.deferred'.format(DEFERRED_EVENTS_DIR)) + + +def read_event_file(file_name): + """Read a file and return the corresponding objects. + + :param file_name: Name of file to read. + :type file_name: str + :returns: ServiceEvent from file. + :rtype: ServiceEvent + """ + with open(file_name, 'r') as f: + contents = yaml.safe_load(f) + event = ServiceEvent( + contents['timestamp'], + contents['service'], + contents['reason'], + contents['action'], + policy_requestor_name=contents.get('policy_requestor_name'), + policy_requestor_type=contents.get('policy_requestor_type')) + return event + + +def deferred_events(): + """Get list of deferred events. + + List of deferred events. Events are represented by dicts of the form: + + { + action: restart, + policy_requestor_name: neutron-openvswitch, + policy_requestor_type: charm, + reason: 'Pkg update', + service: openvswitch-switch, + time: 1614328743} + + :returns: List of deferred events. + :rtype: List[ServiceEvent] + """ + events = [] + for defer_file in deferred_events_files(): + events.append((defer_file, read_event_file(defer_file))) + return events + + +def duplicate_event_files(event): + """Get list of event files that have equivalent deferred events. + + :param event: Event to compare + :type event: ServiceEvent + :returns: List of event files + :rtype: List[str] + """ + duplicates = [] + for event_file, existing_event in deferred_events(): + if event.matching_request(existing_event): + duplicates.append(event_file) + return duplicates + + +def get_event_record_file(policy_requestor_type, policy_requestor_name): + """Generate filename for storing a new event. + + :param policy_requestor_type: System that blocked event + :type policy_requestor_type: str + :param policy_requestor_name: Name of application that blocked event + :type policy_requestor_name: str + :returns: File name + :rtype: str + """ + file_name = '{}/{}-{}-{}.deferred'.format( + DEFERRED_EVENTS_DIR, + policy_requestor_type, + policy_requestor_name, + uuid.uuid1()) + return file_name + + +def save_event(event): + """Write deferred events to backend. + + :param event: Event to save + :type event: ServiceEvent + """ + requestor_name = hookenv.service_name() + requestor_type = 'charm' + init_policy_log_dir() + if duplicate_event_files(event): + hookenv.log( + "Not writing new event, existing event found. {} {} {}".format( + event.service, + event.action, + event.reason), + level="DEBUG") + else: + record_file = get_event_record_file( + policy_requestor_type=requestor_type, + policy_requestor_name=requestor_name) + + with open(record_file, 'w') as f: + data = { + 'timestamp': event.timestamp, + 'service': event.service, + 'action': event.action, + 'reason': event.reason, + 'policy_requestor_type': requestor_type, + 'policy_requestor_name': requestor_name} + yaml.dump(data, f) + + +def clear_deferred_events(svcs, action): + """Remove any outstanding deferred events. + + Remove a deferred event if its service is in the services list and its + action matches. + + :param svcs: List of services to remove. + :type svcs: List[str] + :param action: Action to remove + :type action: str + """ + # XXX This function is not currently processing the action. It needs to + # match the action and also take account of try-restart and the + # equivalnce of stop-start and restart. + for defer_file in deferred_events_files(): + deferred_event = read_event_file(defer_file) + if deferred_event.service in svcs: + os.remove(defer_file) + + +def init_policy_log_dir(): + """Ensure directory to store events exists.""" + if not os.path.exists(DEFERRED_EVENTS_DIR): + os.mkdir(DEFERRED_EVENTS_DIR) + + +def get_deferred_events(): + """Return a list of deferred events requested by the charm and packages. + + :returns: List of deferred events + :rtype: List[ServiceEvent] + """ + events = [] + for _, event in deferred_events(): + events.append(event) + return events + + +def get_deferred_restarts(): + """List of deferred restart events requested by the charm and packages. + + :returns: List of deferred restarts + :rtype: List[ServiceEvent] + """ + return [e for e in get_deferred_events() if e.action == 'restart'] + + +def clear_deferred_restarts(services): + """Clear deferred restart events targeted at `services`. + + :param services: Services with deferred actions to clear. + :type services: List[str] + """ + clear_deferred_events(services, 'restart') + + +def process_svc_restart(service): + """Respond to a service restart having occurred. + + :param service: Services that the action was performed against. + :type service: str + """ + clear_deferred_restarts([service]) + + +def is_restart_permitted(): + """Check whether restarts are permitted. + + :returns: Whether restarts are permitted + :rtype: bool + """ + if hookenv.config('enable-auto-restarts') is None: + return True + return hookenv.config('enable-auto-restarts') + + +def check_and_record_restart_request(service, changed_files): + """Check if restarts are permitted, if they are not log the request. + + :param service: Service to be restarted + :type service: str + :param changed_files: Files that have changed to trigger restarts. + :type changed_files: List[str] + :returns: Whether restarts are permitted + :rtype: bool + """ + changed_files = sorted(list(set(changed_files))) + permitted = is_restart_permitted() + if not permitted: + save_event(ServiceEvent( + timestamp=round(time.time()), + service=service, + reason='File(s) changed: {}'.format( + ', '.join(changed_files)), + action='restart')) + return permitted + + +def deferrable_svc_restart(service, reason=None): + """Restarts service if permitted, if not defer it. + + :param service: Service to be restarted + :type service: str + :param reason: Reason for restart + :type reason: Union[str, None] + """ + if is_restart_permitted(): + host.service_restart(service) + else: + save_event(ServiceEvent( + timestamp=round(time.time()), + service=service, + reason=reason, + action='restart')) + + +def configure_deferred_restarts(services): + """Setup deferred restarts. + + :param services: Services to block restarts of. + :type services: List[str] + """ + policy_rcd.install_policy_rcd() + if is_restart_permitted(): + policy_rcd.remove_policy_file() + else: + blocked_actions = ['stop', 'restart', 'try-restart'] + for svc in services: + policy_rcd.add_policy_block(svc, blocked_actions) + + +def get_service_start_time(service): + """Find point in time when the systemd unit transitioned to active state. + + :param service: Services to check timetsamp of. + :type service: str + """ + start_time = None + out = subprocess.check_output( + [ + 'systemctl', + 'show', + service, + '--property=ActiveEnterTimestamp']) + str_time = out.decode().rstrip().replace('ActiveEnterTimestamp=', '') + if str_time: + start_time = datetime.datetime.strptime( + str_time, + '%a %Y-%m-%d %H:%M:%S %Z') + return start_time + + +def check_restart_timestamps(): + """Check deferred restarts against systemd units start time. + + Check if a service has a deferred event and clear it if it has been + subsequently restarted. + """ + for event in get_deferred_restarts(): + start_time = get_service_start_time(event.service) + deferred_restart_time = datetime.datetime.fromtimestamp( + event.timestamp) + if start_time and start_time < deferred_restart_time: + hookenv.log( + ("Restart still required, {} was started at {}, restart was " + "requested after that at {}").format( + event.service, + start_time, + deferred_restart_time), + level='DEBUG') + else: + clear_deferred_restarts([event.service]) + + +def set_deferred_hook(hookname): + """Record that a hook has been deferred. + + :param hookname: Name of hook that was deferred. + :type hookname: str + """ + with unitdata.HookData()() as t: + kv = t[0] + deferred_hooks = kv.get('deferred-hooks', []) + if hookname not in deferred_hooks: + deferred_hooks.append(hookname) + kv.set('deferred-hooks', sorted(list(set(deferred_hooks)))) + + +def get_deferred_hooks(): + """Get a list of deferred hooks. + + :returns: List of hook names. + :rtype: List[str] + """ + with unitdata.HookData()() as t: + kv = t[0] + return kv.get('deferred-hooks', []) + + +def clear_deferred_hooks(): + """Clear any deferred hooks.""" + with unitdata.HookData()() as t: + kv = t[0] + kv.set('deferred-hooks', []) + + +def clear_deferred_hook(hookname): + """Clear a specific deferred hooks. + + :param hookname: Name of hook to remove. + :type hookname: str + """ + with unitdata.HookData()() as t: + kv = t[0] + deferred_hooks = kv.get('deferred-hooks', []) + if hookname in deferred_hooks: + deferred_hooks.remove(hookname) + kv.set('deferred-hooks', deferred_hooks) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/exceptions.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/exceptions.py new file mode 100644 index 0000000..b233063 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/exceptions.py @@ -0,0 +1,26 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class OSContextError(Exception): + """Raised when an error occurs during context generation. + + This exception is principally used in contrib.openstack.context + """ + pass + + +class ServiceActionError(Exception): + """Raised when a service action (stop/start/ etc) failed.""" + pass diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/__init__.py new file mode 100644 index 0000000..9df5f74 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy.sh b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy.sh new file mode 100755 index 0000000..1df55db --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +CRITICAL=0 +NOTACTIVE='' +LOGFILE=/var/log/nagios/check_haproxy.log +AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}') + +typeset -i N_INSTANCES=0 +for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg) +do + N_INSTANCES=N_INSTANCES+1 + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' --regex=",${appserver},.*,UP.*" -e ' 200 OK') + if [ $? != 0 ]; then + date >> $LOGFILE + echo $output >> $LOGFILE + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v | grep ",${appserver}," >> $LOGFILE 2>&1 + CRITICAL=1 + NOTACTIVE="${NOTACTIVE} $appserver" + fi +done + +if [ $CRITICAL = 1 ]; then + echo "CRITICAL:${NOTACTIVE}" + exit 2 +fi + +echo "OK: All haproxy instances ($N_INSTANCES) looking good" +exit 0 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh new file mode 100755 index 0000000..91ce024 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh @@ -0,0 +1,30 @@ +#!/bin/bash +#-------------------------------------------- +# This file is managed by Juju +#-------------------------------------------- +# +# Copyright 2009,2012 Canonical Ltd. +# Author: Tom Haddon + +# These should be config options at some stage +CURRQthrsh=0 +MAXQthrsh=100 + +AUTH=$(grep -r "stats auth" /etc/haproxy/haproxy.cfg | awk 'NR=1{print $3}') + +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) + +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') +do + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) + + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" + exit 2 + fi +done + +echo "OK: All haproxy queue depths looking good" +exit 0 + diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/policy_rc_d_script.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/policy_rc_d_script.py new file mode 100755 index 0000000..431e972 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/files/policy_rc_d_script.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +"""This script is an implementation of policy-rc.d + +For further information on policy-rc.d see *1 + +*1 https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt +""" +import collections +import glob +import os +import logging +import sys +import time +import uuid +import yaml + + +SystemPolicy = collections.namedtuple( + 'SystemPolicy', + [ + 'policy_requestor_name', + 'policy_requestor_type', + 'service', + 'blocked_actions']) + +DEFAULT_POLICY_CONFIG_DIR = '/etc/policy-rc.d' +DEFAULT_POLICY_LOG_DIR = '/var/lib/policy-rc.d' + + +def read_policy_file(policy_file): + """Return system policies from given file. + + :param file_name: Name of file to read. + :type file_name: str + :returns: Policy + :rtype: List[SystemPolicy] + """ + policies = [] + if os.path.exists(policy_file): + with open(policy_file, 'r') as f: + policy = yaml.safe_load(f) + for service, actions in policy['blocked_actions'].items(): + service = service.replace('.service', '') + policies.append(SystemPolicy( + policy_requestor_name=policy['policy_requestor_name'], + policy_requestor_type=policy['policy_requestor_type'], + service=service, + blocked_actions=actions)) + return policies + + +def get_policies(policy_config_dir): + """Return all system policies in policy_config_dir. + + :param policy_config_dir: Name of file to read. + :type policy_config_dir: str + :returns: Policy + :rtype: List[SystemPolicy] + """ + _policy = [] + for f in glob.glob('{}/*.policy'.format(policy_config_dir)): + _policy.extend(read_policy_file(f)) + return _policy + + +def record_blocked_action(service, action, blocking_policies, policy_log_dir): + """Record that an action was requested but deniedl + + :param service: Service that was blocked + :type service: str + :param action: Action that was blocked. + :type action: str + :param blocking_policies: Policies that blocked the action on the service. + :type blocking_policies: List[SystemPolicy] + :param policy_log_dir: Directory to place the blocking action record. + :type policy_log_dir: str + """ + if not os.path.exists(policy_log_dir): + os.mkdir(policy_log_dir) + seconds = round(time.time()) + for policy in blocking_policies: + if not os.path.exists(policy_log_dir): + os.mkdir(policy_log_dir) + file_name = '{}/{}-{}-{}.deferred'.format( + policy_log_dir, + policy.policy_requestor_type, + policy.policy_requestor_name, + uuid.uuid1()) + with open(file_name, 'w') as f: + data = { + 'timestamp': seconds, + 'service': service, + 'action': action, + 'reason': 'Package update', + 'policy_requestor_type': policy.policy_requestor_type, + 'policy_requestor_name': policy.policy_requestor_name} + yaml.dump(data, f) + + +def get_blocking_policies(service, action, policy_config_dir): + """Record that an action was requested but deniedl + + :param service: Service that action is requested against. + :type service: str + :param action: Action that is requested. + :type action: str + :param policy_config_dir: Directory that stores policy files. + :type policy_config_dir: str + :returns: Policies + :rtype: List[SystemPolicy] + """ + service = service.replace('.service', '') + blocking_policies = [ + policy + for policy in get_policies(policy_config_dir) + if policy.service == service and action in policy.blocked_actions] + return blocking_policies + + +def process_action_request(service, action, policy_config_dir, policy_log_dir): + """Take the requested action against service and check if it is permitted. + + :param service: Service that action is requested against. + :type service: str + :param action: Action that is requested. + :type action: str + :param policy_config_dir: Directory that stores policy files. + :type policy_config_dir: str + :param policy_log_dir: Directory that stores policy files. + :type policy_log_dir: str + :returns: Tuple of whether the action is permitted and explanation. + :rtype: (boolean, str) + """ + blocking_policies = get_blocking_policies( + service, + action, + policy_config_dir) + if blocking_policies: + policy_msg = [ + '{} {}'.format(p.policy_requestor_type, p.policy_requestor_name) + for p in sorted(blocking_policies)] + message = '{} of {} blocked by {}'.format( + action, + service, + ', '.join(policy_msg)) + record_blocked_action( + service, + action, + blocking_policies, + policy_log_dir) + action_permitted = False + else: + message = "Permitting {} {}".format(service, action) + action_permitted = True + return action_permitted, message + + +def main(): + logging.basicConfig( + filename='/var/log/policy-rc.d.log', + level=logging.DEBUG, + format='%(asctime)s %(message)s') + + service = sys.argv[1] + action = sys.argv[2] + + permitted, message = process_action_request( + service, + action, + DEFAULT_POLICY_CONFIG_DIR, + DEFAULT_POLICY_LOG_DIR) + logging.info(message) + + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + # Exit status codes: + # 0 - action allowed + # 1 - unknown action (therefore, undefined policy) + # 100 - unknown initscript id + # 101 - action forbidden by policy + # 102 - subsystem error + # 103 - syntax error + # 104 - [reserved] + # 105 - behaviour uncertain, policy undefined. + # 106 - action not allowed. Use the returned fallback actions + # (which are implied to be "allowed") instead. + + if permitted: + return 0 + else: + return 101 + + +if __name__ == "__main__": + rc = main() + sys.exit(rc) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/__init__.py new file mode 100644 index 0000000..9b088de --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/utils.py new file mode 100644 index 0000000..a5cbdf5 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ha/utils.py @@ -0,0 +1,348 @@ +# Copyright 2014-2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copyright 2016 Canonical Ltd. +# +# Authors: +# Openstack Charmers < +# + +""" +Helpers for high availability. +""" + +import hashlib +import json + +import re + +from charmhelpers.core.hookenv import ( + expected_related_units, + log, + relation_set, + charm_name, + config, + status_set, + DEBUG, +) + +from charmhelpers.core.host import ( + lsb_release +) + +from charmhelpers.contrib.openstack.ip import ( + resolve_address, + is_ipv6, +) + +from charmhelpers.contrib.network.ip import ( + get_iface_for_address, + get_netmask_for_address, +) + +from charmhelpers.contrib.hahelpers.cluster import ( + get_hacluster_config +) + +JSON_ENCODE_OPTIONS = dict( + sort_keys=True, + allow_nan=False, + indent=None, + separators=(',', ':'), +) + +VIP_GROUP_NAME = 'grp_{service}_vips' +DNSHA_GROUP_NAME = 'grp_{service}_hostnames' + + +class DNSHAException(Exception): + """Raised when an error occurs setting up DNS HA + """ + + pass + + +def update_dns_ha_resource_params(resources, resource_params, + relation_id=None, + crm_ocf='ocf:maas:dns'): + """ Configure DNS-HA resources based on provided configuration and + update resource dictionaries for the HA relation. + + @param resources: Pointer to dictionary of resources. + Usually instantiated in ha_joined(). + @param resource_params: Pointer to dictionary of resource parameters. + Usually instantiated in ha_joined() + @param relation_id: Relation ID of the ha relation + @param crm_ocf: Corosync Open Cluster Framework resource agent to use for + DNS HA + """ + _relation_data = {'resources': {}, 'resource_params': {}} + update_hacluster_dns_ha(charm_name(), + _relation_data, + crm_ocf) + resources.update(_relation_data['resources']) + resource_params.update(_relation_data['resource_params']) + relation_set(relation_id=relation_id, groups=_relation_data['groups']) + + +def assert_charm_supports_dns_ha(): + """Validate prerequisites for DNS HA + The MAAS client is only available on Xenial or greater + + :raises DNSHAException: if release is < 16.04 + """ + if lsb_release().get('DISTRIB_RELEASE') < '16.04': + msg = ('DNS HA is only supported on 16.04 and greater ' + 'versions of Ubuntu.') + status_set('blocked', msg) + raise DNSHAException(msg) + return True + + +def expect_ha(): + """ Determine if the unit expects to be in HA + + Check juju goal-state if ha relation is expected, check for VIP or dns-ha + settings which indicate the unit should expect to be related to hacluster. + + @returns boolean + """ + ha_related_units = [] + try: + ha_related_units = list(expected_related_units(reltype='ha')) + except (NotImplementedError, KeyError): + pass + return len(ha_related_units) > 0 or config('vip') or config('dns-ha') + + +def generate_ha_relation_data(service, + extra_settings=None, + haproxy_enabled=True): + """ Generate relation data for ha relation + + Based on configuration options and unit interfaces, generate a json + encoded dict of relation data items for the hacluster relation, + providing configuration for DNS HA or VIP's + haproxy clone sets. + + Example of supplying additional settings:: + + COLO_CONSOLEAUTH = 'inf: res_nova_consoleauth grp_nova_vips' + AGENT_CONSOLEAUTH = 'ocf:openstack:nova-consoleauth' + AGENT_CA_PARAMS = 'op monitor interval="5s"' + + ha_console_settings = { + 'colocations': {'vip_consoleauth': COLO_CONSOLEAUTH}, + 'init_services': {'res_nova_consoleauth': 'nova-consoleauth'}, + 'resources': {'res_nova_consoleauth': AGENT_CONSOLEAUTH}, + 'resource_params': {'res_nova_consoleauth': AGENT_CA_PARAMS}) + generate_ha_relation_data('nova', extra_settings=ha_console_settings) + + + @param service: Name of the service being configured + @param extra_settings: Dict of additional resource data + @returns dict: json encoded data for use with relation_set + """ + _relation_data = {'resources': {}, 'resource_params': {}} + + if haproxy_enabled: + _meta = 'meta migration-threshold="INFINITY" failure-timeout="5s"' + _haproxy_res = 'res_{}_haproxy'.format(service) + _relation_data['resources'] = {_haproxy_res: 'lsb:haproxy'} + _relation_data['resource_params'] = { + _haproxy_res: '{} op monitor interval="5s"'.format(_meta) + } + _relation_data['init_services'] = {_haproxy_res: 'haproxy'} + _relation_data['clones'] = { + 'cl_{}_haproxy'.format(service): _haproxy_res + } + + if extra_settings: + for k, v in extra_settings.items(): + if _relation_data.get(k): + _relation_data[k].update(v) + else: + _relation_data[k] = v + + if config('dns-ha'): + update_hacluster_dns_ha(service, _relation_data) + else: + update_hacluster_vip(service, _relation_data) + + return { + 'json_{}'.format(k): json.dumps(v, **JSON_ENCODE_OPTIONS) + for k, v in _relation_data.items() if v + } + + +def update_hacluster_dns_ha(service, relation_data, + crm_ocf='ocf:maas:dns'): + """ Configure DNS-HA resources based on provided configuration + + @param service: Name of the service being configured + @param relation_data: Pointer to dictionary of relation data. + @param crm_ocf: Corosync Open Cluster Framework resource agent to use for + DNS HA + """ + # Validate the charm environment for DNS HA + assert_charm_supports_dns_ha() + + settings = ['os-admin-hostname', 'os-internal-hostname', + 'os-public-hostname', 'os-access-hostname'] + + # Check which DNS settings are set and update dictionaries + hostname_group = [] + for setting in settings: + hostname = config(setting) + if hostname is None: + log('DNS HA: Hostname setting {} is None. Ignoring.' + ''.format(setting), + DEBUG) + continue + m = re.search('os-(.+?)-hostname', setting) + if m: + endpoint_type = m.group(1) + # resolve_address's ADDRESS_MAP uses 'int' not 'internal' + if endpoint_type == 'internal': + endpoint_type = 'int' + else: + msg = ('Unexpected DNS hostname setting: {}. ' + 'Cannot determine endpoint_type name' + ''.format(setting)) + status_set('blocked', msg) + raise DNSHAException(msg) + + hostname_key = 'res_{}_{}_hostname'.format(service, endpoint_type) + if hostname_key in hostname_group: + log('DNS HA: Resource {}: {} already exists in ' + 'hostname group - skipping'.format(hostname_key, hostname), + DEBUG) + continue + + hostname_group.append(hostname_key) + relation_data['resources'][hostname_key] = crm_ocf + relation_data['resource_params'][hostname_key] = ( + 'params fqdn="{}" ip_address="{}"' + .format(hostname, resolve_address(endpoint_type=endpoint_type, + override=False))) + + if len(hostname_group) >= 1: + log('DNS HA: Hostname group is set with {} as members. ' + 'Informing the ha relation'.format(' '.join(hostname_group)), + DEBUG) + relation_data['groups'] = { + DNSHA_GROUP_NAME.format(service=service): ' '.join(hostname_group) + } + else: + msg = 'DNS HA: Hostname group has no members.' + status_set('blocked', msg) + raise DNSHAException(msg) + + +def get_vip_settings(vip): + """Calculate which nic is on the correct network for the given vip. + + If nic or netmask discovery fail then fallback to using charm supplied + config. If fallback is used this is indicated via the fallback variable. + + @param vip: VIP to lookup nic and cidr for. + @returns (str, str, bool): eg (iface, netmask, fallback) + """ + iface = get_iface_for_address(vip) + netmask = get_netmask_for_address(vip) + fallback = False + if iface is None: + iface = config('vip_iface') + fallback = True + if netmask is None: + netmask = config('vip_cidr') + fallback = True + return iface, netmask, fallback + + +def update_hacluster_vip(service, relation_data): + """ Configure VIP resources based on provided configuration + + @param service: Name of the service being configured + @param relation_data: Pointer to dictionary of relation data. + """ + cluster_config = get_hacluster_config() + vip_group = [] + vips_to_delete = [] + for vip in cluster_config['vip'].split(): + if is_ipv6(vip): + res_vip = 'ocf:heartbeat:IPv6addr' + vip_params = 'ipv6addr' + else: + res_vip = 'ocf:heartbeat:IPaddr2' + vip_params = 'ip' + + iface, netmask, fallback = get_vip_settings(vip) + + vip_monitoring = 'op monitor timeout="20s" interval="10s" depth="0"' + if iface is not None: + # NOTE(jamespage): Delete old VIP resources + # Old style naming encoding iface in name + # does not work well in environments where + # interface/subnet wiring is not consistent + vip_key = 'res_{}_{}_vip'.format(service, iface) + if vip_key in vips_to_delete: + vip_key = '{}_{}'.format(vip_key, vip_params) + vips_to_delete.append(vip_key) + + vip_key = 'res_{}_{}_vip'.format( + service, + hashlib.sha1(vip.encode('UTF-8')).hexdigest()[:7]) + + relation_data['resources'][vip_key] = res_vip + # NOTE(jamespage): + # Use option provided vip params if these where used + # instead of auto-detected values + if fallback: + relation_data['resource_params'][vip_key] = ( + 'params {ip}="{vip}" cidr_netmask="{netmask}" ' + 'nic="{iface}" {vip_monitoring}'.format( + ip=vip_params, + vip=vip, + iface=iface, + netmask=netmask, + vip_monitoring=vip_monitoring)) + else: + # NOTE(jamespage): + # let heartbeat figure out which interface and + # netmask to configure, which works nicely + # when network interface naming is not + # consistent across units. + relation_data['resource_params'][vip_key] = ( + 'params {ip}="{vip}" {vip_monitoring}'.format( + ip=vip_params, + vip=vip, + vip_monitoring=vip_monitoring)) + + vip_group.append(vip_key) + + if vips_to_delete: + try: + relation_data['delete_resources'].extend(vips_to_delete) + except KeyError: + relation_data['delete_resources'] = vips_to_delete + + if len(vip_group) >= 1: + key = VIP_GROUP_NAME.format(service=service) + try: + relation_data['groups'][key] = ' '.join(vip_group) + except KeyError: + relation_data['groups'] = { + key: ' '.join(vip_group) + } diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ip.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ip.py new file mode 100644 index 0000000..b8c94c5 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ip.py @@ -0,0 +1,235 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + NoNetworkBinding, + config, + unit_get, + service_name, + network_get_primary_address, +) +from charmhelpers.contrib.network.ip import ( + get_address_in_network, + is_address_in_network, + is_ipv6, + get_ipv6_addr, + resolve_network_cidr, +) +from charmhelpers.contrib.hahelpers.cluster import is_clustered + +PUBLIC = 'public' +INTERNAL = 'int' +ADMIN = 'admin' +ACCESS = 'access' + +# TODO: reconcile 'int' vs 'internal' binding names +ADDRESS_MAP = { + PUBLIC: { + 'binding': 'public', + 'config': 'os-public-network', + 'fallback': 'public-address', + 'override': 'os-public-hostname', + }, + INTERNAL: { + 'binding': 'internal', + 'config': 'os-internal-network', + 'fallback': 'private-address', + 'override': 'os-internal-hostname', + }, + ADMIN: { + 'binding': 'admin', + 'config': 'os-admin-network', + 'fallback': 'private-address', + 'override': 'os-admin-hostname', + }, + ACCESS: { + 'binding': 'access', + 'config': 'access-network', + 'fallback': 'private-address', + 'override': 'os-access-hostname', + }, + # Note (thedac) bridge to begin the reconciliation between 'int' vs + # 'internal' binding names + 'internal': { + 'binding': 'internal', + 'config': 'os-internal-network', + 'fallback': 'private-address', + 'override': 'os-internal-hostname', + }, +} + + +def canonical_url(configs, endpoint_type=PUBLIC): + """Returns the correct HTTP URL to this host given the state of HTTPS + configuration, hacluster and charm configuration. + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :param endpoint_type: str endpoint type to resolve. + :param returns: str base URL for services on the current service unit. + """ + scheme = _get_scheme(configs) + + address = resolve_address(endpoint_type) + if is_ipv6(address): + address = "[{}]".format(address) + + return '%s://%s' % (scheme, address) + + +def _get_scheme(configs): + """Returns the scheme to use for the url (either http or https) + depending upon whether https is in the configs value. + + :param configs: OSTemplateRenderer config templating object to inspect + for a complete https context. + :returns: either 'http' or 'https' depending on whether https is + configured within the configs context. + """ + scheme = 'http' + if configs and 'https' in configs.complete_contexts(): + scheme = 'https' + return scheme + + +def _get_address_override(endpoint_type=PUBLIC): + """Returns any address overrides that the user has defined based on the + endpoint type. + + Note: this function allows for the service name to be inserted into the + address if the user specifies {service_name}.somehost.org. + + :param endpoint_type: the type of endpoint to retrieve the override + value for. + :returns: any endpoint address or hostname that the user has overridden + or None if an override is not present. + """ + override_key = ADDRESS_MAP[endpoint_type]['override'] + addr_override = config(override_key) + if not addr_override: + return None + else: + return addr_override.format(service_name=service_name()) + + +def local_address(unit_get_fallback='public-address'): + """Return a network address for this unit. + + Attempt to retrieve a 'default' IP address for this unit + from network-get. If this is running with an old version of Juju then + fallback to unit_get. + + Note on juju < 2.9 the binding to juju-info may not exist, so fall back to + the unit-get. + + :param unit_get_fallback: Either 'public-address' or 'private-address'. + Only used with old versions of Juju. + :type unit_get_fallback: str + :returns: IP Address + :rtype: str + """ + try: + return network_get_primary_address('juju-info') + except (NotImplementedError, NoNetworkBinding): + return unit_get(unit_get_fallback) + + +def resolve_address(endpoint_type=PUBLIC, override=True): + """Return unit address depending on net config. + + If unit is clustered with vip(s) and has net splits defined, return vip on + correct network. If clustered with no nets defined, return primary vip. + + If not clustered, return unit address ensuring address is on configured net + split if one is configured, or a Juju 2.0 extra-binding has been used. + + :param endpoint_type: Network endpoing type + :param override: Accept hostname overrides or not + """ + resolved_address = None + if override: + resolved_address = _get_address_override(endpoint_type) + if resolved_address: + return resolved_address + + vips = config('vip') + if vips: + vips = vips.split() + + net_type = ADDRESS_MAP[endpoint_type]['config'] + net_addr = config(net_type) + net_fallback = ADDRESS_MAP[endpoint_type]['fallback'] + binding = ADDRESS_MAP[endpoint_type]['binding'] + clustered = is_clustered() + + if clustered and vips: + if net_addr: + for vip in vips: + if is_address_in_network(net_addr, vip): + resolved_address = vip + break + else: + # NOTE: endeavour to check vips against network space + # bindings + try: + bound_cidr = resolve_network_cidr( + network_get_primary_address(binding) + ) + for vip in vips: + if is_address_in_network(bound_cidr, vip): + resolved_address = vip + break + except (NotImplementedError, NoNetworkBinding): + # If no net-splits configured and no support for extra + # bindings/network spaces so we expect a single vip + resolved_address = vips[0] + else: + if config('prefer-ipv6'): + fallback_addr = get_ipv6_addr(exc_list=vips)[0] + else: + fallback_addr = local_address(unit_get_fallback=net_fallback) + + if net_addr: + resolved_address = get_address_in_network(net_addr, fallback_addr) + else: + # NOTE: only try to use extra bindings if legacy network + # configuration is not in use + try: + resolved_address = network_get_primary_address(binding) + except (NotImplementedError, NoNetworkBinding): + resolved_address = fallback_addr + + if resolved_address is None: + raise ValueError("Unable to resolve a suitable IP address based on " + "charm state and configuration. (net_type=%s, " + "clustered=%s)" % (net_type, clustered)) + + return resolved_address + + +def get_vip_in_network(network): + matching_vip = None + vips = config('vip') + if vips: + for vip in vips.split(): + if is_address_in_network(network, vip): + matching_vip = vip + return matching_vip + + +def get_default_api_bindings(): + _default_bindings = [] + for binding in [INTERNAL, ADMIN, PUBLIC]: + _default_bindings.append(ADDRESS_MAP[binding]['binding']) + return _default_bindings diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/keystone.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/keystone.py new file mode 100644 index 0000000..d7e02cc --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/keystone.py @@ -0,0 +1,178 @@ +#!/usr/bin/python +# +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +from charmhelpers.fetch import apt_install +from charmhelpers.contrib.openstack.context import IdentityServiceContext +from charmhelpers.core.hookenv import ( + log, + ERROR, +) + + +def get_api_suffix(api_version): + """Return the formatted api suffix for the given version + @param api_version: version of the keystone endpoint + @returns the api suffix formatted according to the given api + version + """ + return 'v2.0' if api_version in (2, "2", "2.0") else 'v3' + + +def format_endpoint(schema, addr, port, api_version): + """Return a formatted keystone endpoint + @param schema: http or https + @param addr: ipv4/ipv6 host of the keystone service + @param port: port of the keystone service + @param api_version: 2 or 3 + @returns a fully formatted keystone endpoint + """ + return '{}://{}:{}/{}/'.format(schema, addr, port, + get_api_suffix(api_version)) + + +def get_keystone_manager(endpoint, api_version, **kwargs): + """Return a keystonemanager for the correct API version + + @param endpoint: the keystone endpoint to point client at + @param api_version: version of the keystone api the client should use + @param kwargs: token or username/tenant/password information + @returns keystonemanager class used for interrogating keystone + """ + if api_version == 2: + return KeystoneManager2(endpoint, **kwargs) + if api_version == 3: + return KeystoneManager3(endpoint, **kwargs) + raise ValueError('No manager found for api version {}'.format(api_version)) + + +def get_keystone_manager_from_identity_service_context(): + """Return a keystonmanager generated from a + instance of charmhelpers.contrib.openstack.context.IdentityServiceContext + @returns keystonamenager instance + """ + context = IdentityServiceContext()() + if not context: + msg = "Identity service context cannot be generated" + log(msg, level=ERROR) + raise ValueError(msg) + + endpoint = format_endpoint(context['service_protocol'], + context['service_host'], + context['service_port'], + context['api_version']) + + if context['api_version'] in (2, "2.0"): + api_version = 2 + else: + api_version = 3 + + return get_keystone_manager(endpoint, api_version, + username=context['admin_user'], + password=context['admin_password'], + tenant_name=context['admin_tenant_name']) + + +class KeystoneManager(object): + + def resolve_service_id(self, service_name=None, service_type=None): + """Find the service_id of a given service""" + services = [s._info for s in self.api.services.list()] + + service_name = service_name.lower() + for s in services: + name = s['name'].lower() + if service_type and service_name: + if (service_name == name and service_type == s['type']): + return s['id'] + elif service_name and service_name == name: + return s['id'] + elif service_type and service_type == s['type']: + return s['id'] + return None + + def service_exists(self, service_name=None, service_type=None): + """Determine if the given service exists on the service list""" + return self.resolve_service_id(service_name, service_type) is not None + + +class KeystoneManager2(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v2_0 import client + from keystoneclient.auth.identity import v2 + from keystoneclient import session + + self.api_version = 2 + + token = kwargs.get("token", None) + if token: + api = client.Client(endpoint=endpoint, token=token) + else: + auth = v2.Password(username=kwargs.get("username"), + password=kwargs.get("password"), + tenant_name=kwargs.get("tenant_name"), + auth_url=endpoint) + sess = session.Session(auth=auth) + api = client.Client(session=sess) + + self.api = api + + +class KeystoneManager3(KeystoneManager): + + def __init__(self, endpoint, **kwargs): + try: + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + except ImportError: + if six.PY2: + apt_install(["python-keystoneclient"], fatal=True) + else: + apt_install(["python3-keystoneclient"], fatal=True) + + from keystoneclient.v3 import client + from keystoneclient.auth import token_endpoint + from keystoneclient import session + from keystoneclient.auth.identity import v3 + + self.api_version = 3 + + token = kwargs.get("token", None) + if token: + auth = token_endpoint.Token(endpoint=endpoint, + token=token) + sess = session.Session(auth=auth) + else: + auth = v3.Password(auth_url=endpoint, + user_id=kwargs.get("username"), + password=kwargs.get("password"), + project_id=kwargs.get("tenant_name")) + sess = session.Session(auth=auth) + + self.api = client.Client(session=sess) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/neutron.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/neutron.py new file mode 100644 index 0000000..b41314c --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/neutron.py @@ -0,0 +1,359 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Various utilities for dealing with Neutron and the renaming from Quantum. + +import six +from subprocess import check_output + +from charmhelpers.core.hookenv import ( + config, + log, + ERROR, +) + +from charmhelpers.contrib.openstack.utils import ( + os_release, + CompareOpenStackReleases, +) + + +def headers_package(): + """Ensures correct linux-headers for running kernel are installed, + for building DKMS package""" + kver = check_output(['uname', '-r']).decode('UTF-8').strip() + return 'linux-headers-%s' % kver + + +QUANTUM_CONF_DIR = '/etc/quantum' + + +def kernel_version(): + """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """ + kver = check_output(['uname', '-r']).decode('UTF-8').strip() + kver = kver.split('.') + return (int(kver[0]), int(kver[1])) + + +def determine_dkms_package(): + """ Determine which DKMS package should be used based on kernel version """ + # NOTE: 3.13 kernels have support for GRE and VXLAN native + if kernel_version() >= (3, 13): + return [] + else: + return [headers_package(), 'openvswitch-datapath-dkms'] + + +# legacy + + +def quantum_plugins(): + return { + 'ovs': { + 'config': '/etc/quantum/plugins/openvswitch/' + 'ovs_quantum_plugin.ini', + 'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.' + 'OVSQuantumPluginV2', + 'contexts': [], + 'services': ['quantum-plugin-openvswitch-agent'], + 'packages': [determine_dkms_package(), + ['quantum-plugin-openvswitch-agent']], + 'server_packages': ['quantum-server', + 'quantum-plugin-openvswitch'], + 'server_services': ['quantum-server'] + }, + 'nvp': { + 'config': '/etc/quantum/plugins/nicira/nvp.ini', + 'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.' + 'QuantumPlugin.NvpPluginV2', + 'contexts': [], + 'services': [], + 'packages': [], + 'server_packages': ['quantum-server', + 'quantum-plugin-nicira'], + 'server_services': ['quantum-server'] + } + } + + +NEUTRON_CONF_DIR = '/etc/neutron' + + +def neutron_plugins(): + release = os_release('nova-common') + plugins = { + 'ovs': { + 'config': '/etc/neutron/plugins/openvswitch/' + 'ovs_neutron_plugin.ini', + 'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.' + 'OVSNeutronPluginV2', + 'contexts': [], + 'services': ['neutron-plugin-openvswitch-agent'], + 'packages': [determine_dkms_package(), + ['neutron-plugin-openvswitch-agent']], + 'server_packages': ['neutron-server', + 'neutron-plugin-openvswitch'], + 'server_services': ['neutron-server'] + }, + 'nvp': { + 'config': '/etc/neutron/plugins/nicira/nvp.ini', + 'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.' + 'NeutronPlugin.NvpPluginV2', + 'contexts': [], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', + 'neutron-plugin-nicira'], + 'server_services': ['neutron-server'] + }, + 'nsx': { + 'config': '/etc/neutron/plugins/vmware/nsx.ini', + 'driver': 'vmware', + 'contexts': [], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', + 'neutron-plugin-vmware'], + 'server_services': ['neutron-server'] + }, + 'n1kv': { + 'config': '/etc/neutron/plugins/cisco/cisco_plugins.ini', + 'driver': 'neutron.plugins.cisco.network_plugin.PluginV2', + 'contexts': [], + 'services': [], + 'packages': [determine_dkms_package(), + ['neutron-plugin-cisco']], + 'server_packages': ['neutron-server', + 'neutron-plugin-cisco'], + 'server_services': ['neutron-server'] + }, + 'Calico': { + 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini', + 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin', + 'contexts': [], + 'services': ['calico-felix', + 'bird', + 'neutron-dhcp-agent', + 'nova-api-metadata', + 'etcd'], + 'packages': [determine_dkms_package(), + ['calico-compute', + 'bird', + 'neutron-dhcp-agent', + 'nova-api-metadata', + 'etcd']], + 'server_packages': ['neutron-server', 'calico-control', 'etcd'], + 'server_services': ['neutron-server', 'etcd'] + }, + 'vsp': { + 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini', + 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin', + 'contexts': [], + 'services': [], + 'packages': [], + 'server_packages': ['neutron-server', 'neutron-plugin-nuage'], + 'server_services': ['neutron-server'] + }, + 'plumgrid': { + 'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini', + 'driver': ('neutron.plugins.plumgrid.plumgrid_plugin' + '.plumgrid_plugin.NeutronPluginPLUMgridV2'), + 'contexts': [], + 'services': [], + 'packages': ['plumgrid-lxc', + 'iovisor-dkms'], + 'server_packages': ['neutron-server', + 'neutron-plugin-plumgrid'], + 'server_services': ['neutron-server'] + }, + 'midonet': { + 'config': '/etc/neutron/plugins/midonet/midonet.ini', + 'driver': 'midonet.neutron.plugin.MidonetPluginV2', + 'contexts': [], + 'services': [], + 'packages': [determine_dkms_package()], + 'server_packages': ['neutron-server', + 'python-neutron-plugin-midonet'], + 'server_services': ['neutron-server'] + } + } + if CompareOpenStackReleases(release) >= 'icehouse': + # NOTE: patch in ml2 plugin for icehouse onwards + plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' + plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' + plugins['ovs']['server_packages'] = ['neutron-server', + 'neutron-plugin-ml2'] + # NOTE: patch in vmware renames nvp->nsx for icehouse onwards + plugins['nvp'] = plugins['nsx'] + if CompareOpenStackReleases(release) >= 'kilo': + plugins['midonet']['driver'] = ( + 'neutron.plugins.midonet.plugin.MidonetPluginV2') + if CompareOpenStackReleases(release) >= 'liberty': + plugins['midonet']['driver'] = ( + 'midonet.neutron.plugin_v1.MidonetPluginV2') + plugins['midonet']['server_packages'].remove( + 'python-neutron-plugin-midonet') + plugins['midonet']['server_packages'].append( + 'python-networking-midonet') + plugins['plumgrid']['driver'] = ( + 'networking_plumgrid.neutron.plugins' + '.plugin.NeutronPluginPLUMgridV2') + plugins['plumgrid']['server_packages'].remove( + 'neutron-plugin-plumgrid') + if CompareOpenStackReleases(release) >= 'mitaka': + plugins['nsx']['server_packages'].remove('neutron-plugin-vmware') + plugins['nsx']['server_packages'].append('python-vmware-nsx') + plugins['nsx']['config'] = '/etc/neutron/nsx.ini' + plugins['vsp']['driver'] = ( + 'nuage_neutron.plugins.nuage.plugin.NuagePlugin') + if CompareOpenStackReleases(release) >= 'newton': + plugins['vsp']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' + plugins['vsp']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' + plugins['vsp']['server_packages'] = ['neutron-server', + 'neutron-plugin-ml2'] + return plugins + + +def neutron_plugin_attribute(plugin, attr, net_manager=None): + manager = net_manager or network_manager() + if manager == 'quantum': + plugins = quantum_plugins() + elif manager == 'neutron': + plugins = neutron_plugins() + else: + log("Network manager '%s' does not support plugins." % (manager), + level=ERROR) + raise Exception + + try: + _plugin = plugins[plugin] + except KeyError: + log('Unrecognised plugin for %s: %s' % (manager, plugin), level=ERROR) + raise Exception + + try: + return _plugin[attr] + except KeyError: + return None + + +def network_manager(): + ''' + Deals with the renaming of Quantum to Neutron in H and any situations + that require compatibility (eg, deploying H with network-manager=quantum, + upgrading from G). + ''' + release = os_release('nova-common') + manager = config('network-manager').lower() + + if manager not in ['quantum', 'neutron']: + return manager + + if release in ['essex']: + # E does not support neutron + log('Neutron networking not supported in Essex.', level=ERROR) + raise Exception + elif release in ['folsom', 'grizzly']: + # neutron is named quantum in F and G + return 'quantum' + else: + # ensure accurate naming for all releases post-H + return 'neutron' + + +def parse_mappings(mappings, key_rvalue=False): + """By default mappings are lvalue keyed. + + If key_rvalue is True, the mapping will be reversed to allow multiple + configs for the same lvalue. + """ + parsed = {} + if mappings: + mappings = mappings.split() + for m in mappings: + p = m.partition(':') + + if key_rvalue: + key_index = 2 + val_index = 0 + # if there is no rvalue skip to next + if not p[1]: + continue + else: + key_index = 0 + val_index = 2 + + key = p[key_index].strip() + parsed[key] = p[val_index].strip() + + return parsed + + +def parse_bridge_mappings(mappings): + """Parse bridge mappings. + + Mappings must be a space-delimited list of provider:bridge mappings. + + Returns dict of the form {provider:bridge}. + """ + return parse_mappings(mappings) + + +def parse_data_port_mappings(mappings, default_bridge='br-data'): + """Parse data port mappings. + + Mappings must be a space-delimited list of bridge:port. + + Returns dict of the form {port:bridge} where ports may be mac addresses or + interface names. + """ + + # NOTE(dosaboy): we use rvalue for key to allow multiple values to be + # proposed for since it may be a mac address which will differ + # across units this allowing first-known-good to be chosen. + _mappings = parse_mappings(mappings, key_rvalue=True) + if not _mappings or list(_mappings.values()) == ['']: + if not mappings: + return {} + + # For backwards-compatibility we need to support port-only provided in + # config. + _mappings = {mappings.split()[0]: default_bridge} + + ports = _mappings.keys() + if len(set(ports)) != len(ports): + raise Exception("It is not allowed to have the same port configured " + "on more than one bridge") + + return _mappings + + +def parse_vlan_range_mappings(mappings): + """Parse vlan range mappings. + + Mappings must be a space-delimited list of provider:start:end mappings. + + The start:end range is optional and may be omitted. + + Returns dict of the form {provider: (start, end)}. + """ + _mappings = parse_mappings(mappings) + if not _mappings: + return {} + + mappings = {} + for p, r in six.iteritems(_mappings): + mappings[p] = tuple(r.split(':')) + + return mappings diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policy_rcd.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policy_rcd.py new file mode 100644 index 0000000..ecffbc6 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policy_rcd.py @@ -0,0 +1,173 @@ +# Copyright 2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for managing policy-rc.d script and associated files. + +This module manages the installation of /usr/sbin/policy-rc.d, the +policy files and the event files. When a package update occurs the +packaging system calls: + +policy-rc.d [options] + +The return code of the script determines if the packaging system +will perform that action on the given service. The policy-rc.d +implementation installed by this module checks if an action is +permitted by checking policy files placed in /etc/policy-rc.d. +If a policy file exists which denies the requested action then +this is recorded in an event file which is placed in +/var/lib/policy-rc.d. +""" + +import os +import shutil +import tempfile +import yaml + +import charmhelpers.contrib.openstack.files as os_files +import charmhelpers.contrib.openstack.alternatives as alternatives +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as host + +POLICY_HEADER = """# Managed by juju\n""" +POLICY_DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d' +POLICY_CONFIG_DIR = '/etc/policy-rc.d' + + +def get_policy_file_name(): + """Get the name of the policy file for this application. + + :returns: Policy file name + :rtype: str + """ + application_name = hookenv.service_name() + return '{}/charm-{}.policy'.format(POLICY_CONFIG_DIR, application_name) + + +def read_default_policy_file(): + """Return the policy file. + + A policy is in the form: + blocked_actions: + neutron-dhcp-agent: [restart, stop, try-restart] + neutron-l3-agent: [restart, stop, try-restart] + neutron-metadata-agent: [restart, stop, try-restart] + neutron-openvswitch-agent: [restart, stop, try-restart] + openvswitch-switch: [restart, stop, try-restart] + ovs-vswitchd: [restart, stop, try-restart] + ovs-vswitchd-dpdk: [restart, stop, try-restart] + ovsdb-server: [restart, stop, try-restart] + policy_requestor_name: neutron-openvswitch + policy_requestor_type: charm + + :returns: Policy + :rtype: Dict[str, Union[str, Dict[str, List[str]]] + """ + policy = {} + policy_file = get_policy_file_name() + if os.path.exists(policy_file): + with open(policy_file, 'r') as f: + policy = yaml.safe_load(f) + return policy + + +def write_policy_file(policy_file, policy): + """Write policy to disk. + + :param policy_file: Name of policy file + :type policy_file: str + :param policy: Policy + :type policy: Dict[str, Union[str, Dict[str, List[str]]]] + """ + with tempfile.NamedTemporaryFile('w', delete=False) as f: + f.write(POLICY_HEADER) + yaml.dump(policy, f) + tmp_file_name = f.name + shutil.move(tmp_file_name, policy_file) + + +def remove_policy_file(): + """Remove policy file.""" + try: + os.remove(get_policy_file_name()) + except FileNotFoundError: + pass + + +def install_policy_rcd(): + """Install policy-rc.d components.""" + source_file_dir = os.path.dirname(os.path.abspath(os_files.__file__)) + policy_rcd_exec = "/var/lib/charm/{}/policy-rc.d".format( + hookenv.service_name()) + host.mkdir(os.path.dirname(policy_rcd_exec)) + shutil.copy2( + '{}/policy_rc_d_script.py'.format(source_file_dir), + policy_rcd_exec) + # policy-rc.d must be installed via the alternatives system: + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + if not os.path.exists('/usr/sbin/policy-rc.d'): + alternatives.install_alternative( + 'policy-rc.d', + '/usr/sbin/policy-rc.d', + policy_rcd_exec) + host.mkdir(POLICY_CONFIG_DIR) + + +def get_default_policy(): + """Return the default policy structure. + + :returns: Policy + :rtype: Dict[str, Union[str, Dict[str, List[str]]] + """ + policy = { + 'policy_requestor_name': hookenv.service_name(), + 'policy_requestor_type': 'charm', + 'blocked_actions': {}} + return policy + + +def add_policy_block(service, blocked_actions): + """Update a policy file with new list of actions. + + :param service: Service name + :type service: str + :param blocked_actions: Action to block + :type blocked_actions: List[str] + """ + policy = read_default_policy_file() or get_default_policy() + policy_file = get_policy_file_name() + if policy['blocked_actions'].get(service): + policy['blocked_actions'][service].extend(blocked_actions) + else: + policy['blocked_actions'][service] = blocked_actions + policy['blocked_actions'][service] = sorted( + list(set(policy['blocked_actions'][service]))) + write_policy_file(policy_file, policy) + + +def remove_policy_block(service, unblocked_actions): + """Remove list of actions from policy file. + + :param service: Service name + :type service: str + :param unblocked_actions: Action to unblock + :type unblocked_actions: List[str] + """ + policy_file = get_policy_file_name() + policy = read_default_policy_file() + for action in unblocked_actions: + try: + policy['blocked_actions'][service].remove(action) + except (KeyError, ValueError): + continue + write_policy_file(policy_file, policy) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policyd.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policyd.py new file mode 100644 index 0000000..6fa06f2 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/policyd.py @@ -0,0 +1,801 @@ +# Copyright 2019-2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import contextlib +import os +import six +import shutil +import yaml +import zipfile + +import charmhelpers +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as ch_host + +# Features provided by this module: + +""" +Policy.d helper functions +========================= + +The functions in this module are designed, as a set, to provide an easy-to-use +set of hooks for classic charms to add in /etc//policy.d/ +directory override YAML files. + +(For charms.openstack charms, a mixin class is provided for this +functionality). + +In order to "hook" this functionality into a (classic) charm, two functions are +provided: + + maybe_do_policyd_overrides(openstack_release, + service, + blacklist_paths=none, + blacklist_keys=none, + template_function=none, + restart_handler=none) + + maybe_do_policyd_overrides_on_config_changed(openstack_release, + service, + blacklist_paths=None, + blacklist_keys=None, + template_function=None, + restart_handler=None + +(See the docstrings for details on the parameters) + +The functions should be called from the install and upgrade hooks in the charm. +The `maybe_do_policyd_overrides_on_config_changed` function is designed to be +called on the config-changed hook, in that it does an additional check to +ensure that an already overridden policy.d in an upgrade or install hooks isn't +repeated. + +In order the *enable* this functionality, the charm's install, config_changed, +and upgrade_charm hooks need to be modified, and a new config option (see +below) needs to be added. The README for the charm should also be updated. + +Examples from the keystone charm are: + +@hooks.hook('install.real') +@harden() +def install(): + ... + # call the policy overrides handler which will install any policy overrides + maybe_do_policyd_overrides(os_release('keystone'), 'keystone') + + +@hooks.hook('config-changed') +@restart_on_change(restart_map(), restart_functions=restart_function_map()) +@harden() +def config_changed(): + ... + # call the policy overrides handler which will install any policy overrides + maybe_do_policyd_overrides_on_config_changed(os_release('keystone'), + 'keystone') + +@hooks.hook('upgrade-charm') +@restart_on_change(restart_map(), stopstart=True) +@harden() +def upgrade_charm(): + ... + # call the policy overrides handler which will install any policy overrides + maybe_do_policyd_overrides(os_release('keystone'), 'keystone') + +Status Line +=========== + +The workload status code in charm-helpers has been modified to detect if +policy.d override code has been incorporated into the charm by checking for the +new config variable (in the config.yaml). If it has been, then the workload +status line will automatically show "PO:" at the beginning of the workload +status for that unit/service if the config option is set. If the policy +override is broken, the "PO (broken):" will be shown. No changes to the charm +(apart from those already mentioned) are needed to enable this functionality. +(charms.openstack charms also get this functionality, but please see that +library for further details). +""" + +# The config.yaml for the charm should contain the following for the config +# option: + +""" + use-policyd-override: + type: boolean + default: False + description: | + If True then use the resource file named 'policyd-override' to install + override YAML files in the service's policy.d directory. The resource + file should be a ZIP file containing at least one yaml file with a .yaml + or .yml extension. If False then remove the overrides. +""" + +# The metadata.yaml for the charm should contain the following: +""" +resources: + policyd-override: + type: file + filename: policyd-override.zip + description: The policy.d overrides file +""" + +# The README for the charm should contain the following: +""" +Policy Overrides +---------------- + +This feature allows for policy overrides using the `policy.d` directory. This +is an **advanced** feature and the policies that the OpenStack service supports +should be clearly and unambiguously understood before trying to override, or +add to, the default policies that the service uses. The charm also has some +policy defaults. They should also be understood before being overridden. + +> **Caution**: It is possible to break the system (for tenants and other + services) if policies are incorrectly applied to the service. + +Policy overrides are YAML files that contain rules that will add to, or +override, existing policy rules in the service. The `policy.d` directory is +a place to put the YAML override files. This charm owns the +`/etc/keystone/policy.d` directory, and as such, any manual changes to it will +be overwritten on charm upgrades. + +Overrides are provided to the charm using a Juju resource called +`policyd-override`. The resource is a ZIP file. This file, say +`overrides.zip`, is attached to the charm by: + + + juju attach-resource policyd-override=overrides.zip + +The policy override is enabled in the charm using: + + juju config use-policyd-override=true + +When `use-policyd-override` is `True` the status line of the charm will be +prefixed with `PO:` indicating that policies have been overridden. If the +installation of the policy override YAML files failed for any reason then the +status line will be prefixed with `PO (broken):`. The log file for the charm +will indicate the reason. No policy override files are installed if the `PO +(broken):` is shown. The status line indicates that the overrides are broken, +not that the policy for the service has failed. The policy will be the defaults +for the charm and service. + +Policy overrides on one service may affect the functionality of another +service. Therefore, it may be necessary to provide policy overrides for +multiple service charms to achieve a consistent set of policies across the +OpenStack system. The charms for the other services that may need overrides +should be checked to ensure that they support overrides before proceeding. +""" + +POLICYD_VALID_EXTS = ['.yaml', '.yml', '.j2', '.tmpl', '.tpl'] +POLICYD_TEMPLATE_EXTS = ['.j2', '.tmpl', '.tpl'] +POLICYD_RESOURCE_NAME = "policyd-override" +POLICYD_CONFIG_NAME = "use-policyd-override" +POLICYD_SUCCESS_FILENAME = "policyd-override-success" +POLICYD_LOG_LEVEL_DEFAULT = hookenv.INFO +POLICYD_ALWAYS_BLACKLISTED_KEYS = ("admin_required", "cloud_admin") + + +class BadPolicyZipFile(Exception): + + def __init__(self, log_message): + self.log_message = log_message + + def __str__(self): + return self.log_message + + +class BadPolicyYamlFile(Exception): + + def __init__(self, log_message): + self.log_message = log_message + + def __str__(self): + return self.log_message + + +if six.PY2: + BadZipFile = zipfile.BadZipfile +else: + BadZipFile = zipfile.BadZipFile + + +def is_policyd_override_valid_on_this_release(openstack_release): + """Check that the charm is running on at least Ubuntu Xenial, and at + least the queens release. + + :param openstack_release: the release codename that is installed. + :type openstack_release: str + :returns: True if okay + :rtype: bool + """ + # NOTE(ajkavanagh) circular import! This is because the status message + # generation code in utils has to call into this module, but this function + # needs the CompareOpenStackReleases() function. The only way to solve + # this is either to put ALL of this module into utils, or refactor one or + # other of the CompareOpenStackReleases or status message generation code + # into a 3rd module. + import charmhelpers.contrib.openstack.utils as ch_utils + return ch_utils.CompareOpenStackReleases(openstack_release) >= 'queens' + + +def maybe_do_policyd_overrides(openstack_release, + service, + blacklist_paths=None, + blacklist_keys=None, + template_function=None, + restart_handler=None, + user=None, + group=None, + config_changed=False): + """If the config option is set, get the resource file and process it to + enable the policy.d overrides for the service passed. + + The param `openstack_release` is required as the policyd overrides feature + is only supported on openstack_release "queens" or later, and on ubuntu + "xenial" or later. Prior to these versions, this feature is a NOP. + + The optional template_function is a function that accepts a string and has + an opportunity to modify the loaded file prior to it being read by + yaml.safe_load(). This allows the charm to perform "templating" using + charm derived data. + + The param blacklist_paths are paths (that are in the service's policy.d + directory that should not be touched). + + The param blacklist_keys are keys that must not appear in the yaml file. + If they do, then the whole policy.d file fails. + + The yaml file extracted from the resource_file (which is a zipped file) has + its file path reconstructed. This, also, must not match any path in the + black list. + + The param restart_handler is an optional Callable that is called to perform + the service restart if the policy.d file is changed. This should normally + be None as oslo.policy automatically picks up changes in the policy.d + directory. However, for any services where this is buggy then a + restart_handler can be used to force the policy.d files to be read. + + If the config_changed param is True, then the handling is slightly + different: It will only perform the policyd overrides if the config is True + and the success file doesn't exist. Otherwise, it does nothing as the + resource file has already been processed. + + :param openstack_release: The openstack release that is installed. + :type openstack_release: str + :param service: the service name to construct the policy.d directory for. + :type service: str + :param blacklist_paths: optional list of paths to leave alone + :type blacklist_paths: Union[None, List[str]] + :param blacklist_keys: optional list of keys that mustn't appear in the + yaml file's + :type blacklist_keys: Union[None, List[str]] + :param template_function: Optional function that can modify the string + prior to being processed as a Yaml document. + :type template_function: Union[None, Callable[[str], str]] + :param restart_handler: The function to call if the service should be + restarted. + :type restart_handler: Union[None, Callable[]] + :param user: The user to create/write files/directories as + :type user: Union[None, str] + :param group: the group to create/write files/directories as + :type group: Union[None, str] + :param config_changed: Set to True for config_changed hook. + :type config_changed: bool + """ + _user = service if user is None else user + _group = service if group is None else group + if not is_policyd_override_valid_on_this_release(openstack_release): + return + hookenv.log("Running maybe_do_policyd_overrides", + level=POLICYD_LOG_LEVEL_DEFAULT) + config = hookenv.config() + try: + if not config.get(POLICYD_CONFIG_NAME, False): + clean_policyd_dir_for(service, + blacklist_paths, + user=_user, + group=_group) + if (os.path.isfile(_policy_success_file()) and + restart_handler is not None and + callable(restart_handler)): + restart_handler() + remove_policy_success_file() + return + except Exception as e: + hookenv.log("... ERROR: Exception is: {}".format(str(e)), + level=POLICYD_CONFIG_NAME) + import traceback + hookenv.log(traceback.format_exc(), level=POLICYD_LOG_LEVEL_DEFAULT) + return + # if the policyd overrides have been performed when doing config_changed + # just return + if config_changed and is_policy_success_file_set(): + hookenv.log("... already setup, so skipping.", + level=POLICYD_LOG_LEVEL_DEFAULT) + return + # from now on it should succeed; if it doesn't then status line will show + # broken. + resource_filename = get_policy_resource_filename() + restart = process_policy_resource_file( + resource_filename, service, blacklist_paths, blacklist_keys, + template_function) + if restart and restart_handler is not None and callable(restart_handler): + restart_handler() + + +@charmhelpers.deprecate("Use maybe_do_policyd_overrides instead") +def maybe_do_policyd_overrides_on_config_changed(*args, **kwargs): + """This function is designed to be called from the config changed hook. + + DEPRECATED: please use maybe_do_policyd_overrides() with the param + `config_changed` as `True`. + + See maybe_do_policyd_overrides() for more details on the params. + """ + if 'config_changed' not in kwargs.keys(): + kwargs['config_changed'] = True + return maybe_do_policyd_overrides(*args, **kwargs) + + +def get_policy_resource_filename(): + """Function to extract the policy resource filename + + :returns: The filename of the resource, if set, otherwise, if an error + occurs, then None is returned. + :rtype: Union[str, None] + """ + try: + return hookenv.resource_get(POLICYD_RESOURCE_NAME) + except Exception: + return None + + +@contextlib.contextmanager +def open_and_filter_yaml_files(filepath, has_subdirs=False): + """Validate that the filepath provided is a zip file and contains at least + one (.yaml|.yml) file, and that the files are not duplicated when the zip + file is flattened. Note that the yaml files are not checked. This is the + first stage in validating the policy zipfile; individual yaml files are not + checked for validity or black listed keys. + + If the has_subdirs param is True, then the files are flattened to the first + directory, and the files in the root are ignored. + + An example of use is: + + with open_and_filter_yaml_files(some_path) as zfp, g: + for zipinfo in g: + # do something with zipinfo ... + + :param filepath: a filepath object that can be opened by zipfile + :type filepath: Union[AnyStr, os.PathLike[AntStr]] + :param has_subdirs: Keep first level of subdirectories in yaml file. + :type has_subdirs: bool + :returns: (zfp handle, + a generator of the (name, filename, ZipInfo object) tuples) as a + tuple. + :rtype: ContextManager[(zipfile.ZipFile, + Generator[(name, str, str, zipfile.ZipInfo)])] + :raises: zipfile.BadZipFile + :raises: BadPolicyZipFile if duplicated yaml or missing + :raises: IOError if the filepath is not found + """ + with zipfile.ZipFile(filepath, 'r') as zfp: + # first pass through; check for duplicates and at least one yaml file. + names = collections.defaultdict(int) + yamlfiles = _yamlfiles(zfp, has_subdirs) + for name, _, _, _ in yamlfiles: + names[name] += 1 + # There must be at least 1 yaml file. + if len(names.keys()) == 0: + raise BadPolicyZipFile("contains no yaml files with {} extensions." + .format(", ".join(POLICYD_VALID_EXTS))) + # There must be no duplicates + duplicates = [n for n, c in names.items() if c > 1] + if duplicates: + raise BadPolicyZipFile("{} have duplicates in the zip file." + .format(", ".join(duplicates))) + # Finally, let's yield the generator + yield (zfp, yamlfiles) + + +def _yamlfiles(zipfile, has_subdirs=False): + """Helper to get a yaml file (according to POLICYD_VALID_EXTS extensions) + and the infolist item from a zipfile. + + If the `has_subdirs` param is True, the the only yaml files that have a + directory component are read, and then first part of the directory + component is kept, along with the filename in the name. e.g. an entry with + a filename of: + + compute/someotherdir/override.yaml + + is returned as: + + compute/override, yaml, override.yaml, + + This is to help with the special, additional, processing that the dashboard + charm requires. + + :param zipfile: the zipfile to read zipinfo items from + :type zipfile: zipfile.ZipFile + :param has_subdirs: Keep first level of subdirectories in yaml file. + :type has_subdirs: bool + :returns: generator of (name, ext, filename, info item) for each + self-identified yaml file. + :rtype: List[(str, str, str, zipfile.ZipInfo)] + """ + files = [] + for infolist_item in zipfile.infolist(): + try: + if infolist_item.is_dir(): + continue + except AttributeError: + # fallback to "old" way to determine dir entry for pre-py36 + if infolist_item.filename.endswith('/'): + continue + _dir, name_ext = os.path.split(infolist_item.filename) + name, ext = os.path.splitext(name_ext) + if has_subdirs and _dir != "": + name = os.path.join(_dir.split(os.path.sep)[0], name) + ext = ext.lower() + if ext and ext in POLICYD_VALID_EXTS: + files.append((name, ext, name_ext, infolist_item)) + return files + + +def read_and_validate_yaml(stream_or_doc, blacklist_keys=None): + """Read, validate and return the (first) yaml document from the stream. + + The doc is read, and checked for a yaml file. The the top-level keys are + checked against the blacklist_keys provided. If there are problems then an + Exception is raised. Otherwise the yaml document is returned as a Python + object that can be dumped back as a yaml file on the system. + + The yaml file must only consist of a str:str mapping, and if not then the + yaml file is rejected. + + :param stream_or_doc: the file object to read the yaml from + :type stream_or_doc: Union[AnyStr, IO[AnyStr]] + :param blacklist_keys: Any keys, which if in the yaml file, should cause + and error. + :type blacklisted_keys: Union[None, List[str]] + :returns: the yaml file as a python document + :rtype: Dict[str, str] + :raises: yaml.YAMLError if there is a problem with the document + :raises: BadPolicyYamlFile if file doesn't look right or there are + blacklisted keys in the file. + """ + blacklist_keys = blacklist_keys or [] + blacklist_keys.append(POLICYD_ALWAYS_BLACKLISTED_KEYS) + doc = yaml.safe_load(stream_or_doc) + if not isinstance(doc, dict): + raise BadPolicyYamlFile("doesn't look like a policy file?") + keys = set(doc.keys()) + blacklisted_keys_present = keys.intersection(blacklist_keys) + if blacklisted_keys_present: + raise BadPolicyYamlFile("blacklisted keys {} present." + .format(", ".join(blacklisted_keys_present))) + if not all(isinstance(k, six.string_types) for k in keys): + raise BadPolicyYamlFile("keys in yaml aren't all strings?") + # check that the dictionary looks like a mapping of str to str + if not all(isinstance(v, six.string_types) for v in doc.values()): + raise BadPolicyYamlFile("values in yaml aren't all strings?") + return doc + + +def policyd_dir_for(service): + """Return the policy directory for the named service. + + :param service: str + :returns: the policy.d override directory. + :rtype: os.PathLike[str] + """ + return os.path.join("/", "etc", service, "policy.d") + + +def clean_policyd_dir_for(service, keep_paths=None, user=None, group=None): + """Clean out the policyd directory except for items that should be kept. + + The keep_paths, if used, should be set to the full path of the files that + should be kept in the policyd directory for the service. Note that the + service name is passed in, and then the policyd_dir_for() function is used. + This is so that a coding error doesn't result in a sudden deletion of the + charm (say). + + :param service: the service name to use to construct the policy.d dir. + :type service: str + :param keep_paths: optional list of paths to not delete. + :type keep_paths: Union[None, List[str]] + :param user: The user to create/write files/directories as + :type user: Union[None, str] + :param group: the group to create/write files/directories as + :type group: Union[None, str] + """ + _user = service if user is None else user + _group = service if group is None else group + keep_paths = keep_paths or [] + path = policyd_dir_for(service) + hookenv.log("Cleaning path: {}".format(path), level=hookenv.DEBUG) + if not os.path.exists(path): + ch_host.mkdir(path, owner=_user, group=_group, perms=0o775) + _scanner = os.scandir if hasattr(os, 'scandir') else _fallback_scandir + for direntry in _scanner(path): + # see if the path should be kept. + if direntry.path in keep_paths: + continue + # we remove any directories; it's ours and there shouldn't be any + if direntry.is_dir(): + shutil.rmtree(direntry.path) + else: + os.remove(direntry.path) + + +def maybe_create_directory_for(path, user, group): + """For the filename 'path', ensure that the directory for that path exists. + + Note that if the directory already exists then the permissions are NOT + changed. + + :param path: the filename including the path to it. + :type path: str + :param user: the user to create the directory as + :param group: the group to create the directory as + """ + _dir, _ = os.path.split(path) + if not os.path.exists(_dir): + ch_host.mkdir(_dir, owner=user, group=group, perms=0o775) + + +@contextlib.contextmanager +def _fallback_scandir(path): + """Fallback os.scandir implementation. + + provide a fallback implementation of os.scandir if this module ever gets + used in a py2 or py34 charm. Uses os.listdir() to get the names in the path, + and then mocks the is_dir() function using os.path.isdir() to check for + directory. + + :param path: the path to list the directories for + :type path: str + :returns: Generator that provides _FBDirectory objects + :rtype: ContextManager[_FBDirectory] + """ + for f in os.listdir(path): + yield _FBDirectory(f) + + +class _FBDirectory(object): + """Mock a scandir Directory object with enough to use in + clean_policyd_dir_for + """ + + def __init__(self, path): + self.path = path + + def is_dir(self): + return os.path.isdir(self.path) + + +def path_for_policy_file(service, name): + """Return the full path for a policy.d file that will be written to the + service's policy.d directory. + + It is constructed using policyd_dir_for(), the name and the ".yaml" + extension. + + For horizon, for example, it's a bit more complicated. The name param is + actually "override_service_dir/a_name", where target_service needs to be + one the allowed horizon override services. This translation and check is + done in the _yamlfiles() function. + + :param service: the service name + :type service: str + :param name: the name for the policy override + :type name: str + :returns: the full path name for the file + :rtype: os.PathLike[str] + """ + return os.path.join(policyd_dir_for(service), name + ".yaml") + + +def _policy_success_file(): + """Return the file name for a successful drop of policy.d overrides + + :returns: the path name for the file. + :rtype: str + """ + return os.path.join(hookenv.charm_dir(), POLICYD_SUCCESS_FILENAME) + + +def remove_policy_success_file(): + """Remove the file that indicates successful policyd override.""" + try: + os.remove(_policy_success_file()) + except Exception: + pass + + +def set_policy_success_file(): + """Set the file that indicates successful policyd override.""" + open(_policy_success_file(), "w").close() + + +def is_policy_success_file_set(): + """Returns True if the policy success file has been set. + + This indicates that policies are overridden and working properly. + + :returns: True if the policy file is set + :rtype: bool + """ + return os.path.isfile(_policy_success_file()) + + +def policyd_status_message_prefix(): + """Return the prefix str for the status line. + + "PO:" indicating that the policy overrides are in place, or "PO (broken):" + if the policy is supposed to be working but there is no success file. + + :returns: the prefix + :rtype: str + """ + if is_policy_success_file_set(): + return "PO:" + return "PO (broken):" + + +def process_policy_resource_file(resource_file, + service, + blacklist_paths=None, + blacklist_keys=None, + template_function=None, + preserve_topdir=False, + preprocess_filename=None, + user=None, + group=None): + """Process the resource file (which should contain at least one yaml file) + and write those files to the service's policy.d directory. + + The optional template_function is a function that accepts a python + string and has an opportunity to modify the document + prior to it being read by the yaml.safe_load() function and written to + disk. Note that this function does *not* say how the templating is done - + this is up to the charm to implement its chosen method. + + The param blacklist_paths are paths (that are in the service's policy.d + directory that should not be touched). + + The param blacklist_keys are keys that must not appear in the yaml file. + If they do, then the whole policy.d file fails. + + The yaml file extracted from the resource_file (which is a zipped file) has + its file path reconstructed. This, also, must not match any path in the + black list. + + The yaml filename can be modified in two ways. If the `preserve_topdir` + param is True, then files will be flattened to the top dir. This allows + for creating sets of files that can be grouped into a single level tree + structure. + + Secondly, if the `preprocess_filename` param is not None and callable() + then the name is passed to that function for preprocessing before being + converted to the end location. This is to allow munging of the filename + prior to being tested for a blacklist path. + + If any error occurs, then the policy.d directory is cleared, the error is + written to the log, and the status line will eventually show as failed. + + :param resource_file: The zipped file to open and extract yaml files form. + :type resource_file: Union[AnyStr, os.PathLike[AnyStr]] + :param service: the service name to construct the policy.d directory for. + :type service: str + :param blacklist_paths: optional list of paths to leave alone + :type blacklist_paths: Union[None, List[str]] + :param blacklist_keys: optional list of keys that mustn't appear in the + yaml file's + :type blacklist_keys: Union[None, List[str]] + :param template_function: Optional function that can modify the yaml + document. + :type template_function: Union[None, Callable[[AnyStr], AnyStr]] + :param preserve_topdir: Keep the toplevel subdir + :type preserve_topdir: bool + :param preprocess_filename: Optional function to use to process filenames + extracted from the resource file. + :type preprocess_filename: Union[None, Callable[[AnyStr]. AnyStr]] + :param user: The user to create/write files/directories as + :type user: Union[None, str] + :param group: the group to create/write files/directories as + :type group: Union[None, str] + :returns: True if the processing was successful, False if not. + :rtype: boolean + """ + hookenv.log("Running process_policy_resource_file", level=hookenv.DEBUG) + blacklist_paths = blacklist_paths or [] + completed = False + _preprocess = None + if preprocess_filename is not None and callable(preprocess_filename): + _preprocess = preprocess_filename + _user = service if user is None else user + _group = service if group is None else group + try: + with open_and_filter_yaml_files( + resource_file, preserve_topdir) as (zfp, gen): + # first clear out the policy.d directory and clear success + remove_policy_success_file() + clean_policyd_dir_for(service, + blacklist_paths, + user=_user, + group=_group) + for name, ext, filename, zipinfo in gen: + # See if the name should be preprocessed. + if _preprocess is not None: + name = _preprocess(name) + # construct a name for the output file. + yaml_filename = path_for_policy_file(service, name) + if yaml_filename in blacklist_paths: + raise BadPolicyZipFile("policy.d name {} is blacklisted" + .format(yaml_filename)) + with zfp.open(zipinfo) as fp: + doc = fp.read() + # if template_function is not None, then offer the document + # to the template function + if ext in POLICYD_TEMPLATE_EXTS: + if (template_function is None or not + callable(template_function)): + raise BadPolicyZipFile( + "Template {} but no template_function is " + "available".format(filename)) + doc = template_function(doc) + yaml_doc = read_and_validate_yaml(doc, blacklist_keys) + # we may have to create the directory + maybe_create_directory_for(yaml_filename, _user, _group) + ch_host.write_file(yaml_filename, + yaml.dump(yaml_doc).encode('utf-8'), + _user, + _group) + # Every thing worked, so we mark up a success. + completed = True + except (BadZipFile, BadPolicyZipFile, BadPolicyYamlFile) as e: + hookenv.log("Processing {} failed: {}".format(resource_file, str(e)), + level=POLICYD_LOG_LEVEL_DEFAULT) + except IOError as e: + # technically this shouldn't happen; it would be a programming error as + # the filename comes from Juju and thus, should exist. + hookenv.log( + "File {} failed with IOError. This really shouldn't happen" + " -- error: {}".format(resource_file, str(e)), + level=POLICYD_LOG_LEVEL_DEFAULT) + except Exception as e: + import traceback + hookenv.log("General Exception({}) during policyd processing" + .format(str(e)), + level=POLICYD_LOG_LEVEL_DEFAULT) + hookenv.log(traceback.format_exc()) + finally: + if not completed: + hookenv.log("Processing {} failed: cleaning policy.d directory" + .format(resource_file), + level=POLICYD_LOG_LEVEL_DEFAULT) + clean_policyd_dir_for(service, + blacklist_paths, + user=_user, + group=_group) + else: + # touch the success filename + hookenv.log("policy.d overrides installed.", + level=POLICYD_LOG_LEVEL_DEFAULT) + set_policy_success_file() + return completed diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ssh_migrations.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ssh_migrations.py new file mode 100644 index 0000000..96b9f71 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/ssh_migrations.py @@ -0,0 +1,412 @@ +# Copyright 2018 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess + +from charmhelpers.core.hookenv import ( + ERROR, + log, + relation_get, +) +from charmhelpers.contrib.network.ip import ( + is_ipv6, + ns_query, +) +from charmhelpers.contrib.openstack.utils import ( + get_hostname, + get_host_ip, + is_ip, +) + +NOVA_SSH_DIR = '/etc/nova/compute_ssh/' + + +def ssh_directory_for_unit(application_name, user=None): + """Return the directory used to store ssh assets for the application. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Fully qualified directory path. + :rtype: str + """ + if user: + application_name = "{}_{}".format(application_name, user) + _dir = os.path.join(NOVA_SSH_DIR, application_name) + for d in [NOVA_SSH_DIR, _dir]: + if not os.path.isdir(d): + os.mkdir(d) + for f in ['authorized_keys', 'known_hosts']: + f = os.path.join(_dir, f) + if not os.path.isfile(f): + open(f, 'w').close() + return _dir + + +def known_hosts(application_name, user=None): + """Return the known hosts file for the application. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Fully qualified path to file. + :rtype: str + """ + return os.path.join( + ssh_directory_for_unit(application_name, user), + 'known_hosts') + + +def authorized_keys(application_name, user=None): + """Return the authorized keys file for the application. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Fully qualified path to file. + :rtype: str + """ + return os.path.join( + ssh_directory_for_unit(application_name, user), + 'authorized_keys') + + +def ssh_known_host_key(host, application_name, user=None): + """Return the first entry in known_hosts for host. + + :param host: hostname to lookup in file. + :type host: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Host key + :rtype: str or None + """ + cmd = [ + 'ssh-keygen', + '-f', known_hosts(application_name, user), + '-H', + '-F', + host] + try: + # The first line of output is like '# Host xx found: line 1 type RSA', + # which should be excluded. + output = subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + # RC of 1 seems to be legitimate for most ssh-keygen -F calls. + if e.returncode == 1: + output = e.output + else: + raise + output = output.strip() + + if output: + # Bug #1500589 cmd has 0 rc on precise if entry not present + lines = output.split('\n') + if len(lines) >= 1: + return lines[0] + + return None + + +def remove_known_host(host, application_name, user=None): + """Remove the entry in known_hosts for host. + + :param host: hostname to lookup in file. + :type host: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + log('Removing SSH known host entry for compute host at %s' % host) + cmd = ['ssh-keygen', '-f', known_hosts(application_name, user), '-R', host] + subprocess.check_call(cmd) + + +def is_same_key(key_1, key_2): + """Extract the key from two host entries and compare them. + + :param key_1: Host key + :type key_1: str + :param key_2: Host key + :type key_2: str + """ + # The key format get will be like '|1|2rUumCavEXWVaVyB5uMl6m85pZo=|Cp' + # 'EL6l7VTY37T/fg/ihhNb/GPgs= ssh-rsa AAAAB', we only need to compare + # the part start with 'ssh-rsa' followed with '= ', because the hash + # value in the beginning will change each time. + k_1 = key_1.split('= ')[1] + k_2 = key_2.split('= ')[1] + return k_1 == k_2 + + +def add_known_host(host, application_name, user=None): + """Add the given host key to the known hosts file. + + :param host: host name + :type host: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host] + try: + remote_key = subprocess.check_output(cmd).strip() + except Exception as e: + log('Could not obtain SSH host key from %s' % host, level=ERROR) + raise e + + current_key = ssh_known_host_key(host, application_name, user) + if current_key and remote_key: + if is_same_key(remote_key, current_key): + log('Known host key for compute host %s up to date.' % host) + return + else: + remove_known_host(host, application_name, user) + + log('Adding SSH host key to known hosts for compute node at %s.' % host) + with open(known_hosts(application_name, user), 'a') as out: + out.write("{}\n".format(remote_key)) + + +def ssh_authorized_key_exists(public_key, application_name, user=None): + """Check if given key is in the authorized_key file. + + :param public_key: Public key. + :type public_key: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Whether given key is in the authorized_key file. + :rtype: boolean + """ + with open(authorized_keys(application_name, user)) as keys: + return ('%s' % public_key) in keys.read() + + +def add_authorized_key(public_key, application_name, user=None): + """Add given key to the authorized_key file. + + :param public_key: Public key. + :type public_key: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + with open(authorized_keys(application_name, user), 'a') as keys: + keys.write("{}\n".format(public_key)) + + +def ssh_compute_add_host_and_key(public_key, hostname, private_address, + application_name, user=None): + """Add a compute nodes ssh details to local cache. + + Collect various hostname variations and add the corresponding host keys to + the local known hosts file. Finally, add the supplied public key to the + authorized_key file. + + :param public_key: Public key. + :type public_key: str + :param hostname: Hostname to collect host keys from. + :type hostname: str + :param private_address:aCorresponding private address for hostname + :type private_address: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + # If remote compute node hands us a hostname, ensure we have a + # known hosts entry for its IP, hostname and FQDN. + hosts = [private_address] + + if not is_ipv6(private_address): + if hostname: + hosts.append(hostname) + + if is_ip(private_address): + hn = get_hostname(private_address) + if hn: + hosts.append(hn) + short = hn.split('.')[0] + if ns_query(short): + hosts.append(short) + else: + hosts.append(get_host_ip(private_address)) + short = private_address.split('.')[0] + if ns_query(short): + hosts.append(short) + + for host in list(set(hosts)): + add_known_host(host, application_name, user) + + if not ssh_authorized_key_exists(public_key, application_name, user): + log('Saving SSH authorized key for compute host at %s.' % + private_address) + add_authorized_key(public_key, application_name, user) + + +def ssh_compute_add(public_key, application_name, rid=None, unit=None, + user=None): + """Add a compute nodes ssh details to local cache. + + Collect various hostname variations and add the corresponding host keys to + the local known hosts file. Finally, add the supplied public key to the + authorized_key file. + + :param public_key: Public key. + :type public_key: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param rid: Relation id of the relation between this charm and the app. If + none is supplied it is assumed its the relation relating to + the current hook context. + :type rid: str + :param unit: Unit to add ssh asserts for if none is supplied it is assumed + its the unit relating to the current hook context. + :type unit: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + relation_data = relation_get(rid=rid, unit=unit) + ssh_compute_add_host_and_key( + public_key, + relation_data.get('hostname'), + relation_data.get('private-address'), + application_name, + user=user) + + +def ssh_known_hosts_lines(application_name, user=None): + """Return contents of known_hosts file for given application. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + known_hosts_list = [] + with open(known_hosts(application_name, user)) as hosts: + for hosts_line in hosts: + if hosts_line.rstrip(): + known_hosts_list.append(hosts_line.rstrip()) + return(known_hosts_list) + + +def ssh_authorized_keys_lines(application_name, user=None): + """Return contents of authorized_keys file for given application. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + authorized_keys_list = [] + + with open(authorized_keys(application_name, user)) as keys: + for authkey_line in keys: + if authkey_line.rstrip(): + authorized_keys_list.append(authkey_line.rstrip()) + return(authorized_keys_list) + + +def ssh_compute_remove(public_key, application_name, user=None): + """Remove given public key from authorized_keys file. + + :param public_key: Public key. + :type public_key: str + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + """ + if not (os.path.isfile(authorized_keys(application_name, user)) or + os.path.isfile(known_hosts(application_name, user))): + return + + keys = ssh_authorized_keys_lines(application_name, user=None) + keys = [k.strip() for k in keys] + + if public_key not in keys: + return + + [keys.remove(key) for key in keys if key == public_key] + + with open(authorized_keys(application_name, user), 'w') as _keys: + keys = '\n'.join(keys) + if not keys.endswith('\n'): + keys += '\n' + _keys.write(keys) + + +def get_ssh_settings(application_name, user=None): + """Retrieve the known host entries and public keys for application + + Retrieve the known host entries and public keys for application for all + units of the given application related to this application for the + app + user combination. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :param user: The user that the ssh asserts are for. + :type user: str + :returns: Public keys + host keys for all units for app + user combination. + :rtype: dict + """ + settings = {} + keys = {} + prefix = '' + if user: + prefix = '{}_'.format(user) + + for i, line in enumerate(ssh_known_hosts_lines( + application_name=application_name, user=user)): + settings['{}known_hosts_{}'.format(prefix, i)] = line + if settings: + settings['{}known_hosts_max_index'.format(prefix)] = len( + settings.keys()) + + for i, line in enumerate(ssh_authorized_keys_lines( + application_name=application_name, user=user)): + keys['{}authorized_keys_{}'.format(prefix, i)] = line + if keys: + keys['{}authorized_keys_max_index'.format(prefix)] = len(keys.keys()) + settings.update(keys) + return settings + + +def get_all_user_ssh_settings(application_name): + """Retrieve the known host entries and public keys for application + + Retrieve the known host entries and public keys for application for all + units of the given application related to this application for root user + and nova user. + + :param application_name: Name of application eg nova-compute-something + :type application_name: str + :returns: Public keys + host keys for all units for app + user combination. + :rtype: dict + """ + settings = get_ssh_settings(application_name) + settings.update(get_ssh_settings(application_name, user='nova')) + return settings diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/__init__.py new file mode 100644 index 0000000..9df5f74 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# dummy __init__.py to fool syncer into thinking this is a syncable python +# module diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/ceph.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/ceph.conf new file mode 100644 index 0000000..c0f2236 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/ceph.conf @@ -0,0 +1,28 @@ +############################################################################### +# [ WARNING ] +# ceph configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### +[global] +{% if auth -%} +auth_supported = {{ auth }} +keyring = /etc/ceph/$cluster.$name.keyring +mon host = {{ mon_hosts }} +{% endif -%} +log to syslog = {{ use_syslog }} +err to syslog = {{ use_syslog }} +clog to syslog = {{ use_syslog }} +{% if rbd_features %} +rbd default features = {{ rbd_features }} +{% endif %} + +[client] +{% if rbd_client_cache_settings -%} +{% for key, value in rbd_client_cache_settings.items() -%} +{{ key }} = {{ value }} +{% endfor -%} +{%- endif %} + +{% if rbd_default_data_pool -%} +rbd default data pool = {{ rbd_default_data_pool }} +{% endif %} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/git.upstart b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/git.upstart new file mode 100644 index 0000000..4bed404 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/git.upstart @@ -0,0 +1,17 @@ +description "{{ service_description }}" +author "Juju {{ service_name }} Charm " + +start on runlevel [2345] +stop on runlevel [!2345] + +respawn + +exec start-stop-daemon --start --chuid {{ user_name }} \ + --chdir {{ start_dir }} --name {{ process_name }} \ + --exec {{ executable_name }} -- \ + {% for config_file in config_files -%} + --config-file={{ config_file }} \ + {% endfor -%} + {% if log_file -%} + --log-file={{ log_file }} + {% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/haproxy.cfg b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/haproxy.cfg new file mode 100644 index 0000000..875e139 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -0,0 +1,89 @@ +global + # NOTE: on startup haproxy chroot's to /var/lib/haproxy. + # + # Unfortunately the program will open some files prior to the call to + # chroot never to reopen them, and some after. So looking at the on-disk + # layout of haproxy resources you will find some resources relative to / + # such as the admin socket, and some relative to /var/lib/haproxy such as + # the log socket. + # + # The logging socket is (re-)opened after the chroot and must be relative + # to /var/lib/haproxy. + log /dev/log local0 + log /dev/log local1 notice + maxconn 20000 + user haproxy + group haproxy + spread-checks 0 + # The admin socket is opened prior to the chroot never to be reopened, so + # it lives outside the chroot directory in the filesystem. + stats socket /var/run/haproxy/admin.sock mode 600 level admin + stats timeout 2m + +defaults + log global + mode tcp + option tcplog + option dontlognull + retries 3 +{%- if haproxy_queue_timeout %} + timeout queue {{ haproxy_queue_timeout }} +{%- else %} + timeout queue 9000 +{%- endif %} +{%- if haproxy_connect_timeout %} + timeout connect {{ haproxy_connect_timeout }} +{%- else %} + timeout connect 9000 +{%- endif %} +{%- if haproxy_client_timeout %} + timeout client {{ haproxy_client_timeout }} +{%- else %} + timeout client 90000 +{%- endif %} +{%- if haproxy_server_timeout %} + timeout server {{ haproxy_server_timeout }} +{%- else %} + timeout server 90000 +{%- endif %} + +listen stats + bind {{ local_host }}:{{ stat_port }} + mode http + stats enable + stats hide-version + stats realm Haproxy\ Statistics + stats uri / + stats auth admin:{{ stat_password }} + +{% if frontends -%} +{% for service, ports in service_ports.items() -%} +frontend tcp-in_{{ service }} + bind *:{{ ports[0] }} + {% if ipv6_enabled -%} + bind :::{{ ports[0] }} + {% endif -%} + {% for frontend in frontends -%} + acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} + use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} + {% endfor -%} + default_backend {{ service }}_{{ default_backend }} + +{% for frontend in frontends -%} +backend {{ service }}_{{ frontend }} + balance leastconn + {% if backend_options -%} + {% if backend_options[service] -%} + {% for option in backend_options[service] -%} + {% for key, value in option.items() -%} + {{ key }} {{ value }} + {% endfor -%} + {% endfor -%} + {% endif -%} + {% endif -%} + {% for unit, address in frontends[frontend]['backends'].items() -%} + server {{ unit }} {{ address }}:{{ ports[1] }} check + {% endfor %} +{% endfor -%} +{% endfor -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/logrotate b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/logrotate new file mode 100644 index 0000000..b2900d0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/logrotate @@ -0,0 +1,9 @@ +/var/log/{{ logrotate_logs_location }}/*.log { + {{ logrotate_interval }} + {{ logrotate_count }} + compress + delaycompress + missingok + notifempty + copytruncate +} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/memcached.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/memcached.conf new file mode 100644 index 0000000..26cb037 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/memcached.conf @@ -0,0 +1,53 @@ +############################################################################### +# [ WARNING ] +# memcached configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### + +# memcached default config file +# 2003 - Jay Bonci +# This configuration file is read by the start-memcached script provided as +# part of the Debian GNU/Linux distribution. + +# Run memcached as a daemon. This command is implied, and is not needed for the +# daemon to run. See the README.Debian that comes with this package for more +# information. +-d + +# Log memcached's output to /var/log/memcached +logfile /var/log/memcached.log + +# Be verbose +# -v + +# Be even more verbose (print client commands as well) +# -vv + +# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default +# Note that the daemon will grow to this size, but does not start out holding this much +# memory +-m 64 + +# Default connection port is 11211 +-p {{ memcache_port }} + +# Run the daemon as root. The start-memcached will default to running as root if no +# -u command is present in this config file +-u memcache + +# Specify which IP address to listen on. The default is to listen on all IP addresses +# This parameter is one of the only security measures that memcached has, so make sure +# it's listening on a firewalled interface. +-l {{ memcache_server }} + +# Limit the number of simultaneous incoming connections. The daemon default is 1024 +# -c 1024 + +# Lock down all paged memory. Consult with the README and homepage before you do this +# -k + +# Return error when memory is exhausted (rather than removing items) +# -M + +# Maximize core file limit +# -r diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend new file mode 100644 index 0000000..530719e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend @@ -0,0 +1,35 @@ +{% if endpoints -%} +{% for ext_port in ext_ports -%} +Listen {{ ext_port }} +{% endfor -%} +{% for address, endpoint, ext, int in endpoints -%} + + ServerName {{ endpoint }} + SSLEngine on + + # This section is based on Mozilla's recommendation + # as the "intermediate" profile as of July 7th, 2020. + # https://wiki.mozilla.org/Security/Server_Side_TLS + SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 + SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + SSLHonorCipherOrder off + + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} + # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8 + SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} + ProxyPass / http://localhost:{{ int }}/ + ProxyPassReverse / http://localhost:{{ int }}/ + ProxyPreserveHost on + RequestHeader set X-Forwarded-Proto "https" + +{% endfor -%} + + Order deny,allow + Allow from all + + + Order allow,deny + Allow from all + +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf new file mode 120000 index 0000000..9a2f6f2 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf @@ -0,0 +1 @@ +openstack_https_frontend \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression new file mode 100644 index 0000000..a643010 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression @@ -0,0 +1,28 @@ +{# section header omitted as options can belong to multiple sections #} +{% if bluestore_compression_algorithm -%} +bluestore compression algorithm = {{ bluestore_compression_algorithm }} +{% endif -%} +{% if bluestore_compression_mode -%} +bluestore compression mode = {{ bluestore_compression_mode }} +{% endif -%} +{% if bluestore_compression_required_ratio -%} +bluestore compression required ratio = {{ bluestore_compression_required_ratio }} +{% endif -%} +{% if bluestore_compression_min_blob_size -%} +bluestore compression min blob size = {{ bluestore_compression_min_blob_size }} +{% endif -%} +{% if bluestore_compression_min_blob_size_hdd -%} +bluestore compression min blob size hdd = {{ bluestore_compression_min_blob_size_hdd }} +{% endif -%} +{% if bluestore_compression_min_blob_size_ssd -%} +bluestore compression min blob size ssd = {{ bluestore_compression_min_blob_size_ssd }} +{% endif -%} +{% if bluestore_compression_max_blob_size -%} +bluestore compression max blob size = {{ bluestore_compression_max_blob_size }} +{% endif -%} +{% if bluestore_compression_max_blob_size_hdd -%} +bluestore compression max blob size hdd = {{ bluestore_compression_max_blob_size_hdd }} +{% endif -%} +{% if bluestore_compression_max_blob_size_ssd -%} +bluestore compression max blob size ssd = {{ bluestore_compression_max_blob_size_ssd }} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken new file mode 100644 index 0000000..5dcebe7 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken @@ -0,0 +1,12 @@ +{% if auth_host -%} +[keystone_authtoken] +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +auth_plugin = password +project_domain_id = default +user_domain_id = default +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy new file mode 100644 index 0000000..9356b2b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy @@ -0,0 +1,10 @@ +{% if auth_host -%} +[keystone_authtoken] +# Juno specific config (Bug #1557223) +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} +identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +admin_tenant_name = {{ admin_tenant_name }} +admin_user = {{ admin_user }} +admin_password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka new file mode 100644 index 0000000..c281868 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka @@ -0,0 +1,22 @@ +{% if auth_host -%} +[keystone_authtoken] +auth_type = password +{% if api_version == "3" -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3 +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3 +project_domain_name = {{ admin_domain_name }} +user_domain_name = {{ admin_domain_name }} +{% else -%} +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +project_domain_name = default +user_domain_name = default +{% endif -%} +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +signing_dir = {{ signing_dir }} +{% if use_memcache == true %} +memcached_servers = {{ memcache_url }} +{% endif -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-v3only b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-v3only new file mode 100644 index 0000000..d26a91f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-v3only @@ -0,0 +1,9 @@ +{% if auth_host -%} +[keystone_authtoken] +{% for option_name, option_value in keystone_authtoken.items() -%} +{{ option_name }} = {{ option_value }} +{% endfor -%} +{% if use_memcache == true %} +memcached_servers = {{ memcache_url }} +{% endif -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-cache b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-cache new file mode 100644 index 0000000..e056a32 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-cache @@ -0,0 +1,6 @@ +[cache] +{% if memcache_url %} +enabled = true +backend = oslo_cache.memcache_pool +memcache_servers = {{ memcache_url }} +{% endif %} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit new file mode 100644 index 0000000..bed2216 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit @@ -0,0 +1,10 @@ +[oslo_messaging_rabbit] +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +{% endif -%} +{% if rabbit_ssl_port -%} +ssl = True +{% endif -%} +{% if rabbit_ssl_ca -%} +ssl_ca_file = {{ rabbit_ssl_ca }} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit-ocata b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit-ocata new file mode 100644 index 0000000..365f437 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-messaging-rabbit-ocata @@ -0,0 +1,10 @@ +[oslo_messaging_rabbit] +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +{% endif -%} +{% if rabbit_ssl_ca -%} +ssl_ca_file = {{ rabbit_ssl_ca }} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-middleware b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-middleware new file mode 100644 index 0000000..dd73230 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-middleware @@ -0,0 +1,5 @@ +[oslo_middleware] + +# Bug #1758675 +enable_proxy_headers_parsing = true + diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-notifications b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-notifications new file mode 100644 index 0000000..71c7eb0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-oslo-notifications @@ -0,0 +1,15 @@ +{% if transport_url -%} +[oslo_messaging_notifications] +driver = {{ oslo_messaging_driver }} +transport_url = {{ transport_url }} +{% if send_notifications_to_logs %} +driver = log +{% endif %} +{% if notification_topics -%} +topics = {{ notification_topics }} +{% endif -%} +{% if notification_format -%} +[notifications] +notification_format = {{ notification_format }} +{% endif -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-placement b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-placement new file mode 100644 index 0000000..8c224ec --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-placement @@ -0,0 +1,20 @@ +[placement] +{% if auth_host -%} +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} +auth_type = password +{% if api_version == "3" -%} +project_domain_name = {{ admin_domain_name }} +user_domain_name = {{ admin_domain_name }} +{% else -%} +project_domain_name = default +user_domain_name = default +{% endif -%} +project_name = {{ admin_tenant_name }} +username = {{ admin_user }} +password = {{ admin_password }} +{% endif -%} +{% if region -%} +os_region_name = {{ region }} +region_name = {{ region }} +{% endif -%} +randomize_allocation_candidates = true diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo new file mode 100644 index 0000000..b444c9c --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo @@ -0,0 +1,22 @@ +{% if rabbitmq_host or rabbitmq_hosts -%} +[oslo_messaging_rabbit] +rabbit_userid = {{ rabbitmq_user }} +rabbit_virtual_host = {{ rabbitmq_virtual_host }} +rabbit_password = {{ rabbitmq_password }} +{% if rabbitmq_hosts -%} +rabbit_hosts = {{ rabbitmq_hosts }} +{% if rabbitmq_ha_queues -%} +rabbit_ha_queues = True +rabbit_durable_queues = False +{% endif -%} +{% else -%} +rabbit_host = {{ rabbitmq_host }} +{% endif -%} +{% if rabbit_ssl_port -%} +rabbit_use_ssl = True +rabbit_port = {{ rabbit_ssl_port }} +{% if rabbit_ssl_ca -%} +kombu_ssl_ca_certs = {{ rabbit_ssl_ca }} +{% endif -%} +{% endif -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-zeromq b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-zeromq new file mode 100644 index 0000000..95f1a76 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/section-zeromq @@ -0,0 +1,14 @@ +{% if zmq_host -%} +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) +rpc_backend = zmq +rpc_zmq_host = {{ zmq_host }} +{% if zmq_redis_address -%} +rpc_zmq_matchmaker = redis +matchmaker_heartbeat_freq = 15 +matchmaker_heartbeat_ttl = 30 +[matchmaker_redis] +host = {{ zmq_redis_address }} +{% else -%} +rpc_zmq_matchmaker = ring +{% endif -%} +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/vendor_data.json b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/vendor_data.json new file mode 100644 index 0000000..904f612 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/vendor_data.json @@ -0,0 +1 @@ +{{ vendor_data_json }} \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf new file mode 100644 index 0000000..b9ca396 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf @@ -0,0 +1,91 @@ +# Configuration file maintained by Juju. Local changes may be overwritten. + +{% if port -%} +Listen {{ port }} +{% endif -%} + +{% if admin_port -%} +Listen {{ admin_port }} +{% endif -%} + +{% if public_port -%} +Listen {{ public_port }} +{% endif -%} + +{% if port -%} + + WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ user }} group={{ group }} \ + display-name=%{GROUP} lang=C.UTF-8 locale=C.UTF-8 + WSGIProcessGroup {{ service_name }} + WSGIScriptAlias / {{ script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} + +{% if admin_port -%} + + WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ user }} group={{ group }} \ + display-name=%{GROUP} lang=C.UTF-8 locale=C.UTF-8 + WSGIProcessGroup {{ service_name }}-admin + WSGIScriptAlias / {{ admin_script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} + +{% if public_port -%} + + WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ user }} group={{ group }} \ + display-name=%{GROUP} lang=C.UTF-8 locale=C.UTF-8 + WSGIProcessGroup {{ service_name }}-public + WSGIScriptAlias / {{ public_script }} + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /var/log/apache2/{{ service_name }}_error.log + CustomLog /var/log/apache2/{{ service_name }}_access.log combined + + + = 2.4> + Require all granted + + + Order allow,deny + Allow from all + + + +{% endif -%} diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf new file mode 120000 index 0000000..2387e5d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templates/wsgi-openstack-metadata.conf @@ -0,0 +1 @@ +wsgi-openstack-api.conf \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templating.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templating.py new file mode 100644 index 0000000..050f8af --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/templating.py @@ -0,0 +1,379 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import six + +from charmhelpers.fetch import apt_install, apt_update +from charmhelpers.core.hookenv import ( + log, + ERROR, + INFO, + TRACE +) +from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES + +try: + from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) + from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions + + +class OSConfigException(Exception): + pass + + +def get_loader(templates_dir, os_release): + """ + Create a jinja2.ChoiceLoader containing template dirs up to + and including os_release. If directory template directory + is missing at templates_dir, it will be omitted from the loader. + templates_dir is added to the bottom of the search list as a base + loading dir. + + A charm may also ship a templates dir with this module + and it will be appended to the bottom of the search list, eg:: + + hooks/charmhelpers/contrib/openstack/templates + + :param templates_dir (str): Base template directory containing release + sub-directories. + :param os_release (str): OpenStack release codename to construct template + loader. + :returns: jinja2.ChoiceLoader constructed with a list of + jinja2.FilesystemLoaders, ordered in descending + order by OpenStack release. + """ + tmpl_dirs = [(rel, os.path.join(templates_dir, rel)) + for rel in six.itervalues(OPENSTACK_CODENAMES)] + + if not os.path.isdir(templates_dir): + log('Templates directory not found @ %s.' % templates_dir, + level=ERROR) + raise OSConfigException + + # the bottom contains tempaltes_dir and possibly a common templates dir + # shipped with the helper. + loaders = [FileSystemLoader(templates_dir)] + helper_templates = os.path.join(os.path.dirname(__file__), 'templates') + if os.path.isdir(helper_templates): + loaders.append(FileSystemLoader(helper_templates)) + + for rel, tmpl_dir in tmpl_dirs: + if os.path.isdir(tmpl_dir): + loaders.insert(0, FileSystemLoader(tmpl_dir)) + if rel == os_release: + break + # demote this log to the lowest level; we don't really need to see these + # lots in production even when debugging. + log('Creating choice loader with dirs: %s' % + [l.searchpath for l in loaders], level=TRACE) + return ChoiceLoader(loaders) + + +class OSConfigTemplate(object): + """ + Associates a config file template with a list of context generators. + Responsible for constructing a template context based on those generators. + """ + + def __init__(self, config_file, contexts, config_template=None): + self.config_file = config_file + + if hasattr(contexts, '__call__'): + self.contexts = [contexts] + else: + self.contexts = contexts + + self._complete_contexts = [] + + self.config_template = config_template + + def context(self): + ctxt = {} + for context in self.contexts: + _ctxt = context() + if _ctxt: + ctxt.update(_ctxt) + # track interfaces for every complete context. + [self._complete_contexts.append(interface) + for interface in context.interfaces + if interface not in self._complete_contexts] + return ctxt + + def complete_contexts(self): + ''' + Return a list of interfaces that have satisfied contexts. + ''' + if self._complete_contexts: + return self._complete_contexts + self.context() + return self._complete_contexts + + @property + def is_string_template(self): + """:returns: Boolean if this instance is a template initialised with a string""" + return self.config_template is not None + + +class OSConfigRenderer(object): + """ + This class provides a common templating system to be used by OpenStack + charms. It is intended to help charms share common code and templates, + and ease the burden of managing config templates across multiple OpenStack + releases. + + Basic usage:: + + # import some common context generates from charmhelpers + from charmhelpers.contrib.openstack import context + + # Create a renderer object for a specific OS release. + configs = OSConfigRenderer(templates_dir='/tmp/templates', + openstack_release='folsom') + # register some config files with context generators. + configs.register(config_file='/etc/nova/nova.conf', + contexts=[context.SharedDBContext(), + context.AMQPContext()]) + configs.register(config_file='/etc/nova/api-paste.ini', + contexts=[context.IdentityServiceContext()]) + configs.register(config_file='/etc/haproxy/haproxy.conf', + contexts=[context.HAProxyContext()]) + configs.register(config_file='/etc/keystone/policy.d/extra.cfg', + contexts=[context.ExtraPolicyContext() + context.KeystoneContext()], + config_template=hookenv.config('extra-policy')) + # write out a single config + configs.write('/etc/nova/nova.conf') + # write out all registered configs + configs.write_all() + + **OpenStack Releases and template loading** + + When the object is instantiated, it is associated with a specific OS + release. This dictates how the template loader will be constructed. + + The constructed loader attempts to load the template from several places + in the following order: + - from the most recent OS release-specific template dir (if one exists) + - the base templates_dir + - a template directory shipped in the charm with this helper file. + + For the example above, '/tmp/templates' contains the following structure:: + + /tmp/templates/nova.conf + /tmp/templates/api-paste.ini + /tmp/templates/grizzly/api-paste.ini + /tmp/templates/havana/api-paste.ini + + Since it was registered with the grizzly release, it first searches + the grizzly directory for nova.conf, then the templates dir. + + When writing api-paste.ini, it will find the template in the grizzly + directory. + + If the object were created with folsom, it would fall back to the + base templates dir for its api-paste.ini template. + + This system should help manage changes in config files through + openstack releases, allowing charms to fall back to the most recently + updated config template for a given release + + The haproxy.conf, since it is not shipped in the templates dir, will + be loaded from the module directory's template directory, eg + $CHARM/hooks/charmhelpers/contrib/openstack/templates. This allows + us to ship common templates (haproxy, apache) with the helpers. + + **Context generators** + + Context generators are used to generate template contexts during hook + execution. Doing so may require inspecting service relations, charm + config, etc. When registered, a config file is associated with a list + of generators. When a template is rendered and written, all context + generates are called in a chain to generate the context dictionary + passed to the jinja2 template. See context.py for more info. + """ + def __init__(self, templates_dir, openstack_release): + if not os.path.isdir(templates_dir): + log('Could not locate templates dir %s' % templates_dir, + level=ERROR) + raise OSConfigException + + self.templates_dir = templates_dir + self.openstack_release = openstack_release + self.templates = {} + self._tmpl_env = None + + if None in [Environment, ChoiceLoader, FileSystemLoader]: + # if this code is running, the object is created pre-install hook. + # jinja2 shouldn't get touched until the module is reloaded on next + # hook execution, with proper jinja2 bits successfully imported. + if six.PY2: + apt_install('python-jinja2') + else: + apt_install('python3-jinja2') + + def register(self, config_file, contexts, config_template=None): + """ + Register a config file with a list of context generators to be called + during rendering. + config_template can be used to load a template from a string instead of + using template loaders and template files. + :param config_file (str): a path where a config file will be rendered + :param contexts (list): a list of context dictionaries with kv pairs + :param config_template (str): an optional template string to use + """ + self.templates[config_file] = OSConfigTemplate( + config_file=config_file, + contexts=contexts, + config_template=config_template + ) + log('Registered config file: {}'.format(config_file), + level=INFO) + + def _get_tmpl_env(self): + if not self._tmpl_env: + loader = get_loader(self.templates_dir, self.openstack_release) + self._tmpl_env = Environment(loader=loader) + + def _get_template(self, template): + self._get_tmpl_env() + template = self._tmpl_env.get_template(template) + log('Loaded template from {}'.format(template.filename), + level=INFO) + return template + + def _get_template_from_string(self, ostmpl): + ''' + Get a jinja2 template object from a string. + :param ostmpl: OSConfigTemplate to use as a data source. + ''' + self._get_tmpl_env() + template = self._tmpl_env.from_string(ostmpl.config_template) + log('Loaded a template from a string for {}'.format( + ostmpl.config_file), + level=INFO) + return template + + def render(self, config_file): + if config_file not in self.templates: + log('Config not registered: {}'.format(config_file), level=ERROR) + raise OSConfigException + + ostmpl = self.templates[config_file] + ctxt = ostmpl.context() + + if ostmpl.is_string_template: + template = self._get_template_from_string(ostmpl) + log('Rendering from a string template: ' + '{}'.format(config_file), + level=INFO) + else: + _tmpl = os.path.basename(config_file) + try: + template = self._get_template(_tmpl) + except exceptions.TemplateNotFound: + # if no template is found with basename, try looking + # for it using a munged full path, eg: + # /etc/apache2/apache2.conf -> etc_apache2_apache2.conf + _tmpl = '_'.join(config_file.split('/')[1:]) + try: + template = self._get_template(_tmpl) + except exceptions.TemplateNotFound as e: + log('Could not load template from {} by {} or {}.' + ''.format( + self.templates_dir, + os.path.basename(config_file), + _tmpl + ), + level=ERROR) + raise e + + log('Rendering from template: {}'.format(config_file), + level=INFO) + return template.render(ctxt) + + def write(self, config_file): + """ + Write a single config file, raises if config file is not registered. + """ + if config_file not in self.templates: + log('Config not registered: %s' % config_file, level=ERROR) + raise OSConfigException + + _out = self.render(config_file) + if six.PY3: + _out = _out.encode('UTF-8') + + with open(config_file, 'wb') as out: + out.write(_out) + + log('Wrote template %s.' % config_file, level=INFO) + + def write_all(self): + """ + Write out all registered config files. + """ + [self.write(k) for k in six.iterkeys(self.templates)] + + def set_release(self, openstack_release): + """ + Resets the template environment and generates a new template loader + based on a the new openstack release. + """ + self._tmpl_env = None + self.openstack_release = openstack_release + self._get_tmpl_env() + + def complete_contexts(self): + ''' + Returns a list of context interfaces that yield a complete context. + ''' + interfaces = [] + [interfaces.extend(i.complete_contexts()) + for i in six.itervalues(self.templates)] + return interfaces + + def get_incomplete_context_data(self, interfaces): + ''' + Return dictionary of relation status of interfaces and any missing + required context data. Example: + {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True}, + 'zeromq-configuration': {'related': False}} + ''' + incomplete_context_data = {} + + for i in six.itervalues(self.templates): + for context in i.contexts: + for interface in interfaces: + related = False + if interface in context.interfaces: + related = context.get_related() + missing_data = context.missing_data + if missing_data: + incomplete_context_data[interface] = {'missing_data': missing_data} + if related: + if incomplete_context_data.get(interface): + incomplete_context_data[interface].update({'related': True}) + else: + incomplete_context_data[interface] = {'related': True} + else: + incomplete_context_data[interface] = {'related': False} + return incomplete_context_data diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/utils.py new file mode 100644 index 0000000..a26dbcc --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/utils.py @@ -0,0 +1,2660 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common python helper functions used for OpenStack charms. +from collections import OrderedDict, namedtuple +from functools import partial, wraps + +import subprocess +import json +import operator +import os +import sys +import re +import itertools +import functools + +import six +import traceback +import uuid +import yaml + +from charmhelpers import deprecate + +from charmhelpers.contrib.network import ip + +from charmhelpers.core import decorators, unitdata + +import charmhelpers.contrib.openstack.deferred_events as deferred_events + +from charmhelpers.core.hookenv import ( + WORKLOAD_STATES, + action_fail, + action_get, + action_set, + config, + expected_peer_units, + expected_related_units, + log as juju_log, + charm_dir, + INFO, + ERROR, + metadata, + related_units, + relation_get, + relation_id, + relation_ids, + relation_set, + service_name as ch_service_name, + status_set, + hook_name, + application_version_set, + cached, + leader_set, + leader_get, + local_unit, +) + +from charmhelpers.core.strutils import ( + BasicStringComparator, + bool_from_string, +) + +from charmhelpers.contrib.storage.linux.lvm import ( + deactivate_lvm_volume_group, + is_lvm_physical_volume, + remove_lvm_physical_volume, +) + +from charmhelpers.contrib.network.ip import ( + get_ipv6_addr, + is_ipv6, + port_has_listener, +) + +from charmhelpers.core.host import ( + lsb_release, + mounts, + umount, + service_running, + service_pause, + service_resume, + service_stop, + service_start, + restart_on_change_helper, +) + +from charmhelpers.fetch import ( + apt_cache, + apt_install, + import_key as fetch_import_key, + add_source as fetch_add_source, + SourceConfigError, + GPGKeyError, + get_upstream_version, + filter_installed_packages, + filter_missing_packages, + ubuntu_apt_pkg as apt, + OPENSTACK_RELEASES, + UBUNTU_OPENSTACK_RELEASE, +) + +from charmhelpers.fetch.snap import ( + snap_install, + snap_refresh, + valid_snap_channel, +) + +from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk +from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device +from charmhelpers.contrib.openstack.exceptions import OSContextError, ServiceActionError +from charmhelpers.contrib.openstack.policyd import ( + policyd_status_message_prefix, + POLICYD_CONFIG_NAME, +) + +from charmhelpers.contrib.openstack.ha.utils import ( + expect_ha, +) + +CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" +CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' + +DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed ' + 'restricted main multiverse universe') + +OPENSTACK_CODENAMES = OrderedDict([ + # NOTE(lourot): 'yyyy.i' isn't actually mapping with any real version + # number. This just means the i-th version of the year yyyy. + ('2011.2', 'diablo'), + ('2012.1', 'essex'), + ('2012.2', 'folsom'), + ('2013.1', 'grizzly'), + ('2013.2', 'havana'), + ('2014.1', 'icehouse'), + ('2014.2', 'juno'), + ('2015.1', 'kilo'), + ('2015.2', 'liberty'), + ('2016.1', 'mitaka'), + ('2016.2', 'newton'), + ('2017.1', 'ocata'), + ('2017.2', 'pike'), + ('2018.1', 'queens'), + ('2018.2', 'rocky'), + ('2019.1', 'stein'), + ('2019.2', 'train'), + ('2020.1', 'ussuri'), + ('2020.2', 'victoria'), + ('2021.1', 'wallaby'), + ('2021.2', 'xena'), + ('2022.1', 'yoga'), +]) + +# The ugly duckling - must list releases oldest to newest +SWIFT_CODENAMES = OrderedDict([ + ('diablo', + ['1.4.3']), + ('essex', + ['1.4.8']), + ('folsom', + ['1.7.4']), + ('grizzly', + ['1.7.6', '1.7.7', '1.8.0']), + ('havana', + ['1.9.0', '1.9.1', '1.10.0']), + ('icehouse', + ['1.11.0', '1.12.0', '1.13.0', '1.13.1']), + ('juno', + ['2.0.0', '2.1.0', '2.2.0']), + ('kilo', + ['2.2.1', '2.2.2']), + ('liberty', + ['2.3.0', '2.4.0', '2.5.0']), + ('mitaka', + ['2.5.0', '2.6.0', '2.7.0']), + ('newton', + ['2.8.0', '2.9.0', '2.10.0']), + ('ocata', + ['2.11.0', '2.12.0', '2.13.0']), + ('pike', + ['2.13.0', '2.15.0']), + ('queens', + ['2.16.0', '2.17.0']), + ('rocky', + ['2.18.0', '2.19.0']), + ('stein', + ['2.20.0', '2.21.0']), + ('train', + ['2.22.0', '2.23.0']), + ('ussuri', + ['2.24.0', '2.25.0']), + ('victoria', + ['2.25.0', '2.26.0']), +]) + +# >= Liberty version->codename mapping +PACKAGE_CODENAMES = { + 'nova-common': OrderedDict([ + ('12', 'liberty'), + ('13', 'mitaka'), + ('14', 'newton'), + ('15', 'ocata'), + ('16', 'pike'), + ('17', 'queens'), + ('18', 'rocky'), + ('19', 'stein'), + ('20', 'train'), + ('21', 'ussuri'), + ('22', 'victoria'), + ]), + 'neutron-common': OrderedDict([ + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), + ('11', 'pike'), + ('12', 'queens'), + ('13', 'rocky'), + ('14', 'stein'), + ('15', 'train'), + ('16', 'ussuri'), + ('17', 'victoria'), + ]), + 'cinder-common': OrderedDict([ + ('7', 'liberty'), + ('8', 'mitaka'), + ('9', 'newton'), + ('10', 'ocata'), + ('11', 'pike'), + ('12', 'queens'), + ('13', 'rocky'), + ('14', 'stein'), + ('15', 'train'), + ('16', 'ussuri'), + ('17', 'victoria'), + ]), + 'keystone': OrderedDict([ + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), + ('12', 'pike'), + ('13', 'queens'), + ('14', 'rocky'), + ('15', 'stein'), + ('16', 'train'), + ('17', 'ussuri'), + ('18', 'victoria'), + ]), + 'horizon-common': OrderedDict([ + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), + ('12', 'pike'), + ('13', 'queens'), + ('14', 'rocky'), + ('15', 'stein'), + ('16', 'train'), + ('18', 'ussuri'), # Note this was actually 17.0 - 18.3 + ('19', 'victoria'), # Note this is really 18.6 + ]), + 'ceilometer-common': OrderedDict([ + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), + ('9', 'pike'), + ('10', 'queens'), + ('11', 'rocky'), + ('12', 'stein'), + ('13', 'train'), + ('14', 'ussuri'), + ('15', 'victoria'), + ]), + 'heat-common': OrderedDict([ + ('5', 'liberty'), + ('6', 'mitaka'), + ('7', 'newton'), + ('8', 'ocata'), + ('9', 'pike'), + ('10', 'queens'), + ('11', 'rocky'), + ('12', 'stein'), + ('13', 'train'), + ('14', 'ussuri'), + ('15', 'victoria'), + ]), + 'glance-common': OrderedDict([ + ('11', 'liberty'), + ('12', 'mitaka'), + ('13', 'newton'), + ('14', 'ocata'), + ('15', 'pike'), + ('16', 'queens'), + ('17', 'rocky'), + ('18', 'stein'), + ('19', 'train'), + ('20', 'ussuri'), + ('21', 'victoria'), + ]), + 'openstack-dashboard': OrderedDict([ + ('8', 'liberty'), + ('9', 'mitaka'), + ('10', 'newton'), + ('11', 'ocata'), + ('12', 'pike'), + ('13', 'queens'), + ('14', 'rocky'), + ('15', 'stein'), + ('16', 'train'), + ('18', 'ussuri'), + ('19', 'victoria'), + ]), +} + +DEFAULT_LOOPBACK_SIZE = '5G' + +DB_SERIES_UPGRADING_KEY = 'cluster-series-upgrading' + +DB_MAINTENANCE_KEYS = [DB_SERIES_UPGRADING_KEY] + + +class CompareOpenStackReleases(BasicStringComparator): + """Provide comparisons of OpenStack releases. + + Use in the form of + + if CompareOpenStackReleases(release) > 'mitaka': + # do something with mitaka + """ + _list = OPENSTACK_RELEASES + + +def error_out(msg): + juju_log("FATAL ERROR: %s" % msg, level='ERROR') + sys.exit(1) + + +def get_installed_semantic_versioned_packages(): + '''Get a list of installed packages which have OpenStack semantic versioning + + :returns List of installed packages + :rtype: [pkg1, pkg2, ...] + ''' + return filter_missing_packages(PACKAGE_CODENAMES.keys()) + + +def get_os_codename_install_source(src): + '''Derive OpenStack release codename from a given installation source.''' + ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] + rel = '' + if src is None: + return rel + if src in ['distro', 'distro-proposed', 'proposed']: + try: + rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel] + except KeyError: + e = 'Could not derive openstack release for '\ + 'this Ubuntu release: %s' % ubuntu_rel + error_out(e) + return rel + + if src.startswith('cloud:'): + ca_rel = src.split(':')[1] + ca_rel = ca_rel.split('-')[1].split('/')[0] + return ca_rel + + # Best guess match based on deb string provided + if (src.startswith('deb') or + src.startswith('ppa') or + src.startswith('snap')): + for v in OPENSTACK_CODENAMES.values(): + if v in src: + return v + + +def get_os_version_install_source(src): + codename = get_os_codename_install_source(src) + return get_os_version_codename(codename) + + +def get_os_codename_version(vers): + '''Determine OpenStack codename from version number.''' + try: + return OPENSTACK_CODENAMES[vers] + except KeyError: + e = 'Could not determine OpenStack codename for version %s' % vers + error_out(e) + + +def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): + '''Determine OpenStack version number from codename.''' + for k, v in six.iteritems(version_map): + if v == codename: + return k + e = 'Could not derive OpenStack version for '\ + 'codename: %s' % codename + error_out(e) + + +def get_os_version_codename_swift(codename): + '''Determine OpenStack version number of swift from codename.''' + for k, v in six.iteritems(SWIFT_CODENAMES): + if k == codename: + return v[-1] + e = 'Could not derive swift version for '\ + 'codename: %s' % codename + error_out(e) + + +def get_swift_codename(version): + '''Determine OpenStack codename that corresponds to swift version.''' + codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + + if len(codenames) > 1: + # If more than one release codename contains this version we determine + # the actual codename based on the highest available install source. + for codename in reversed(codenames): + releases = UBUNTU_OPENSTACK_RELEASE + release = [k for k, v in six.iteritems(releases) if codename in v] + ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) + if six.PY3: + ret = ret.decode('UTF-8') + if codename in ret or release[0] in ret: + return codename + elif len(codenames) == 1: + return codenames[0] + + # NOTE: fallback - attempt to match with just major.minor version + match = re.match(r'^(\d+)\.(\d+)', version) + if match: + major_minor_version = match.group(0) + for codename, versions in six.iteritems(SWIFT_CODENAMES): + for release_version in versions: + if release_version.startswith(major_minor_version): + return codename + + return None + + +def get_os_codename_package(package, fatal=True): + """Derive OpenStack release codename from an installed package. + + Initially, see if the openstack-release pkg is available (by trying to + install it) and use it instead. + + If it isn't then it falls back to the existing method of checking the + version of the package passed and then resolving the version from that + using lookup tables. + + Note: if possible, charms should use get_installed_os_version() to + determine the version of the "openstack-release" pkg. + + :param package: the package to test for version information. + :type package: str + :param fatal: If True (default), then die via error_out() + :type fatal: bool + :returns: the OpenStack release codename (e.g. ussuri) + :rtype: str + """ + + codename = get_installed_os_version() + if codename: + return codename + + if snap_install_requested(): + cmd = ['snap', 'list', package] + try: + out = subprocess.check_output(cmd) + if six.PY3: + out = out.decode('UTF-8') + except subprocess.CalledProcessError: + return None + lines = out.split('\n') + for line in lines: + if package in line: + # Second item in list is Version + return line.split()[1] + + cache = apt_cache() + + try: + pkg = cache[package] + except Exception: + if not fatal: + return None + # the package is unknown to the current apt cache. + e = 'Could not determine version of package with no installation '\ + 'candidate: %s' % package + error_out(e) + + if not pkg.current_ver: + if not fatal: + return None + # package is known, but no version is currently installed. + e = 'Could not determine version of uninstalled package: %s' % package + error_out(e) + + vers = apt.upstream_version(pkg.current_ver.ver_str) + if 'swift' in pkg.name: + # Fully x.y.z match for swift versions + match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers) + else: + # x.y match only for 20XX.X + # and ignore patch level for other packages + match = re.match(r'^(\d+)\.(\d+)', vers) + + if match: + vers = match.group(0) + + # Generate a major version number for newer semantic + # versions of openstack projects + major_vers = vers.split('.')[0] + # >= Liberty independent project versions + if (package in PACKAGE_CODENAMES and + major_vers in PACKAGE_CODENAMES[package]): + return PACKAGE_CODENAMES[package][major_vers] + else: + # < Liberty co-ordinated project versions + try: + if 'swift' in pkg.name: + return get_swift_codename(vers) + else: + return OPENSTACK_CODENAMES[vers] + except KeyError: + if not fatal: + return None + e = 'Could not determine OpenStack codename for version %s' % vers + error_out(e) + + +def get_os_version_package(pkg, fatal=True): + '''Derive OpenStack version number from an installed package.''' + codename = get_os_codename_package(pkg, fatal=fatal) + + if not codename: + return None + + if 'swift' in pkg: + vers_map = SWIFT_CODENAMES + for cname, version in six.iteritems(vers_map): + if cname == codename: + return version[-1] + else: + vers_map = OPENSTACK_CODENAMES + for version, cname in six.iteritems(vers_map): + if cname == codename: + return version + # e = "Could not determine OpenStack version for package: %s" % pkg + # error_out(e) + + +def get_installed_os_version(): + """Determine the OpenStack release code name from openstack-release pkg. + + This uses the "openstack-release" pkg (if it exists) to return the + OpenStack release codename (e.g. usurri, mitaka, ocata, etc.) + + Note, it caches the result so that it is only done once per hook. + + :returns: the OpenStack release codename, if available + :rtype: Optional[str] + """ + @cached + def _do_install(): + apt_install(filter_installed_packages(['openstack-release']), + fatal=False, quiet=True) + + _do_install() + return openstack_release().get('OPENSTACK_CODENAME') + + +@cached +def openstack_release(): + """Return /etc/os-release in a dict.""" + d = {} + try: + with open('/etc/openstack-release', 'r') as lsb: + for l in lsb: + s = l.split('=') + if len(s) != 2: + continue + d[s[0].strip()] = s[1].strip() + except FileNotFoundError: + pass + return d + + +# Module local cache variable for the os_release. +_os_rel = None + + +def reset_os_release(): + '''Unset the cached os_release version''' + global _os_rel + _os_rel = None + + +def os_release(package, base=None, reset_cache=False, source_key=None): + """Returns OpenStack release codename from a cached global. + + If reset_cache then unset the cached os_release version and return the + freshly determined version. + + If the codename can not be determined from either an installed package or + the installation source, the earliest release supported by the charm should + be returned. + + :param package: Name of package to determine release from + :type package: str + :param base: Fallback codename if endavours to determine from package fail + :type base: Optional[str] + :param reset_cache: Reset any cached codename value + :type reset_cache: bool + :param source_key: Name of source configuration option + (default: 'openstack-origin') + :type source_key: Optional[str] + :returns: OpenStack release codename + :rtype: str + """ + source_key = source_key or 'openstack-origin' + if not base: + base = UBUNTU_OPENSTACK_RELEASE[lsb_release()['DISTRIB_CODENAME']] + global _os_rel + if reset_cache: + reset_os_release() + if _os_rel: + return _os_rel + _os_rel = ( + get_os_codename_package(package, fatal=False) or + get_os_codename_install_source(config(source_key)) or + base) + return _os_rel + + +@deprecate("moved to charmhelpers.fetch.import_key()", "2017-07", log=juju_log) +def import_key(keyid): + """Import a key, either ASCII armored, or a GPG key id. + + @param keyid: the key in ASCII armor format, or a GPG key id. + @raises SystemExit() via sys.exit() on failure. + """ + try: + return fetch_import_key(keyid) + except GPGKeyError as e: + error_out("Could not import key: {}".format(str(e))) + + +def get_source_and_pgp_key(source_and_key): + """Look for a pgp key ID or ascii-armor key in the given input. + + :param source_and_key: String, "source_spec|keyid" where '|keyid' is + optional. + :returns (source_spec, key_id OR None) as a tuple. Returns None for key_id + if there was no '|' in the source_and_key string. + """ + try: + source, key = source_and_key.split('|', 2) + return source, key or None + except ValueError: + return source_and_key, None + + +@deprecate("use charmhelpers.fetch.add_source() instead.", + "2017-07", log=juju_log) +def configure_installation_source(source_plus_key): + """Configure an installation source. + + The functionality is provided by charmhelpers.fetch.add_source() + The difference between the two functions is that add_source() signature + requires the key to be passed directly, whereas this function passes an + optional key by appending '|' to the end of the source specification + 'source'. + + Another difference from add_source() is that the function calls sys.exit(1) + if the configuration fails, whereas add_source() raises + SourceConfigurationError(). Another difference, is that add_source() + silently fails (with a juju_log command) if there is no matching source to + configure, whereas this function fails with a sys.exit(1) + + :param source: String_plus_key -- see above for details. + + Note that the behaviour on error is to log the error to the juju log and + then call sys.exit(1). + """ + if source_plus_key.startswith('snap'): + # Do nothing for snap installs + return + # extract the key if there is one, denoted by a '|' in the rel + source, key = get_source_and_pgp_key(source_plus_key) + + # handle the ordinary sources via add_source + try: + fetch_add_source(source, key, fail_invalid=True) + except SourceConfigError as se: + error_out(str(se)) + + +def config_value_changed(option): + """ + Determine if config value changed since last call to this function. + """ + hook_data = unitdata.HookData() + with hook_data(): + db = unitdata.kv() + current = config(option) + saved = db.get(option) + db.set(option, current) + if saved is None: + return False + return current != saved + + +def get_endpoint_key(service_name, relation_id, unit_name): + """Return the key used to refer to an ep changed notification from a unit. + + :param service_name: Service name eg nova, neutron, placement etc + :type service_name: str + :param relation_id: The id of the relation the unit is on. + :type relation_id: str + :param unit_name: The name of the unit publishing the notification. + :type unit_name: str + :returns: The key used to refer to an ep changed notification from a unit + :rtype: str + """ + return '{}-{}-{}'.format( + service_name, + relation_id.replace(':', '_'), + unit_name.replace('/', '_')) + + +def get_endpoint_notifications(service_names, rel_name='identity-service'): + """Return all notifications for the given services. + + :param service_names: List of service name. + :type service_name: List + :param rel_name: Name of the relation to query + :type rel_name: str + :returns: A dict containing the source of the notification and its nonce. + :rtype: Dict[str, str] + """ + notifications = {} + for rid in relation_ids(rel_name): + for unit in related_units(relid=rid): + ep_changed_json = relation_get( + rid=rid, + unit=unit, + attribute='ep_changed') + if ep_changed_json: + ep_changed = json.loads(ep_changed_json) + for service in service_names: + if ep_changed.get(service): + key = get_endpoint_key(service, rid, unit) + notifications[key] = ep_changed[service] + return notifications + + +def endpoint_changed(service_name, rel_name='identity-service'): + """Whether a new notification has been received for an endpoint. + + :param service_name: Service name eg nova, neutron, placement etc + :type service_name: str + :param rel_name: Name of the relation to query + :type rel_name: str + :returns: Whether endpoint has changed + :rtype: bool + """ + changed = False + with unitdata.HookData()() as t: + db = t[0] + notifications = get_endpoint_notifications( + [service_name], + rel_name=rel_name) + for key, nonce in notifications.items(): + if db.get(key) != nonce: + juju_log(('New endpoint change notification found: ' + '{}={}').format(key, nonce), + 'INFO') + changed = True + break + return changed + + +def save_endpoint_changed_triggers(service_names, rel_name='identity-service'): + """Save the endpoint triggers in db so it can be tracked if they changed. + + :param service_names: List of service name. + :type service_name: List + :param rel_name: Name of the relation to query + :type rel_name: str + """ + with unitdata.HookData()() as t: + db = t[0] + notifications = get_endpoint_notifications( + service_names, + rel_name=rel_name) + for key, nonce in notifications.items(): + db.set(key, nonce) + + +def save_script_rc(script_path="scripts/scriptrc", **env_vars): + """ + Write an rc file in the charm-delivered directory containing + exported environment variables provided by env_vars. Any charm scripts run + outside the juju hook environment can source this scriptrc to obtain + updated config information necessary to perform health checks or + service changes. + """ + juju_rc_path = "%s/%s" % (charm_dir(), script_path) + if not os.path.exists(os.path.dirname(juju_rc_path)): + os.mkdir(os.path.dirname(juju_rc_path)) + with open(juju_rc_path, 'wt') as rc_script: + rc_script.write( + "#!/bin/bash\n") + [rc_script.write('export %s=%s\n' % (u, p)) + for u, p in six.iteritems(env_vars) if u != "script_path"] + + +def openstack_upgrade_available(package): + """ + Determines if an OpenStack upgrade is available from installation + source, based on version of installed package. + + :param package: str: Name of installed package. + + :returns: bool: : Returns True if configured installation source offers + a newer version of package. + """ + + src = config('openstack-origin') + cur_vers = get_os_version_package(package) + if not cur_vers: + # The package has not been installed yet do not attempt upgrade + return False + if "swift" in package: + codename = get_os_codename_install_source(src) + avail_vers = get_os_version_codename_swift(codename) + else: + try: + avail_vers = get_os_version_install_source(src) + except Exception: + avail_vers = cur_vers + apt.init() + return apt.version_compare(avail_vers, cur_vers) >= 1 + + +def ensure_block_device(block_device): + ''' + Confirm block_device, create as loopback if necessary. + + :param block_device: str: Full path of block device to ensure. + + :returns: str: Full path of ensured block device. + ''' + _none = ['None', 'none', None] + if (block_device in _none): + error_out('prepare_storage(): Missing required input: block_device=%s.' + % block_device) + + if block_device.startswith('/dev/'): + bdev = block_device + elif block_device.startswith('/'): + _bd = block_device.split('|') + if len(_bd) == 2: + bdev, size = _bd + else: + bdev = block_device + size = DEFAULT_LOOPBACK_SIZE + bdev = ensure_loopback_device(bdev, size) + else: + bdev = '/dev/%s' % block_device + + if not is_block_device(bdev): + error_out('Failed to locate valid block device at %s' % bdev) + + return bdev + + +def clean_storage(block_device): + ''' + Ensures a block device is clean. That is: + - unmounted + - any lvm volume groups are deactivated + - any lvm physical device signatures removed + - partition table wiped + + :param block_device: str: Full path to block device to clean. + ''' + for mp, d in mounts(): + if d == block_device: + juju_log('clean_storage(): %s is mounted @ %s, unmounting.' % + (d, mp), level=INFO) + umount(mp, persist=True) + + if is_lvm_physical_volume(block_device): + deactivate_lvm_volume_group(block_device) + remove_lvm_physical_volume(block_device) + else: + zap_disk(block_device) + + +is_ip = ip.is_ip +ns_query = ip.ns_query +get_host_ip = ip.get_host_ip +get_hostname = ip.get_hostname + + +def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'): + mm_map = {} + if os.path.isfile(mm_file): + with open(mm_file, 'r') as f: + mm_map = json.load(f) + return mm_map + + +def sync_db_with_multi_ipv6_addresses(database, database_user, + relation_prefix=None): + hosts = get_ipv6_addr(dynamic_only=False) + + if config('vip'): + vips = config('vip').split() + for vip in vips: + if vip and is_ipv6(vip): + hosts.append(vip) + + kwargs = {'database': database, + 'username': database_user, + 'hostname': json.dumps(hosts)} + + if relation_prefix: + for key in list(kwargs.keys()): + kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key] + del kwargs[key] + + for rid in relation_ids('shared-db'): + relation_set(relation_id=rid, **kwargs) + + +def os_requires_version(ostack_release, pkg): + """ + Decorator for hook to specify minimum supported release + """ + def wrap(f): + @wraps(f) + def wrapped_f(*args): + if os_release(pkg) < ostack_release: + raise Exception("This hook is not supported on releases" + " before %s" % ostack_release) + f(*args) + return wrapped_f + return wrap + + +def os_workload_status(configs, required_interfaces, charm_func=None): + """ + Decorator to set workload status based on complete contexts + """ + def wrap(f): + @wraps(f) + def wrapped_f(*args, **kwargs): + # Run the original function first + f(*args, **kwargs) + # Set workload status now that contexts have been + # acted on + set_os_workload_status(configs, required_interfaces, charm_func) + return wrapped_f + return wrap + + +def set_os_workload_status(configs, required_interfaces, charm_func=None, + services=None, ports=None): + """Set the state of the workload status for the charm. + + This calls _determine_os_workload_status() to get the new state, message + and sets the status using status_set() + + @param configs: a templating.OSConfigRenderer() object + @param required_interfaces: {generic: [specific, specific2, ...]} + @param charm_func: a callable function that returns state, message. The + signature is charm_func(configs) -> (state, message) + @param services: list of strings OR dictionary specifying services/ports + @param ports: OPTIONAL list of port numbers. + @returns state, message: the new workload status, user message + """ + state, message = _determine_os_workload_status( + configs, required_interfaces, charm_func, services, ports) + status_set(state, message) + + +def _determine_os_workload_status( + configs, required_interfaces, charm_func=None, + services=None, ports=None): + """Determine the state of the workload status for the charm. + + This function returns the new workload status for the charm based + on the state of the interfaces, the paused state and whether the + services are actually running and any specified ports are open. + + This checks: + + 1. if the unit should be paused, that it is actually paused. If so the + state is 'maintenance' + message, else 'broken'. + 2. that the interfaces/relations are complete. If they are not then + it sets the state to either 'broken' or 'waiting' and an appropriate + message. + 3. If all the relation data is set, then it checks that the actual + services really are running. If not it sets the state to 'broken'. + + If everything is okay then the state returns 'active'. + + @param configs: a templating.OSConfigRenderer() object + @param required_interfaces: {generic: [specific, specific2, ...]} + @param charm_func: a callable function that returns state, message. The + signature is charm_func(configs) -> (state, message) + @param services: list of strings OR dictionary specifying services/ports + @param ports: OPTIONAL list of port numbers. + @returns state, message: the new workload status, user message + """ + state, message = _ows_check_if_paused(services, ports) + + if state is None: + state, message = _ows_check_generic_interfaces( + configs, required_interfaces) + + if state != 'maintenance' and charm_func: + # _ows_check_charm_func() may modify the state, message + state, message = _ows_check_charm_func( + state, message, lambda: charm_func(configs)) + + if state is None: + state, message = _ows_check_services_running(services, ports) + + if state is None: + state = 'active' + message = "Unit is ready" + juju_log(message, 'INFO') + + try: + if config(POLICYD_CONFIG_NAME): + message = "{} {}".format(policyd_status_message_prefix(), message) + # Get deferred restarts events that have been triggered by a policy + # written by this charm. + deferred_restarts = list(set( + [e.service + for e in deferred_events.get_deferred_restarts() + if e.policy_requestor_name == ch_service_name()])) + if deferred_restarts: + svc_msg = "Services queued for restart: {}".format( + ', '.join(sorted(deferred_restarts))) + message = "{}. {}".format(message, svc_msg) + deferred_hooks = deferred_events.get_deferred_hooks() + if deferred_hooks: + svc_msg = "Hooks skipped due to disabled auto restarts: {}".format( + ', '.join(sorted(deferred_hooks))) + message = "{}. {}".format(message, svc_msg) + + except Exception: + pass + + return state, message + + +def _ows_check_if_paused(services=None, ports=None): + """Check if the unit is supposed to be paused, and if so check that the + services/ports (if passed) are actually stopped/not being listened to. + + If the unit isn't supposed to be paused, just return None, None + + If the unit is performing a series upgrade, return a message indicating + this. + + @param services: OPTIONAL services spec or list of service names. + @param ports: OPTIONAL list of port numbers. + @returns state, message or None, None + """ + if is_unit_upgrading_set(): + state, message = check_actually_paused(services=services, + ports=ports) + if state is None: + # we're paused okay, so set maintenance and return + state = "blocked" + message = ("Ready for do-release-upgrade and reboot. " + "Set complete when finished.") + return state, message + + if is_unit_paused_set(): + state, message = check_actually_paused(services=services, + ports=ports) + if state is None: + # we're paused okay, so set maintenance and return + state = "maintenance" + message = "Paused. Use 'resume' action to resume normal service." + return state, message + return None, None + + +def _ows_check_generic_interfaces(configs, required_interfaces): + """Check the complete contexts to determine the workload status. + + - Checks for missing or incomplete contexts + - juju log details of missing required data. + - determines the correct workload status + - creates an appropriate message for status_set(...) + + if there are no problems then the function returns None, None + + @param configs: a templating.OSConfigRenderer() object + @params required_interfaces: {generic_interface: [specific_interface], } + @returns state, message or None, None + """ + incomplete_rel_data = incomplete_relation_data(configs, + required_interfaces) + state = None + message = None + missing_relations = set() + incomplete_relations = set() + + for generic_interface, relations_states in incomplete_rel_data.items(): + related_interface = None + missing_data = {} + # Related or not? + for interface, relation_state in relations_states.items(): + if relation_state.get('related'): + related_interface = interface + missing_data = relation_state.get('missing_data') + break + # No relation ID for the generic_interface? + if not related_interface: + juju_log("{} relation is missing and must be related for " + "functionality. ".format(generic_interface), 'WARN') + state = 'blocked' + missing_relations.add(generic_interface) + else: + # Relation ID eists but no related unit + if not missing_data: + # Edge case - relation ID exists but departings + _hook_name = hook_name() + if (('departed' in _hook_name or 'broken' in _hook_name) and + related_interface in _hook_name): + state = 'blocked' + missing_relations.add(generic_interface) + juju_log("{} relation's interface, {}, " + "relationship is departed or broken " + "and is required for functionality." + "".format(generic_interface, related_interface), + "WARN") + # Normal case relation ID exists but no related unit + # (joining) + else: + juju_log("{} relations's interface, {}, is related but has" + " no units in the relation." + "".format(generic_interface, related_interface), + "INFO") + # Related unit exists and data missing on the relation + else: + juju_log("{} relation's interface, {}, is related awaiting " + "the following data from the relationship: {}. " + "".format(generic_interface, related_interface, + ", ".join(missing_data)), "INFO") + if state != 'blocked': + state = 'waiting' + if generic_interface not in missing_relations: + incomplete_relations.add(generic_interface) + + if missing_relations: + message = "Missing relations: {}".format(", ".join(missing_relations)) + if incomplete_relations: + message += "; incomplete relations: {}" \ + "".format(", ".join(incomplete_relations)) + state = 'blocked' + elif incomplete_relations: + message = "Incomplete relations: {}" \ + "".format(", ".join(incomplete_relations)) + state = 'waiting' + + return state, message + + +def _ows_check_charm_func(state, message, charm_func_with_configs): + """Run a custom check function for the charm to see if it wants to + change the state. This is only run if not in 'maintenance' and + tests to see if the new state is more important that the previous + one determined by the interfaces/relations check. + + @param state: the previously determined state so far. + @param message: the user orientated message so far. + @param charm_func: a callable function that returns state, message + @returns state, message strings. + """ + if charm_func_with_configs: + charm_state, charm_message = charm_func_with_configs() + if (charm_state != 'active' and + charm_state != 'unknown' and + charm_state is not None): + state = workload_state_compare(state, charm_state) + if message: + charm_message = charm_message.replace("Incomplete relations: ", + "") + message = "{}, {}".format(message, charm_message) + else: + message = charm_message + return state, message + + +def _ows_check_services_running(services, ports): + """Check that the services that should be running are actually running + and that any ports specified are being listened to. + + @param services: list of strings OR dictionary specifying services/ports + @param ports: list of ports + @returns state, message: strings or None, None + """ + messages = [] + state = None + if services is not None: + services = _extract_services_list_helper(services) + services_running, running = _check_running_services(services) + if not all(running): + messages.append( + "Services not running that should be: {}" + .format(", ".join(_filter_tuples(services_running, False)))) + state = 'blocked' + # also verify that the ports that should be open are open + # NB, that ServiceManager objects only OPTIONALLY have ports + map_not_open, ports_open = ( + _check_listening_on_services_ports(services)) + if not all(ports_open): + # find which service has missing ports. They are in service + # order which makes it a bit easier. + message_parts = {service: ", ".join([str(v) for v in open_ports]) + for service, open_ports in map_not_open.items()} + message = ", ".join( + ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) + messages.append( + "Services with ports not open that should be: {}" + .format(message)) + state = 'blocked' + + if ports is not None: + # and we can also check ports which we don't know the service for + ports_open, ports_open_bools = _check_listening_on_ports_list(ports) + if not all(ports_open_bools): + messages.append( + "Ports which should be open, but are not: {}" + .format(", ".join([str(p) for p, v in ports_open + if not v]))) + state = 'blocked' + + if state is not None: + message = "; ".join(messages) + return state, message + + return None, None + + +def _extract_services_list_helper(services): + """Extract a OrderedDict of {service: [ports]} of the supplied services + for use by the other functions. + + The services object can either be: + - None : no services were passed (an empty dict is returned) + - a list of strings + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} + - An array of [{'service': service_name, ...}, ...] + + @param services: see above + @returns OrderedDict(service: [ports], ...) + """ + if services is None: + return {} + if isinstance(services, dict): + services = services.values() + # either extract the list of services from the dictionary, or if + # it is a simple string, use that. i.e. works with mixed lists. + _s = OrderedDict() + for s in services: + if isinstance(s, dict) and 'service' in s: + _s[s['service']] = s.get('ports', []) + if isinstance(s, str): + _s[s] = [] + return _s + + +def _check_running_services(services): + """Check that the services dict provided is actually running and provide + a list of (service, boolean) tuples for each service. + + Returns both a zipped list of (service, boolean) and a list of booleans + in the same order as the services. + + @param services: OrderedDict of strings: [ports], one for each service to + check. + @returns [(service, boolean), ...], : results for checks + [boolean] : just the result of the service checks + """ + services_running = [service_running(s) for s in services] + return list(zip(services, services_running)), services_running + + +def _check_listening_on_services_ports(services, test=False): + """Check that the unit is actually listening (has the port open) on the + ports that the service specifies are open. If test is True then the + function returns the services with ports that are open rather than + closed. + + Returns an OrderedDict of service: ports and a list of booleans + + @param services: OrderedDict(service: [port, ...], ...) + @param test: default=False, if False, test for closed, otherwise open. + @returns OrderedDict(service: [port-not-open, ...]...), [boolean] + """ + test = not(not(test)) # ensure test is True or False + all_ports = list(itertools.chain(*services.values())) + ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports] + map_ports = OrderedDict() + matched_ports = [p for p, opened in zip(all_ports, ports_states) + if opened == test] # essentially opened xor test + for service, ports in services.items(): + set_ports = set(ports).intersection(matched_ports) + if set_ports: + map_ports[service] = set_ports + return map_ports, ports_states + + +def _check_listening_on_ports_list(ports): + """Check that the ports list given are being listened to + + Returns a list of ports being listened to and a list of the + booleans. + + @param ports: LIST of port numbers. + @returns [(port_num, boolean), ...], [boolean] + """ + ports_open = [port_has_listener('0.0.0.0', p) for p in ports] + return zip(ports, ports_open), ports_open + + +def _filter_tuples(services_states, state): + """Return a simple list from a list of tuples according to the condition + + @param services_states: LIST of (string, boolean): service and running + state. + @param state: Boolean to match the tuple against. + @returns [LIST of strings] that matched the tuple RHS. + """ + return [s for s, b in services_states if b == state] + + +def workload_state_compare(current_workload_state, workload_state): + """ Return highest priority of two states""" + hierarchy = {'unknown': -1, + 'active': 0, + 'maintenance': 1, + 'waiting': 2, + 'blocked': 3, + } + + if hierarchy.get(workload_state) is None: + workload_state = 'unknown' + if hierarchy.get(current_workload_state) is None: + current_workload_state = 'unknown' + + # Set workload_state based on hierarchy of statuses + if hierarchy.get(current_workload_state) > hierarchy.get(workload_state): + return current_workload_state + else: + return workload_state + + +def incomplete_relation_data(configs, required_interfaces): + """Check complete contexts against required_interfaces + Return dictionary of incomplete relation data. + + configs is an OSConfigRenderer object with configs registered + + required_interfaces is a dictionary of required general interfaces + with dictionary values of possible specific interfaces. + Example: + required_interfaces = {'database': ['shared-db', 'pgsql-db']} + + The interface is said to be satisfied if anyone of the interfaces in the + list has a complete context. + + Return dictionary of incomplete or missing required contexts with relation + status of interfaces and any missing data points. Example: + {'message': + {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True}, + 'zeromq-configuration': {'related': False}}, + 'identity': + {'identity-service': {'related': False}}, + 'database': + {'pgsql-db': {'related': False}, + 'shared-db': {'related': True}}} + """ + complete_ctxts = configs.complete_contexts() + incomplete_relations = [ + svc_type + for svc_type, interfaces in required_interfaces.items() + if not set(interfaces).intersection(complete_ctxts)] + return { + i: configs.get_incomplete_context_data(required_interfaces[i]) + for i in incomplete_relations} + + +def do_action_openstack_upgrade(package, upgrade_callback, configs): + """Perform action-managed OpenStack upgrade. + + Upgrades packages to the configured openstack-origin version and sets + the corresponding action status as a result. + + If the charm was installed from source we cannot upgrade it. + For backwards compatibility a config flag (action-managed-upgrade) must + be set for this code to run, otherwise a full service level upgrade will + fire on config-changed. + + @param package: package name for determining if upgrade available + @param upgrade_callback: function callback to charm's upgrade function + @param configs: templating object derived from OSConfigRenderer class + + @return: True if upgrade successful; False if upgrade failed or skipped + """ + ret = False + + if openstack_upgrade_available(package): + if config('action-managed-upgrade'): + juju_log('Upgrading OpenStack release') + + try: + upgrade_callback(configs=configs) + action_set({'outcome': 'success, upgrade completed.'}) + ret = True + except Exception: + action_set({'outcome': 'upgrade failed, see traceback.'}) + action_set({'traceback': traceback.format_exc()}) + action_fail('do_openstack_upgrade resulted in an ' + 'unexpected error') + else: + action_set({'outcome': 'action-managed-upgrade config is ' + 'False, skipped upgrade.'}) + else: + action_set({'outcome': 'no upgrade available.'}) + + return ret + + +def remote_restart(rel_name, remote_service=None): + trigger = { + 'restart-trigger': str(uuid.uuid4()), + } + if remote_service: + trigger['remote-service'] = remote_service + for rid in relation_ids(rel_name): + # This subordinate can be related to two separate services using + # different subordinate relations so only issue the restart if + # the principle is connected down the relation we think it is + if related_units(relid=rid): + relation_set(relation_id=rid, + relation_settings=trigger, + ) + + +def check_actually_paused(services=None, ports=None): + """Check that services listed in the services object and ports + are actually closed (not listened to), to verify that the unit is + properly paused. + + @param services: See _extract_services_list_helper + @returns status, : string for status (None if okay) + message : string for problem for status_set + """ + state = None + message = None + messages = [] + if services is not None: + services = _extract_services_list_helper(services) + services_running, services_states = _check_running_services(services) + if any(services_states): + # there shouldn't be any running so this is a problem + messages.append("these services running: {}" + .format(", ".join( + _filter_tuples(services_running, True)))) + state = "blocked" + ports_open, ports_open_bools = ( + _check_listening_on_services_ports(services, True)) + if any(ports_open_bools): + message_parts = {service: ", ".join([str(v) for v in open_ports]) + for service, open_ports in ports_open.items()} + message = ", ".join( + ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) + messages.append( + "these service:ports are open: {}".format(message)) + state = 'blocked' + if ports is not None: + ports_open, bools = _check_listening_on_ports_list(ports) + if any(bools): + messages.append( + "these ports which should be closed, but are open: {}" + .format(", ".join([str(p) for p, v in ports_open if v]))) + state = 'blocked' + if messages: + message = ("Services should be paused but {}" + .format(", ".join(messages))) + return state, message + + +def set_unit_paused(): + """Set the unit to a paused state in the local kv() store. + This does NOT actually pause the unit + """ + with unitdata.HookData()() as t: + kv = t[0] + kv.set('unit-paused', True) + + +def clear_unit_paused(): + """Clear the unit from a paused state in the local kv() store + This does NOT actually restart any services - it only clears the + local state. + """ + with unitdata.HookData()() as t: + kv = t[0] + kv.set('unit-paused', False) + + +def is_unit_paused_set(): + """Return the state of the kv().get('unit-paused'). + This does NOT verify that the unit really is paused. + + To help with units that don't have HookData() (testing) + if it excepts, return False + """ + try: + with unitdata.HookData()() as t: + kv = t[0] + # transform something truth-y into a Boolean. + return not(not(kv.get('unit-paused'))) + except Exception: + return False + + +def is_hook_allowed(hookname, check_deferred_restarts=True): + """Check if hook can run. + + :param hookname: Name of hook to check.. + :type hookname: str + :param check_deferred_restarts: Whether to check deferred restarts. + :type check_deferred_restarts: bool + """ + permitted = True + reasons = [] + if is_unit_paused_set(): + reasons.append( + "Unit is pause or upgrading. Skipping {}".format(hookname)) + permitted = False + + if check_deferred_restarts: + if deferred_events.is_restart_permitted(): + permitted = True + deferred_events.clear_deferred_hook(hookname) + else: + if not config().changed('enable-auto-restarts'): + deferred_events.set_deferred_hook(hookname) + reasons.append("auto restarts are disabled") + permitted = False + return permitted, " and ".join(reasons) + + +def manage_payload_services(action, services=None, charm_func=None): + """Run an action against all services. + + An optional charm_func() can be called. It should raise an Exception to + indicate that the function failed. If it was successful it should return + None or an optional message. + + The signature for charm_func is: + charm_func() -> message: str + + charm_func() is executed after any services are stopped, if supplied. + + The services object can either be: + - None : no services were passed (an empty dict is returned) + - a list of strings + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} + - An array of [{'service': service_name, ...}, ...] + + :param action: Action to run: pause, resume, start or stop. + :type action: str + :param services: See above + :type services: See above + :param charm_func: function to run for custom charm pausing. + :type charm_func: f() + :returns: Status boolean and list of messages + :rtype: (bool, []) + :raises: RuntimeError + """ + actions = { + 'pause': service_pause, + 'resume': service_resume, + 'start': service_start, + 'stop': service_stop} + action = action.lower() + if action not in actions.keys(): + raise RuntimeError( + "action: {} must be one of: {}".format(action, + ', '.join(actions.keys()))) + services = _extract_services_list_helper(services) + messages = [] + success = True + if services: + for service in services.keys(): + rc = actions[action](service) + if not rc: + success = False + messages.append("{} didn't {} cleanly.".format(service, + action)) + if charm_func: + try: + message = charm_func() + if message: + messages.append(message) + except Exception as e: + success = False + messages.append(str(e)) + return success, messages + + +def make_wait_for_ports_barrier(ports, retry_count=5): + """Make a function to wait for port shutdowns. + + Create a function which closes over the provided ports. The function will + retry probing ports until they are closed or the retry count has been reached. + + """ + @decorators.retry_on_predicate(retry_count, operator.not_, base_delay=0.1) + def retry_port_check(): + _, ports_states = _check_listening_on_ports_list(ports) + juju_log("Probe ports {}, result: {}".format(ports, ports_states), level="DEBUG") + return any(ports_states) + return retry_port_check + + +def pause_unit(assess_status_func, services=None, ports=None, + charm_func=None): + """Pause a unit by stopping the services and setting 'unit-paused' + in the local kv() store. + + Also checks that the services have stopped and ports are no longer + being listened to. + + An optional charm_func() can be called that can either raise an + Exception or return non None, None to indicate that the unit + didn't pause cleanly. + + The signature for charm_func is: + charm_func() -> message: string + + charm_func() is executed after any services are stopped, if supplied. + + The services object can either be: + - None : no services were passed (an empty dict is returned) + - a list of strings + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} + - An array of [{'service': service_name, ...}, ...] + + @param assess_status_func: (f() -> message: string | None) or None + @param services: OPTIONAL see above + @param ports: OPTIONAL list of port + @param charm_func: function to run for custom charm pausing. + @returns None + @raises Exception(message) on an error for action_fail(). + """ + _, messages = manage_payload_services( + 'pause', + services=services, + charm_func=charm_func) + set_unit_paused() + + if assess_status_func: + message = assess_status_func() + if message: + messages.append(message) + if messages and not is_unit_upgrading_set(): + raise Exception("Couldn't pause: {}".format("; ".join(messages))) + + +def resume_unit(assess_status_func, services=None, ports=None, + charm_func=None): + """Resume a unit by starting the services and clearning 'unit-paused' + in the local kv() store. + + Also checks that the services have started and ports are being listened to. + + An optional charm_func() can be called that can either raise an + Exception or return non None to indicate that the unit + didn't resume cleanly. + + The signature for charm_func is: + charm_func() -> message: string + + charm_func() is executed after any services are started, if supplied. + + The services object can either be: + - None : no services were passed (an empty dict is returned) + - a list of strings + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} + - An array of [{'service': service_name, ...}, ...] + + @param assess_status_func: (f() -> message: string | None) or None + @param services: OPTIONAL see above + @param ports: OPTIONAL list of port + @param charm_func: function to run for custom charm resuming. + @returns None + @raises Exception(message) on an error for action_fail(). + """ + _, messages = manage_payload_services( + 'resume', + services=services, + charm_func=charm_func) + clear_unit_paused() + if assess_status_func: + message = assess_status_func() + if message: + messages.append(message) + if messages: + raise Exception("Couldn't resume: {}".format("; ".join(messages))) + + +def restart_services_action(services=None, when_all_stopped_func=None, + deferred_only=None): + """Manage a service restart request via charm action. + + :param services: Services to be restarted + :type model_name: List[str] + :param when_all_stopped_func: Function to call when all services are + stopped. + :type when_all_stopped_func: Callable[] + :param model_name: Only restart services which have a deferred restart + event. + :type model_name: bool + """ + if services and deferred_only: + raise ValueError( + "services and deferred_only are mutually exclusive") + if deferred_only: + services = list(set( + [a.service for a in deferred_events.get_deferred_restarts()])) + _, messages = manage_payload_services( + 'stop', + services=services, + charm_func=when_all_stopped_func) + if messages: + raise ServiceActionError( + "Error processing service stop request: {}".format( + "; ".join(messages))) + _, messages = manage_payload_services( + 'start', + services=services) + if messages: + raise ServiceActionError( + "Error processing service start request: {}".format( + "; ".join(messages))) + deferred_events.clear_deferred_restarts(services) + + +def make_assess_status_func(*args, **kwargs): + """Creates an assess_status_func() suitable for handing to pause_unit() + and resume_unit(). + + This uses the _determine_os_workload_status(...) function to determine + what the workload_status should be for the unit. If the unit is + not in maintenance or active states, then the message is returned to + the caller. This is so an action that doesn't result in either a + complete pause or complete resume can signal failure with an action_fail() + """ + def _assess_status_func(): + state, message = _determine_os_workload_status(*args, **kwargs) + status_set(state, message) + if state not in ['maintenance', 'active']: + return message + return None + + return _assess_status_func + + +def pausable_restart_on_change(restart_map, stopstart=False, + restart_functions=None, + can_restart_now_f=None, + post_svc_restart_f=None, + pre_restarts_wait_f=None): + """A restart_on_change decorator that checks to see if the unit is + paused. If it is paused then the decorated function doesn't fire. + + This is provided as a helper, as the @restart_on_change(...) decorator + is in core.host, yet the openstack specific helpers are in this file + (contrib.openstack.utils). Thus, this needs to be an optional feature + for openstack charms (or charms that wish to use the openstack + pause/resume type features). + + It is used as follows: + + from contrib.openstack.utils import ( + pausable_restart_on_change as restart_on_change) + + @restart_on_change(restart_map, stopstart=) + def some_hook(...): + pass + + see core.utils.restart_on_change() for more details. + + Note restart_map can be a callable, in which case, restart_map is only + evaluated at runtime. This means that it is lazy and the underlying + function won't be called if the decorated function is never called. Note, + retains backwards compatibility for passing a non-callable dictionary. + + :param f: function to decorate. + :type f: Callable + :param restart_map: Optionally callable, which then returns the restart_map or + the restart map {conf_file: [services]} + :type restart_map: Union[Callable[[],], Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function called before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + :returns: decorator to use a restart_on_change with pausability + :rtype: decorator + + + """ + def wrap(f): + # py27 compatible nonlocal variable. When py3 only, replace with + # nonlocal keyword + __restart_map_cache = {'cache': None} + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + if is_unit_paused_set(): + return f(*args, **kwargs) + if __restart_map_cache['cache'] is None: + __restart_map_cache['cache'] = restart_map() \ + if callable(restart_map) else restart_map + # otherwise, normal restart_on_change functionality + return restart_on_change_helper( + (lambda: f(*args, **kwargs)), + __restart_map_cache['cache'], + stopstart, + restart_functions, + can_restart_now_f, + post_svc_restart_f, + pre_restarts_wait_f) + return wrapped_f + return wrap + + +def ordered(orderme): + """Converts the provided dictionary into a collections.OrderedDict. + + The items in the returned OrderedDict will be inserted based on the + natural sort order of the keys. Nested dictionaries will also be sorted + in order to ensure fully predictable ordering. + + :param orderme: the dict to order + :return: collections.OrderedDict + :raises: ValueError: if `orderme` isn't a dict instance. + """ + if not isinstance(orderme, dict): + raise ValueError('argument must be a dict type') + + result = OrderedDict() + for k, v in sorted(six.iteritems(orderme), key=lambda x: x[0]): + if isinstance(v, dict): + result[k] = ordered(v) + else: + result[k] = v + + return result + + +def config_flags_parser(config_flags): + """Parses config flags string into dict. + + This parsing method supports a few different formats for the config + flag values to be parsed: + + 1. A string in the simple format of key=value pairs, with the possibility + of specifying multiple key value pairs within the same string. For + example, a string in the format of 'key1=value1, key2=value2' will + return a dict of: + + {'key1': 'value1', 'key2': 'value2'}. + + 2. A string in the above format, but supporting a comma-delimited list + of values for the same key. For example, a string in the format of + 'key1=value1, key2=value3,value4,value5' will return a dict of: + + {'key1': 'value1', 'key2': 'value2,value3,value4'} + + 3. A string containing a colon character (:) prior to an equal + character (=) will be treated as yaml and parsed as such. This can be + used to specify more complex key value pairs. For example, + a string in the format of 'key1: subkey1=value1, subkey2=value2' will + return a dict of: + + {'key1', 'subkey1=value1, subkey2=value2'} + + The provided config_flags string may be a list of comma-separated values + which themselves may be comma-separated list of values. + """ + # If we find a colon before an equals sign then treat it as yaml. + # Note: limit it to finding the colon first since this indicates assignment + # for inline yaml. + colon = config_flags.find(':') + equals = config_flags.find('=') + if colon > 0: + if colon < equals or equals < 0: + return ordered(yaml.safe_load(config_flags)) + + if config_flags.find('==') >= 0: + juju_log("config_flags is not in expected format (key=value)", + level=ERROR) + raise OSContextError + + # strip the following from each value. + post_strippers = ' ,' + # we strip any leading/trailing '=' or ' ' from the string then + # split on '='. + split = config_flags.strip(' =').split('=') + limit = len(split) + flags = OrderedDict() + for i in range(0, limit - 1): + current = split[i] + next = split[i + 1] + vindex = next.rfind(',') + if (i == limit - 2) or (vindex < 0): + value = next + else: + value = next[:vindex] + + if i == 0: + key = current + else: + # if this not the first entry, expect an embedded key. + index = current.rfind(',') + if index < 0: + juju_log("Invalid config value(s) at index %s" % (i), + level=ERROR) + raise OSContextError + key = current[index + 1:] + + # Add to collection. + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) + + return flags + + +def os_application_version_set(package): + '''Set version of application for Juju 2.0 and later''' + application_version = get_upstream_version(package) + # NOTE(jamespage) if not able to figure out package version, fallback to + # openstack codename version detection. + if not application_version: + application_version_set(os_release(package)) + else: + application_version_set(application_version) + + +def os_application_status_set(check_function): + """Run the supplied function and set the application status accordingly. + + :param check_function: Function to run to get app states and messages. + :type check_function: function + """ + state, message = check_function() + status_set(state, message, application=True) + + +def enable_memcache(source=None, release=None, package=None): + """Determine if memcache should be enabled on the local unit + + @param release: release of OpenStack currently deployed + @param package: package to derive OpenStack version deployed + @returns boolean Whether memcache should be enabled + """ + _release = None + if release: + _release = release + else: + _release = os_release(package) + if not _release: + _release = get_os_codename_install_source(source) + + return CompareOpenStackReleases(_release) >= 'mitaka' + + +def token_cache_pkgs(source=None, release=None): + """Determine additional packages needed for token caching + + @param source: source string for charm + @param release: release of OpenStack currently deployed + @returns List of package to enable token caching + """ + packages = [] + if enable_memcache(source=source, release=release): + packages.extend(['memcached', 'python-memcache']) + return packages + + +def update_json_file(filename, items): + """Updates the json `filename` with a given dict. + :param filename: path to json file (e.g. /etc/glance/policy.json) + :param items: dict of items to update + """ + if not items: + return + + with open(filename) as fd: + policy = json.load(fd) + + # Compare before and after and if nothing has changed don't write the file + # since that could cause unnecessary service restarts. + before = json.dumps(policy, indent=4, sort_keys=True) + policy.update(items) + after = json.dumps(policy, indent=4, sort_keys=True) + if before == after: + return + + with open(filename, "w") as fd: + fd.write(after) + + +@cached +def snap_install_requested(): + """ Determine if installing from snaps + + If openstack-origin is of the form snap:track/channel[/branch] + and channel is in SNAPS_CHANNELS return True. + """ + origin = config('openstack-origin') or "" + if not origin.startswith('snap:'): + return False + + _src = origin[5:] + if '/' in _src: + channel = _src.split('/')[1] + else: + # Handle snap:track with no channel + channel = 'stable' + return valid_snap_channel(channel) + + +def get_snaps_install_info_from_origin(snaps, src, mode='classic'): + """Generate a dictionary of snap install information from origin + + @param snaps: List of snaps + @param src: String of openstack-origin or source of the form + snap:track/channel + @param mode: String classic, devmode or jailmode + @returns: Dictionary of snaps with channels and modes + """ + + if not src.startswith('snap:'): + juju_log("Snap source is not a snap origin", 'WARN') + return {} + + _src = src[5:] + channel = '--channel={}'.format(_src) + + return {snap: {'channel': channel, 'mode': mode} + for snap in snaps} + + +def install_os_snaps(snaps, refresh=False): + """Install OpenStack snaps from channel and with mode + + @param snaps: Dictionary of snaps with channels and modes of the form: + {'snap_name': {'channel': 'snap_channel', + 'mode': 'snap_mode'}} + Where channel is a snapstore channel and mode is --classic, --devmode + or --jailmode. + @param post_snap_install: Callback function to run after snaps have been + installed + """ + + def _ensure_flag(flag): + if flag.startswith('--'): + return flag + return '--{}'.format(flag) + + if refresh: + for snap in snaps.keys(): + snap_refresh(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + else: + for snap in snaps.keys(): + snap_install(snap, + _ensure_flag(snaps[snap]['channel']), + _ensure_flag(snaps[snap]['mode'])) + + +def set_unit_upgrading(): + """Set the unit to a upgrading state in the local kv() store. + """ + with unitdata.HookData()() as t: + kv = t[0] + kv.set('unit-upgrading', True) + + +def clear_unit_upgrading(): + """Clear the unit from a upgrading state in the local kv() store + """ + with unitdata.HookData()() as t: + kv = t[0] + kv.set('unit-upgrading', False) + + +def is_unit_upgrading_set(): + """Return the state of the kv().get('unit-upgrading'). + + To help with units that don't have HookData() (testing) + if it excepts, return False + """ + try: + with unitdata.HookData()() as t: + kv = t[0] + # transform something truth-y into a Boolean. + return not(not(kv.get('unit-upgrading'))) + except Exception: + return False + + +def series_upgrade_prepare(pause_unit_helper=None, configs=None): + """ Run common series upgrade prepare tasks. + + :param pause_unit_helper: function: Function to pause unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + set_unit_upgrading() + if pause_unit_helper and configs: + if not is_unit_paused_set(): + pause_unit_helper(configs) + + +def series_upgrade_complete(resume_unit_helper=None, configs=None): + """ Run common series upgrade complete tasks. + + :param resume_unit_helper: function: Function to resume unit + :param configs: OSConfigRenderer object: Configurations + :returns None: + """ + clear_unit_paused() + clear_unit_upgrading() + if configs: + configs.write_all() + if resume_unit_helper: + resume_unit_helper(configs) + + +def is_db_initialised(): + """Check leader storage to see if database has been initialised. + + :returns: Whether DB has been initialised + :rtype: bool + """ + db_initialised = None + if leader_get('db-initialised') is None: + juju_log( + 'db-initialised key missing, assuming db is not initialised', + 'DEBUG') + db_initialised = False + else: + db_initialised = bool_from_string(leader_get('db-initialised')) + juju_log('Database initialised: {}'.format(db_initialised), 'DEBUG') + return db_initialised + + +def set_db_initialised(): + """Add flag to leader storage to indicate database has been initialised. + """ + juju_log('Setting db-initialised to True', 'DEBUG') + leader_set({'db-initialised': True}) + + +def is_db_maintenance_mode(relid=None): + """Check relation data from notifications of db in maintenance mode. + + :returns: Whether db has notified it is in maintenance mode. + :rtype: bool + """ + juju_log('Checking for maintenance notifications', 'DEBUG') + if relid: + r_ids = [relid] + else: + r_ids = relation_ids('shared-db') + rids_units = [(r, u) for r in r_ids for u in related_units(r)] + notifications = [] + for r_id, unit in rids_units: + settings = relation_get(unit=unit, rid=r_id) + for key, value in settings.items(): + if value and key in DB_MAINTENANCE_KEYS: + juju_log( + 'Unit: {}, Key: {}, Value: {}'.format(unit, key, value), + 'DEBUG') + try: + notifications.append(bool_from_string(value)) + except ValueError: + juju_log( + 'Could not discern bool from {}'.format(value), + 'WARN') + pass + return True in notifications + + +@cached +def container_scoped_relations(): + """Get all the container scoped relations + + :returns: List of relation names + :rtype: List + """ + md = metadata() + relations = [] + for relation_type in ('provides', 'requires', 'peers'): + for relation in md.get(relation_type, []): + if md[relation_type][relation].get('scope') == 'container': + relations.append(relation) + return relations + + +def container_scoped_relation_get(attribute=None): + """Get relation data from all container scoped relations. + + :param attribute: Name of attribute to get + :type attribute: Optional[str] + :returns: Iterator with relation data + :rtype: Iterator[Optional[any]] + """ + for endpoint_name in container_scoped_relations(): + for rid in relation_ids(endpoint_name): + for unit in related_units(rid): + yield relation_get( + attribute=attribute, + unit=unit, + rid=rid) + + +def is_db_ready(use_current_context=False, rel_name=None): + """Check remote database is ready to be used. + + Database relations are expected to provide a list of 'allowed' units to + confirm that the database is ready for use by those units. + + If db relation has provided this information and local unit is a member, + returns True otherwise False. + + :param use_current_context: Whether to limit checks to current hook + context. + :type use_current_context: bool + :param rel_name: Name of relation to check + :type rel_name: string + :returns: Whether remote db is ready. + :rtype: bool + :raises: Exception + """ + key = 'allowed_units' + + rel_name = rel_name or 'shared-db' + this_unit = local_unit() + + if use_current_context: + if relation_id() in relation_ids(rel_name): + rids_units = [(None, None)] + else: + raise Exception("use_current_context=True but not in {} " + "rel hook contexts (currently in {})." + .format(rel_name, relation_id())) + else: + rids_units = [(r_id, u) + for r_id in relation_ids(rel_name) + for u in related_units(r_id)] + + for rid, unit in rids_units: + allowed_units = relation_get(rid=rid, unit=unit, attribute=key) + if allowed_units and this_unit in allowed_units.split(): + juju_log("This unit ({}) is in allowed unit list from {}".format( + this_unit, + unit), 'DEBUG') + return True + + juju_log("This unit was not found in any allowed unit list") + return False + + +def is_expected_scale(peer_relation_name='cluster'): + """Query juju goal-state to determine whether our peer- and dependency- + relations are at the expected scale. + + Useful for deferring per unit per relation housekeeping work until we are + ready to complete it successfully and without unnecessary repetiton. + + Always returns True if version of juju used does not support goal-state. + + :param peer_relation_name: Name of peer relation + :type rel_name: string + :returns: True or False + :rtype: bool + """ + def _get_relation_id(rel_type): + return next((rid for rid in relation_ids(reltype=rel_type)), None) + + Relation = namedtuple('Relation', 'rel_type rel_id') + peer_rid = _get_relation_id(peer_relation_name) + # Units with no peers should still have a peer relation. + if not peer_rid: + juju_log('Not at expected scale, no peer relation found', 'DEBUG') + return False + expected_relations = [ + Relation(rel_type='shared-db', rel_id=_get_relation_id('shared-db'))] + if expect_ha(): + expected_relations.append( + Relation( + rel_type='ha', + rel_id=_get_relation_id('ha'))) + juju_log( + 'Checking scale of {} relations'.format( + ','.join([r.rel_type for r in expected_relations])), + 'DEBUG') + try: + if (len(related_units(relid=peer_rid)) < + len(list(expected_peer_units()))): + return False + for rel in expected_relations: + if not rel.rel_id: + juju_log( + 'Expected to find {} relation, but it is missing'.format( + rel.rel_type), + 'DEBUG') + return False + # Goal state returns every unit even for container scoped + # relations but the charm only ever has a relation with + # the local unit. + if rel.rel_type in container_scoped_relations(): + expected_count = 1 + else: + expected_count = len( + list(expected_related_units(reltype=rel.rel_type))) + if len(related_units(relid=rel.rel_id)) < expected_count: + juju_log( + ('Not at expected scale, not enough units on {} ' + 'relation'.format(rel.rel_type)), + 'DEBUG') + return False + except NotImplementedError: + return True + juju_log('All checks have passed, unit is at expected scale', 'DEBUG') + return True + + +def get_peer_key(unit_name): + """Get the peer key for this unit. + + The peer key is the key a unit uses to publish its status down the peer + relation + + :param unit_name: Name of unit + :type unit_name: string + :returns: Peer key for given unit + :rtype: string + """ + return 'unit-state-{}'.format(unit_name.replace('/', '-')) + + +UNIT_READY = 'READY' +UNIT_NOTREADY = 'NOTREADY' +UNIT_UNKNOWN = 'UNKNOWN' +UNIT_STATES = [UNIT_READY, UNIT_NOTREADY, UNIT_UNKNOWN] + + +def inform_peers_unit_state(state, relation_name='cluster'): + """Inform peers of the state of this unit. + + :param state: State of unit to publish + :type state: string + :param relation_name: Name of relation to publish state on + :type relation_name: string + """ + if state not in UNIT_STATES: + raise ValueError( + "Setting invalid state {} for unit".format(state)) + this_unit = local_unit() + for r_id in relation_ids(relation_name): + juju_log('Telling peer behind relation {} that {} is {}'.format( + r_id, this_unit, state), 'DEBUG') + relation_set(relation_id=r_id, + relation_settings={ + get_peer_key(this_unit): state}) + + +def get_peers_unit_state(relation_name='cluster'): + """Get the state of all peers. + + :param relation_name: Name of relation to check peers on. + :type relation_name: string + :returns: Unit states keyed on unit name. + :rtype: dict + :raises: ValueError + """ + r_ids = relation_ids(relation_name) + rids_units = [(r, u) for r in r_ids for u in related_units(r)] + unit_states = {} + for r_id, unit in rids_units: + settings = relation_get(unit=unit, rid=r_id) + unit_states[unit] = settings.get(get_peer_key(unit), UNIT_UNKNOWN) + if unit_states[unit] not in UNIT_STATES: + raise ValueError( + "Unit in unknown state {}".format(unit_states[unit])) + return unit_states + + +def are_peers_ready(relation_name='cluster'): + """Check if all peers are ready. + + :param relation_name: Name of relation to check peers on. + :type relation_name: string + :returns: Whether all units are ready. + :rtype: bool + """ + unit_states = get_peers_unit_state(relation_name).values() + juju_log('{} peers are in the following states: {}'.format( + relation_name, unit_states), 'DEBUG') + return all(state == UNIT_READY for state in unit_states) + + +def inform_peers_if_ready(check_unit_ready_func, relation_name='cluster'): + """Inform peers if this unit is ready. + + The check function should return a tuple (state, message). A state + of 'READY' indicates the unit is READY. + + :param check_unit_ready_func: Function to run to check readiness + :type check_unit_ready_func: function + :param relation_name: Name of relation to check peers on. + :type relation_name: string + """ + unit_ready, msg = check_unit_ready_func() + if unit_ready: + state = UNIT_READY + else: + state = UNIT_NOTREADY + juju_log('Telling peers this unit is: {}'.format(state), 'DEBUG') + inform_peers_unit_state(state, relation_name) + + +def check_api_unit_ready(check_db_ready=True): + """Check if this unit is ready. + + :param check_db_ready: Include checks of database readiness. + :type check_db_ready: bool + :returns: Whether unit state is ready and status message + :rtype: (bool, str) + """ + unit_state, msg = get_api_unit_status(check_db_ready=check_db_ready) + return unit_state == WORKLOAD_STATES.ACTIVE, msg + + +def get_api_unit_status(check_db_ready=True): + """Return a workload status and message for this unit. + + :param check_db_ready: Include checks of database readiness. + :type check_db_ready: bool + :returns: Workload state and message + :rtype: (bool, str) + """ + unit_state = WORKLOAD_STATES.ACTIVE + msg = 'Unit is ready' + if is_db_maintenance_mode(): + unit_state = WORKLOAD_STATES.MAINTENANCE + msg = 'Database in maintenance mode.' + elif is_unit_paused_set(): + unit_state = WORKLOAD_STATES.BLOCKED + msg = 'Unit paused.' + elif check_db_ready and not is_db_ready(): + unit_state = WORKLOAD_STATES.WAITING + msg = 'Allowed_units list provided but this unit not present' + elif not is_db_initialised(): + unit_state = WORKLOAD_STATES.WAITING + msg = 'Database not initialised' + elif not is_expected_scale(): + unit_state = WORKLOAD_STATES.WAITING + msg = 'Charm and its dependencies not yet at expected scale' + juju_log(msg, 'DEBUG') + return unit_state, msg + + +def check_api_application_ready(): + """Check if this application is ready. + + :returns: Whether application state is ready and status message + :rtype: (bool, str) + """ + app_state, msg = get_api_application_status() + return app_state == WORKLOAD_STATES.ACTIVE, msg + + +def get_api_application_status(): + """Return a workload status and message for this application. + + :returns: Workload state and message + :rtype: (bool, str) + """ + app_state, msg = get_api_unit_status() + if app_state == WORKLOAD_STATES.ACTIVE: + if are_peers_ready(): + msg = 'Application Ready' + else: + app_state = WORKLOAD_STATES.WAITING + msg = 'Some units are not ready' + juju_log(msg, 'DEBUG') + return app_state, msg + + +def sequence_status_check_functions(*functions): + """Sequence the functions passed so that they all get a chance to run as + the charm status check functions. + + :param *functions: a list of functions that return (state, message) + :type *functions: List[Callable[[OSConfigRender], (str, str)]] + :returns: the Callable that takes configs and returns (state, message) + :rtype: Callable[[OSConfigRender], (str, str)] + """ + def _inner_sequenced_functions(configs): + state, message = 'unknown', '' + for f in functions: + new_state, new_message = f(configs) + state = workload_state_compare(state, new_state) + if message: + message = "{}, {}".format(message, new_message) + else: + message = new_message + return state, message + + return _inner_sequenced_functions + + +SubordinatePackages = namedtuple('SubordinatePackages', ['install', 'purge']) + + +def get_subordinate_release_packages(os_release, package_type='deb'): + """Iterate over subordinate relations and get package information. + + :param os_release: OpenStack release to look for + :type os_release: str + :param package_type: Package type (one of 'deb' or 'snap') + :type package_type: str + :returns: Packages to install and packages to purge or None + :rtype: SubordinatePackages[set,set] + """ + install = set() + purge = set() + + for rdata in container_scoped_relation_get('releases-packages-map'): + rp_map = json.loads(rdata or '{}') + # The map provided by subordinate has OpenStack release name as key. + # Find package information from subordinate matching requested release + # or the most recent release prior to requested release by sorting the + # keys in reverse order. This follows established patterns in our + # charms for templates and reactive charm implementations, i.e. as long + # as nothing has changed the definitions for the prior OpenStack + # release is still valid. + for release in sorted(rp_map.keys(), reverse=True): + if (CompareOpenStackReleases(release) <= os_release and + package_type in rp_map[release]): + for name, container in ( + ('install', install), + ('purge', purge)): + for pkg in rp_map[release][package_type].get(name, []): + container.add(pkg) + break + return SubordinatePackages(install, purge) + + +def get_subordinate_services(): + """Iterate over subordinate relations and get service information. + + In a similar fashion as with get_subordinate_release_packages(), + principle charms can retrieve a list of services advertised by their + subordinate charms. This is useful to know about subordinate services when + pausing, resuming or upgrading a principle unit. + + :returns: Name of all services advertised by all subordinates + :rtype: Set[str] + """ + services = set() + for rdata in container_scoped_relation_get('services'): + services |= set(json.loads(rdata or '[]')) + return services + + +os_restart_on_change = partial( + pausable_restart_on_change, + can_restart_now_f=deferred_events.check_and_record_restart_request, + post_svc_restart_f=deferred_events.process_svc_restart) + + +def restart_services_action_helper(all_services): + """Helper to run the restart-services action. + + NOTE: all_services is all services that could be restarted but + depending on the action arguments it may be a subset of + these that are actually restarted. + + :param all_services: All services that could be restarted + :type all_services: List[str] + """ + deferred_only = action_get("deferred-only") + services = action_get("services") + if services: + services = services.split() + else: + services = all_services + if deferred_only: + restart_services_action(deferred_only=True) + else: + restart_services_action(services=services) + + +def show_deferred_events_action_helper(): + """Helper to run the show-deferred-restarts action.""" + restarts = [] + for event in deferred_events.get_deferred_events(): + restarts.append('{} {} {}'.format( + str(event.timestamp), + event.service.ljust(40), + event.reason)) + restarts.sort() + output = { + 'restarts': restarts, + 'hooks': deferred_events.get_deferred_hooks()} + action_set({'output': "{}".format( + yaml.dump(output, default_flow_style=False))}) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/vaultlocker.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/vaultlocker.py new file mode 100644 index 0000000..e5418c3 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/openstack/vaultlocker.py @@ -0,0 +1,179 @@ +# Copyright 2018-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os + +import charmhelpers.contrib.openstack.alternatives as alternatives +import charmhelpers.contrib.openstack.context as context + +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as host +import charmhelpers.core.templating as templating +import charmhelpers.core.unitdata as unitdata + +VAULTLOCKER_BACKEND = 'charm-vaultlocker' + + +class VaultKVContext(context.OSContextGenerator): + """Vault KV context for interaction with vault-kv interfaces""" + interfaces = ['secrets-storage'] + + def __init__(self, secret_backend=None): + super(context.OSContextGenerator, self).__init__() + self.secret_backend = ( + secret_backend or 'charm-{}'.format(hookenv.service_name()) + ) + + def __call__(self): + try: + import hvac + except ImportError: + # BUG: #1862085 - if the relation is made to vault, but the + # 'encrypt' option is not made, then the charm errors with an + # import warning. This catches that, logs a warning, and returns + # with an empty context. + hookenv.log("VaultKVContext: trying to use hvac pythong module " + "but it's not available. Is secrets-stroage relation " + "made, but encrypt option not set?", + level=hookenv.WARNING) + # return an empty context on hvac import error + return {} + ctxt = {} + # NOTE(hopem): see https://bugs.launchpad.net/charm-helpers/+bug/1849323 + db = unitdata.kv() + # currently known-good secret-id + secret_id = db.get('secret-id') + + for relation_id in hookenv.relation_ids(self.interfaces[0]): + for unit in hookenv.related_units(relation_id): + data = hookenv.relation_get(unit=unit, + rid=relation_id) + vault_url = data.get('vault_url') + role_id = data.get('{}_role_id'.format(hookenv.local_unit())) + token = data.get('{}_token'.format(hookenv.local_unit())) + + if all([vault_url, role_id, token]): + token = json.loads(token) + vault_url = json.loads(vault_url) + + # Tokens may change when secret_id's are being + # reissued - if so use token to get new secret_id + token_success = False + try: + secret_id = retrieve_secret_id( + url=vault_url, + token=token + ) + token_success = True + except hvac.exceptions.InvalidRequest: + # Try next + pass + + if token_success: + db.set('secret-id', secret_id) + db.flush() + + ctxt['vault_url'] = vault_url + ctxt['role_id'] = json.loads(role_id) + ctxt['secret_id'] = secret_id + ctxt['secret_backend'] = self.secret_backend + vault_ca = data.get('vault_ca') + if vault_ca: + ctxt['vault_ca'] = json.loads(vault_ca) + + self.complete = True + break + else: + if secret_id: + ctxt['vault_url'] = vault_url + ctxt['role_id'] = json.loads(role_id) + ctxt['secret_id'] = secret_id + ctxt['secret_backend'] = self.secret_backend + vault_ca = data.get('vault_ca') + if vault_ca: + ctxt['vault_ca'] = json.loads(vault_ca) + + if self.complete: + break + + if ctxt: + self.complete = True + + return ctxt + + +def write_vaultlocker_conf(context, priority=100): + """Write vaultlocker configuration to disk and install alternative + + :param context: Dict of data from vault-kv relation + :ptype: context: dict + :param priority: Priority of alternative configuration + :ptype: priority: int""" + charm_vl_path = "/var/lib/charm/{}/vaultlocker.conf".format( + hookenv.service_name() + ) + host.mkdir(os.path.dirname(charm_vl_path), perms=0o700) + templating.render(source='vaultlocker.conf.j2', + target=charm_vl_path, + context=context, perms=0o600), + alternatives.install_alternative('vaultlocker.conf', + '/etc/vaultlocker/vaultlocker.conf', + charm_vl_path, priority) + + +def vault_relation_complete(backend=None): + """Determine whether vault relation is complete + + :param backend: Name of secrets backend requested + :ptype backend: string + :returns: whether the relation to vault is complete + :rtype: bool""" + try: + import hvac + except ImportError: + return False + try: + vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND) + vault_kv() + return vault_kv.complete + except hvac.exceptions.InvalidRequest: + return False + + +# TODO: contrib a high level unwrap method to hvac that works +def retrieve_secret_id(url, token): + """Retrieve a response-wrapped secret_id from Vault + + :param url: URL to Vault Server + :ptype url: str + :param token: One shot Token to use + :ptype token: str + :returns: secret_id to use for Vault Access + :rtype: str""" + import hvac + try: + # hvac 0.10.1 changed default adapter to JSONAdapter + client = hvac.Client(url=url, token=token, adapter=hvac.adapters.Request) + except AttributeError: + # hvac < 0.6.2 doesn't have adapter but uses the same response interface + client = hvac.Client(url=url, token=token) + else: + # hvac < 0.9.2 assumes adapter is an instance, so doesn't instantiate + if not isinstance(client.adapter, hvac.adapters.Request): + client.adapter = hvac.adapters.Request(base_uri=url, token=token) + response = client._post('/v1/sys/wrapping/unwrap') + if response.status_code == 200: + data = response.json() + return data['data']['secret_id'] diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/peerstorage/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/peerstorage/__init__.py new file mode 100644 index 0000000..a8fa60c --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/peerstorage/__init__.py @@ -0,0 +1,267 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import six + +from charmhelpers.core.hookenv import relation_id as current_relation_id +from charmhelpers.core.hookenv import ( + is_relation_made, + relation_ids, + relation_get as _relation_get, + local_unit, + relation_set as _relation_set, + leader_get as _leader_get, + leader_set, + is_leader, +) + + +""" +This helper provides functions to support use of a peer relation +for basic key/value storage, with the added benefit that all storage +can be replicated across peer units. + +Requirement to use: + +To use this, the "peer_echo()" method has to be called form the peer +relation's relation-changed hook: + +@hooks.hook("cluster-relation-changed") # Adapt the to your peer relation name +def cluster_relation_changed(): + peer_echo() + +Once this is done, you can use peer storage from anywhere: + +@hooks.hook("some-hook") +def some_hook(): + # You can store and retrieve key/values this way: + if is_relation_made("cluster"): # from charmhelpers.core.hookenv + # There are peers available so we can work with peer storage + peer_store("mykey", "myvalue") + value = peer_retrieve("mykey") + print value + else: + print "No peers joind the relation, cannot share key/values :(" +""" + + +def leader_get(attribute=None, rid=None): + """Wrapper to ensure that settings are migrated from the peer relation. + + This is to support upgrading an environment that does not support + Juju leadership election to one that does. + + If a setting is not extant in the leader-get but is on the relation-get + peer rel, it is migrated and marked as such so that it is not re-migrated. + """ + migration_key = '__leader_get_migrated_settings__' + if not is_leader(): + return _leader_get(attribute=attribute) + + settings_migrated = False + leader_settings = _leader_get(attribute=attribute) + previously_migrated = _leader_get(attribute=migration_key) + + if previously_migrated: + migrated = set(json.loads(previously_migrated)) + else: + migrated = set([]) + + try: + if migration_key in leader_settings: + del leader_settings[migration_key] + except TypeError: + pass + + if attribute: + if attribute in migrated: + return leader_settings + + # If attribute not present in leader db, check if this unit has set + # the attribute in the peer relation + if not leader_settings: + peer_setting = _relation_get(attribute=attribute, unit=local_unit(), + rid=rid) + if peer_setting: + leader_set(settings={attribute: peer_setting}) + leader_settings = peer_setting + + if leader_settings: + settings_migrated = True + migrated.add(attribute) + else: + r_settings = _relation_get(unit=local_unit(), rid=rid) + if r_settings: + for key in set(r_settings.keys()).difference(migrated): + # Leader setting wins + if not leader_settings.get(key): + leader_settings[key] = r_settings[key] + + settings_migrated = True + migrated.add(key) + + if settings_migrated: + leader_set(**leader_settings) + + if migrated and settings_migrated: + migrated = json.dumps(list(migrated)) + leader_set(settings={migration_key: migrated}) + + return leader_settings + + +def relation_set(relation_id=None, relation_settings=None, **kwargs): + """Attempt to use leader-set if supported in the current version of Juju, + otherwise falls back on relation-set. + + Note that we only attempt to use leader-set if the provided relation_id is + a peer relation id or no relation id is provided (in which case we assume + we are within the peer relation context). + """ + try: + if relation_id in relation_ids('cluster'): + return leader_set(settings=relation_settings, **kwargs) + else: + raise NotImplementedError + except NotImplementedError: + return _relation_set(relation_id=relation_id, + relation_settings=relation_settings, **kwargs) + + +def relation_get(attribute=None, unit=None, rid=None): + """Attempt to use leader-get if supported in the current version of Juju, + otherwise falls back on relation-get. + + Note that we only attempt to use leader-get if the provided rid is a peer + relation id or no relation id is provided (in which case we assume we are + within the peer relation context). + """ + try: + if rid in relation_ids('cluster'): + return leader_get(attribute, rid) + else: + raise NotImplementedError + except NotImplementedError: + return _relation_get(attribute=attribute, rid=rid, unit=unit) + + +def peer_retrieve(key, relation_name='cluster'): + """Retrieve a named key from peer relation `relation_name`.""" + cluster_rels = relation_ids(relation_name) + if len(cluster_rels) > 0: + cluster_rid = cluster_rels[0] + return relation_get(attribute=key, rid=cluster_rid, + unit=local_unit()) + else: + raise ValueError('Unable to detect' + 'peer relation {}'.format(relation_name)) + + +def peer_retrieve_by_prefix(prefix, relation_name='cluster', delimiter='_', + inc_list=None, exc_list=None): + """ Retrieve k/v pairs given a prefix and filter using {inc,exc}_list """ + inc_list = inc_list if inc_list else [] + exc_list = exc_list if exc_list else [] + peerdb_settings = peer_retrieve('-', relation_name=relation_name) + matched = {} + if peerdb_settings is None: + return matched + for k, v in peerdb_settings.items(): + full_prefix = prefix + delimiter + if k.startswith(full_prefix): + new_key = k.replace(full_prefix, '') + if new_key in exc_list: + continue + if new_key in inc_list or len(inc_list) == 0: + matched[new_key] = v + return matched + + +def peer_store(key, value, relation_name='cluster'): + """Store the key/value pair on the named peer relation `relation_name`.""" + cluster_rels = relation_ids(relation_name) + if len(cluster_rels) > 0: + cluster_rid = cluster_rels[0] + relation_set(relation_id=cluster_rid, + relation_settings={key: value}) + else: + raise ValueError('Unable to detect ' + 'peer relation {}'.format(relation_name)) + + +def peer_echo(includes=None, force=False): + """Echo filtered attributes back onto the same relation for storage. + + This is a requirement to use the peerstorage module - it needs to be called + from the peer relation's changed hook. + + If Juju leader support exists this will be a noop unless force is True. + """ + try: + is_leader() + except NotImplementedError: + pass + else: + if not force: + return # NOOP if leader-election is supported + + # Use original non-leader calls + relation_get = _relation_get + relation_set = _relation_set + + rdata = relation_get() + echo_data = {} + if includes is None: + echo_data = rdata.copy() + for ex in ['private-address', 'public-address']: + if ex in echo_data: + echo_data.pop(ex) + else: + for attribute, value in six.iteritems(rdata): + for include in includes: + if include in attribute: + echo_data[attribute] = value + if len(echo_data) > 0: + relation_set(relation_settings=echo_data) + + +def peer_store_and_set(relation_id=None, peer_relation_name='cluster', + peer_store_fatal=False, relation_settings=None, + delimiter='_', **kwargs): + """Store passed-in arguments both in argument relation and in peer storage. + + It functions like doing relation_set() and peer_store() at the same time, + with the same data. + + @param relation_id: the id of the relation to store the data on. Defaults + to the current relation. + @param peer_store_fatal: Set to True, the function will raise an exception + should the peer storage not be available.""" + + relation_settings = relation_settings if relation_settings else {} + relation_set(relation_id=relation_id, + relation_settings=relation_settings, + **kwargs) + if is_relation_made(peer_relation_name): + for key, value in six.iteritems(dict(list(kwargs.items()) + + list(relation_settings.items()))): + key_prefix = relation_id or current_relation_id() + peer_store(key_prefix + delimiter + key, + value, + relation_name=peer_relation_name) + else: + if peer_store_fatal: + raise ValueError('Unable to detect ' + 'peer relation {}'.format(peer_relation_name)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/python.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/python.py new file mode 100644 index 0000000..84cba8c --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/python.py @@ -0,0 +1,21 @@ +# Copyright 2014-2019 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +# deprecated aliases for backwards compatibility +from charmhelpers.fetch.python import debug # noqa +from charmhelpers.fetch.python import packages # noqa +from charmhelpers.fetch.python import rpdb # noqa +from charmhelpers.fetch.python import version # noqa diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/saltstack/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/saltstack/__init__.py new file mode 100644 index 0000000..d74f403 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/saltstack/__init__.py @@ -0,0 +1,116 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Charm Helpers saltstack - declare the state of your machines. + +This helper enables you to declare your machine state, rather than +program it procedurally (and have to test each change to your procedures). +Your install hook can be as simple as:: + + {{{ + from charmhelpers.contrib.saltstack import ( + install_salt_support, + update_machine_state, + ) + + + def install(): + install_salt_support() + update_machine_state('machine_states/dependencies.yaml') + update_machine_state('machine_states/installed.yaml') + }}} + +and won't need to change (nor will its tests) when you change the machine +state. + +It's using a python package called salt-minion which allows various formats for +specifying resources, such as:: + + {{{ + /srv/{{ basedir }}: + file.directory: + - group: ubunet + - user: ubunet + - require: + - user: ubunet + - recurse: + - user + - group + + ubunet: + group.present: + - gid: 1500 + user.present: + - uid: 1500 + - gid: 1500 + - createhome: False + - require: + - group: ubunet + }}} + +The docs for all the different state definitions are at: + http://docs.saltstack.com/ref/states/all/ + + +TODO: + * Add test helpers which will ensure that machine state definitions + are functionally (but not necessarily logically) correct (ie. getting + salt to parse all state defs. + * Add a link to a public bootstrap charm example / blogpost. + * Find a way to obviate the need to use the grains['charm_dir'] syntax + in templates. +""" +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +import subprocess + +import charmhelpers.contrib.templating.contexts +import charmhelpers.core.host +import charmhelpers.core.hookenv + + +salt_grains_path = '/etc/salt/grains' + + +def install_salt_support(from_ppa=True): + """Installs the salt-minion helper for machine state. + + By default the salt-minion package is installed from + the saltstack PPA. If from_ppa is False you must ensure + that the salt-minion package is available in the apt cache. + """ + if from_ppa: + subprocess.check_call([ + '/usr/bin/add-apt-repository', + '--yes', + 'ppa:saltstack/salt', + ]) + subprocess.check_call(['/usr/bin/apt-get', 'update']) + # We install salt-common as salt-minion would run the salt-minion + # daemon. + charmhelpers.fetch.apt_install('salt-common') + + +def update_machine_state(state_path): + """Update the machine state using the provided state declaration.""" + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + salt_grains_path) + subprocess.check_call([ + 'salt-call', + '--local', + 'state.template', + state_path, + ]) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/__init__.py new file mode 100644 index 0000000..0a2aad2 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/__init__.py @@ -0,0 +1,92 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +from charmhelpers.core import hookenv + + +def generate_selfsigned(keyfile, certfile, keysize="1024", config=None, subject=None, cn=None): + """Generate selfsigned SSL key pair + + You must provide one of the 3 optional arguments: + config, subject or cn + If more than one is provided the leftmost will be used + + Arguments: + keyfile -- (required) full path to the keyfile to be created + certfile -- (required) full path to the certfile to be created + keysize -- (optional) SSL key length + config -- (optional) openssl configuration file + subject -- (optional) dictionary with SSL subject variables + cn -- (optional) cerfificate common name + + Required keys in subject dict: + cn -- Common name (eq. FQDN) + + Optional keys in subject dict + country -- Country Name (2 letter code) + state -- State or Province Name (full name) + locality -- Locality Name (eg, city) + organization -- Organization Name (eg, company) + organizational_unit -- Organizational Unit Name (eg, section) + email -- Email Address + """ + + cmd = [] + if config: + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-config", config] + elif subject: + ssl_subject = "" + if "country" in subject: + ssl_subject = ssl_subject + "/C={}".format(subject["country"]) + if "state" in subject: + ssl_subject = ssl_subject + "/ST={}".format(subject["state"]) + if "locality" in subject: + ssl_subject = ssl_subject + "/L={}".format(subject["locality"]) + if "organization" in subject: + ssl_subject = ssl_subject + "/O={}".format(subject["organization"]) + if "organizational_unit" in subject: + ssl_subject = ssl_subject + "/OU={}".format(subject["organizational_unit"]) + if "cn" in subject: + ssl_subject = ssl_subject + "/CN={}".format(subject["cn"]) + else: + hookenv.log("When using \"subject\" argument you must " + "provide \"cn\" field at very least") + return False + if "email" in subject: + ssl_subject = ssl_subject + "/emailAddress={}".format(subject["email"]) + + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-subj", ssl_subject] + elif cn: + cmd = ["/usr/bin/openssl", "req", "-new", "-newkey", + "rsa:{}".format(keysize), "-days", "365", "-nodes", "-x509", + "-keyout", keyfile, + "-out", certfile, "-subj", "/CN={}".format(cn)] + + if not cmd: + hookenv.log("No config, subject or cn provided," + "unable to generate self signed SSL certificates") + return False + try: + subprocess.check_call(cmd) + return True + except Exception as e: + print("Execution of openssl command failed:\n{}".format(e)) + return False diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/service.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/service.py new file mode 100644 index 0000000..06b534f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/ssl/service.py @@ -0,0 +1,277 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from os.path import join as path_join +from os.path import exists +import subprocess + +from charmhelpers.core.hookenv import log, DEBUG + +STD_CERT = "standard" + +# Mysql server is fairly picky about cert creation +# and types, spec its creation separately for now. +MYSQL_CERT = "mysql" + + +class ServiceCA(object): + + default_expiry = str(365 * 2) + default_ca_expiry = str(365 * 6) + + def __init__(self, name, ca_dir, cert_type=STD_CERT): + self.name = name + self.ca_dir = ca_dir + self.cert_type = cert_type + + ############### + # Hook Helper API + @staticmethod + def get_ca(type=STD_CERT): + service_name = os.environ['JUJU_UNIT_NAME'].split('/')[0] + ca_path = os.path.join(os.environ['CHARM_DIR'], 'ca') + ca = ServiceCA(service_name, ca_path, type) + ca.init() + return ca + + @classmethod + def get_service_cert(cls, type=STD_CERT): + service_name = os.environ['JUJU_UNIT_NAME'].split('/')[0] + ca = cls.get_ca() + crt, key = ca.get_or_create_cert(service_name) + return crt, key, ca.get_ca_bundle() + + ############### + + def init(self): + log("initializing service ca", level=DEBUG) + if not exists(self.ca_dir): + self._init_ca_dir(self.ca_dir) + self._init_ca() + + @property + def ca_key(self): + return path_join(self.ca_dir, 'private', 'cacert.key') + + @property + def ca_cert(self): + return path_join(self.ca_dir, 'cacert.pem') + + @property + def ca_conf(self): + return path_join(self.ca_dir, 'ca.cnf') + + @property + def signing_conf(self): + return path_join(self.ca_dir, 'signing.cnf') + + def _init_ca_dir(self, ca_dir): + os.mkdir(ca_dir) + for i in ['certs', 'crl', 'newcerts', 'private']: + sd = path_join(ca_dir, i) + if not exists(sd): + os.mkdir(sd) + + if not exists(path_join(ca_dir, 'serial')): + with open(path_join(ca_dir, 'serial'), 'w') as fh: + fh.write('02\n') + + if not exists(path_join(ca_dir, 'index.txt')): + with open(path_join(ca_dir, 'index.txt'), 'w') as fh: + fh.write('') + + def _init_ca(self): + """Generate the root ca's cert and key. + """ + if not exists(path_join(self.ca_dir, 'ca.cnf')): + with open(path_join(self.ca_dir, 'ca.cnf'), 'w') as fh: + fh.write( + CA_CONF_TEMPLATE % (self.get_conf_variables())) + + if not exists(path_join(self.ca_dir, 'signing.cnf')): + with open(path_join(self.ca_dir, 'signing.cnf'), 'w') as fh: + fh.write( + SIGNING_CONF_TEMPLATE % (self.get_conf_variables())) + + if exists(self.ca_cert) or exists(self.ca_key): + raise RuntimeError("Initialized called when CA already exists") + cmd = ['openssl', 'req', '-config', self.ca_conf, + '-x509', '-nodes', '-newkey', 'rsa', + '-days', self.default_ca_expiry, + '-keyout', self.ca_key, '-out', self.ca_cert, + '-outform', 'PEM'] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + log("CA Init:\n %s" % output, level=DEBUG) + + def get_conf_variables(self): + return dict( + org_name="juju", + org_unit_name="%s service" % self.name, + common_name=self.name, + ca_dir=self.ca_dir) + + def get_or_create_cert(self, common_name): + if common_name in self: + return self.get_certificate(common_name) + return self.create_certificate(common_name) + + def create_certificate(self, common_name): + if common_name in self: + return self.get_certificate(common_name) + key_p = path_join(self.ca_dir, "certs", "%s.key" % common_name) + crt_p = path_join(self.ca_dir, "certs", "%s.crt" % common_name) + csr_p = path_join(self.ca_dir, "certs", "%s.csr" % common_name) + self._create_certificate(common_name, key_p, csr_p, crt_p) + return self.get_certificate(common_name) + + def get_certificate(self, common_name): + if common_name not in self: + raise ValueError("No certificate for %s" % common_name) + key_p = path_join(self.ca_dir, "certs", "%s.key" % common_name) + crt_p = path_join(self.ca_dir, "certs", "%s.crt" % common_name) + with open(crt_p) as fh: + crt = fh.read() + with open(key_p) as fh: + key = fh.read() + return crt, key + + def __contains__(self, common_name): + crt_p = path_join(self.ca_dir, "certs", "%s.crt" % common_name) + return exists(crt_p) + + def _create_certificate(self, common_name, key_p, csr_p, crt_p): + template_vars = self.get_conf_variables() + template_vars['common_name'] = common_name + subj = '/O=%(org_name)s/OU=%(org_unit_name)s/CN=%(common_name)s' % ( + template_vars) + + log("CA Create Cert %s" % common_name, level=DEBUG) + cmd = ['openssl', 'req', '-sha1', '-newkey', 'rsa:2048', + '-nodes', '-days', self.default_expiry, + '-keyout', key_p, '-out', csr_p, '-subj', subj] + subprocess.check_call(cmd, stderr=subprocess.PIPE) + cmd = ['openssl', 'rsa', '-in', key_p, '-out', key_p] + subprocess.check_call(cmd, stderr=subprocess.PIPE) + + log("CA Sign Cert %s" % common_name, level=DEBUG) + if self.cert_type == MYSQL_CERT: + cmd = ['openssl', 'x509', '-req', + '-in', csr_p, '-days', self.default_expiry, + '-CA', self.ca_cert, '-CAkey', self.ca_key, + '-set_serial', '01', '-out', crt_p] + else: + cmd = ['openssl', 'ca', '-config', self.signing_conf, + '-extensions', 'req_extensions', + '-days', self.default_expiry, '-notext', + '-in', csr_p, '-out', crt_p, '-subj', subj, '-batch'] + log("running %s" % " ".join(cmd), level=DEBUG) + subprocess.check_call(cmd, stderr=subprocess.PIPE) + + def get_ca_bundle(self): + with open(self.ca_cert) as fh: + return fh.read() + + +CA_CONF_TEMPLATE = """ +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = %(ca_dir)s +policy = policy_match +database = $dir/index.txt +serial = $dir/serial +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +certificate = $dir/cacert.pem +private_key = $dir/private/cacert.key +RANDFILE = $dir/private/.rand +default_md = default + +[ req ] +default_bits = 1024 +default_md = sha1 + +prompt = no +distinguished_name = ca_distinguished_name + +x509_extensions = ca_extensions + +[ ca_distinguished_name ] +organizationName = %(org_name)s +organizationalUnitName = %(org_unit_name)s Certificate Authority + + +[ policy_match ] +countryName = optional +stateOrProvinceName = optional +organizationName = match +organizationalUnitName = optional +commonName = supplied + +[ ca_extensions ] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +keyUsage = cRLSign, keyCertSign +""" + + +SIGNING_CONF_TEMPLATE = """ +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = %(ca_dir)s +policy = policy_match +database = $dir/index.txt +serial = $dir/serial +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +certificate = $dir/cacert.pem +private_key = $dir/private/cacert.key +RANDFILE = $dir/private/.rand +default_md = default + +[ req ] +default_bits = 1024 +default_md = sha1 + +prompt = no +distinguished_name = req_distinguished_name + +x509_extensions = req_extensions + +[ req_distinguished_name ] +organizationName = %(org_name)s +organizationalUnitName = %(org_unit_name)s machine resources +commonName = %(common_name)s + +[ policy_match ] +countryName = optional +stateOrProvinceName = optional +organizationName = match +organizationalUnitName = optional +commonName = supplied + +[ req_extensions ] +basicConstraints = CA:false +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +keyUsage = digitalSignature, keyEncipherment, keyAgreement +extendedKeyUsage = serverAuth, clientAuth +""" diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/bcache.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/bcache.py new file mode 100644 index 0000000..605991e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/bcache.py @@ -0,0 +1,74 @@ +# Copyright 2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import json + +from charmhelpers.core.hookenv import log + +stats_intervals = ['stats_day', 'stats_five_minute', + 'stats_hour', 'stats_total'] + +SYSFS = '/sys' + + +class Bcache(object): + """Bcache behaviour + """ + + def __init__(self, cachepath): + self.cachepath = cachepath + + @classmethod + def fromdevice(cls, devname): + return cls('{}/block/{}/bcache'.format(SYSFS, devname)) + + def __str__(self): + return self.cachepath + + def get_stats(self, interval): + """Get cache stats + """ + intervaldir = 'stats_{}'.format(interval) + path = "{}/{}".format(self.cachepath, intervaldir) + out = dict() + for elem in os.listdir(path): + out[elem] = open('{}/{}'.format(path, elem)).read().strip() + return out + + +def get_bcache_fs(): + """Return all cache sets + """ + cachesetroot = "{}/fs/bcache".format(SYSFS) + try: + dirs = os.listdir(cachesetroot) + except OSError: + log("No bcache fs found") + return [] + cacheset = set([Bcache('{}/{}'.format(cachesetroot, d)) for d in dirs if not d.startswith('register')]) + return cacheset + + +def get_stats_action(cachespec, interval): + """Action for getting bcache statistics for a given cachespec. + Cachespec can either be a device name, eg. 'sdb', which will retrieve + cache stats for the given device, or 'global', which will retrieve stats + for all cachesets + """ + if cachespec == 'global': + caches = get_bcache_fs() + else: + caches = [Bcache.fromdevice(cachespec)] + res = dict((c.cachepath, c.get_stats(interval)) for c in caches) + return json.dumps(res, indent=4, separators=(',', ': ')) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/ceph.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/ceph.py new file mode 100644 index 0000000..3eb46d7 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/ceph.py @@ -0,0 +1,2378 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is sourced from lp:openstack-charm-helpers +# +# Authors: +# James Page +# Adam Gandelman +# + +import collections +import errno +import hashlib +import math +import six + +import os +import shutil +import json +import time +import uuid + +from subprocess import ( + check_call, + check_output, + CalledProcessError, +) +from charmhelpers import deprecate +from charmhelpers.core.hookenv import ( + application_name, + config, + service_name, + local_unit, + relation_get, + relation_ids, + relation_set, + related_units, + log, + DEBUG, + INFO, + WARNING, + ERROR, +) +from charmhelpers.core.host import ( + mount, + mounts, + service_start, + service_stop, + service_running, + umount, + cmp_pkgrevno, +) +from charmhelpers.fetch import ( + apt_install, +) +from charmhelpers.core.unitdata import kv + +from charmhelpers.core.kernel import modprobe +from charmhelpers.contrib.openstack.utils import config_flags_parser + +KEYRING = '/etc/ceph/ceph.client.{}.keyring' +KEYFILE = '/etc/ceph/ceph.client.{}.key' + +CEPH_CONF = """[global] +auth supported = {auth} +keyring = {keyring} +mon host = {mon_hosts} +log to syslog = {use_syslog} +err to syslog = {use_syslog} +clog to syslog = {use_syslog} +""" + +# The number of placement groups per OSD to target for placement group +# calculations. This number is chosen as 100 due to the ceph PG Calc +# documentation recommending to choose 100 for clusters which are not +# expected to increase in the foreseeable future. Since the majority of the +# calculations are done on deployment, target the case of non-expanding +# clusters as the default. +DEFAULT_PGS_PER_OSD_TARGET = 100 +DEFAULT_POOL_WEIGHT = 10.0 +LEGACY_PG_COUNT = 200 +DEFAULT_MINIMUM_PGS = 2 +AUTOSCALER_DEFAULT_PGS = 32 + + +class OsdPostUpgradeError(Exception): + """Error class for OSD post-upgrade operations.""" + pass + + +class OSDSettingConflict(Exception): + """Error class for conflicting osd setting requests.""" + pass + + +class OSDSettingNotAllowed(Exception): + """Error class for a disallowed setting.""" + pass + + +OSD_SETTING_EXCEPTIONS = (OSDSettingConflict, OSDSettingNotAllowed) + +OSD_SETTING_WHITELIST = [ + 'osd heartbeat grace', + 'osd heartbeat interval', +] + + +def _order_dict_by_key(rdict): + """Convert a dictionary into an OrderedDict sorted by key. + + :param rdict: Dictionary to be ordered. + :type rdict: dict + :returns: Ordered Dictionary. + :rtype: collections.OrderedDict + """ + return collections.OrderedDict(sorted(rdict.items(), key=lambda k: k[0])) + + +def get_osd_settings(relation_name): + """Consolidate requested osd settings from all clients. + + Consolidate requested osd settings from all clients. Check that the + requested setting is on the whitelist and it does not conflict with + any other requested settings. + + :returns: Dictionary of settings + :rtype: dict + + :raises: OSDSettingNotAllowed + :raises: OSDSettingConflict + """ + rel_ids = relation_ids(relation_name) + osd_settings = {} + for relid in rel_ids: + for unit in related_units(relid): + unit_settings = relation_get('osd-settings', unit, relid) or '{}' + unit_settings = json.loads(unit_settings) + for key, value in unit_settings.items(): + if key not in OSD_SETTING_WHITELIST: + msg = 'Illegal settings "{}"'.format(key) + raise OSDSettingNotAllowed(msg) + if key in osd_settings: + if osd_settings[key] != unit_settings[key]: + msg = 'Conflicting settings for "{}"'.format(key) + raise OSDSettingConflict(msg) + else: + osd_settings[key] = value + return _order_dict_by_key(osd_settings) + + +def send_application_name(relid=None): + """Send the application name down the relation. + + :param relid: Relation id to set application name in. + :type relid: str + """ + relation_set( + relation_id=relid, + relation_settings={'application-name': application_name()}) + + +def send_osd_settings(): + """Pass on requested OSD settings to osd units.""" + try: + settings = get_osd_settings('client') + except OSD_SETTING_EXCEPTIONS as e: + # There is a problem with the settings, not passing them on. Update + # status will notify the user. + log(e, level=ERROR) + return + data = { + 'osd-settings': json.dumps(settings, sort_keys=True)} + for relid in relation_ids('osd'): + relation_set(relation_id=relid, + relation_settings=data) + + +def validator(value, valid_type, valid_range=None): + """Helper function for type validation. + + Used to validate these: + https://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values + https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression + + Example input: + validator(value=1, + valid_type=int, + valid_range=[0, 2]) + + This says I'm testing value=1. It must be an int inclusive in [0,2] + + :param value: The value to validate. + :type value: any + :param valid_type: The type that value should be. + :type valid_type: any + :param valid_range: A range of values that value can assume. + :type valid_range: Optional[Union[List,Tuple]] + :raises: AssertionError, ValueError + """ + assert isinstance(value, valid_type), ( + "{} is not a {}".format(value, valid_type)) + if valid_range is not None: + assert isinstance( + valid_range, list) or isinstance(valid_range, tuple), ( + "valid_range must be of type List or Tuple, " + "was given {} of type {}" + .format(valid_range, type(valid_range))) + # If we're dealing with strings + if isinstance(value, six.string_types): + assert value in valid_range, ( + "{} is not in the list {}".format(value, valid_range)) + # Integer, float should have a min and max + else: + if len(valid_range) != 2: + raise ValueError( + "Invalid valid_range list of {} for {}. " + "List must be [min,max]".format(valid_range, value)) + assert value >= valid_range[0], ( + "{} is less than minimum allowed value of {}" + .format(value, valid_range[0])) + assert value <= valid_range[1], ( + "{} is greater than maximum allowed value of {}" + .format(value, valid_range[1])) + + +class PoolCreationError(Exception): + """A custom exception to inform the caller that a pool creation failed. + + Provides an error message + """ + + def __init__(self, message): + super(PoolCreationError, self).__init__(message) + + +class BasePool(object): + """An object oriented approach to Ceph pool creation. + + This base class is inherited by ReplicatedPool and ErasurePool. Do not call + create() on this base class as it will raise an exception. + + Instantiate a child class and call create(). + """ + # Dictionary that maps pool operation properties to Tuples with valid type + # and valid range + op_validation_map = { + 'compression-algorithm': (str, ('lz4', 'snappy', 'zlib', 'zstd')), + 'compression-mode': (str, ('none', 'passive', 'aggressive', 'force')), + 'compression-required-ratio': (float, None), + 'compression-min-blob-size': (int, None), + 'compression-min-blob-size-hdd': (int, None), + 'compression-min-blob-size-ssd': (int, None), + 'compression-max-blob-size': (int, None), + 'compression-max-blob-size-hdd': (int, None), + 'compression-max-blob-size-ssd': (int, None), + 'rbd-mirroring-mode': (str, ('image', 'pool')) + } + + def __init__(self, service, name=None, percent_data=None, app_name=None, + op=None): + """Initialize BasePool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + :param service: The Ceph user name to run commands under. + :type service: str + :param name: Name of pool to operate on. + :type name: str + :param percent_data: The expected pool size in relation to all + available resources in the Ceph cluster. Will be + used to set the ``target_size_ratio`` pool + property. (default: 10.0) + :type percent_data: Optional[float] + :param app_name: Ceph application name, usually one of: + ('cephfs', 'rbd', 'rgw') (default: 'unknown') + :type app_name: Optional[str] + :param op: Broker request Op to compile pool data from. + :type op: Optional[Dict[str,any]] + :raises: KeyError + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. + self.service = service + self.nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 + self.op = op or {} + + if op: + # When initializing from op the `name` attribute is required and we + # will fail with KeyError if it is not provided. + self.name = op['name'] + self.percent_data = op.get('weight') + self.app_name = op.get('app-name') + else: + self.name = name + self.percent_data = percent_data + self.app_name = app_name + + # Set defaults for these if they are not provided + self.percent_data = self.percent_data or 10.0 + self.app_name = self.app_name or 'unknown' + + def validate(self): + """Check that value of supplied operation parameters are valid. + + :raises: ValueError + """ + for op_key, op_value in self.op.items(): + if op_key in self.op_validation_map and op_value is not None: + valid_type, valid_range = self.op_validation_map[op_key] + try: + validator(op_value, valid_type, valid_range) + except (AssertionError, ValueError) as e: + # Normalize on ValueError, also add information about which + # variable we had an issue with. + raise ValueError("'{}': {}".format(op_key, str(e))) + + def _create(self): + """Perform the pool creation, method MUST be overridden by child class. + """ + raise NotImplementedError + + def _post_create(self): + """Perform common post pool creation tasks. + + Note that pool properties subject to change during the lifetime of a + pool / deployment should go into the ``update`` method. + + Do not add calls for a specific pool type here, those should go into + one of the pool specific classes. + """ + if self.nautilus_or_later: + # Ensure we set the expected pool ratio + update_pool( + client=self.service, + pool=self.name, + settings={ + 'target_size_ratio': str( + self.percent_data / 100.0), + }) + try: + set_app_name_for_pool(client=self.service, + pool=self.name, + name=self.app_name) + except CalledProcessError: + log('Could not set app name for pool {}' + .format(self.name), + level=WARNING) + if 'pg_autoscaler' in enabled_manager_modules(): + try: + enable_pg_autoscale(self.service, self.name) + except CalledProcessError as e: + log('Could not configure auto scaling for pool {}: {}' + .format(self.name, e), + level=WARNING) + + def create(self): + """Create pool and perform any post pool creation tasks. + + To allow for sharing of common code among pool specific classes the + processing has been broken out into the private methods ``_create`` + and ``_post_create``. + + Do not add any pool type specific handling here, that should go into + one of the pool specific classes. + """ + if not pool_exists(self.service, self.name): + self.validate() + self._create() + self._post_create() + self.update() + + def set_quota(self): + """Set a quota if requested. + + :raises: CalledProcessError + """ + max_bytes = self.op.get('max-bytes') + max_objects = self.op.get('max-objects') + if max_bytes or max_objects: + set_pool_quota(service=self.service, pool_name=self.name, + max_bytes=max_bytes, max_objects=max_objects) + + def set_compression(self): + """Set compression properties if requested. + + :raises: CalledProcessError + """ + compression_properties = { + key.replace('-', '_'): value + for key, value in self.op.items() + if key in ( + 'compression-algorithm', + 'compression-mode', + 'compression-required-ratio', + 'compression-min-blob-size', + 'compression-min-blob-size-hdd', + 'compression-min-blob-size-ssd', + 'compression-max-blob-size', + 'compression-max-blob-size-hdd', + 'compression-max-blob-size-ssd') and value} + if compression_properties: + update_pool(self.service, self.name, compression_properties) + + def update(self): + """Update properties for an already existing pool. + + Do not add calls for a specific pool type here, those should go into + one of the pool specific classes. + """ + self.validate() + self.set_quota() + self.set_compression() + + def add_cache_tier(self, cache_pool, mode): + """Adds a new cache tier to an existing pool. + + :param cache_pool: The cache tier pool name to add. + :type cache_pool: str + :param mode: The caching mode to use for this pool. + valid range = ["readonly", "writeback"] + :type mode: str + """ + # Check the input types and values + validator(value=cache_pool, valid_type=six.string_types) + validator( + value=mode, valid_type=six.string_types, + valid_range=["readonly", "writeback"]) + + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'add', self.name, cache_pool, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'cache-mode', cache_pool, mode, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'set-overlay', self.name, cache_pool, + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom', + ]) + + def remove_cache_tier(self, cache_pool): + """Removes a cache tier from Ceph. + + Flushes all dirty objects from writeback pools and waits for that to + complete. + + :param cache_pool: The cache tier pool name to remove. + :type cache_pool: str + """ + # read-only is easy, writeback is much harder + mode = get_cache_mode(self.service, cache_pool) + if mode == 'readonly': + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'cache-mode', cache_pool, 'none' + ]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove', self.name, cache_pool, + ]) + + elif mode == 'writeback': + pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier', + 'cache-mode', cache_pool, 'forward'] + if cmp_pkgrevno('ceph-common', '10.1') >= 0: + # Jewel added a mandatory flag + pool_forward_cmd.append('--yes-i-really-mean-it') + + check_call(pool_forward_cmd) + # Flush the cache and wait for it to return + check_call([ + 'rados', '--id', self.service, + '-p', cache_pool, 'cache-flush-evict-all']) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove-overlay', self.name]) + check_call([ + 'ceph', '--id', self.service, + 'osd', 'tier', 'remove', self.name, cache_pool]) + + def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT, + device_class=None): + """Return the number of placement groups to use when creating the pool. + + Returns the number of placement groups which should be specified when + creating the pool. This is based upon the calculation guidelines + provided by the Ceph Placement Group Calculator (located online at + http://ceph.com/pgcalc/). + + The number of placement groups are calculated using the following: + + (Target PGs per OSD) * (OSD #) * (%Data) + ---------------------------------------- + (Pool size) + + Per the upstream guidelines, the OSD # should really be considered + based on the number of OSDs which are eligible to be selected by the + pool. Since the pool creation doesn't specify any of CRUSH set rules, + the default rule will be dependent upon the type of pool being + created (replicated or erasure). + + This code makes no attempt to determine the number of OSDs which can be + selected for the specific rule, rather it is left to the user to tune + in the form of 'expected-osd-count' config option. + + :param pool_size: pool_size is either the number of replicas for + replicated pools or the K+M sum for erasure coded pools + :type pool_size: int + :param percent_data: the percentage of data that is expected to + be contained in the pool for the specific OSD set. Default value + is to assume 10% of the data is for this pool, which is a + relatively low % of the data but allows for the pg_num to be + increased. NOTE: the default is primarily to handle the scenario + where related charms requiring pools has not been upgraded to + include an update to indicate their relative usage of the pools. + :type percent_data: float + :param device_class: class of storage to use for basis of pgs + calculation; ceph supports nvme, ssd and hdd by default based + on presence of devices of each type in the deployment. + :type device_class: str + :returns: The number of pgs to use. + :rtype: int + """ + + # Note: This calculation follows the approach that is provided + # by the Ceph PG Calculator located at http://ceph.com/pgcalc/. + validator(value=pool_size, valid_type=int) + + # Ensure that percent data is set to something - even with a default + # it can be set to None, which would wreak havoc below. + if percent_data is None: + percent_data = DEFAULT_POOL_WEIGHT + + # If the expected-osd-count is specified, then use the max between + # the expected-osd-count and the actual osd_count + osd_list = get_osds(self.service, device_class) + expected = config('expected-osd-count') or 0 + + if osd_list: + if device_class: + osd_count = len(osd_list) + else: + osd_count = max(expected, len(osd_list)) + + # Log a message to provide some insight if the calculations claim + # to be off because someone is setting the expected count and + # there are more OSDs in reality. Try to make a proper guess + # based upon the cluster itself. + if not device_class and expected and osd_count != expected: + log("Found more OSDs than provided expected count. " + "Using the actual count instead", INFO) + elif expected: + # Use the expected-osd-count in older ceph versions to allow for + # a more accurate pg calculations + osd_count = expected + else: + # NOTE(james-page): Default to 200 for older ceph versions + # which don't support OSD query from cli + return LEGACY_PG_COUNT + + percent_data /= 100.0 + target_pgs_per_osd = config( + 'pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET + num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size + + # NOTE: ensure a sane minimum number of PGS otherwise we don't get any + # reasonable data distribution in minimal OSD configurations + if num_pg < DEFAULT_MINIMUM_PGS: + num_pg = DEFAULT_MINIMUM_PGS + + # The CRUSH algorithm has a slight optimization for placement groups + # with powers of 2 so find the nearest power of 2. If the nearest + # power of 2 is more than 25% below the original value, the next + # highest value is used. To do this, find the nearest power of 2 such + # that 2^n <= num_pg, check to see if its within the 25% tolerance. + exponent = math.floor(math.log(num_pg, 2)) + nearest = 2 ** exponent + if (num_pg - nearest) > (num_pg * 0.25): + # Choose the next highest power of 2 since the nearest is more + # than 25% below the original value. + return int(nearest * 2) + else: + return int(nearest) + + +class Pool(BasePool): + """Compatibility shim for any descendents external to this library.""" + + @deprecate( + 'The ``Pool`` baseclass has been replaced by ``BasePool`` class.') + def __init__(self, service, name): + super(Pool, self).__init__(service, name=name) + + def create(self): + pass + + +class ReplicatedPool(BasePool): + def __init__(self, service, name=None, pg_num=None, replicas=None, + percent_data=None, app_name=None, op=None): + """Initialize ReplicatedPool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + Please refer to the docstring of the ``BasePool`` class for + documentation of the common parameters. + + :param pg_num: Express wish for number of Placement Groups (this value + is subject to validation against a running cluster prior + to use to avoid creating a pool with too many PGs) + :type pg_num: int + :param replicas: Number of copies there should be of each object added + to this replicated pool. + :type replicas: int + :raises: KeyError + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. + + # The common parameters are handled in our parents initializer + super(ReplicatedPool, self).__init__( + service=service, name=name, percent_data=percent_data, + app_name=app_name, op=op) + + if op: + # When initializing from op `replicas` is a required attribute, and + # we will fail with KeyError if it is not provided. + self.replicas = op['replicas'] + self.pg_num = op.get('pg_num') + else: + self.replicas = replicas or 2 + self.pg_num = pg_num + + def _create(self): + # Do extra validation on pg_num with data from live cluster + if self.pg_num: + # Since the number of placement groups were specified, ensure + # that there aren't too many created. + max_pgs = self.get_pgs(self.replicas, 100.0) + self.pg_num = min(self.pg_num, max_pgs) + else: + self.pg_num = self.get_pgs(self.replicas, self.percent_data) + + # Create it + if self.nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, self.pg_num) + ), + self.name, str(self.pg_num) + ] + else: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(self.pg_num) + ] + check_call(cmd) + + def _post_create(self): + # Set the pool replica size + update_pool(client=self.service, + pool=self.name, + settings={'size': str(self.replicas)}) + # Perform other common post pool creation tasks + super(ReplicatedPool, self)._post_create() + + +class ErasurePool(BasePool): + """Default jerasure erasure coded pool.""" + + def __init__(self, service, name=None, erasure_code_profile=None, + percent_data=None, app_name=None, op=None, + allow_ec_overwrites=False): + """Initialize ReplicatedPool object. + + Pool information is either initialized from individual keyword + arguments or from a individual CephBrokerRq operation Dict. + + Please refer to the docstring of the ``BasePool`` class for + documentation of the common parameters. + + :param erasure_code_profile: EC Profile to use (default: 'default') + :type erasure_code_profile: Optional[str] + """ + # NOTE: Do not perform initialization steps that require live data from + # a running cluster here. The *Pool classes may be used for validation. + + # The common parameters are handled in our parents initializer + super(ErasurePool, self).__init__( + service=service, name=name, percent_data=percent_data, + app_name=app_name, op=op) + + if op: + # Note that the different default when initializing from op stems + # from different handling of this in the `charms.ceph` library. + self.erasure_code_profile = op.get('erasure-profile', + 'default-canonical') + self.allow_ec_overwrites = op.get('allow-ec-overwrites') + else: + # We keep the class default when initialized from keyword arguments + # to not break the API for any other consumers. + self.erasure_code_profile = erasure_code_profile or 'default' + self.allow_ec_overwrites = allow_ec_overwrites + + def _create(self): + # Try to find the erasure profile information in order to properly + # size the number of placement groups. The size of an erasure + # coded placement group is calculated as k+m. + erasure_profile = get_erasure_profile(self.service, + self.erasure_code_profile) + + # Check for errors + if erasure_profile is None: + msg = ("Failed to discover erasure profile named " + "{}".format(self.erasure_code_profile)) + log(msg, level=ERROR) + raise PoolCreationError(msg) + if 'k' not in erasure_profile or 'm' not in erasure_profile: + # Error + msg = ("Unable to find k (data chunks) or m (coding chunks) " + "in erasure profile {}".format(erasure_profile)) + log(msg, level=ERROR) + raise PoolCreationError(msg) + + k = int(erasure_profile['k']) + m = int(erasure_profile['m']) + pgs = self.get_pgs(k + m, self.percent_data) + self.nautilus_or_later = cmp_pkgrevno('ceph-common', '14.2.0') >= 0 + # Create it + if self.nautilus_or_later: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + '--pg-num-min={}'.format( + min(AUTOSCALER_DEFAULT_PGS, pgs) + ), + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + else: + cmd = [ + 'ceph', '--id', self.service, 'osd', 'pool', 'create', + self.name, str(pgs), str(pgs), + 'erasure', self.erasure_code_profile + ] + check_call(cmd) + + def _post_create(self): + super(ErasurePool, self)._post_create() + if self.allow_ec_overwrites: + update_pool(self.service, self.name, + {'allow_ec_overwrites': 'true'}) + + +def enabled_manager_modules(): + """Return a list of enabled manager modules. + + :rtype: List[str] + """ + cmd = ['ceph', 'mgr', 'module', 'ls'] + try: + modules = check_output(cmd) + if six.PY3: + modules = modules.decode('UTF-8') + except CalledProcessError as e: + log("Failed to list ceph modules: {}".format(e), WARNING) + return [] + modules = json.loads(modules) + return modules['enabled_modules'] + + +def enable_pg_autoscale(service, pool_name): + """Enable Ceph's PG autoscaler for the specified pool. + + :param service: The Ceph user name to run the command under + :type service: str + :param pool_name: The name of the pool to enable sutoscaling on + :type pool_name: str + :raises: CalledProcessError if the command fails + """ + check_call([ + 'ceph', '--id', service, + 'osd', 'pool', 'set', pool_name, 'pg_autoscale_mode', 'on']) + + +def get_mon_map(service): + """Return the current monitor map. + + :param service: The Ceph user name to run the command under + :type service: str + :returns: Dictionary with monitor map data + :rtype: Dict[str,any] + :raises: ValueError if the monmap fails to parse, CalledProcessError if our + ceph command fails. + """ + try: + mon_status = check_output(['ceph', '--id', service, + 'mon_status', '--format=json']) + if six.PY3: + mon_status = mon_status.decode('UTF-8') + try: + return json.loads(mon_status) + except ValueError as v: + log("Unable to parse mon_status json: {}. Error: {}" + .format(mon_status, str(v))) + raise + except CalledProcessError as e: + log("mon_status command failed with message: {}" + .format(str(e))) + raise + + +def hash_monitor_names(service): + """Get a sorted list of monitor hashes in ascending order. + + Uses the get_mon_map() function to get information about the monitor + cluster. Hash the name of each monitor. + + :param service: The Ceph user name to run the command under. + :type service: str + :returns: a sorted list of monitor hashes in an ascending order. + :rtype : List[str] + :raises: CalledProcessError, ValueError + """ + try: + hash_list = [] + monitor_list = get_mon_map(service=service) + if monitor_list['monmap']['mons']: + for mon in monitor_list['monmap']['mons']: + hash_list.append( + hashlib.sha224(mon['name'].encode('utf-8')).hexdigest()) + return sorted(hash_list) + else: + return None + except (ValueError, CalledProcessError): + raise + + +def monitor_key_delete(service, key): + """Delete a key and value pair from the monitor cluster. + + Deletes a key value pair on the monitor cluster. + + :param service: The Ceph user name to run the command under + :type service: str + :param key: The key to delete. + :type key: str + :raises: CalledProcessError + """ + try: + check_output( + ['ceph', '--id', service, + 'config-key', 'del', str(key)]) + except CalledProcessError as e: + log("Monitor config-key put failed with message: {}" + .format(e.output)) + raise + + +def monitor_key_set(service, key, value): + """Set a key value pair on the monitor cluster. + + :param service: The Ceph user name to run the command under. + :type service str + :param key: The key to set. + :type key: str + :param value: The value to set. This will be coerced into a string. + :type value: str + :raises: CalledProcessError + """ + try: + check_output( + ['ceph', '--id', service, + 'config-key', 'put', str(key), str(value)]) + except CalledProcessError as e: + log("Monitor config-key put failed with message: {}" + .format(e.output)) + raise + + +def monitor_key_get(service, key): + """Get the value of an existing key in the monitor cluster. + + :param service: The Ceph user name to run the command under + :type service: str + :param key: The key to search for. + :type key: str + :return: Returns the value of that key or None if not found. + :rtype: Optional[str] + """ + try: + output = check_output( + ['ceph', '--id', service, + 'config-key', 'get', str(key)]).decode('UTF-8') + return output + except CalledProcessError as e: + log("Monitor config-key get failed with message: {}" + .format(e.output)) + return None + + +def monitor_key_exists(service, key): + """Search for existence of key in the monitor cluster. + + :param service: The Ceph user name to run the command under. + :type service: str + :param key: The key to search for. + :type key: str + :return: Returns True if the key exists, False if not. + :rtype: bool + :raises: CalledProcessError if an unknown error occurs. + """ + try: + check_call( + ['ceph', '--id', service, + 'config-key', 'exists', str(key)]) + # I can return true here regardless because Ceph returns + # ENOENT if the key wasn't found + return True + except CalledProcessError as e: + if e.returncode == errno.ENOENT: + return False + else: + log("Unknown error from ceph config-get exists: {} {}" + .format(e.returncode, e.output)) + raise + + +def get_erasure_profile(service, name): + """Get an existing erasure code profile if it exists. + + :param service: The Ceph user name to run the command under. + :type service: str + :param name: Name of profile. + :type name: str + :returns: Dictionary with profile data. + :rtype: Optional[Dict[str]] + """ + try: + out = check_output(['ceph', '--id', service, + 'osd', 'erasure-code-profile', 'get', + name, '--format=json']) + if six.PY3: + out = out.decode('UTF-8') + return json.loads(out) + except (CalledProcessError, OSError, ValueError): + return None + + +def pool_set(service, pool_name, key, value): + """Sets a value for a RADOS pool in ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to set property on. + :type pool_name: str + :param key: Property key. + :type key: str + :param value: Value, will be coerced into str and shifted to lowercase. + :type value: str + :raises: CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set', pool_name, key, str(value).lower()] + check_call(cmd) + + +def snapshot_pool(service, pool_name, snapshot_name): + """Snapshots a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to snapshot. + :type pool_name: str + :param snapshot_name: Name of snapshot to create. + :type snapshot_name: str + :raises: CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'mksnap', pool_name, snapshot_name] + check_call(cmd) + + +def remove_pool_snapshot(service, pool_name, snapshot_name): + """Remove a snapshot from a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to remove snapshot from. + :type pool_name: str + :param snapshot_name: Name of snapshot to remove. + :type snapshot_name: str + :raises: CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] + check_call(cmd) + + +def set_pool_quota(service, pool_name, max_bytes=None, max_objects=None): + """Set byte quota on a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under + :type service: str + :param pool_name: Name of pool + :type pool_name: str + :param max_bytes: Maximum bytes quota to apply + :type max_bytes: int + :param max_objects: Maximum objects quota to apply + :type max_objects: int + :raises: subprocess.CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set-quota', pool_name] + if max_bytes: + cmd = cmd + ['max_bytes', str(max_bytes)] + if max_objects: + cmd = cmd + ['max_objects', str(max_objects)] + check_call(cmd) + + +def remove_pool_quota(service, pool_name): + """Remove byte quota on a RADOS pool in Ceph. + + :param service: The Ceph user name to run the command under. + :type service: str + :param pool_name: Name of pool to remove quota from. + :type pool_name: str + :raises: CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'set-quota', pool_name, 'max_bytes', '0'] + check_call(cmd) + + +def remove_erasure_profile(service, profile_name): + """Remove erasure code profile. + + :param service: The Ceph user name to run the command under + :type service: str + :param profile_name: Name of profile to remove. + :type profile_name: str + :raises: CalledProcessError + """ + cmd = [ + 'ceph', '--id', service, + 'osd', 'erasure-code-profile', 'rm', profile_name] + check_call(cmd) + + +def create_erasure_profile(service, profile_name, + erasure_plugin_name='jerasure', + failure_domain=None, + data_chunks=2, coding_chunks=1, + locality=None, durability_estimator=None, + helper_chunks=None, + scalar_mds=None, + crush_locality=None, + device_class=None, + erasure_plugin_technique=None): + """Create a new erasure code profile if one does not already exist for it. + + Profiles are considered immutable so will not be updated if the named + profile already exists. + + Please refer to [0] for more details. + + 0: http://docs.ceph.com/docs/master/rados/operations/erasure-code-profile/ + + :param service: The Ceph user name to run the command under. + :type service: str + :param profile_name: Name of profile. + :type profile_name: str + :param erasure_plugin_name: Erasure code plugin. + :type erasure_plugin_name: str + :param failure_domain: Failure domain, one of: + ('chassis', 'datacenter', 'host', 'osd', 'pdu', + 'pod', 'rack', 'region', 'room', 'root', 'row'). + :type failure_domain: str + :param data_chunks: Number of data chunks. + :type data_chunks: int + :param coding_chunks: Number of coding chunks. + :type coding_chunks: int + :param locality: Locality. + :type locality: int + :param durability_estimator: Durability estimator. + :type durability_estimator: int + :param helper_chunks: int + :type helper_chunks: int + :param device_class: Restrict placement to devices of specific class. + :type device_class: str + :param scalar_mds: one of ['isa', 'jerasure', 'shec'] + :type scalar_mds: str + :param crush_locality: LRC locality faulure domain, one of: + ('chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', + 'rack', 'region', 'room', 'root', 'row') or unset. + :type crush_locaity: str + :param erasure_plugin_technique: Coding technique for EC plugin + :type erasure_plugin_technique: str + :return: None. Can raise CalledProcessError, ValueError or AssertionError + """ + if erasure_profile_exists(service, profile_name): + log('EC profile {} exists, skipping update'.format(profile_name), + level=WARNING) + return + + plugin_techniques = { + 'jerasure': [ + 'reed_sol_van', + 'reed_sol_r6_op', + 'cauchy_orig', + 'cauchy_good', + 'liberation', + 'blaum_roth', + 'liber8tion' + ], + 'lrc': [], + 'isa': [ + 'reed_sol_van', + 'cauchy', + ], + 'shec': [ + 'single', + 'multiple' + ], + 'clay': [], + } + failure_domains = [ + 'chassis', 'datacenter', + 'host', 'osd', + 'pdu', 'pod', + 'rack', 'region', + 'room', 'root', + 'row', + ] + device_classes = [ + 'ssd', + 'hdd', + 'nvme' + ] + + validator(erasure_plugin_name, six.string_types, + list(plugin_techniques.keys())) + + cmd = [ + 'ceph', '--id', service, + 'osd', 'erasure-code-profile', 'set', profile_name, + 'plugin={}'.format(erasure_plugin_name), + 'k={}'.format(str(data_chunks)), + 'm={}'.format(str(coding_chunks)), + ] + + if erasure_plugin_technique: + validator(erasure_plugin_technique, six.string_types, + plugin_techniques[erasure_plugin_name]) + cmd.append('technique={}'.format(erasure_plugin_technique)) + + luminous_or_later = cmp_pkgrevno('ceph-common', '12.0.0') >= 0 + + # Set failure domain from options if not provided in args + if not failure_domain and config('customize-failure-domain'): + # Defaults to 'host' so just need to deal with + # setting 'rack' if feature is enabled + failure_domain = 'rack' + + if failure_domain: + validator(failure_domain, six.string_types, failure_domains) + # failure_domain changed in luminous + if luminous_or_later: + cmd.append('crush-failure-domain={}'.format(failure_domain)) + else: + cmd.append('ruleset-failure-domain={}'.format(failure_domain)) + + # device class new in luminous + if luminous_or_later and device_class: + validator(device_class, six.string_types, device_classes) + cmd.append('crush-device-class={}'.format(device_class)) + else: + log('Skipping device class configuration (ceph < 12.0.0)', + level=DEBUG) + + # Add plugin specific information + if erasure_plugin_name == 'lrc': + # LRC mandatory configuration + if locality: + cmd.append('l={}'.format(str(locality))) + else: + raise ValueError("locality must be provided for lrc plugin") + # LRC optional configuration + if crush_locality: + validator(crush_locality, six.string_types, failure_domains) + cmd.append('crush-locality={}'.format(crush_locality)) + + if erasure_plugin_name == 'shec': + # SHEC optional configuration + if durability_estimator: + cmd.append('c={}'.format((durability_estimator))) + + if erasure_plugin_name == 'clay': + # CLAY optional configuration + if helper_chunks: + cmd.append('d={}'.format(str(helper_chunks))) + if scalar_mds: + cmd.append('scalar-mds={}'.format(scalar_mds)) + + check_call(cmd) + + +def rename_pool(service, old_name, new_name): + """Rename a Ceph pool from old_name to new_name. + + :param service: The Ceph user name to run the command under. + :type service: str + :param old_name: Name of pool subject to rename. + :type old_name: str + :param new_name: Name to rename pool to. + :type new_name: str + """ + validator(value=old_name, valid_type=six.string_types) + validator(value=new_name, valid_type=six.string_types) + + cmd = [ + 'ceph', '--id', service, + 'osd', 'pool', 'rename', old_name, new_name] + check_call(cmd) + + +def erasure_profile_exists(service, name): + """Check to see if an Erasure code profile already exists. + + :param service: The Ceph user name to run the command under + :type service: str + :param name: Name of profile to look for. + :type name: str + :returns: True if it exists, False otherwise. + :rtype: bool + """ + validator(value=name, valid_type=six.string_types) + try: + check_call(['ceph', '--id', service, + 'osd', 'erasure-code-profile', 'get', + name]) + return True + except CalledProcessError: + return False + + +def get_cache_mode(service, pool_name): + """Find the current caching mode of the pool_name given. + + :param service: The Ceph user name to run the command under + :type service: str + :param pool_name: Name of pool. + :type pool_name: str + :returns: Current cache mode. + :rtype: Optional[int] + """ + validator(value=service, valid_type=six.string_types) + validator(value=pool_name, valid_type=six.string_types) + out = check_output(['ceph', '--id', service, + 'osd', 'dump', '--format=json']) + if six.PY3: + out = out.decode('UTF-8') + try: + osd_json = json.loads(out) + for pool in osd_json['pools']: + if pool['pool_name'] == pool_name: + return pool['cache_mode'] + return None + except ValueError: + raise + + +def pool_exists(service, name): + """Check to see if a RADOS pool already exists.""" + try: + out = check_output(['rados', '--id', service, 'lspools']) + if six.PY3: + out = out.decode('UTF-8') + except CalledProcessError: + return False + + return name in out.split() + + +def get_osds(service, device_class=None): + """Return a list of all Ceph Object Storage Daemons currently in the + cluster (optionally filtered by storage device class). + + :param device_class: Class of storage device for OSD's + :type device_class: str + """ + luminous_or_later = cmp_pkgrevno('ceph-common', '12.0.0') >= 0 + if luminous_or_later and device_class: + out = check_output(['ceph', '--id', service, + 'osd', 'crush', 'class', + 'ls-osd', device_class, + '--format=json']) + else: + out = check_output(['ceph', '--id', service, + 'osd', 'ls', + '--format=json']) + if six.PY3: + out = out.decode('UTF-8') + return json.loads(out) + + +def install(): + """Basic Ceph client installation.""" + ceph_dir = "/etc/ceph" + if not os.path.exists(ceph_dir): + os.mkdir(ceph_dir) + + apt_install('ceph-common', fatal=True) + + +def rbd_exists(service, pool, rbd_img): + """Check to see if a RADOS block device exists.""" + try: + out = check_output(['rbd', 'list', '--id', + service, '--pool', pool]) + if six.PY3: + out = out.decode('UTF-8') + except CalledProcessError: + return False + + return rbd_img in out + + +def create_rbd_image(service, pool, image, sizemb): + """Create a new RADOS block device.""" + cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service, + '--pool', pool] + check_call(cmd) + + +def update_pool(client, pool, settings): + """Update pool properties. + + :param client: Client/User-name to authenticate with. + :type client: str + :param pool: Name of pool to operate on + :type pool: str + :param settings: Dictionary with key/value pairs to set. + :type settings: Dict[str, str] + :raises: CalledProcessError + """ + cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] + for k, v in six.iteritems(settings): + check_call(cmd + [k, v]) + + +def set_app_name_for_pool(client, pool, name): + """Calls `osd pool application enable` for the specified pool name + + :param client: Name of the ceph client to use + :type client: str + :param pool: Pool to set app name for + :type pool: str + :param name: app name for the specified pool + :type name: str + + :raises: CalledProcessError if ceph call fails + """ + if cmp_pkgrevno('ceph-common', '12.0.0') >= 0: + cmd = ['ceph', '--id', client, 'osd', 'pool', + 'application', 'enable', pool, name] + check_call(cmd) + + +def create_pool(service, name, replicas=3, pg_num=None): + """Create a new RADOS pool.""" + if pool_exists(service, name): + log("Ceph pool {} already exists, skipping creation".format(name), + level=WARNING) + return + + if not pg_num: + # Calculate the number of placement groups based + # on upstream recommended best practices. + osds = get_osds(service) + if osds: + pg_num = (len(osds) * 100 // replicas) + else: + # NOTE(james-page): Default to 200 for older ceph versions + # which don't support OSD query from cli + pg_num = 200 + + cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)] + check_call(cmd) + + update_pool(service, name, settings={'size': str(replicas)}) + + +def delete_pool(service, name): + """Delete a RADOS pool from ceph.""" + cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name, + '--yes-i-really-really-mean-it'] + check_call(cmd) + + +def _keyfile_path(service): + return KEYFILE.format(service) + + +def _keyring_path(service): + return KEYRING.format(service) + + +def add_key(service, key): + """Add a key to a keyring. + + Creates the keyring if it doesn't already exist. + + Logs and returns if the key is already in the keyring. + """ + keyring = _keyring_path(service) + if os.path.exists(keyring): + with open(keyring, 'r') as ring: + if key in ring.read(): + log('Ceph keyring exists at %s and has not changed.' % keyring, + level=DEBUG) + return + log('Updating existing keyring %s.' % keyring, level=DEBUG) + + cmd = ['ceph-authtool', keyring, '--create-keyring', + '--name=client.{}'.format(service), '--add-key={}'.format(key)] + check_call(cmd) + log('Created new ceph keyring at %s.' % keyring, level=DEBUG) + + +def create_keyring(service, key): + """Deprecated. Please use the more accurately named 'add_key'""" + return add_key(service, key) + + +def delete_keyring(service): + """Delete an existing Ceph keyring.""" + keyring = _keyring_path(service) + if not os.path.exists(keyring): + log('Keyring does not exist at %s' % keyring, level=WARNING) + return + + os.remove(keyring) + log('Deleted ring at %s.' % keyring, level=INFO) + + +def create_key_file(service, key): + """Create a file containing key.""" + keyfile = _keyfile_path(service) + if os.path.exists(keyfile): + log('Keyfile exists at %s.' % keyfile, level=WARNING) + return + + with open(keyfile, 'w') as fd: + fd.write(key) + + log('Created new keyfile at %s.' % keyfile, level=INFO) + + +def get_ceph_nodes(relation='ceph'): + """Query named relation to determine current nodes.""" + hosts = [] + for r_id in relation_ids(relation): + for unit in related_units(r_id): + hosts.append(relation_get('private-address', unit=unit, rid=r_id)) + + return hosts + + +def configure(service, key, auth, use_syslog): + """Perform basic configuration of Ceph.""" + add_key(service, key) + create_key_file(service, key) + hosts = get_ceph_nodes() + with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: + ceph_conf.write(CEPH_CONF.format(auth=auth, + keyring=_keyring_path(service), + mon_hosts=",".join(map(str, hosts)), + use_syslog=use_syslog)) + modprobe('rbd') + + +def image_mapped(name): + """Determine whether a RADOS block device is mapped locally.""" + try: + out = check_output(['rbd', 'showmapped']) + if six.PY3: + out = out.decode('UTF-8') + except CalledProcessError: + return False + + return name in out + + +def map_block_storage(service, pool, image): + """Map a RADOS block device for local use.""" + cmd = [ + 'rbd', + 'map', + '{}/{}'.format(pool, image), + '--user', + service, + '--secret', + _keyfile_path(service), + ] + check_call(cmd) + + +def filesystem_mounted(fs): + """Determine whether a filesystem is already mounted.""" + return fs in [f for f, m in mounts()] + + +def make_filesystem(blk_device, fstype='ext4', timeout=10): + """Make a new filesystem on the specified block device.""" + count = 0 + e_noent = errno.ENOENT + while not os.path.exists(blk_device): + if count >= timeout: + log('Gave up waiting on block device %s' % blk_device, + level=ERROR) + raise IOError(e_noent, os.strerror(e_noent), blk_device) + + log('Waiting for block device %s to appear' % blk_device, + level=DEBUG) + count += 1 + time.sleep(1) + else: + log('Formatting block device %s as filesystem %s.' % + (blk_device, fstype), level=INFO) + check_call(['mkfs', '-t', fstype, blk_device]) + + +def place_data_on_block_device(blk_device, data_src_dst): + """Migrate data in data_src_dst to blk_device and then remount.""" + # mount block device into /mnt + mount(blk_device, '/mnt') + # copy data to /mnt + copy_files(data_src_dst, '/mnt') + # umount block device + umount('/mnt') + # Grab user/group ID's from original source + _dir = os.stat(data_src_dst) + uid = _dir.st_uid + gid = _dir.st_gid + # re-mount where the data should originally be + # TODO: persist is currently a NO-OP in core.host + mount(blk_device, data_src_dst, persist=True) + # ensure original ownership of new mount. + os.chown(data_src_dst, uid, gid) + + +def copy_files(src, dst, symlinks=False, ignore=None): + """Copy files from src to dst.""" + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + + +def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point, + blk_device, fstype, system_services=[], + replicas=3): + """NOTE: This function must only be called from a single service unit for + the same rbd_img otherwise data loss will occur. + + Ensures given pool and RBD image exists, is mapped to a block device, + and the device is formatted and mounted at the given mount_point. + + If formatting a device for the first time, data existing at mount_point + will be migrated to the RBD device before being re-mounted. + + All services listed in system_services will be stopped prior to data + migration and restarted when complete. + """ + # Ensure pool, RBD image, RBD mappings are in place. + if not pool_exists(service, pool): + log('Creating new pool {}.'.format(pool), level=INFO) + create_pool(service, pool, replicas=replicas) + + if not rbd_exists(service, pool, rbd_img): + log('Creating RBD image ({}).'.format(rbd_img), level=INFO) + create_rbd_image(service, pool, rbd_img, sizemb) + + if not image_mapped(rbd_img): + log('Mapping RBD Image {} as a Block Device.'.format(rbd_img), + level=INFO) + map_block_storage(service, pool, rbd_img) + + # make file system + # TODO: What happens if for whatever reason this is run again and + # the data is already in the rbd device and/or is mounted?? + # When it is mounted already, it will fail to make the fs + # XXX: This is really sketchy! Need to at least add an fstab entry + # otherwise this hook will blow away existing data if its executed + # after a reboot. + if not filesystem_mounted(mount_point): + make_filesystem(blk_device, fstype) + + for svc in system_services: + if service_running(svc): + log('Stopping services {} prior to migrating data.' + .format(svc), level=DEBUG) + service_stop(svc) + + place_data_on_block_device(blk_device, mount_point) + + for svc in system_services: + log('Starting service {} after migrating data.' + .format(svc), level=DEBUG) + service_start(svc) + + +def ensure_ceph_keyring(service, user=None, group=None, + relation='ceph', key=None): + """Ensures a ceph keyring is created for a named service and optionally + ensures user and group ownership. + + @returns boolean: Flag to indicate whether a key was successfully written + to disk based on either relation data or a supplied key + """ + if not key: + for rid in relation_ids(relation): + for unit in related_units(rid): + key = relation_get('key', rid=rid, unit=unit) + if key: + break + + if not key: + return False + + add_key(service=service, key=key) + keyring = _keyring_path(service) + if user and group: + check_call(['chown', '%s.%s' % (user, group), keyring]) + + return True + + +class CephBrokerRq(object): + """Ceph broker request. + + Multiple operations can be added to a request and sent to the Ceph broker + to be executed. + + Request is json-encoded for sending over the wire. + + The API is versioned and defaults to version 1. + """ + + def __init__(self, api_version=1, request_id=None, raw_request_data=None): + """Initialize CephBrokerRq object. + + Builds a new empty request or rebuilds a request from on-wire JSON + data. + + :param api_version: API version for request (default: 1). + :type api_version: Optional[int] + :param request_id: Unique identifier for request. + (default: string representation of generated UUID) + :type request_id: Optional[str] + :param raw_request_data: JSON-encoded string to build request from. + :type raw_request_data: Optional[str] + :raises: KeyError + """ + if raw_request_data: + request_data = json.loads(raw_request_data) + self.api_version = request_data['api-version'] + self.request_id = request_data['request-id'] + self.set_ops(request_data['ops']) + else: + self.api_version = api_version + if request_id: + self.request_id = request_id + else: + self.request_id = str(uuid.uuid1()) + self.ops = [] + + def add_op(self, op): + """Add an op if it is not already in the list. + + :param op: Operation to add. + :type op: dict + """ + if op not in self.ops: + self.ops.append(op) + + def add_op_request_access_to_group(self, name, namespace=None, + permission=None, key_name=None, + object_prefix_permissions=None): + """ + Adds the requested permissions to the current service's Ceph key, + allowing the key to access only the specified pools or + object prefixes. object_prefix_permissions should be a dictionary + keyed on the permission with the corresponding value being a list + of prefixes to apply that permission to. + { + 'rwx': ['prefix1', 'prefix2'], + 'class-read': ['prefix3']} + """ + self.add_op({ + 'op': 'add-permissions-to-key', 'group': name, + 'namespace': namespace, + 'name': key_name or service_name(), + 'group-permission': permission, + 'object-prefix-permissions': object_prefix_permissions}) + + def add_op_create_pool(self, name, replica_count=3, pg_num=None, + weight=None, group=None, namespace=None, + app_name=None, max_bytes=None, max_objects=None): + """DEPRECATED: Use ``add_op_create_replicated_pool()`` or + ``add_op_create_erasure_pool()`` instead. + """ + return self.add_op_create_replicated_pool( + name, replica_count=replica_count, pg_num=pg_num, weight=weight, + group=group, namespace=namespace, app_name=app_name, + max_bytes=max_bytes, max_objects=max_objects) + + # Use function parameters and docstring to define types in a compatible + # manner. + # + # NOTE: Our caller should always use a kwarg Dict when calling us so + # no need to maintain fixed order/position for parameters. Please keep them + # sorted by name when adding new ones. + def _partial_build_common_op_create(self, + app_name=None, + compression_algorithm=None, + compression_mode=None, + compression_required_ratio=None, + compression_min_blob_size=None, + compression_min_blob_size_hdd=None, + compression_min_blob_size_ssd=None, + compression_max_blob_size=None, + compression_max_blob_size_hdd=None, + compression_max_blob_size_ssd=None, + group=None, + max_bytes=None, + max_objects=None, + namespace=None, + rbd_mirroring_mode='pool', + weight=None): + """Build common part of a create pool operation. + + :param app_name: Tag pool with application name. Note that there is + certain protocols emerging upstream with regard to + meaningful application names to use. + Examples are 'rbd' and 'rgw'. + :type app_name: Optional[str] + :param compression_algorithm: Compressor to use, one of: + ('lz4', 'snappy', 'zlib', 'zstd') + :type compression_algorithm: Optional[str] + :param compression_mode: When to compress data, one of: + ('none', 'passive', 'aggressive', 'force') + :type compression_mode: Optional[str] + :param compression_required_ratio: Minimum compression ratio for data + chunk, if the requested ratio is not + achieved the compressed version will + be thrown away and the original + stored. + :type compression_required_ratio: Optional[float] + :param compression_min_blob_size: Chunks smaller than this are never + compressed (unit: bytes). + :type compression_min_blob_size: Optional[int] + :param compression_min_blob_size_hdd: Chunks smaller than this are not + compressed when destined to + rotational media (unit: bytes). + :type compression_min_blob_size_hdd: Optional[int] + :param compression_min_blob_size_ssd: Chunks smaller than this are not + compressed when destined to flash + media (unit: bytes). + :type compression_min_blob_size_ssd: Optional[int] + :param compression_max_blob_size: Chunks larger than this are broken + into N * compression_max_blob_size + chunks before being compressed + (unit: bytes). + :type compression_max_blob_size: Optional[int] + :param compression_max_blob_size_hdd: Chunks larger than this are + broken into + N * compression_max_blob_size_hdd + chunks before being compressed + when destined for rotational + media (unit: bytes) + :type compression_max_blob_size_hdd: Optional[int] + :param compression_max_blob_size_ssd: Chunks larger than this are + broken into + N * compression_max_blob_size_ssd + chunks before being compressed + when destined for flash media + (unit: bytes). + :type compression_max_blob_size_ssd: Optional[int] + :param group: Group to add pool to + :type group: Optional[str] + :param max_bytes: Maximum bytes quota to apply + :type max_bytes: Optional[int] + :param max_objects: Maximum objects quota to apply + :type max_objects: Optional[int] + :param namespace: Group namespace + :type namespace: Optional[str] + :param rbd_mirroring_mode: Pool mirroring mode used when Ceph RBD + mirroring is enabled. + :type rbd_mirroring_mode: Optional[str] + :param weight: The percentage of data that is expected to be contained + in the pool from the total available space on the OSDs. + Used to calculate number of Placement Groups to create + for pool. + :type weight: Optional[float] + :returns: Dictionary with kwarg name as key. + :rtype: Dict[str,any] + :raises: AssertionError + """ + return { + 'app-name': app_name, + 'compression-algorithm': compression_algorithm, + 'compression-mode': compression_mode, + 'compression-required-ratio': compression_required_ratio, + 'compression-min-blob-size': compression_min_blob_size, + 'compression-min-blob-size-hdd': compression_min_blob_size_hdd, + 'compression-min-blob-size-ssd': compression_min_blob_size_ssd, + 'compression-max-blob-size': compression_max_blob_size, + 'compression-max-blob-size-hdd': compression_max_blob_size_hdd, + 'compression-max-blob-size-ssd': compression_max_blob_size_ssd, + 'group': group, + 'max-bytes': max_bytes, + 'max-objects': max_objects, + 'group-namespace': namespace, + 'rbd-mirroring-mode': rbd_mirroring_mode, + 'weight': weight, + } + + def add_op_create_replicated_pool(self, name, replica_count=3, pg_num=None, + **kwargs): + """Adds an operation to create a replicated pool. + + Refer to docstring for ``_partial_build_common_op_create`` for + documentation of keyword arguments. + + :param name: Name of pool to create + :type name: str + :param replica_count: Number of copies Ceph should keep of your data. + :type replica_count: int + :param pg_num: Request specific number of Placement Groups to create + for pool. + :type pg_num: int + :raises: AssertionError if provided data is of invalid type/range + """ + if pg_num and kwargs.get('weight'): + raise ValueError('pg_num and weight are mutually exclusive') + + op = { + 'op': 'create-pool', + 'name': name, + 'replicas': replica_count, + 'pg_num': pg_num, + } + op.update(self._partial_build_common_op_create(**kwargs)) + + # Initialize Pool-object to validate type and range of ops. + pool = ReplicatedPool('dummy-service', op=op) + pool.validate() + + self.add_op(op) + + def add_op_create_erasure_pool(self, name, erasure_profile=None, + allow_ec_overwrites=False, **kwargs): + """Adds an operation to create a erasure coded pool. + + Refer to docstring for ``_partial_build_common_op_create`` for + documentation of keyword arguments. + + :param name: Name of pool to create + :type name: str + :param erasure_profile: Name of erasure code profile to use. If not + set the ceph-mon unit handling the broker + request will set its default value. + :type erasure_profile: str + :param allow_ec_overwrites: allow EC pools to be overridden + :type allow_ec_overwrites: bool + :raises: AssertionError if provided data is of invalid type/range + """ + op = { + 'op': 'create-pool', + 'name': name, + 'pool-type': 'erasure', + 'erasure-profile': erasure_profile, + 'allow-ec-overwrites': allow_ec_overwrites, + } + op.update(self._partial_build_common_op_create(**kwargs)) + + # Initialize Pool-object to validate type and range of ops. + pool = ErasurePool('dummy-service', op) + pool.validate() + + self.add_op(op) + + def add_op_create_erasure_profile(self, name, + erasure_type='jerasure', + erasure_technique=None, + k=None, m=None, + failure_domain=None, + lrc_locality=None, + shec_durability_estimator=None, + clay_helper_chunks=None, + device_class=None, + clay_scalar_mds=None, + lrc_crush_locality=None): + """Adds an operation to create a erasure coding profile. + + :param name: Name of profile to create + :type name: str + :param erasure_type: Which of the erasure coding plugins should be used + :type erasure_type: string + :param erasure_technique: EC plugin technique to use + :type erasure_technique: string + :param k: Number of data chunks + :type k: int + :param m: Number of coding chunks + :type m: int + :param lrc_locality: Group the coding and data chunks into sets of size locality + (lrc plugin) + :type lrc_locality: int + :param durability_estimator: The number of parity chunks each of which includes + a data chunk in its calculation range (shec plugin) + :type durability_estimator: int + :param helper_chunks: The number of helper chunks to use for recovery operations + (clay plugin) + :type: helper_chunks: int + :param failure_domain: Type of failure domain from Ceph bucket types + to be used + :type failure_domain: string + :param device_class: Device class to use for profile (ssd, hdd) + :type device_class: string + :param clay_scalar_mds: Plugin to use for CLAY layered construction + (jerasure|isa|shec) + :type clay_scaler_mds: string + :param lrc_crush_locality: Type of crush bucket in which set of chunks + defined by lrc_locality will be stored. + :type lrc_crush_locality: string + """ + self.add_op({'op': 'create-erasure-profile', + 'name': name, + 'k': k, + 'm': m, + 'l': lrc_locality, + 'c': shec_durability_estimator, + 'd': clay_helper_chunks, + 'erasure-type': erasure_type, + 'erasure-technique': erasure_technique, + 'failure-domain': failure_domain, + 'device-class': device_class, + 'scalar-mds': clay_scalar_mds, + 'crush-locality': lrc_crush_locality}) + + def set_ops(self, ops): + """Set request ops to provided value. + + Useful for injecting ops that come from a previous request + to allow comparisons to ensure validity. + """ + self.ops = ops + + @property + def request(self): + return json.dumps({'api-version': self.api_version, 'ops': self.ops, + 'request-id': self.request_id}) + + def _ops_equal(self, other): + keys_to_compare = [ + 'replicas', 'name', 'op', 'pg_num', 'group-permission', + 'object-prefix-permissions', + ] + keys_to_compare += list(self._partial_build_common_op_create().keys()) + if len(self.ops) == len(other.ops): + for req_no in range(0, len(self.ops)): + for key in keys_to_compare: + if self.ops[req_no].get(key) != other.ops[req_no].get(key): + return False + else: + return False + return True + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + if self.api_version == other.api_version and \ + self._ops_equal(other): + return True + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class CephBrokerRsp(object): + """Ceph broker response. + + Response is json-decoded and contents provided as methods/properties. + + The API is versioned and defaults to version 1. + """ + + def __init__(self, encoded_rsp): + self.api_version = None + self.rsp = json.loads(encoded_rsp) + + @property + def request_id(self): + return self.rsp.get('request-id') + + @property + def exit_code(self): + return self.rsp.get('exit-code') + + @property + def exit_msg(self): + return self.rsp.get('stderr') + + +# Ceph Broker Conversation: +# If a charm needs an action to be taken by ceph it can create a CephBrokerRq +# and send that request to ceph via the ceph relation. The CephBrokerRq has a +# unique id so that the client can identity which CephBrokerRsp is associated +# with the request. Ceph will also respond to each client unit individually +# creating a response key per client unit eg glance/0 will get a CephBrokerRsp +# via key broker-rsp-glance-0 +# +# To use this the charm can just do something like: +# +# from charmhelpers.contrib.storage.linux.ceph import ( +# send_request_if_needed, +# is_request_complete, +# CephBrokerRq, +# ) +# +# @hooks.hook('ceph-relation-changed') +# def ceph_changed(): +# rq = CephBrokerRq() +# rq.add_op_create_pool(name='poolname', replica_count=3) +# +# if is_request_complete(rq): +# +# else: +# send_request_if_needed(get_ceph_request()) +# +# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example +# of glance having sent a request to ceph which ceph has successfully processed +# 'ceph:8': { +# 'ceph/0': { +# 'auth': 'cephx', +# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}', +# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}', +# 'ceph-public-address': '10.5.44.103', +# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', +# 'private-address': '10.5.44.103', +# }, +# 'glance/0': { +# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", ' +# '"ops": [{"replicas": 3, "name": "glance", ' +# '"op": "create-pool"}]}'), +# 'private-address': '10.5.44.109', +# }, +# } + +def get_previous_request(rid): + """Return the last ceph broker request sent on a given relation + + :param rid: Relation id to query for request + :type rid: str + :returns: CephBrokerRq object or None if relation data not found. + :rtype: Optional[CephBrokerRq] + """ + broker_req = relation_get(attribute='broker_req', rid=rid, + unit=local_unit()) + if broker_req: + return CephBrokerRq(raw_request_data=broker_req) + + +def get_request_states(request, relation='ceph'): + """Return a dict of requests per relation id with their corresponding + completion state. + + This allows a charm, which has a request for ceph, to see whether there is + an equivalent request already being processed and if so what state that + request is in. + + @param request: A CephBrokerRq object + """ + complete = [] + requests = {} + for rid in relation_ids(relation): + complete = False + previous_request = get_previous_request(rid) + if request == previous_request: + sent = True + complete = is_request_complete_for_rid(previous_request, rid) + else: + sent = False + complete = False + + requests[rid] = { + 'sent': sent, + 'complete': complete, + } + + return requests + + +def is_request_sent(request, relation='ceph'): + """Check to see if a functionally equivalent request has already been sent + + Returns True if a similair request has been sent + + @param request: A CephBrokerRq object + """ + states = get_request_states(request, relation=relation) + for rid in states.keys(): + if not states[rid]['sent']: + return False + + return True + + +def is_request_complete(request, relation='ceph'): + """Check to see if a functionally equivalent request has already been + completed + + Returns True if a similair request has been completed + + @param request: A CephBrokerRq object + """ + states = get_request_states(request, relation=relation) + for rid in states.keys(): + if not states[rid]['complete']: + return False + + return True + + +def is_request_complete_for_rid(request, rid): + """Check if a given request has been completed on the given relation + + @param request: A CephBrokerRq object + @param rid: Relation ID + """ + broker_key = get_broker_rsp_key() + for unit in related_units(rid): + rdata = relation_get(rid=rid, unit=unit) + if rdata.get(broker_key): + rsp = CephBrokerRsp(rdata.get(broker_key)) + if rsp.request_id == request.request_id: + if not rsp.exit_code: + return True + else: + # The remote unit sent no reply targeted at this unit so either the + # remote ceph cluster does not support unit targeted replies or it + # has not processed our request yet. + if rdata.get('broker_rsp'): + request_data = json.loads(rdata['broker_rsp']) + if request_data.get('request-id'): + log('Ignoring legacy broker_rsp without unit key as remote ' + 'service supports unit specific replies', level=DEBUG) + else: + log('Using legacy broker_rsp as remote service does not ' + 'supports unit specific replies', level=DEBUG) + rsp = CephBrokerRsp(rdata['broker_rsp']) + if not rsp.exit_code: + return True + + return False + + +def get_broker_rsp_key(): + """Return broker response key for this unit + + This is the key that ceph is going to use to pass request status + information back to this unit + """ + return 'broker-rsp-' + local_unit().replace('/', '-') + + +def send_request_if_needed(request, relation='ceph'): + """Send broker request if an equivalent request has not already been sent + + @param request: A CephBrokerRq object + """ + if is_request_sent(request, relation=relation): + log('Request already sent but not complete, not sending new request', + level=DEBUG) + else: + for rid in relation_ids(relation): + log('Sending request {}'.format(request.request_id), level=DEBUG) + relation_set(relation_id=rid, broker_req=request.request) + relation_set(relation_id=rid, relation_settings={'unit-name': local_unit()}) + + +def has_broker_rsp(rid=None, unit=None): + """Return True if the broker_rsp key is 'truthy' (i.e. set to something) in the relation data. + + :param rid: The relation to check (default of None means current relation) + :type rid: Union[str, None] + :param unit: The remote unit to check (default of None means current unit) + :type unit: Union[str, None] + :returns: True if broker key exists and is set to something 'truthy' + :rtype: bool + """ + rdata = relation_get(rid=rid, unit=unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + return True if broker_rsp else False + + +def is_broker_action_done(action, rid=None, unit=None): + """Check whether broker action has completed yet. + + @param action: name of action to be performed + @returns True if action complete otherwise False + """ + rdata = relation_get(rid=rid, unit=unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return False + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + val = kvstore.get(key=key) + if val and val == rsp.request_id: + return True + + return False + + +def mark_broker_action_done(action, rid=None, unit=None): + """Mark action as having been completed. + + @param action: name of action to be performed + @returns None + """ + rdata = relation_get(rid=rid, unit=unit) or {} + broker_rsp = rdata.get(get_broker_rsp_key()) + if not broker_rsp: + return + + rsp = CephBrokerRsp(broker_rsp) + unit_name = local_unit().partition('/')[2] + key = "unit_{}_ceph_broker_action.{}".format(unit_name, action) + kvstore = kv() + kvstore.set(key=key, value=rsp.request_id) + kvstore.flush() + + +class CephConfContext(object): + """Ceph config (ceph.conf) context. + + Supports user-provided Ceph configuration settings. Use can provide a + dictionary as the value for the config-flags charm option containing + Ceph configuration settings keyede by their section in ceph.conf. + """ + def __init__(self, permitted_sections=None): + self.permitted_sections = permitted_sections or [] + + def __call__(self): + conf = config('config-flags') + if not conf: + return {} + + conf = config_flags_parser(conf) + if not isinstance(conf, dict): + log("Provided config-flags is not a dictionary - ignoring", + level=WARNING) + return {} + + permitted = self.permitted_sections + if permitted: + diff = set(conf.keys()).difference(set(permitted)) + if diff: + log("Config-flags contains invalid keys '%s' - they will be " + "ignored" % (', '.join(diff)), level=WARNING) + + ceph_conf = {} + for key in conf: + if permitted and key not in permitted: + log("Ignoring key '%s'" % key, level=WARNING) + continue + + ceph_conf[key] = conf[key] + return ceph_conf + + +class CephOSDConfContext(CephConfContext): + """Ceph config (ceph.conf) context. + + Consolidates settings from config-flags via CephConfContext with + settings provided by the mons. The config-flag values are preserved in + conf['osd'], settings from the mons which do not clash with config-flag + settings are in conf['osd_from_client'] and finally settings which do + clash are in conf['osd_from_client_conflict']. Rather than silently drop + the conflicting settings they are provided in the context so they can be + rendered commented out to give some visibility to the admin. + """ + + def __init__(self, permitted_sections=None): + super(CephOSDConfContext, self).__init__( + permitted_sections=permitted_sections) + try: + self.settings_from_mons = get_osd_settings('mon') + except OSDSettingConflict: + log( + "OSD settings from mons are inconsistent, ignoring them", + level=WARNING) + self.settings_from_mons = {} + + def filter_osd_from_mon_settings(self): + """Filter settings from client relation against config-flags. + + :returns: A tuple ( + ,config-flag values, + ,client settings which do not conflict with config-flag values, + ,client settings which confilct with config-flag values) + :rtype: (OrderedDict, OrderedDict, OrderedDict) + """ + ceph_conf = super(CephOSDConfContext, self).__call__() + conflicting_entries = {} + clear_entries = {} + for key, value in self.settings_from_mons.items(): + if key in ceph_conf.get('osd', {}): + if ceph_conf['osd'][key] != value: + conflicting_entries[key] = value + else: + clear_entries[key] = value + clear_entries = _order_dict_by_key(clear_entries) + conflicting_entries = _order_dict_by_key(conflicting_entries) + return ceph_conf, clear_entries, conflicting_entries + + def __call__(self): + """Construct OSD config context. + + Standard context with two additional special keys. + osd_from_client_conflict: client settings which confilct with + config-flag values + osd_from_client: settings which do not conflict with config-flag + values + + :returns: OSD config context dict. + :rtype: dict + """ + conf, osd_clear, osd_conflict = self.filter_osd_from_mon_settings() + conf['osd_from_client_conflict'] = osd_conflict + conf['osd_from_client'] = osd_clear + return conf diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/loopback.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/loopback.py new file mode 100644 index 0000000..74bab40 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/loopback.py @@ -0,0 +1,92 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from subprocess import ( + check_call, + check_output, +) + +import six + + +################################################## +# loopback device helpers. +################################################## +def loopback_devices(): + ''' + Parse through 'losetup -a' output to determine currently mapped + loopback devices. Output is expected to look like: + + /dev/loop0: [0807]:961814 (/tmp/my.img) + + or: + + /dev/loop0: [0807]:961814 (/tmp/my.img (deleted)) + + :returns: dict: a dict mapping {loopback_dev: backing_file} + ''' + loopbacks = {} + cmd = ['losetup', '-a'] + output = check_output(cmd) + if six.PY3: + output = output.decode('utf-8') + devs = [d.strip().split(' ', 2) for d in output.splitlines() if d != ''] + for dev, _, f in devs: + loopbacks[dev.replace(':', '')] = re.search(r'\((.+)\)', f).groups()[0] + return loopbacks + + +def create_loopback(file_path): + ''' + Create a loopback device for a given backing file. + + :returns: str: Full path to new loopback device (eg, /dev/loop0) + ''' + file_path = os.path.abspath(file_path) + check_call(['losetup', '--find', file_path]) + for d, f in six.iteritems(loopback_devices()): + if f == file_path: + return d + + +def ensure_loopback_device(path, size): + ''' + Ensure a loopback device exists for a given backing file path and size. + If it a loopback device is not mapped to file, a new one will be created. + + TODO: Confirm size of found loopback device. + + :returns: str: Full path to the ensured loopback device (eg, /dev/loop0) + ''' + for d, f in six.iteritems(loopback_devices()): + if f == path: + return d + + if not os.path.exists(path): + cmd = ['truncate', '--size', size, path] + check_call(cmd) + + return create_loopback(path) + + +def is_mapped_loopback_device(device): + """ + Checks if a given device name is an existing/mapped loopback device. + :param device: str: Full path to the device (eg, /dev/loop1). + :returns: str: Path to the backing file if is a loopback device + empty string otherwise + """ + return loopback_devices().get(device, "") diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/lvm.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/lvm.py new file mode 100644 index 0000000..d0a5721 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/lvm.py @@ -0,0 +1,182 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +from subprocess import ( + CalledProcessError, + check_call, + check_output, + Popen, + PIPE, +) + + +################################################## +# LVM helpers. +################################################## +def deactivate_lvm_volume_group(block_device): + ''' + Deactivate any volume group associated with an LVM physical volume. + + :param block_device: str: Full path to LVM physical volume + ''' + vg = list_lvm_volume_group(block_device) + if vg: + cmd = ['vgchange', '-an', vg] + check_call(cmd) + + +def is_lvm_physical_volume(block_device): + ''' + Determine whether a block device is initialized as an LVM PV. + + :param block_device: str: Full path of block device to inspect. + + :returns: boolean: True if block device is a PV, False if not. + ''' + try: + check_output(['pvdisplay', block_device]) + return True + except CalledProcessError: + return False + + +def remove_lvm_physical_volume(block_device): + ''' + Remove LVM PV signatures from a given block device. + + :param block_device: str: Full path of block device to scrub. + ''' + p = Popen(['pvremove', '-ff', block_device], + stdin=PIPE) + p.communicate(input='y\n') + + +def list_lvm_volume_group(block_device): + ''' + List LVM volume group associated with a given block device. + + Assumes block device is a valid LVM PV. + + :param block_device: str: Full path of block device to inspect. + + :returns: str: Name of volume group associated with block device or None + ''' + vg = None + pvd = check_output(['pvdisplay', block_device]).splitlines() + for lvm in pvd: + lvm = lvm.decode('UTF-8') + if lvm.strip().startswith('VG Name'): + vg = ' '.join(lvm.strip().split()[2:]) + return vg + + +def create_lvm_physical_volume(block_device): + ''' + Initialize a block device as an LVM physical volume. + + :param block_device: str: Full path of block device to initialize. + + ''' + check_call(['pvcreate', block_device]) + + +def create_lvm_volume_group(volume_group, block_device): + ''' + Create an LVM volume group backed by a given block device. + + Assumes block device has already been initialized as an LVM PV. + + :param volume_group: str: Name of volume group to create. + :block_device: str: Full path of PV-initialized block device. + ''' + check_call(['vgcreate', volume_group, block_device]) + + +def list_logical_volumes(select_criteria=None, path_mode=False): + ''' + List logical volumes + + :param select_criteria: str: Limit list to those volumes matching this + criteria (see 'lvs -S help' for more details) + :param path_mode: bool: return logical volume name in 'vg/lv' format, this + format is required for some commands like lvextend + :returns: [str]: List of logical volumes + ''' + lv_diplay_attr = 'lv_name' + if path_mode: + # Parsing output logic relies on the column order + lv_diplay_attr = 'vg_name,' + lv_diplay_attr + cmd = ['lvs', '--options', lv_diplay_attr, '--noheadings'] + if select_criteria: + cmd.extend(['--select', select_criteria]) + lvs = [] + for lv in check_output(cmd).decode('UTF-8').splitlines(): + if not lv: + continue + if path_mode: + lvs.append('/'.join(lv.strip().split())) + else: + lvs.append(lv.strip()) + return lvs + + +list_thin_logical_volume_pools = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^t') + +list_thin_logical_volumes = functools.partial( + list_logical_volumes, + select_criteria='lv_attr =~ ^V') + + +def extend_logical_volume_by_device(lv_name, block_device): + ''' + Extends the size of logical volume lv_name by the amount of free space on + physical volume block_device. + + :param lv_name: str: name of logical volume to be extended (vg/lv format) + :param block_device: str: name of block_device to be allocated to lv_name + ''' + cmd = ['lvextend', lv_name, block_device] + check_call(cmd) + + +def create_logical_volume(lv_name, volume_group, size=None): + ''' + Create a new logical volume in an existing volume group + + :param lv_name: str: name of logical volume to be created. + :param volume_group: str: Name of volume group to use for the new volume. + :param size: str: Size of logical volume to create (100% if not supplied) + :raises subprocess.CalledProcessError: in the event that the lvcreate fails. + ''' + if size: + check_call([ + 'lvcreate', + '--yes', + '-L', + '{}'.format(size), + '-n', lv_name, volume_group + ]) + # create the lv with all the space available, this is needed because the + # system call is different for LVM + else: + check_call([ + 'lvcreate', + '--yes', + '-l', + '100%FREE', + '-n', lv_name, volume_group + ]) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/utils.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/utils.py new file mode 100644 index 0000000..a356176 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/storage/linux/utils.py @@ -0,0 +1,128 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from stat import S_ISBLK + +from subprocess import ( + CalledProcessError, + check_call, + check_output, + call +) + + +def _luks_uuid(dev): + """ + Check to see if dev is a LUKS encrypted volume, returning the UUID + of volume if it is. + + :param: dev: path to block device to check. + :returns: str. UUID of LUKS device or None if not a LUKS device + """ + try: + cmd = ['cryptsetup', 'luksUUID', dev] + return check_output(cmd).decode('UTF-8').strip() + except CalledProcessError: + return None + + +def is_luks_device(dev): + """ + Determine if dev is a LUKS-formatted block device. + + :param: dev: A full path to a block device to check for LUKS header + presence + :returns: boolean: indicates whether a device is used based on LUKS header. + """ + return True if _luks_uuid(dev) else False + + +def is_mapped_luks_device(dev): + """ + Determine if dev is a mapped LUKS device + :param: dev: A full path to a block device to be checked + :returns: boolean: indicates whether a device is mapped + """ + _, dirs, _ = next(os.walk( + '/sys/class/block/{}/holders/' + .format(os.path.basename(os.path.realpath(dev)))) + ) + is_held = len(dirs) > 0 + return is_held and is_luks_device(dev) + + +def is_block_device(path): + ''' + Confirm device at path is a valid block device node. + + :returns: boolean: True if path is a block device, False if not. + ''' + if not os.path.exists(path): + return False + return S_ISBLK(os.stat(path).st_mode) + + +def zap_disk(block_device): + ''' + Clear a block device of partition table. Relies on sgdisk, which is + installed as pat of the 'gdisk' package in Ubuntu. + + :param block_device: str: Full path of block device to clean. + ''' + # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b + # sometimes sgdisk exits non-zero; this is OK, dd will clean up + call(['sgdisk', '--zap-all', '--', block_device]) + call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device]) + dev_end = check_output(['blockdev', '--getsz', + block_device]).decode('UTF-8') + gpt_end = int(dev_end.split()[0]) - 100 + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), + 'bs=1M', 'count=1']) + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), + 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) + + +def is_device_mounted(device): + '''Given a device path, return True if that device is mounted, and False + if it isn't. + + :param device: str: Full path of the device to check. + :returns: boolean: True if the path represents a mounted device, False if + it doesn't. + ''' + try: + out = check_output(['lsblk', '-P', device]).decode('UTF-8') + except Exception: + return False + return bool(re.search(r'MOUNTPOINT=".+"', out)) + + +def mkfs_xfs(device, force=False, inode_size=1024): + """Format device with XFS filesystem. + + By default this should fail if the device already has a filesystem on it. + :param device: Full path to device to format + :ptype device: tr + :param force: Force operation + :ptype: force: boolean + :param inode_size: XFS inode size in bytes + :ptype inode_size: int""" + cmd = ['mkfs.xfs'] + if force: + cmd.append("-f") + + cmd += ['-i', "size={}".format(inode_size), device] + check_call(cmd) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/sysctl/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/sysctl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/sysctl/watermark_scale_factor.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/sysctl/watermark_scale_factor.py new file mode 100644 index 0000000..cfc9534 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/sysctl/watermark_scale_factor.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + ERROR, +) + +import re +from charmhelpers.core.host import get_total_ram + +WMARK_MAX = 1000 +WMARK_DEFAULT = 10 +MEMTOTAL_MIN_BYTES = 17179803648 # 16G +MAX_PAGES = 2500000000 + + +def calculate_watermark_scale_factor(): + """Calculates optimal vm.watermark_scale_factor value + + :returns: watermark_scale_factor + :rtype: int + """ + + memtotal = get_total_ram() + normal_managed_pages = get_normal_managed_pages() + + try: + wmark = min([watermark_scale_factor(memtotal, managed_pages) + for managed_pages in normal_managed_pages]) + except ValueError as e: + log("Failed to calculate watermark_scale_factor from normal managed pages: {}".format(normal_managed_pages), ERROR) + raise e + + log("vm.watermark_scale_factor: {}".format(wmark), DEBUG) + return wmark + + +def get_normal_managed_pages(): + """Parse /proc/zoneinfo for managed pages of the + normal zone on each node + + :returns: normal_managed_pages + :rtype: [int] + """ + try: + normal_managed_pages = [] + with open('/proc/zoneinfo', 'r') as f: + in_zone_normal = False + # regex to search for strings that look like "Node 0, zone Normal" and last string to group 1 + normal_zone_matcher = re.compile(r"^Node\s\d+,\s+zone\s+(\S+)$") + # regex to match to a number at the end of the line. + managed_matcher = re.compile(r"\s+managed\s+(\d+)$") + for line in f.readlines(): + match = normal_zone_matcher.search(line) + if match: + in_zone_normal = match.group(1) == 'Normal' + if in_zone_normal: + # match the number at the end of " managed 3840" into group 1. + managed_match = managed_matcher.search(line) + if managed_match: + normal_managed_pages.append(int(managed_match.group(1))) + in_zone_normal = False + + except OSError as e: + log("Failed to read /proc/zoneinfo in calculating watermark_scale_factor: {}".format(e), ERROR) + raise e + + return normal_managed_pages + + +def watermark_scale_factor(memtotal, managed_pages): + """Calculate a value for vm.watermark_scale_factor + + :param memtotal: Total system memory in KB + :type memtotal: int + :param managed_pages: Number of managed pages + :type managed_pages: int + :returns: normal_managed_pages + :rtype: int + """ + if memtotal <= MEMTOTAL_MIN_BYTES: + return WMARK_DEFAULT + else: + WMARK = int(MAX_PAGES / managed_pages) + if WMARK > WMARK_MAX: + return WMARK_MAX + else: + return WMARK diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/contexts.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/contexts.py new file mode 100644 index 0000000..c1adf94 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/contexts.py @@ -0,0 +1,137 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +"""A helper to create a yaml cache of config with namespaced relation data.""" +import os +import yaml + +import six + +import charmhelpers.core.hookenv + + +charm_dir = os.environ.get('CHARM_DIR', '') + + +def dict_keys_without_hyphens(a_dict): + """Return the a new dict with underscores instead of hyphens in keys.""" + return dict( + (key.replace('-', '_'), val) for key, val in a_dict.items()) + + +def update_relations(context, namespace_separator=':'): + """Update the context with the relation data.""" + # Add any relation data prefixed with the relation type. + relation_type = charmhelpers.core.hookenv.relation_type() + relations = [] + context['current_relation'] = {} + if relation_type is not None: + relation_data = charmhelpers.core.hookenv.relation_get() + context['current_relation'] = relation_data + # Deprecated: the following use of relation data as keys + # directly in the context will be removed. + relation_data = dict( + ("{relation_type}{namespace_separator}{key}".format( + relation_type=relation_type, + key=key, + namespace_separator=namespace_separator), val) + for key, val in relation_data.items()) + relation_data = dict_keys_without_hyphens(relation_data) + context.update(relation_data) + relations = charmhelpers.core.hookenv.relations_of_type(relation_type) + relations = [dict_keys_without_hyphens(rel) for rel in relations] + + context['relations_full'] = charmhelpers.core.hookenv.relations() + + # the hookenv.relations() data structure is effectively unusable in + # templates and other contexts when trying to access relation data other + # than the current relation. So provide a more useful structure that works + # with any hook. + local_unit = charmhelpers.core.hookenv.local_unit() + relations = {} + for rname, rids in context['relations_full'].items(): + relations[rname] = [] + for rid, rdata in rids.items(): + data = rdata.copy() + if local_unit in rdata: + data.pop(local_unit) + for unit_name, rel_data in data.items(): + new_data = {'__relid__': rid, '__unit__': unit_name} + new_data.update(rel_data) + relations[rname].append(new_data) + context['relations'] = relations + + +def juju_state_to_yaml(yaml_path, namespace_separator=':', + allow_hyphens_in_keys=True, mode=None): + """Update the juju config and state in a yaml file. + + This includes any current relation-get data, and the charm + directory. + + This function was created for the ansible and saltstack + support, as those libraries can use a yaml file to supply + context to templates, but it may be useful generally to + create and update an on-disk cache of all the config, including + previous relation data. + + By default, hyphens are allowed in keys as this is supported + by yaml, but for tools like ansible, hyphens are not valid [1]. + + [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name + """ + config = charmhelpers.core.hookenv.config() + + # Add the charm_dir which we will need to refer to charm + # file resources etc. + config['charm_dir'] = charm_dir + config['local_unit'] = charmhelpers.core.hookenv.local_unit() + config['unit_private_address'] = charmhelpers.core.hookenv.unit_private_ip() + config['unit_public_address'] = charmhelpers.core.hookenv.unit_get( + 'public-address' + ) + + # Don't use non-standard tags for unicode which will not + # work when salt uses yaml.load_safe. + yaml.add_representer(six.text_type, + lambda dumper, value: dumper.represent_scalar( + six.u('tag:yaml.org,2002:str'), value)) + + yaml_dir = os.path.dirname(yaml_path) + if not os.path.exists(yaml_dir): + os.makedirs(yaml_dir) + + if os.path.exists(yaml_path): + with open(yaml_path, "r") as existing_vars_file: + existing_vars = yaml.load(existing_vars_file.read()) + else: + with open(yaml_path, "w+"): + pass + existing_vars = {} + + if mode is not None: + os.chmod(yaml_path, mode) + + if not allow_hyphens_in_keys: + config = dict_keys_without_hyphens(config) + existing_vars.update(config) + + update_relations(existing_vars, namespace_separator) + + with open(yaml_path, "w+") as fp: + fp.write(yaml.dump(existing_vars, default_flow_style=False)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/jinja.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/jinja.py new file mode 100644 index 0000000..c6ad9d0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/jinja.py @@ -0,0 +1,51 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Templating using the python-jinja2 package. +""" +import six +from charmhelpers.fetch import apt_install, apt_update +try: + import jinja2 +except ImportError: + apt_update(fatal=True) + if six.PY3: + apt_install(["python3-jinja2"], fatal=True) + else: + apt_install(["python-jinja2"], fatal=True) + import jinja2 + + +DEFAULT_TEMPLATES_DIR = 'templates' + + +def render(template_name, context, template_dir=DEFAULT_TEMPLATES_DIR, + jinja_env_args=None): + """ + Render jinja2 template with provided context. + + :param template_name: name of the jinja template file + :param context: template context + :param template_dir: directory in which the template file is located + :param jinja_env_args: additional arguments passed to the + jinja2.Environment. Expected dict with format + {'arg_name': 'arg_value'} + :return: Rendered template as a string + """ + env_kwargs = jinja_env_args or {} + templates = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir), **env_kwargs) + template = templates.get_template(template_name) + return template.render(context) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/pyformat.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/pyformat.py new file mode 100644 index 0000000..51a24dc --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/templating/pyformat.py @@ -0,0 +1,27 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Templating using standard Python str.format() method. +''' + +from charmhelpers.core import hookenv + + +def render(template, extra={}, **kwargs): + """Return the template rendered using Python's str.format().""" + context = hookenv.execution_environment() + context.update(extra) + context.update(kwargs) + return template.format(**context) diff --git a/nrpe/mod/charmhelpers/charmhelpers/contrib/unison/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/contrib/unison/__init__.py new file mode 100644 index 0000000..0fd71b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/contrib/unison/__init__.py @@ -0,0 +1,316 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Easy file synchronization among peer units using ssh + unison. +# +# For the -joined, -changed, and -departed peer relations, add a call to +# ssh_authorized_peers() describing the peer relation and the desired +# user + group. After all peer relations have settled, all hosts should +# be able to connect to on another via key auth'd ssh as the specified user. +# +# Other hooks are then free to synchronize files and directories using +# sync_to_peers(). +# +# For a peer relation named 'cluster', for example: +# +# cluster-relation-joined: +# ... +# ssh_authorized_peers(peer_interface='cluster', +# user='juju_ssh', group='juju_ssh', +# ensure_local_user=True) +# ... +# +# cluster-relation-changed: +# ... +# ssh_authorized_peers(peer_interface='cluster', +# user='juju_ssh', group='juju_ssh', +# ensure_local_user=True) +# ... +# +# cluster-relation-departed: +# ... +# ssh_authorized_peers(peer_interface='cluster', +# user='juju_ssh', group='juju_ssh', +# ensure_local_user=True) +# ... +# +# Hooks are now free to sync files as easily as: +# +# files = ['/etc/fstab', '/etc/apt.conf.d/'] +# sync_to_peers(peer_interface='cluster', +# user='juju_ssh, paths=[files]) +# +# It is assumed the charm itself has setup permissions on each unit +# such that 'juju_ssh' has read + write permissions. Also assumed +# that the calling charm takes care of leader delegation. +# +# Additionally files can be synchronized only to an specific unit: +# sync_to_peer(slave_address, user='juju_ssh', +# paths=[files], verbose=False) + +import os +import pwd + +from copy import copy +from subprocess import check_call, check_output + +from charmhelpers.core.host import ( + adduser, + add_user_to_group, + pwgen, + remove_password_expiry, +) + +from charmhelpers.core.hookenv import ( + log, + hook_name, + relation_ids, + related_units, + relation_set, + relation_get, + unit_private_ip, + INFO, + ERROR, +) + +BASE_CMD = ['unison', '-auto', '-batch=true', '-confirmbigdel=false', + '-fastcheck=true', '-group=false', '-owner=false', + '-prefer=newer', '-times=true'] + + +def get_homedir(user): + try: + user = pwd.getpwnam(user) + return user.pw_dir + except KeyError: + log('Could not get homedir for user %s: user exists?' % (user), ERROR) + raise Exception + + +def create_private_key(user, priv_key_path, key_type='rsa'): + types_bits = { + 'rsa': '2048', + 'ecdsa': '521', + } + if key_type not in types_bits: + log('Unknown ssh key type {}, using rsa'.format(key_type), ERROR) + key_type = 'rsa' + if not os.path.isfile(priv_key_path): + log('Generating new SSH key for user %s.' % user) + cmd = ['ssh-keygen', '-q', '-N', '', '-t', key_type, + '-b', types_bits[key_type], '-f', priv_key_path] + check_call(cmd) + else: + log('SSH key already exists at %s.' % priv_key_path) + os.chown(priv_key_path, pwd.getpwnam(user).pw_uid, -1) + os.chmod(priv_key_path, 0o600) + + +def create_public_key(user, priv_key_path, pub_key_path): + if not os.path.isfile(pub_key_path): + log('Generating missing ssh public key @ %s.' % pub_key_path) + cmd = ['ssh-keygen', '-y', '-f', priv_key_path] + p = check_output(cmd).strip() + with open(pub_key_path, 'wb') as out: + out.write(p) + os.chown(pub_key_path, pwd.getpwnam(user).pw_uid, -1) + + +def get_keypair(user): + home_dir = get_homedir(user) + ssh_dir = os.path.join(home_dir, '.ssh') + priv_key = os.path.join(ssh_dir, 'id_rsa') + pub_key = '%s.pub' % priv_key + + if not os.path.isdir(ssh_dir): + os.mkdir(ssh_dir) + check_call(['chown', '-R', user, ssh_dir]) + + create_private_key(user, priv_key) + create_public_key(user, priv_key, pub_key) + + with open(priv_key, 'r') as p: + _priv = p.read().strip() + + with open(pub_key, 'r') as p: + _pub = p.read().strip() + + return (_priv, _pub) + + +def write_authorized_keys(user, keys): + home_dir = get_homedir(user) + ssh_dir = os.path.join(home_dir, '.ssh') + auth_keys = os.path.join(ssh_dir, 'authorized_keys') + log('Syncing authorized_keys @ %s.' % auth_keys) + with open(auth_keys, 'w') as out: + for k in keys: + out.write('%s\n' % k) + os.chown(auth_keys, pwd.getpwnam(user).pw_uid, -1) + + +def write_known_hosts(user, hosts): + home_dir = get_homedir(user) + ssh_dir = os.path.join(home_dir, '.ssh') + known_hosts = os.path.join(ssh_dir, 'known_hosts') + khosts = [] + for host in hosts: + cmd = ['ssh-keyscan', host] + remote_key = check_output(cmd, universal_newlines=True).strip() + khosts.append(remote_key) + log('Syncing known_hosts @ %s.' % known_hosts) + with open(known_hosts, 'w') as out: + for host in khosts: + out.write('%s\n' % host) + os.chown(known_hosts, pwd.getpwnam(user).pw_uid, -1) + + +def ensure_user(user, group=None): + adduser(user, pwgen()) + if group: + add_user_to_group(user, group) + # Remove password expiry (Bug #1686085) + remove_password_expiry(user) + + +def ssh_authorized_peers(peer_interface, user, group=None, + ensure_local_user=False): + """ + Main setup function, should be called from both peer -changed and -joined + hooks with the same parameters. + """ + if ensure_local_user: + ensure_user(user, group) + priv_key, pub_key = get_keypair(user) + hook = hook_name() + if hook == '%s-relation-joined' % peer_interface: + relation_set(ssh_pub_key=pub_key) + elif hook == '%s-relation-changed' % peer_interface or \ + hook == '%s-relation-departed' % peer_interface: + hosts = [] + keys = [] + + for r_id in relation_ids(peer_interface): + for unit in related_units(r_id): + ssh_pub_key = relation_get('ssh_pub_key', + rid=r_id, + unit=unit) + priv_addr = relation_get('private-address', + rid=r_id, + unit=unit) + if ssh_pub_key: + keys.append(ssh_pub_key) + hosts.append(priv_addr) + else: + log('ssh_authorized_peers(): ssh_pub_key ' + 'missing for unit %s, skipping.' % unit) + write_authorized_keys(user, keys) + write_known_hosts(user, hosts) + authed_hosts = ':'.join(hosts) + relation_set(ssh_authorized_hosts=authed_hosts) + + +def _run_as_user(user, gid=None): + try: + user = pwd.getpwnam(user) + except KeyError: + log('Invalid user: %s' % user) + raise Exception + uid = user.pw_uid + gid = gid or user.pw_gid + os.environ['HOME'] = user.pw_dir + + def _inner(): + os.setgid(gid) + os.setuid(uid) + return _inner + + +def run_as_user(user, cmd, gid=None): + return check_output(cmd, preexec_fn=_run_as_user(user, gid), cwd='/') + + +def collect_authed_hosts(peer_interface): + '''Iterate through the units on peer interface to find all that + have the calling host in its authorized hosts list''' + hosts = [] + for r_id in (relation_ids(peer_interface) or []): + for unit in related_units(r_id): + private_addr = relation_get('private-address', + rid=r_id, unit=unit) + authed_hosts = relation_get('ssh_authorized_hosts', + rid=r_id, unit=unit) + + if not authed_hosts: + log('Peer %s has not authorized *any* hosts yet, skipping.' % + (unit), level=INFO) + continue + + if unit_private_ip() in authed_hosts.split(':'): + hosts.append(private_addr) + else: + log('Peer %s has not authorized *this* host yet, skipping.' % + (unit), level=INFO) + return hosts + + +def sync_path_to_host(path, host, user, verbose=False, cmd=None, gid=None, + fatal=False): + """Sync path to an specific peer host + + Propagates exception if operation fails and fatal=True. + """ + cmd = cmd or copy(BASE_CMD) + if not verbose: + cmd.append('-silent') + + # removing trailing slash from directory paths, unison + # doesn't like these. + if path.endswith('/'): + path = path[:(len(path) - 1)] + + cmd = cmd + [path, 'ssh://%s@%s/%s' % (user, host, path)] + + try: + log('Syncing local path %s to %s@%s:%s' % (path, user, host, path)) + run_as_user(user, cmd, gid) + except Exception: + log('Error syncing remote files') + if fatal: + raise + + +def sync_to_peer(host, user, paths=None, verbose=False, cmd=None, gid=None, + fatal=False): + """Sync paths to an specific peer host + + Propagates exception if any operation fails and fatal=True. + """ + if paths: + for p in paths: + sync_path_to_host(p, host, user, verbose, cmd, gid, fatal) + + +def sync_to_peers(peer_interface, user, paths=None, verbose=False, cmd=None, + gid=None, fatal=False): + """Sync all hosts to an specific path + + The type of group is integer, it allows user has permissions to + operate a directory have a different group id with the user id. + + Propagates exception if any operation fails and fatal=True. + """ + if paths: + for host in collect_authed_hosts(peer_interface): + sync_to_peer(host, user, paths, verbose, cmd, gid, fatal) diff --git a/nrpe/mod/charmhelpers/charmhelpers/coordinator.py b/nrpe/mod/charmhelpers/charmhelpers/coordinator.py new file mode 100644 index 0000000..59bee3e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/coordinator.py @@ -0,0 +1,606 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +The coordinator module allows you to use Juju's leadership feature to +coordinate operations between units of a service. + +Behavior is defined in subclasses of coordinator.BaseCoordinator. +One implementation is provided (coordinator.Serial), which allows an +operation to be run on a single unit at a time, on a first come, first +served basis. You can trivially define more complex behavior by +subclassing BaseCoordinator or Serial. + +:author: Stuart Bishop + + +Services Framework Usage +======================== + +Ensure a peers relation is defined in metadata.yaml. Instantiate a +BaseCoordinator subclass before invoking ServiceManager.manage(). +Ensure that ServiceManager.manage() is wired up to the leader-elected, +leader-settings-changed, peers relation-changed and peers +relation-departed hooks in addition to any other hooks you need, or your +service will deadlock. + +Ensure calls to acquire() are guarded, so that locks are only requested +when they are really needed (and thus hooks only triggered when necessary). +Failing to do this and calling acquire() unconditionally will put your unit +into a hook loop. Calls to granted() do not need to be guarded. + +For example:: + + from charmhelpers.core import hookenv, services + from charmhelpers import coordinator + + def maybe_restart(servicename): + serial = coordinator.Serial() + if needs_restart(): + serial.acquire('restart') + if serial.granted('restart'): + hookenv.service_restart(servicename) + + services = [dict(service='servicename', + data_ready=[maybe_restart])] + + if __name__ == '__main__': + _ = coordinator.Serial() # Must instantiate before manager.manage() + manager = services.ServiceManager(services) + manager.manage() + + +You can implement a similar pattern using a decorator. If the lock has +not been granted, an attempt to acquire() it will be made if the guard +function returns True. If the lock has been granted, the decorated function +is run as normal:: + + from charmhelpers.core import hookenv, services + from charmhelpers import coordinator + + serial = coordinator.Serial() # Global, instatiated on module import. + + def needs_restart(): + [ ... Introspect state. Return True if restart is needed ... ] + + @serial.require('restart', needs_restart) + def maybe_restart(servicename): + hookenv.service_restart(servicename) + + services = [dict(service='servicename', + data_ready=[maybe_restart])] + + if __name__ == '__main__': + manager = services.ServiceManager(services) + manager.manage() + + +Traditional Usage +================= + +Ensure a peers relation is defined in metadata.yaml. + +If you are using charmhelpers.core.hookenv.Hooks, ensure that a +BaseCoordinator subclass is instantiated before calling Hooks.execute. + +If you are not using charmhelpers.core.hookenv.Hooks, ensure +that a BaseCoordinator subclass is instantiated and its handle() +method called at the start of all your hooks. + +For example:: + + import sys + from charmhelpers.core import hookenv + from charmhelpers import coordinator + + hooks = hookenv.Hooks() + + def maybe_restart(): + serial = coordinator.Serial() + if serial.granted('restart'): + hookenv.service_restart('myservice') + + @hooks.hook + def config_changed(): + update_config() + serial = coordinator.Serial() + if needs_restart(): + serial.acquire('restart'): + maybe_restart() + + # Cluster hooks must be wired up. + @hooks.hook('cluster-relation-changed', 'cluster-relation-departed') + def cluster_relation_changed(): + maybe_restart() + + # Leader hooks must be wired up. + @hooks.hook('leader-elected', 'leader-settings-changed') + def leader_settings_changed(): + maybe_restart() + + [ ... repeat for *all* other hooks you are using ... ] + + if __name__ == '__main__': + _ = coordinator.Serial() # Must instantiate before execute() + hooks.execute(sys.argv) + + +You can also use the require decorator. If the lock has not been granted, +an attempt to acquire() it will be made if the guard function returns True. +If the lock has been granted, the decorated function is run as normal:: + + from charmhelpers.core import hookenv + + hooks = hookenv.Hooks() + serial = coordinator.Serial() # Must instantiate before execute() + + @require('restart', needs_restart) + def maybe_restart(): + hookenv.service_restart('myservice') + + @hooks.hook('install', 'config-changed', 'upgrade-charm', + # Peers and leader hooks must be wired up. + 'cluster-relation-changed', 'cluster-relation-departed', + 'leader-elected', 'leader-settings-changed') + def default_hook(): + [...] + maybe_restart() + + if __name__ == '__main__': + hooks.execute() + + +Details +======= + +A simple API is provided similar to traditional locking APIs. A lock +may be requested using the acquire() method, and the granted() method +may be used do to check if a lock previously requested by acquire() has +been granted. It doesn't matter how many times acquire() is called in a +hook. + +Locks are released at the end of the hook they are acquired in. This may +be the current hook if the unit is leader and the lock is free. It is +more likely a future hook (probably leader-settings-changed, possibly +the peers relation-changed or departed hook, potentially any hook). + +Whenever a charm needs to perform a coordinated action it will acquire() +the lock and perform the action immediately if acquisition is +successful. It will also need to perform the same action in every other +hook if the lock has been granted. + + +Grubby Details +-------------- + +Why do you need to be able to perform the same action in every hook? +If the unit is the leader, then it may be able to grant its own lock +and perform the action immediately in the source hook. If the unit is +the leader and cannot immediately grant the lock, then its only +guaranteed chance of acquiring the lock is in the peers relation-joined, +relation-changed or peers relation-departed hooks when another unit has +released it (the only channel to communicate to the leader is the peers +relation). If the unit is not the leader, then it is unlikely the lock +is granted in the source hook (a previous hook must have also made the +request for this to happen). A non-leader is notified about the lock via +leader settings. These changes may be visible in any hook, even before +the leader-settings-changed hook has been invoked. Or the requesting +unit may be promoted to leader after making a request, in which case the +lock may be granted in leader-elected or in a future peers +relation-changed or relation-departed hook. + +This could be simpler if leader-settings-changed was invoked on the +leader. We could then never grant locks except in +leader-settings-changed hooks giving one place for the operation to be +performed. Unfortunately this is not the case with Juju 1.23 leadership. + +But of course, this doesn't really matter to most people as most people +seem to prefer the Services Framework or similar reset-the-world +approaches, rather than the twisty maze of attempting to deduce what +should be done based on what hook happens to be running (which always +seems to evolve into reset-the-world anyway when the charm grows beyond +the trivial). + +I chose not to implement a callback model, where a callback was passed +to acquire to be executed when the lock is granted, because the callback +may become invalid between making the request and the lock being granted +due to an upgrade-charm being run in the interim. And it would create +restrictions, such no lambdas, callback defined at the top level of a +module, etc. Still, we could implement it on top of what is here, eg. +by adding a defer decorator that stores a pickle of itself to disk and +have BaseCoordinator unpickle and execute them when the locks are granted. +''' +from datetime import datetime +from functools import wraps +import json +import os.path + +from six import with_metaclass + +from charmhelpers.core import hookenv + + +# We make BaseCoordinator and subclasses singletons, so that if we +# need to spill to local storage then only a single instance does so, +# rather than having multiple instances stomp over each other. +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, + **kwargs) + return cls._instances[cls] + + +class BaseCoordinator(with_metaclass(Singleton, object)): + relid = None # Peer relation-id, set by __init__ + relname = None + + grants = None # self.grants[unit][lock] == timestamp + requests = None # self.requests[unit][lock] == timestamp + + def __init__(self, relation_key='coordinator', peer_relation_name=None): + '''Instatiate a Coordinator. + + Data is stored on the peers relation and in leadership storage + under the provided relation_key. + + The peers relation is identified by peer_relation_name, and defaults + to the first one found in metadata.yaml. + ''' + # Most initialization is deferred, since invoking hook tools from + # the constructor makes testing hard. + self.key = relation_key + self.relname = peer_relation_name + hookenv.atstart(self.initialize) + + # Ensure that handle() is called, without placing that burden on + # the charm author. They still need to do this manually if they + # are not using a hook framework. + hookenv.atstart(self.handle) + + def initialize(self): + if self.requests is not None: + return # Already initialized. + + assert hookenv.has_juju_version('1.23'), 'Needs Juju 1.23+' + + if self.relname is None: + self.relname = _implicit_peer_relation_name() + + relids = hookenv.relation_ids(self.relname) + if relids: + self.relid = sorted(relids)[0] + + # Load our state, from leadership, the peer relationship, and maybe + # local state as a fallback. Populates self.requests and self.grants. + self._load_state() + self._emit_state() + + # Save our state if the hook completes successfully. + hookenv.atexit(self._save_state) + + # Schedule release of granted locks for the end of the hook. + # This needs to be the last of our atexit callbacks to ensure + # it will be run first when the hook is complete, because there + # is no point mutating our state after it has been saved. + hookenv.atexit(self._release_granted) + + def acquire(self, lock): + '''Acquire the named lock, non-blocking. + + The lock may be granted immediately, or in a future hook. + + Returns True if the lock has been granted. The lock will be + automatically released at the end of the hook in which it is + granted. + + Do not mindlessly call this method, as it triggers a cascade of + hooks. For example, if you call acquire() every time in your + peers relation-changed hook you will end up with an infinite loop + of hooks. It should almost always be guarded by some condition. + ''' + unit = hookenv.local_unit() + ts = self.requests[unit].get(lock) + if not ts: + # If there is no outstanding request on the peers relation, + # create one. + self.requests.setdefault(lock, {}) + self.requests[unit][lock] = _timestamp() + self.msg('Requested {}'.format(lock)) + + # If the leader has granted the lock, yay. + if self.granted(lock): + self.msg('Acquired {}'.format(lock)) + return True + + # If the unit making the request also happens to be the + # leader, it must handle the request now. Even though the + # request has been stored on the peers relation, the peers + # relation-changed hook will not be triggered. + if hookenv.is_leader(): + return self.grant(lock, unit) + + return False # Can't acquire lock, yet. Maybe next hook. + + def granted(self, lock): + '''Return True if a previously requested lock has been granted''' + unit = hookenv.local_unit() + ts = self.requests[unit].get(lock) + if ts and self.grants.get(unit, {}).get(lock) == ts: + return True + return False + + def requested(self, lock): + '''Return True if we are in the queue for the lock''' + return lock in self.requests[hookenv.local_unit()] + + def request_timestamp(self, lock): + '''Return the timestamp of our outstanding request for lock, or None. + + Returns a datetime.datetime() UTC timestamp, with no tzinfo attribute. + ''' + ts = self.requests[hookenv.local_unit()].get(lock, None) + if ts is not None: + return datetime.strptime(ts, _timestamp_format) + + def handle(self): + if not hookenv.is_leader(): + return # Only the leader can grant requests. + + self.msg('Leader handling coordinator requests') + + # Clear our grants that have been released. + for unit in self.grants.keys(): + for lock, grant_ts in list(self.grants[unit].items()): + req_ts = self.requests.get(unit, {}).get(lock) + if req_ts != grant_ts: + # The request timestamp does not match the granted + # timestamp. Several hooks on 'unit' may have run + # before the leader got a chance to make a decision, + # and 'unit' may have released its lock and attempted + # to reacquire it. This will change the timestamp, + # and we correctly revoke the old grant putting it + # to the end of the queue. + ts = datetime.strptime(self.grants[unit][lock], + _timestamp_format) + del self.grants[unit][lock] + self.released(unit, lock, ts) + + # Grant locks + for unit in self.requests.keys(): + for lock in self.requests[unit]: + self.grant(lock, unit) + + def grant(self, lock, unit): + '''Maybe grant the lock to a unit. + + The decision to grant the lock or not is made for $lock + by a corresponding method grant_$lock, which you may define + in a subclass. If no such method is defined, the default_grant + method is used. See Serial.default_grant() for details. + ''' + if not hookenv.is_leader(): + return False # Not the leader, so we cannot grant. + + # Set of units already granted the lock. + granted = set() + for u in self.grants: + if lock in self.grants[u]: + granted.add(u) + if unit in granted: + return True # Already granted. + + # Ordered list of units waiting for the lock. + reqs = set() + for u in self.requests: + if u in granted: + continue # In the granted set. Not wanted in the req list. + for _lock, ts in self.requests[u].items(): + if _lock == lock: + reqs.add((ts, u)) + queue = [t[1] for t in sorted(reqs)] + if unit not in queue: + return False # Unit has not requested the lock. + + # Locate custom logic, or fallback to the default. + grant_func = getattr(self, 'grant_{}'.format(lock), self.default_grant) + + if grant_func(lock, unit, granted, queue): + # Grant the lock. + self.msg('Leader grants {} to {}'.format(lock, unit)) + self.grants.setdefault(unit, {})[lock] = self.requests[unit][lock] + return True + + return False + + def released(self, unit, lock, timestamp): + '''Called on the leader when it has released a lock. + + By default, does nothing but log messages. Override if you + need to perform additional housekeeping when a lock is released, + for example recording timestamps. + ''' + interval = _utcnow() - timestamp + self.msg('Leader released {} from {}, held {}'.format(lock, unit, + interval)) + + def require(self, lock, guard_func, *guard_args, **guard_kw): + """Decorate a function to be run only when a lock is acquired. + + The lock is requested if the guard function returns True. + + The decorated function is called if the lock has been granted. + """ + def decorator(f): + @wraps(f) + def wrapper(*args, **kw): + if self.granted(lock): + self.msg('Granted {}'.format(lock)) + return f(*args, **kw) + if guard_func(*guard_args, **guard_kw) and self.acquire(lock): + return f(*args, **kw) + return None + return wrapper + return decorator + + def msg(self, msg): + '''Emit a message. Override to customize log spam.''' + hookenv.log('coordinator.{} {}'.format(self._name(), msg), + level=hookenv.INFO) + + def _name(self): + return self.__class__.__name__ + + def _load_state(self): + self.msg('Loading state') + + # All responses must be stored in the leadership settings. + # The leader cannot use local state, as a different unit may + # be leader next time. Which is fine, as the leadership + # settings are always available. + self.grants = json.loads(hookenv.leader_get(self.key) or '{}') + + local_unit = hookenv.local_unit() + + # All requests must be stored on the peers relation. This is + # the only channel units have to communicate with the leader. + # Even the leader needs to store its requests here, as a + # different unit may be leader by the time the request can be + # granted. + if self.relid is None: + # The peers relation is not available. Maybe we are early in + # the units's lifecycle. Maybe this unit is standalone. + # Fallback to using local state. + self.msg('No peer relation. Loading local state') + self.requests = {local_unit: self._load_local_state()} + else: + self.requests = self._load_peer_state() + if local_unit not in self.requests: + # The peers relation has just been joined. Update any state + # loaded from our peers with our local state. + self.msg('New peer relation. Merging local state') + self.requests[local_unit] = self._load_local_state() + + def _emit_state(self): + # Emit this units lock status. + for lock in sorted(self.requests[hookenv.local_unit()].keys()): + if self.granted(lock): + self.msg('Granted {}'.format(lock)) + else: + self.msg('Waiting on {}'.format(lock)) + + def _save_state(self): + self.msg('Publishing state') + if hookenv.is_leader(): + # sort_keys to ensure stability. + raw = json.dumps(self.grants, sort_keys=True) + hookenv.leader_set({self.key: raw}) + + local_unit = hookenv.local_unit() + + if self.relid is None: + # No peers relation yet. Fallback to local state. + self.msg('No peer relation. Saving local state') + self._save_local_state(self.requests[local_unit]) + else: + # sort_keys to ensure stability. + raw = json.dumps(self.requests[local_unit], sort_keys=True) + hookenv.relation_set(self.relid, relation_settings={self.key: raw}) + + def _load_peer_state(self): + requests = {} + units = set(hookenv.related_units(self.relid)) + units.add(hookenv.local_unit()) + for unit in units: + raw = hookenv.relation_get(self.key, unit, self.relid) + if raw: + requests[unit] = json.loads(raw) + return requests + + def _local_state_filename(self): + # Include the class name. We allow multiple BaseCoordinator + # subclasses to be instantiated, and they are singletons, so + # this avoids conflicts (unless someone creates and uses two + # BaseCoordinator subclasses with the same class name, so don't + # do that). + return '.charmhelpers.coordinator.{}'.format(self._name()) + + def _load_local_state(self): + fn = self._local_state_filename() + if os.path.exists(fn): + with open(fn, 'r') as f: + return json.load(f) + return {} + + def _save_local_state(self, state): + fn = self._local_state_filename() + with open(fn, 'w') as f: + json.dump(state, f) + + def _release_granted(self): + # At the end of every hook, release all locks granted to + # this unit. If a hook neglects to make use of what it + # requested, it will just have to make the request again. + # Implicit release is the only way this will work, as + # if the unit is standalone there may be no future triggers + # called to do a manual release. + unit = hookenv.local_unit() + for lock in list(self.requests[unit].keys()): + if self.granted(lock): + self.msg('Released local {} lock'.format(lock)) + del self.requests[unit][lock] + + +class Serial(BaseCoordinator): + def default_grant(self, lock, unit, granted, queue): + '''Default logic to grant a lock to a unit. Unless overridden, + only one unit may hold the lock and it will be granted to the + earliest queued request. + + To define custom logic for $lock, create a subclass and + define a grant_$lock method. + + `unit` is the unit name making the request. + + `granted` is the set of units already granted the lock. It will + never include `unit`. It may be empty. + + `queue` is the list of units waiting for the lock, ordered by time + of request. It will always include `unit`, but `unit` is not + necessarily first. + + Returns True if the lock should be granted to `unit`. + ''' + return unit == queue[0] and not granted + + +def _implicit_peer_relation_name(): + md = hookenv.metadata() + assert 'peers' in md, 'No peer relations in metadata.yaml' + return sorted(md['peers'].keys())[0] + + +# A human readable, sortable UTC timestamp format. +_timestamp_format = '%Y-%m-%d %H:%M:%S.%fZ' + + +def _utcnow(): # pragma: no cover + # This wrapper exists as mocking datetime methods is problematic. + return datetime.utcnow() + + +def _timestamp(): + return _utcnow().strftime(_timestamp_format) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/core/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/decorators.py b/nrpe/mod/charmhelpers/charmhelpers/core/decorators.py new file mode 100644 index 0000000..e7e95d1 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/decorators.py @@ -0,0 +1,93 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copyright 2014 Canonical Ltd. +# +# Authors: +# Edward Hope-Morley +# + +import time + +from charmhelpers.core.hookenv import ( + log, + INFO, +) + + +def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): + """If the decorated function raises exception exc_type, allow num_retries + retry attempts before raise the exception. + """ + def _retry_on_exception_inner_1(f): + def _retry_on_exception_inner_2(*args, **kwargs): + retries = num_retries + multiplier = 1 + while True: + try: + return f(*args, **kwargs) + except exc_type: + if not retries: + raise + + delay = base_delay * multiplier + multiplier += 1 + log("Retrying '%s' %d more times (delay=%s)" % + (f.__name__, retries, delay), level=INFO) + retries -= 1 + if delay: + time.sleep(delay) + + return _retry_on_exception_inner_2 + + return _retry_on_exception_inner_1 + + +def retry_on_predicate(num_retries, predicate_fun, base_delay=0): + """Retry based on return value + + The return value of the decorated function is passed to the given predicate_fun. If the + result of the predicate is False, retry the decorated function up to num_retries times + + An exponential backoff up to base_delay^num_retries seconds can be introduced by setting + base_delay to a nonzero value. The default is to run with a zero (i.e. no) delay + + :param num_retries: Max. number of retries to perform + :type num_retries: int + :param predicate_fun: Predicate function to determine if a retry is necessary + :type predicate_fun: callable + :param base_delay: Starting value in seconds for exponential delay, defaults to 0 (no delay) + :type base_delay: float + """ + def _retry_on_pred_inner_1(f): + def _retry_on_pred_inner_2(*args, **kwargs): + retries = num_retries + multiplier = 1 + delay = base_delay + while True: + result = f(*args, **kwargs) + if predicate_fun(result) or retries <= 0: + return result + delay *= multiplier + multiplier += 1 + log("Result {}, retrying '{}' {} more times (delay={})".format( + result, f.__name__, retries, delay), level=INFO) + retries -= 1 + if delay: + time.sleep(delay) + + return _retry_on_pred_inner_2 + + return _retry_on_pred_inner_1 diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/files.py b/nrpe/mod/charmhelpers/charmhelpers/core/files.py new file mode 100644 index 0000000..fdd82b7 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/files.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'Jorge Niedbalski ' + +import os +import subprocess + + +def sed(filename, before, after, flags='g'): + """ + Search and replaces the given pattern on filename. + + :param filename: relative or absolute file path. + :param before: expression to be replaced (see 'man sed') + :param after: expression to replace with (see 'man sed') + :param flags: sed-compatible regex flags in example, to make + the search and replace case insensitive, specify ``flags="i"``. + The ``g`` flag is always specified regardless, so you do not + need to remember to include it when overriding this parameter. + :returns: If the sed command exit code was zero then return, + otherwise raise CalledProcessError. + """ + expression = r's/{0}/{1}/{2}'.format(before, + after, flags) + + return subprocess.check_call(["sed", "-i", "-r", "-e", + expression, + os.path.expanduser(filename)]) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/fstab.py b/nrpe/mod/charmhelpers/charmhelpers/core/fstab.py new file mode 100644 index 0000000..d9fa915 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/fstab.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os + +__author__ = 'Jorge Niedbalski R. ' + + +class Fstab(io.FileIO): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = int(d) + self.p = int(p) + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + super(Fstab, self).__init__(self._path, 'rb+') + + def _hydrate_entry(self, line): + # NOTE: use split with no arguments to split on any + # whitespace including tabs + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split())) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + line = line.decode('us-ascii') + try: + if line.strip() and not line.strip().startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write((str(entry) + '\n').encode('us-ascii')) + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = [l.decode('us-ascii') for l in self.readlines()] + + found = False + for index, line in enumerate(lines): + if line.strip() and not line.strip().startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines).encode('us-ascii')) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/hookenv.py b/nrpe/mod/charmhelpers/charmhelpers/core/hookenv.py new file mode 100644 index 0000000..e94247a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/hookenv.py @@ -0,0 +1,1639 @@ +# Copyright 2013-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Interactions with the Juju environment" +# +# Authors: +# Charm Helpers Developers + +from __future__ import print_function +import copy +from distutils.version import LooseVersion +from enum import Enum +from functools import wraps +from collections import namedtuple +import glob +import os +import json +import yaml +import re +import subprocess +import sys +import errno +import tempfile +from subprocess import CalledProcessError + +from charmhelpers import deprecate + +import six +if not six.PY3: + from UserDict import UserDict +else: + from collections import UserDict + + +CRITICAL = "CRITICAL" +ERROR = "ERROR" +WARNING = "WARNING" +INFO = "INFO" +DEBUG = "DEBUG" +TRACE = "TRACE" +MARKER = object() +SH_MAX_ARG = 131071 + + +RANGE_WARNING = ('Passing NO_PROXY string that includes a cidr. ' + 'This may not be compatible with software you are ' + 'running in your shell.') + + +class WORKLOAD_STATES(Enum): + ACTIVE = 'active' + BLOCKED = 'blocked' + MAINTENANCE = 'maintenance' + WAITING = 'waiting' + + +cache = {} + + +def cached(func): + """Cache return values for multiple executions of func + args + + For example:: + + @cached + def unit_get(attribute): + pass + + unit_get('test') + + will cache the result of unit_get + 'test' for future calls. + """ + @wraps(func) + def wrapper(*args, **kwargs): + global cache + key = json.dumps((func, args, kwargs), sort_keys=True, default=str) + try: + return cache[key] + except KeyError: + pass # Drop out of the exception handler scope. + res = func(*args, **kwargs) + cache[key] = res + return res + wrapper._wrapped = func + return wrapper + + +def flush(key): + """Flushes any entries from function cache where the + key is found in the function+args """ + flush_list = [] + for item in cache: + if key in item: + flush_list.append(item) + for item in flush_list: + del cache[item] + + +def log(message, level=None): + """Write a message to the juju log""" + command = ['juju-log'] + if level: + command += ['-l', level] + if not isinstance(message, six.string_types): + message = repr(message) + command += [message[:SH_MAX_ARG]] + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise + + +def function_log(message): + """Write a function progress message""" + command = ['function-log'] + if not isinstance(message, six.string_types): + message = repr(message) + command += [message[:SH_MAX_ARG]] + # Missing function-log should not cause failures in unit tests + # Send function_log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + message = "function-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise + + +class Serializable(UserDict): + """Wrapper, an object that can be serialized to yaml or json""" + + def __init__(self, obj): + # wrap the object + UserDict.__init__(self) + self.data = obj + + def __getattr__(self, attr): + # See if this object has attribute. + if attr in ("json", "yaml", "data"): + return self.__dict__[attr] + # Check for attribute in wrapped object. + got = getattr(self.data, attr, MARKER) + if got is not MARKER: + return got + # Proxy to the wrapped object via dict interface. + try: + return self.data[attr] + except KeyError: + raise AttributeError(attr) + + def __getstate__(self): + # Pickle as a standard dictionary. + return self.data + + def __setstate__(self, state): + # Unpickle into our wrapper. + self.data = state + + def json(self): + """Serialize the object to json""" + return json.dumps(self.data) + + def yaml(self): + """Serialize the object to yaml""" + return yaml.dump(self.data) + + +def execution_environment(): + """A convenient bundling of the current execution context""" + context = {} + context['conf'] = config() + if relation_id(): + context['reltype'] = relation_type() + context['relid'] = relation_id() + context['rel'] = relation_get() + context['unit'] = local_unit() + context['rels'] = relations() + context['env'] = os.environ + return context + + +def in_relation_hook(): + """Determine whether we're running in a relation hook""" + return 'JUJU_RELATION' in os.environ + + +def relation_type(): + """The scope for the current relation hook""" + return os.environ.get('JUJU_RELATION', None) + + +@cached +def relation_id(relation_name=None, service_or_unit=None): + """The relation ID for the current or a specified relation""" + if not relation_name and not service_or_unit: + return os.environ.get('JUJU_RELATION_ID', None) + elif relation_name and service_or_unit: + service_name = service_or_unit.split('/')[0] + for relid in relation_ids(relation_name): + remote_service = remote_service_name(relid) + if remote_service == service_name: + return relid + else: + raise ValueError('Must specify neither or both of relation_name and service_or_unit') + + +def departing_unit(): + """The departing unit for the current relation hook. + + Available since juju 2.8. + + :returns: the departing unit, or None if the information isn't available. + :rtype: Optional[str] + """ + return os.environ.get('JUJU_DEPARTING_UNIT', None) + + +def local_unit(): + """Local unit ID""" + return os.environ['JUJU_UNIT_NAME'] + + +def remote_unit(): + """The remote unit for the current relation hook""" + return os.environ.get('JUJU_REMOTE_UNIT', None) + + +def application_name(): + """ + The name of the deployed application this unit belongs to. + """ + return local_unit().split('/')[0] + + +def service_name(): + """ + .. deprecated:: 0.19.1 + Alias for :func:`application_name`. + """ + return application_name() + + +def model_name(): + """ + Name of the model that this unit is deployed in. + """ + return os.environ['JUJU_MODEL_NAME'] + + +def model_uuid(): + """ + UUID of the model that this unit is deployed in. + """ + return os.environ['JUJU_MODEL_UUID'] + + +def principal_unit(): + """Returns the principal unit of this unit, otherwise None""" + # Juju 2.2 and above provides JUJU_PRINCIPAL_UNIT + principal_unit = os.environ.get('JUJU_PRINCIPAL_UNIT', None) + # If it's empty, then this unit is the principal + if principal_unit == '': + return os.environ['JUJU_UNIT_NAME'] + elif principal_unit is not None: + return principal_unit + # For Juju 2.1 and below, let's try work out the principle unit by + # the various charms' metadata.yaml. + for reltype in relation_types(): + for rid in relation_ids(reltype): + for unit in related_units(rid): + md = _metadata_unit(unit) + if not md: + continue + subordinate = md.pop('subordinate', None) + if not subordinate: + return unit + return None + + +@cached +def remote_service_name(relid=None): + """The remote service name for a given relation-id (or the current relation)""" + if relid is None: + unit = remote_unit() + else: + units = related_units(relid) + unit = units[0] if units else None + return unit.split('/')[0] if unit else None + + +def hook_name(): + """The name of the currently executing hook""" + return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0])) + + +class Config(dict): + """A dictionary representation of the charm's config.yaml, with some + extra features: + + - See which values in the dictionary have changed since the previous hook. + - For values that have changed, see what the previous value was. + - Store arbitrary data for use in a later hook. + + NOTE: Do not instantiate this object directly - instead call + ``hookenv.config()``, which will return an instance of :class:`Config`. + + Example usage:: + + >>> # inside a hook + >>> from charmhelpers.core import hookenv + >>> config = hookenv.config() + >>> config['foo'] + 'bar' + >>> # store a new key/value for later use + >>> config['mykey'] = 'myval' + + + >>> # user runs `juju set mycharm foo=baz` + >>> # now we're inside subsequent config-changed hook + >>> config = hookenv.config() + >>> config['foo'] + 'baz' + >>> # test to see if this val has changed since last hook + >>> config.changed('foo') + True + >>> # what was the previous value? + >>> config.previous('foo') + 'bar' + >>> # keys/values that we add are preserved across hooks + >>> config['mykey'] + 'myval' + + """ + CONFIG_FILE_NAME = '.juju-persistent-config' + + def __init__(self, *args, **kw): + super(Config, self).__init__(*args, **kw) + self.implicit_save = True + self._prev_dict = None + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) + if os.path.exists(self.path) and os.stat(self.path).st_size: + self.load_previous() + atexit(self._implicit_save) + + def load_previous(self, path=None): + """Load previous copy of config from disk. + + In normal usage you don't need to call this method directly - it + is called automatically at object initialization. + + :param path: + + File path from which to load the previous config. If `None`, + config is loaded from the default location. If `path` is + specified, subsequent `save()` calls will write to the same + path. + + """ + self.path = path or self.path + with open(self.path) as f: + try: + self._prev_dict = json.load(f) + except ValueError as e: + log('Found but was unable to parse previous config data, ' + 'ignoring which will report all values as changed - {}' + .format(str(e)), level=ERROR) + return + for k, v in copy.deepcopy(self._prev_dict).items(): + if k not in self: + self[k] = v + + def changed(self, key): + """Return True if the current value for this key is different from + the previous value. + + """ + if self._prev_dict is None: + return True + return self.previous(key) != self.get(key) + + def previous(self, key): + """Return previous value for this key, or None if there + is no previous value. + + """ + if self._prev_dict: + return self._prev_dict.get(key) + return None + + def save(self): + """Save this config to disk. + + If the charm is using the :mod:`Services Framework ` + or :meth:'@hook ' decorator, this + is called automatically at the end of successful hook execution. + Otherwise, it should be called directly by user code. + + To disable automatic saves, set ``implicit_save=False`` on this + instance. + + """ + with open(self.path, 'w') as f: + os.fchmod(f.fileno(), 0o600) + json.dump(self, f) + + def _implicit_save(self): + if self.implicit_save: + self.save() + + +_cache_config = None + + +def config(scope=None): + """ + Get the juju charm configuration (scope==None) or individual key, + (scope=str). The returned value is a Python data structure loaded as + JSON from the Juju config command. + + :param scope: If set, return the value for the specified key. + :type scope: Optional[str] + :returns: Either the whole config as a Config, or a key from it. + :rtype: Any + """ + global _cache_config + config_cmd_line = ['config-get', '--all', '--format=json'] + try: + # JSON Decode Exception for Python3.5+ + exc_json = json.decoder.JSONDecodeError + except AttributeError: + # JSON Decode Exception for Python2.7 through Python3.4 + exc_json = ValueError + try: + if _cache_config is None: + config_data = json.loads( + subprocess.check_output(config_cmd_line).decode('UTF-8')) + _cache_config = Config(config_data) + if scope is not None: + return _cache_config.get(scope) + return _cache_config + except (exc_json, UnicodeDecodeError) as e: + log('Unable to parse output from config-get: config_cmd_line="{}" ' + 'message="{}"' + .format(config_cmd_line, str(e)), level=ERROR) + return None + + +@cached +def relation_get(attribute=None, unit=None, rid=None, app=None): + """Get relation information""" + _args = ['relation-get', '--format=json'] + if app is not None: + if unit is not None: + raise ValueError("Cannot use both 'unit' and 'app'") + _args.append('--app') + if rid: + _args.append('-r') + _args.append(rid) + _args.append(attribute or '-') + # unit or application name + if unit or app: + _args.append(unit or app) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + except CalledProcessError as e: + if e.returncode == 2: + return None + raise + + +def relation_set(relation_id=None, relation_settings=None, app=False, **kwargs): + """Set relation information for the current unit""" + relation_settings = relation_settings if relation_settings else {} + relation_cmd_line = ['relation-set'] + accepts_file = "--file" in subprocess.check_output( + relation_cmd_line + ["--help"], universal_newlines=True) + if app: + relation_cmd_line.append('--app') + if relation_id is not None: + relation_cmd_line.extend(('-r', relation_id)) + settings = relation_settings.copy() + settings.update(kwargs) + for key, value in settings.items(): + # Force value to be a string: it always should, but some call + # sites pass in things like dicts or numbers. + if value is not None: + settings[key] = "{}".format(value) + if accepts_file: + # --file was introduced in Juju 1.23.2. Use it by default if + # available, since otherwise we'll break if the relation data is + # too big. Ideally we should tell relation-set to read the data from + # stdin, but that feature is broken in 1.23.2: Bug #1454678. + with tempfile.NamedTemporaryFile(delete=False) as settings_file: + settings_file.write(yaml.safe_dump(settings).encode("utf-8")) + subprocess.check_call( + relation_cmd_line + ["--file", settings_file.name]) + os.remove(settings_file.name) + else: + for key, value in settings.items(): + if value is None: + relation_cmd_line.append('{}='.format(key)) + else: + relation_cmd_line.append('{}={}'.format(key, value)) + subprocess.check_call(relation_cmd_line) + # Flush cache of any relation-gets for local unit + flush(local_unit()) + + +def relation_clear(r_id=None): + ''' Clears any relation data already set on relation r_id ''' + settings = relation_get(rid=r_id, + unit=local_unit()) + for setting in settings: + if setting not in ['public-address', 'private-address']: + settings[setting] = None + relation_set(relation_id=r_id, + **settings) + + +@cached +def relation_ids(reltype=None): + """A list of relation_ids""" + reltype = reltype or relation_type() + relid_cmd_line = ['relation-ids', '--format=json'] + if reltype is not None: + relid_cmd_line.append(reltype) + return json.loads( + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] + return [] + + +@cached +def related_units(relid=None): + """A list of related units""" + relid = relid or relation_id() + units_cmd_line = ['relation-list', '--format=json'] + if relid is not None: + units_cmd_line.extend(('-r', relid)) + return json.loads( + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] + + +def expected_peer_units(): + """Get a generator for units we expect to join peer relation based on + goal-state. + + The local unit is excluded from the result to make it easy to gauge + completion of all peers joining the relation with existing hook tools. + + Example usage: + log('peer {} of {} joined peer relation' + .format(len(related_units()), + len(list(expected_peer_units())))) + + This function will raise NotImplementedError if used with juju versions + without goal-state support. + + :returns: iterator + :rtype: types.GeneratorType + :raises: NotImplementedError + """ + if not has_juju_version("2.4.0"): + # goal-state first appeared in 2.4.0. + raise NotImplementedError("goal-state") + _goal_state = goal_state() + return (key for key in _goal_state['units'] + if '/' in key and key != local_unit()) + + +def expected_related_units(reltype=None): + """Get a generator for units we expect to join relation based on + goal-state. + + Note that you can not use this function for the peer relation, take a look + at expected_peer_units() for that. + + This function will raise KeyError if you request information for a + relation type for which juju goal-state does not have information. It will + raise NotImplementedError if used with juju versions without goal-state + support. + + Example usage: + log('participant {} of {} joined relation {}' + .format(len(related_units()), + len(list(expected_related_units())), + relation_type())) + + :param reltype: Relation type to list data for, default is to list data for + the relation type we are currently executing a hook for. + :type reltype: str + :returns: iterator + :rtype: types.GeneratorType + :raises: KeyError, NotImplementedError + """ + if not has_juju_version("2.4.4"): + # goal-state existed in 2.4.0, but did not list individual units to + # join a relation in 2.4.1 through 2.4.3. (LP: #1794739) + raise NotImplementedError("goal-state relation unit count") + reltype = reltype or relation_type() + _goal_state = goal_state() + return (key for key in _goal_state['relations'][reltype] if '/' in key) + + +@cached +def relation_for_unit(unit=None, rid=None): + """Get the json representation of a unit's relation""" + unit = unit or remote_unit() + relation = relation_get(unit=unit, rid=rid) + for key in relation: + if key.endswith('-list'): + relation[key] = relation[key].split() + relation['__unit__'] = unit + return relation + + +@cached +def relations_for_id(relid=None): + """Get relations of a specific relation ID""" + relation_data = [] + relid = relid or relation_ids() + for unit in related_units(relid): + unit_data = relation_for_unit(unit, relid) + unit_data['__relid__'] = relid + relation_data.append(unit_data) + return relation_data + + +@cached +def relations_of_type(reltype=None): + """Get relations of a specific type""" + relation_data = [] + reltype = reltype or relation_type() + for relid in relation_ids(reltype): + for relation in relations_for_id(relid): + relation['__relid__'] = relid + relation_data.append(relation) + return relation_data + + +@cached +def metadata(): + """Get the current charm metadata.yaml contents as a python object""" + with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: + return yaml.safe_load(md) + + +def _metadata_unit(unit): + """Given the name of a unit (e.g. apache2/0), get the unit charm's + metadata.yaml. Very similar to metadata() but allows us to inspect + other units. Unit needs to be co-located, such as a subordinate or + principal/primary. + + :returns: metadata.yaml as a python object. + + """ + basedir = os.sep.join(charm_dir().split(os.sep)[:-2]) + unitdir = 'unit-{}'.format(unit.replace(os.sep, '-')) + joineddir = os.path.join(basedir, unitdir, 'charm', 'metadata.yaml') + if not os.path.exists(joineddir): + return None + with open(joineddir) as md: + return yaml.safe_load(md) + + +@cached +def relation_types(): + """Get a list of relation types supported by this charm""" + rel_types = [] + md = metadata() + for key in ('provides', 'requires', 'peers'): + section = md.get(key) + if section: + rel_types.extend(section.keys()) + return rel_types + + +@cached +def peer_relation_id(): + '''Get the peers relation id if a peers relation has been joined, else None.''' + md = metadata() + section = md.get('peers') + if section: + for key in section: + relids = relation_ids(key) + if relids: + return relids[0] + return None + + +@cached +def relation_to_interface(relation_name): + """ + Given the name of a relation, return the interface that relation uses. + + :returns: The interface name, or ``None``. + """ + return relation_to_role_and_interface(relation_name)[1] + + +@cached +def relation_to_role_and_interface(relation_name): + """ + Given the name of a relation, return the role and the name of the interface + that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). + + :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. + """ + _metadata = metadata() + for role in ('provides', 'requires', 'peers'): + interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') + if interface: + return role, interface + return None, None + + +@cached +def role_and_interface_to_relations(role, interface_name): + """ + Given a role and interface name, return a list of relation names for the + current charm that use that interface under that role (where role is one + of ``provides``, ``requires``, or ``peers``). + + :returns: A list of relation names. + """ + _metadata = metadata() + results = [] + for relation_name, relation in _metadata.get(role, {}).items(): + if relation['interface'] == interface_name: + results.append(relation_name) + return results + + +@cached +def interface_to_relations(interface_name): + """ + Given an interface, return a list of relation names for the current + charm that use that interface. + + :returns: A list of relation names. + """ + results = [] + for role in ('provides', 'requires', 'peers'): + results.extend(role_and_interface_to_relations(role, interface_name)) + return results + + +@cached +def charm_name(): + """Get the name of the current charm as is specified on metadata.yaml""" + return metadata().get('name') + + +@cached +def relations(): + """Get a nested dictionary of relation data for all related units""" + rels = {} + for reltype in relation_types(): + relids = {} + for relid in relation_ids(reltype): + units = {local_unit(): relation_get(unit=local_unit(), rid=relid)} + for unit in related_units(relid): + reldata = relation_get(unit=unit, rid=relid) + units[unit] = reldata + relids[relid] = units + rels[reltype] = relids + return rels + + +@cached +def is_relation_made(relation, keys='private-address'): + ''' + Determine whether a relation is established by checking for + presence of key(s). If a list of keys is provided, they + must all be present for the relation to be identified as made + ''' + if isinstance(keys, str): + keys = [keys] + for r_id in relation_ids(relation): + for unit in related_units(r_id): + context = {} + for k in keys: + context[k] = relation_get(k, rid=r_id, + unit=unit) + if None not in context.values(): + return True + return False + + +def _port_op(op_name, port, protocol="TCP"): + """Open or close a service network port""" + _args = [op_name] + icmp = protocol.upper() == "ICMP" + if icmp: + _args.append(protocol) + else: + _args.append('{}/{}'.format(port, protocol)) + try: + subprocess.check_call(_args) + except subprocess.CalledProcessError: + # Older Juju pre 2.3 doesn't support ICMP + # so treat it as a no-op if it fails. + if not icmp: + raise + + +def open_port(port, protocol="TCP"): + """Open a service network port""" + _port_op('open-port', port, protocol) + + +def close_port(port, protocol="TCP"): + """Close a service network port""" + _port_op('close-port', port, protocol) + + +def open_ports(start, end, protocol="TCP"): + """Opens a range of service network ports""" + _args = ['open-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +def close_ports(start, end, protocol="TCP"): + """Close a range of service network ports""" + _args = ['close-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +def opened_ports(): + """Get the opened ports + + *Note that this will only show ports opened in a previous hook* + + :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']`` + """ + _args = ['opened-ports', '--format=json'] + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + + +@cached +def unit_get(attribute): + """Get the unit ID for the remote unit""" + _args = ['unit-get', '--format=json', attribute] + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + + +def unit_public_ip(): + """Get this unit's public IP address""" + return unit_get('public-address') + + +def unit_private_ip(): + """Get this unit's private IP address""" + return unit_get('private-address') + + +@cached +def storage_get(attribute=None, storage_id=None): + """Get storage attributes""" + _args = ['storage-get', '--format=json'] + if storage_id: + _args.extend(('-s', storage_id)) + if attribute: + _args.append(attribute) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + + +@cached +def storage_list(storage_name=None): + """List the storage IDs for the unit""" + _args = ['storage-list', '--format=json'] + if storage_name: + _args.append(storage_name) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + except OSError as e: + import errno + if e.errno == errno.ENOENT: + # storage-list does not exist + return [] + raise + + +class UnregisteredHookError(Exception): + """Raised when an undefined hook is called""" + pass + + +class Hooks(object): + """A convenient handler for hook functions. + + Example:: + + hooks = Hooks() + + # register a hook, taking its name from the function name + @hooks.hook() + def install(): + pass # your code here + + # register a hook, providing a custom hook name + @hooks.hook("config-changed") + def config_changed(): + pass # your code here + + if __name__ == "__main__": + # execute a hook based on the name the program is called by + hooks.execute(sys.argv) + """ + + def __init__(self, config_save=None): + super(Hooks, self).__init__() + self._hooks = {} + + # For unknown reasons, we allow the Hooks constructor to override + # config().implicit_save. + if config_save is not None: + config().implicit_save = config_save + + def register(self, name, function): + """Register a hook""" + self._hooks[name] = function + + def execute(self, args): + """Execute a registered hook based on args[0]""" + _run_atstart() + hook_name = os.path.basename(args[0]) + if hook_name in self._hooks: + try: + self._hooks[hook_name]() + except SystemExit as x: + if x.code is None or x.code == 0: + _run_atexit() + raise + _run_atexit() + else: + raise UnregisteredHookError(hook_name) + + def hook(self, *hook_names): + """Decorator, registering them as hooks""" + def wrapper(decorated): + for hook_name in hook_names: + self.register(hook_name, decorated) + else: + self.register(decorated.__name__, decorated) + if '_' in decorated.__name__: + self.register( + decorated.__name__.replace('_', '-'), decorated) + return decorated + return wrapper + + +class NoNetworkBinding(Exception): + pass + + +def charm_dir(): + """Return the root directory of the current charm""" + d = os.environ.get('JUJU_CHARM_DIR') + if d is not None: + return d + return os.environ.get('CHARM_DIR') + + +def cmd_exists(cmd): + """Return True if the specified cmd exists in the path""" + return any( + os.access(os.path.join(path, cmd), os.X_OK) + for path in os.environ["PATH"].split(os.pathsep) + ) + + +@cached +@deprecate("moved to function_get()", log=log) +def action_get(key=None): + """ + .. deprecated:: 0.20.7 + Alias for :func:`function_get`. + + Gets the value of an action parameter, or all key/value param pairs. + """ + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +@cached +def function_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['function-get'] + # Fallback for older charms. + if not cmd_exists('function-get'): + cmd = ['action-get'] + + if key is not None: + cmd.append(key) + cmd.append('--format=json') + function_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return function_data + + +@deprecate("moved to function_set()", log=log) +def action_set(values): + """ + .. deprecated:: 0.20.7 + Alias for :func:`function_set`. + + Sets the values to be returned after the action finishes. + """ + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def function_set(values): + """Sets the values to be returned after the function finishes""" + cmd = ['function-set'] + # Fallback for older charms. + if not cmd_exists('function-get'): + cmd = ['action-set'] + + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +@deprecate("moved to function_fail()", log=log) +def action_fail(message): + """ + .. deprecated:: 0.20.7 + Alias for :func:`function_fail`. + + Sets the action status to failed and sets the error message. + + The results set by action_set are preserved. + """ + subprocess.check_call(['action-fail', message]) + + +def function_fail(message): + """Sets the function status to failed and sets the error message. + + The results set by function_set are preserved.""" + cmd = ['function-fail'] + # Fallback for older charms. + if not cmd_exists('function-fail'): + cmd = ['action-fail'] + cmd.append(message) + + subprocess.check_call(cmd) + + +def action_name(): + """Get the name of the currently executing action.""" + return os.environ.get('JUJU_ACTION_NAME') + + +def function_name(): + """Get the name of the currently executing function.""" + return os.environ.get('JUJU_FUNCTION_NAME') or action_name() + + +def action_uuid(): + """Get the UUID of the currently executing action.""" + return os.environ.get('JUJU_ACTION_UUID') + + +def function_id(): + """Get the ID of the currently executing function.""" + return os.environ.get('JUJU_FUNCTION_ID') or action_uuid() + + +def action_tag(): + """Get the tag for the currently executing action.""" + return os.environ.get('JUJU_ACTION_TAG') + + +def function_tag(): + """Get the tag for the currently executing function.""" + return os.environ.get('JUJU_FUNCTION_TAG') or action_tag() + + +def status_set(workload_state, message, application=False): + """Set the workload state with a message + + Use status-set to set the workload state with a message which is visible + to the user via juju status. If the status-set command is not found then + assume this is juju < 1.23 and juju-log the message instead. + + workload_state -- valid juju workload state. str or WORKLOAD_STATES + message -- status update message + application -- Whether this is an application state set + """ + bad_state_msg = '{!r} is not a valid workload state' + + if isinstance(workload_state, str): + try: + # Convert string to enum. + workload_state = WORKLOAD_STATES[workload_state.upper()] + except KeyError: + raise ValueError(bad_state_msg.format(workload_state)) + + if workload_state not in WORKLOAD_STATES: + raise ValueError(bad_state_msg.format(workload_state)) + + cmd = ['status-set'] + if application: + cmd.append('--application') + cmd.extend([workload_state.value, message]) + try: + ret = subprocess.call(cmd) + if ret == 0: + return + except OSError as e: + if e.errno != errno.ENOENT: + raise + log_message = 'status-set failed: {} {}'.format(workload_state.value, + message) + log(log_message, level='INFO') + + +def status_get(): + """Retrieve the previously set juju workload state and message + + If the status-get command is not found then assume this is juju < 1.23 and + return 'unknown', "" + + """ + cmd = ['status-get', "--format=json", "--include-data"] + try: + raw_status = subprocess.check_output(cmd) + except OSError as e: + if e.errno == errno.ENOENT: + return ('unknown', "") + else: + raise + else: + status = json.loads(raw_status.decode("UTF-8")) + return (status["status"], status["message"]) + + +def translate_exc(from_exc, to_exc): + def inner_translate_exc1(f): + @wraps(f) + def inner_translate_exc2(*args, **kwargs): + try: + return f(*args, **kwargs) + except from_exc: + raise to_exc + + return inner_translate_exc2 + + return inner_translate_exc1 + + +def application_version_set(version): + """Charm authors may trigger this command from any hook to output what + version of the application is running. This could be a package version, + for instance postgres version 9.5. It could also be a build number or + version control revision identifier, for instance git sha 6fb7ba68. """ + + cmd = ['application-version-set'] + cmd.append(version) + try: + subprocess.check_call(cmd) + except OSError: + log("Application Version: {}".format(version)) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +@cached +def goal_state(): + """Juju goal state values""" + cmd = ['goal-state', '--format=json'] + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def is_leader(): + """Does the current unit hold the juju leadership + + Uses juju to determine whether the current unit is the leader of its peers + """ + cmd = ['is-leader', '--format=json'] + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def leader_get(attribute=None): + """Juju leader get value(s)""" + cmd = ['leader-get', '--format=json'] + [attribute or '-'] + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def leader_set(settings=None, **kwargs): + """Juju leader set value(s)""" + # Don't log secrets. + # log("Juju leader-set '%s'" % (settings), level=DEBUG) + cmd = ['leader-set'] + settings = settings or {} + settings.update(kwargs) + for k, v in settings.items(): + if v is None: + cmd.append('{}='.format(k)) + else: + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_register(ptype, klass, pid): + """ is used while a hook is running to let Juju know that a + payload has been started.""" + cmd = ['payload-register'] + for x in [ptype, klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_unregister(klass, pid): + """ is used while a hook is running to let Juju know + that a payload has been manually stopped. The and provided + must match a payload that has been previously registered with juju using + payload-register.""" + cmd = ['payload-unregister'] + for x in [klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_status_set(klass, pid, status): + """is used to update the current status of a registered payload. + The and provided must match a payload that has been previously + registered with juju using payload-register. The must be one of the + follow: starting, started, stopping, stopped""" + cmd = ['payload-status-set'] + for x in [klass, pid, status]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def resource_get(name): + """used to fetch the resource path of the given name. + + must match a name of defined resource in metadata.yaml + + returns either a path or False if resource not available + """ + if not name: + return False + + cmd = ['resource-get', name] + try: + return subprocess.check_output(cmd).decode('UTF-8') + except subprocess.CalledProcessError: + return False + + +@cached +def juju_version(): + """Full version string (eg. '1.23.3.1-trusty-amd64')""" + # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1 + jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0] + return subprocess.check_output([jujud, 'version'], + universal_newlines=True).strip() + + +def has_juju_version(minimum_version): + """Return True if the Juju version is at least the provided version""" + return LooseVersion(juju_version()) >= LooseVersion(minimum_version) + + +_atexit = [] +_atstart = [] + + +def atstart(callback, *args, **kwargs): + '''Schedule a callback to run before the main hook. + + Callbacks are run in the order they were added. + + This is useful for modules and classes to perform initialization + and inject behavior. In particular: + + - Run common code before all of your hooks, such as logging + the hook name or interesting relation data. + - Defer object or module initialization that requires a hook + context until we know there actually is a hook context, + making testing easier. + - Rather than requiring charm authors to include boilerplate to + invoke your helper's behavior, have it run automatically if + your object is instantiated or module imported. + + This is not at all useful after your hook framework as been launched. + ''' + global _atstart + _atstart.append((callback, args, kwargs)) + + +def atexit(callback, *args, **kwargs): + '''Schedule a callback to run on successful hook completion. + + Callbacks are run in the reverse order that they were added.''' + _atexit.append((callback, args, kwargs)) + + +def _run_atstart(): + '''Hook frameworks must invoke this before running the main hook body.''' + global _atstart + for callback, args, kwargs in _atstart: + callback(*args, **kwargs) + del _atstart[:] + + +def _run_atexit(): + '''Hook frameworks must invoke this after the main hook body has + successfully completed. Do not invoke it if the hook fails.''' + global _atexit + for callback, args, kwargs in reversed(_atexit): + callback(*args, **kwargs) + del _atexit[:] + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def network_get_primary_address(binding): + ''' + Deprecated since Juju 2.3; use network_get() + + Retrieve the primary network address for a named binding + + :param binding: string. The name of a relation of extra-binding + :return: string. The primary IP address for the named binding + :raise: NotImplementedError if run on Juju < 2.0 + ''' + cmd = ['network-get', '--primary-address', binding] + try: + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + except CalledProcessError as e: + if 'no network config found for binding' in e.output.decode('UTF-8'): + raise NoNetworkBinding("No network binding for {}" + .format(binding)) + else: + raise + return response + + +def network_get(endpoint, relation_id=None): + """ + Retrieve the network details for a relation endpoint + + :param endpoint: string. The name of a relation endpoint + :param relation_id: int. The ID of the relation for the current context. + :return: dict. The loaded YAML output of the network-get query. + :raise: NotImplementedError if request not supported by the Juju version. + """ + if not has_juju_version('2.2'): + raise NotImplementedError(juju_version()) # earlier versions require --primary-address + if relation_id and not has_juju_version('2.3'): + raise NotImplementedError # 2.3 added the -r option + + cmd = ['network-get', endpoint, '--format', 'yaml'] + if relation_id: + cmd.append('-r') + cmd.append(relation_id) + response = subprocess.check_output( + cmd, + stderr=subprocess.STDOUT).decode('UTF-8').strip() + return yaml.safe_load(response) + + +def add_metric(*args, **kwargs): + """Add metric values. Values may be expressed with keyword arguments. For + metric names containing dashes, these may be expressed as one or more + 'key=value' positional arguments. May only be called from the collect-metrics + hook.""" + _args = ['add-metric'] + _kvpairs = [] + _kvpairs.extend(args) + _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) + _args.extend(sorted(_kvpairs)) + try: + subprocess.check_call(_args) + return + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) + log(log_message, level='INFO') + + +def meter_status(): + """Get the meter status, if running in the meter-status-changed hook.""" + return os.environ.get('JUJU_METER_STATUS') + + +def meter_info(): + """Get the meter status information, if running in the meter-status-changed + hook.""" + return os.environ.get('JUJU_METER_INFO') + + +def iter_units_for_relation_name(relation_name): + """Iterate through all units in a relation + + Generator that iterates through all the units in a relation and yields + a named tuple with rid and unit field names. + + Usage: + data = [(u.rid, u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param relation_name: string relation name + :yield: Named Tuple with rid and unit field names + """ + RelatedUnit = namedtuple('RelatedUnit', 'rid, unit') + for rid in relation_ids(relation_name): + for unit in related_units(rid): + yield RelatedUnit(rid, unit) + + +def ingress_address(rid=None, unit=None): + """ + Retrieve the ingress-address from a relation when available. + Otherwise, return the private-address. + + When used on the consuming side of the relation (unit is a remote + unit), the ingress-address is the IP address that this unit needs + to use to reach the provided service on the remote unit. + + When used on the providing side of the relation (unit == local_unit()), + the ingress-address is the IP address that is advertised to remote + units on this relation. Remote units need to use this address to + reach the local provided service on this unit. + + Note that charms may document some other method to use in + preference to the ingress_address(), such as an address provided + on a different relation attribute or a service discovery mechanism. + This allows charms to redirect inbound connections to their peers + or different applications such as load balancers. + + Usage: + addresses = [ingress_address(rid=u.rid, unit=u.unit) + for u in iter_units_for_relation_name(relation_name)] + + :param rid: string relation id + :param unit: string unit name + :side effect: calls relation_get + :return: string IP address + """ + settings = relation_get(rid=rid, unit=unit) + return (settings.get('ingress-address') or + settings.get('private-address')) + + +def egress_subnets(rid=None, unit=None): + """ + Retrieve the egress-subnets from a relation. + + This function is to be used on the providing side of the + relation, and provides the ranges of addresses that client + connections may come from. The result is uninteresting on + the consuming side of a relation (unit == local_unit()). + + Returns a stable list of subnets in CIDR format. + eg. ['192.168.1.0/24', '2001::F00F/128'] + + If egress-subnets is not available, falls back to using the published + ingress-address, or finally private-address. + + :param rid: string relation id + :param unit: string unit name + :side effect: calls relation_get + :return: list of subnets in CIDR format. eg. ['192.168.1.0/24', '2001::F00F/128'] + """ + def _to_range(addr): + if re.search(r'^(?:\d{1,3}\.){3}\d{1,3}$', addr) is not None: + addr += '/32' + elif ':' in addr and '/' not in addr: # IPv6 + addr += '/128' + return addr + + settings = relation_get(rid=rid, unit=unit) + if 'egress-subnets' in settings: + return [n.strip() for n in settings['egress-subnets'].split(',') if n.strip()] + if 'ingress-address' in settings: + return [_to_range(settings['ingress-address'])] + if 'private-address' in settings: + return [_to_range(settings['private-address'])] + return [] # Should never happen + + +def unit_doomed(unit=None): + """Determines if the unit is being removed from the model + + Requires Juju 2.4.1. + + :param unit: string unit name, defaults to local_unit + :side effect: calls goal_state + :side effect: calls local_unit + :side effect: calls has_juju_version + :return: True if the unit is being removed, already gone, or never existed + """ + if not has_juju_version("2.4.1"): + # We cannot risk blindly returning False for 'we don't know', + # because that could cause data loss; if call sites don't + # need an accurate answer, they likely don't need this helper + # at all. + # goal-state existed in 2.4.0, but did not handle removals + # correctly until 2.4.1. + raise NotImplementedError("is_doomed") + if unit is None: + unit = local_unit() + gs = goal_state() + units = gs.get('units', {}) + if unit not in units: + return True + # I don't think 'dead' units ever show up in the goal-state, but + # check anyway in addition to 'dying'. + return units[unit]['status'] in ('dying', 'dead') + + +def env_proxy_settings(selected_settings=None): + """Get proxy settings from process environment variables. + + Get charm proxy settings from environment variables that correspond to + juju-http-proxy, juju-https-proxy juju-no-proxy (available as of 2.4.2, see + lp:1782236) and juju-ftp-proxy in a format suitable for passing to an + application that reacts to proxy settings passed as environment variables. + Some applications support lowercase or uppercase notation (e.g. curl), some + support only lowercase (e.g. wget), there are also subjectively rare cases + of only uppercase notation support. no_proxy CIDR and wildcard support also + varies between runtimes and applications as there is no enforced standard. + + Some applications may connect to multiple destinations and expose config + options that would affect only proxy settings for a specific destination + these should be handled in charms in an application-specific manner. + + :param selected_settings: format only a subset of possible settings + :type selected_settings: list + :rtype: Option(None, dict[str, str]) + """ + SUPPORTED_SETTINGS = { + 'http': 'HTTP_PROXY', + 'https': 'HTTPS_PROXY', + 'no_proxy': 'NO_PROXY', + 'ftp': 'FTP_PROXY' + } + if selected_settings is None: + selected_settings = SUPPORTED_SETTINGS + + selected_vars = [v for k, v in SUPPORTED_SETTINGS.items() + if k in selected_settings] + proxy_settings = {} + for var in selected_vars: + var_val = os.getenv(var) + if var_val: + proxy_settings[var] = var_val + proxy_settings[var.lower()] = var_val + # Now handle juju-prefixed environment variables. The legacy vs new + # environment variable usage is mutually exclusive + charm_var_val = os.getenv('JUJU_CHARM_{}'.format(var)) + if charm_var_val: + proxy_settings[var] = charm_var_val + proxy_settings[var.lower()] = charm_var_val + if 'no_proxy' in proxy_settings: + if _contains_range(proxy_settings['no_proxy']): + log(RANGE_WARNING, level=WARNING) + return proxy_settings if proxy_settings else None + + +def _contains_range(addresses): + """Check for cidr or wildcard domain in a string. + + Given a string comprising a comma separated list of ip addresses + and domain names, determine whether the string contains IP ranges + or wildcard domains. + + :param addresses: comma separated list of domains and ip addresses. + :type addresses: str + """ + return ( + # Test for cidr (e.g. 10.20.20.0/24) + "/" in addresses or + # Test for wildcard domains (*.foo.com or .foo.com) + "*" in addresses or + addresses.startswith(".") or + ",." in addresses or + " ." in addresses) + + +def is_subordinate(): + """Check whether charm is subordinate in unit metadata. + + :returns: True if unit is subordniate, False otherwise. + :rtype: bool + """ + return metadata().get('subordinate') is True diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/host.py b/nrpe/mod/charmhelpers/charmhelpers/core/host.py new file mode 100644 index 0000000..994ec8a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/host.py @@ -0,0 +1,1279 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for working with the host system""" +# Copyright 2012 Canonical Ltd. +# +# Authors: +# Nick Moffitt +# Matthew Wedgwood + +import errno +import os +import re +import pwd +import glob +import grp +import random +import string +import subprocess +import hashlib +import functools +import itertools +import six + +from contextlib import contextmanager +from collections import OrderedDict, defaultdict +from .hookenv import log, INFO, DEBUG, local_unit, charm_name +from .fstab import Fstab +from charmhelpers.osplatform import get_platform + +__platform__ = get_platform() +if __platform__ == "ubuntu": + from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401 + service_available, + add_new_group, + lsb_release, + cmp_pkgrevno, + CompareHostReleases, + get_distrib_codename, + arch + ) # flake8: noqa -- ignore F401 for this import +elif __platform__ == "centos": + from charmhelpers.core.host_factory.centos import ( # NOQA:F401 + service_available, + add_new_group, + lsb_release, + cmp_pkgrevno, + CompareHostReleases, + ) # flake8: noqa -- ignore F401 for this import + +UPDATEDB_PATH = '/etc/updatedb.conf' +CA_CERT_DIR = '/usr/local/share/ca-certificates' + + +def service_start(service_name, **kwargs): + """Start a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('start', service_name, **kwargs) + + +def service_stop(service_name, **kwargs): + """Stop a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('stop', service_name, **kwargs) + + +def service_restart(service_name, **kwargs): + """Restart a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be restarted. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_restart('ceph-osd', id=4) + + :param service_name: the name of the service to restart + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + return service('restart', service_name) + + +def service_reload(service_name, restart_on_failure=False, **kwargs): + """Reload a system service, optionally falling back to restart if + reload fails. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_reload('ceph-osd', id=4) + + :param service_name: the name of the service to reload + :param restart_on_failure: boolean indicating whether to fallback to a + restart if the reload fails. + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + service_result = service('reload', service_name, **kwargs) + if not service_result and restart_on_failure: + service_result = service('restart', service_name, **kwargs) + return service_result + + +def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", + **kwargs): + """Pause a system service. + + Stop it, and prevent it from starting again at boot. + + :param service_name: the name of the service to pause + :param init_dir: path to the upstart init directory + :param initd_dir: path to the sysv init directory + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems which do not support + key=value arguments via the commandline. + """ + stopped = True + if service_running(service_name, **kwargs): + stopped = service_stop(service_name, **kwargs) + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) + sysv_file = os.path.join(initd_dir, service_name) + if init_is_systemd(service_name=service_name): + service('disable', service_name) + service('mask', service_name) + elif os.path.exists(upstart_file): + override_path = os.path.join( + init_dir, '{}.override'.format(service_name)) + with open(override_path, 'w') as fh: + fh.write("manual\n") + elif os.path.exists(sysv_file): + subprocess.check_call(["update-rc.d", service_name, "disable"]) + else: + raise ValueError( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( + service_name, upstart_file, sysv_file)) + return stopped + + +def service_resume(service_name, init_dir="/etc/init", + initd_dir="/etc/init.d", **kwargs): + """Resume a system service. + + Re-enable starting again at boot. Start the service. + + :param service_name: the name of the service to resume + :param init_dir: the path to the init dir + :param initd dir: the path to the initd dir + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) + sysv_file = os.path.join(initd_dir, service_name) + if init_is_systemd(service_name=service_name): + service('unmask', service_name) + service('enable', service_name) + elif os.path.exists(upstart_file): + override_path = os.path.join( + init_dir, '{}.override'.format(service_name)) + if os.path.exists(override_path): + os.unlink(override_path) + elif os.path.exists(sysv_file): + subprocess.check_call(["update-rc.d", service_name, "enable"]) + else: + raise ValueError( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( + service_name, upstart_file, sysv_file)) + started = service_running(service_name, **kwargs) + + if not started: + started = service_start(service_name, **kwargs) + return started + + +def service(action, service_name, **kwargs): + """Control a system service. + + :param action: the action to take on the service + :param service_name: the name of the service to perform th action on + :param **kwargs: additional params to be passed to the service command in + the form of key=value. + """ + if init_is_systemd(service_name=service_name): + cmd = ['systemctl', action, service_name] + else: + cmd = ['service', service_name, action] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) + return subprocess.call(cmd) == 0 + + +_UPSTART_CONF = "/etc/init/{}.conf" +_INIT_D_CONF = "/etc/init.d/{}" + + +def service_running(service_name, **kwargs): + """Determine whether a system service is running. + + :param service_name: the name of the service + :param **kwargs: additional args to pass to the service command. This is + used to pass additional key=value arguments to the + service command line for managing specific instance + units (e.g. service ceph-osd status id=2). The kwargs + are ignored in systemd services. + """ + if init_is_systemd(service_name=service_name): + return service('is-active', service_name) + else: + if os.path.exists(_UPSTART_CONF.format(service_name)): + try: + cmd = ['status', service_name] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) + output = subprocess.check_output( + cmd, stderr=subprocess.STDOUT).decode('UTF-8') + except subprocess.CalledProcessError: + return False + else: + # This works for upstart scripts where the 'service' command + # returns a consistent string to represent running + # 'start/running' + if ("start/running" in output or + "is running" in output or + "up and running" in output): + return True + elif os.path.exists(_INIT_D_CONF.format(service_name)): + # Check System V scripts init script return codes + return service('status', service_name) + return False + + +SYSTEMD_SYSTEM = '/run/systemd/system' + + +def init_is_systemd(service_name=None): + """ + Returns whether the host uses systemd for the specified service. + + @param Optional[str] service_name: specific name of service + """ + if str(service_name).startswith("snap."): + return True + if lsb_release()['DISTRIB_CODENAME'] == 'trusty': + return False + return os.path.isdir(SYSTEMD_SYSTEM) + + +def adduser(username, password=None, shell='/bin/bash', + system_user=False, primary_group=None, + secondary_groups=None, uid=None, home_dir=None): + """Add a user to the system. + + Will log but otherwise succeed if the user already exists. + + :param str username: Username to create + :param str password: Password for user; if ``None``, create a system user + :param str shell: The default shell for the user + :param bool system_user: Whether to create a login or system user + :param str primary_group: Primary group for user; defaults to username + :param list secondary_groups: Optional list of additional groups + :param int uid: UID for user being created + :param str home_dir: Home directory for user + + :returns: The password database entry struct, as returned by `pwd.getpwnam` + """ + try: + user_info = pwd.getpwnam(username) + log('user {0} already exists!'.format(username)) + if uid: + user_info = pwd.getpwuid(int(uid)) + log('user with uid {0} already exists!'.format(uid)) + except KeyError: + log('creating user {0}'.format(username)) + cmd = ['useradd'] + if uid: + cmd.extend(['--uid', str(uid)]) + if home_dir: + cmd.extend(['--home', str(home_dir)]) + if system_user or password is None: + cmd.append('--system') + else: + cmd.extend([ + '--create-home', + '--shell', shell, + '--password', password, + ]) + if not primary_group: + try: + grp.getgrnam(username) + primary_group = username # avoid "group exists" error + except KeyError: + pass + if primary_group: + cmd.extend(['-g', primary_group]) + if secondary_groups: + cmd.extend(['-G', ','.join(secondary_groups)]) + cmd.append(username) + subprocess.check_call(cmd) + user_info = pwd.getpwnam(username) + return user_info + + +def user_exists(username): + """Check if a user exists""" + try: + pwd.getpwnam(username) + user_exists = True + except KeyError: + user_exists = False + return user_exists + + +def uid_exists(uid): + """Check if a uid exists""" + try: + pwd.getpwuid(uid) + uid_exists = True + except KeyError: + uid_exists = False + return uid_exists + + +def group_exists(groupname): + """Check if a group exists""" + try: + grp.getgrnam(groupname) + group_exists = True + except KeyError: + group_exists = False + return group_exists + + +def gid_exists(gid): + """Check if a gid exists""" + try: + grp.getgrgid(gid) + gid_exists = True + except KeyError: + gid_exists = False + return gid_exists + + +def add_group(group_name, system_group=False, gid=None): + """Add a group to the system + + Will log but otherwise succeed if the group already exists. + + :param str group_name: group to create + :param bool system_group: Create system group + :param int gid: GID for user being created + + :returns: The password database entry struct, as returned by `grp.getgrnam` + """ + try: + group_info = grp.getgrnam(group_name) + log('group {0} already exists!'.format(group_name)) + if gid: + group_info = grp.getgrgid(gid) + log('group with gid {0} already exists!'.format(gid)) + except KeyError: + log('creating group {0}'.format(group_name)) + add_new_group(group_name, system_group, gid) + group_info = grp.getgrnam(group_name) + return group_info + + +def add_user_to_group(username, group): + """Add a user to a group""" + cmd = ['gpasswd', '-a', username, group] + log("Adding user {} to group {}".format(username, group)) + subprocess.check_call(cmd) + + +def chage(username, lastday=None, expiredate=None, inactive=None, + mindays=None, maxdays=None, root=None, warndays=None): + """Change user password expiry information + + :param str username: User to update + :param str lastday: Set when password was changed in YYYY-MM-DD format + :param str expiredate: Set when user's account will no longer be + accessible in YYYY-MM-DD format. + -1 will remove an account expiration date. + :param str inactive: Set the number of days of inactivity after a password + has expired before the account is locked. + -1 will remove an account's inactivity. + :param str mindays: Set the minimum number of days between password + changes to MIN_DAYS. + 0 indicates the password can be changed anytime. + :param str maxdays: Set the maximum number of days during which a + password is valid. + -1 as MAX_DAYS will remove checking maxdays + :param str root: Apply changes in the CHROOT_DIR directory + :param str warndays: Set the number of days of warning before a password + change is required + :raises subprocess.CalledProcessError: if call to chage fails + """ + cmd = ['chage'] + if root: + cmd.extend(['--root', root]) + if lastday: + cmd.extend(['--lastday', lastday]) + if expiredate: + cmd.extend(['--expiredate', expiredate]) + if inactive: + cmd.extend(['--inactive', inactive]) + if mindays: + cmd.extend(['--mindays', mindays]) + if maxdays: + cmd.extend(['--maxdays', maxdays]) + if warndays: + cmd.extend(['--warndays', warndays]) + cmd.append(username) + subprocess.check_call(cmd) + + +remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') + + +def rsync(from_path, to_path, flags='-r', options=None, timeout=None): + """Replicate the contents of a path""" + options = options or ['--delete', '--executability'] + cmd = ['/usr/bin/rsync', flags] + if timeout: + cmd = ['timeout', str(timeout)] + cmd + cmd.extend(options) + cmd.append(from_path) + cmd.append(to_path) + log(" ".join(cmd)) + return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() + + +def symlink(source, destination): + """Create a symbolic link""" + log("Symlinking {} as {}".format(source, destination)) + cmd = [ + 'ln', + '-sf', + source, + destination, + ] + subprocess.check_call(cmd) + + +def mkdir(path, owner='root', group='root', perms=0o555, force=False): + """Create a directory""" + log("Making dir {} {}:{} {:o}".format(path, owner, group, + perms)) + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + realpath = os.path.abspath(path) + path_exists = os.path.exists(realpath) + if path_exists and force: + if not os.path.isdir(realpath): + log("Removing non-directory file {} prior to mkdir()".format(path)) + os.unlink(realpath) + os.makedirs(realpath, perms) + elif not path_exists: + os.makedirs(realpath, perms) + os.chown(realpath, uid, gid) + os.chmod(realpath, perms) + + +def write_file(path, content, owner='root', group='root', perms=0o444): + """Create or overwrite a file with the contents of a byte string.""" + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + # lets see if we can grab the file and compare the context, to avoid doing + # a write. + existing_content = None + existing_uid, existing_gid, existing_perms = None, None, None + try: + with open(path, 'rb') as target: + existing_content = target.read() + stat = os.stat(path) + existing_uid, existing_gid, existing_perms = ( + stat.st_uid, stat.st_gid, stat.st_mode + ) + except Exception: + pass + if content != existing_content: + log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), + level=DEBUG) + with open(path, 'wb') as target: + os.fchown(target.fileno(), uid, gid) + os.fchmod(target.fileno(), perms) + if six.PY3 and isinstance(content, six.string_types): + content = content.encode('UTF-8') + target.write(content) + return + # the contents were the same, but we might still need to change the + # ownership or permissions. + if existing_uid != uid: + log("Changing uid on already existing content: {} -> {}" + .format(existing_uid, uid), level=DEBUG) + os.chown(path, uid, -1) + if existing_gid != gid: + log("Changing gid on already existing content: {} -> {}" + .format(existing_gid, gid), level=DEBUG) + os.chown(path, -1, gid) + if existing_perms != perms: + log("Changing permissions on existing content: {} -> {}" + .format(existing_perms, perms), level=DEBUG) + os.chmod(path, perms) + + +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab""" + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file""" + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): + """Mount a filesystem at a particular mountpoint""" + cmd_args = ['mount'] + if options is not None: + cmd_args.extend(['-o', options]) + cmd_args.extend([device, mountpoint]) + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) + return False + + if persist: + return fstab_add(device, mountpoint, filesystem, options=options) + return True + + +def umount(mountpoint, persist=False): + """Unmount a filesystem""" + cmd_args = ['umount', mountpoint] + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error unmounting {}\n{}'.format(mountpoint, e.output)) + return False + + if persist: + return fstab_remove(mountpoint) + return True + + +def mounts(): + """Get a list of all mounted volumes as [[mountpoint,device],[...]]""" + with open('/proc/mounts') as f: + # [['/mount/point','/dev/path'],[...]] + system_mounts = [m[1::-1] for m in [l.strip().split() + for l in f.readlines()]] + return system_mounts + + +def fstab_mount(mountpoint): + """Mount filesystem using fstab""" + cmd_args = ['mount', mountpoint] + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error unmounting {}\n{}'.format(mountpoint, e.output)) + return False + return True + + +def file_hash(path, hash_type='md5'): + """Generate a hash checksum of the contents of 'path' or None if not found. + + :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ + if os.path.exists(path): + h = getattr(hashlib, hash_type)() + with open(path, 'rb') as source: + h.update(source.read()) + return h.hexdigest() + else: + return None + + +def path_hash(path): + """Generate a hash checksum of all files matching 'path'. Standard + wildcards like '*' and '?' are supported, see documentation for the 'glob' + module for more information. + + :return: dict: A { filename: hash } dictionary for all matched files. + Empty if none found. + """ + return { + filename: file_hash(filename) + for filename in glob.iglob(path) + } + + +def check_hash(path, checksum, hash_type='md5'): + """Validate a file using a cryptographic checksum. + + :param str checksum: Value of the checksum used to validate the file. + :param str hash_type: Hash algorithm used to generate `checksum`. + Can be any hash algorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + :raises ChecksumError: If the file fails the checksum + + """ + actual_checksum = file_hash(path, hash_type) + if checksum != actual_checksum: + raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) + + +class ChecksumError(ValueError): + """A class derived from Value error to indicate the checksum failed.""" + pass + + +class restart_on_change(object): + """Decorator and context manager to handle restarts. + + Usage: + + @restart_on_change(restart_map, ...) + def function_that_might_trigger_a_restart(...) + ... + + Or: + + with restart_on_change(restart_map, ...): + do_stuff_that_might_trigger_a_restart() + ... + """ + + def __init__(self, restart_map, stopstart=False, restart_functions=None, + can_restart_now_f=None, post_svc_restart_f=None, + pre_restarts_wait_f=None): + """ + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart + services {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function called before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + """ + self.restart_map = restart_map + self.stopstart = stopstart + self.restart_functions = restart_functions + self.can_restart_now_f = can_restart_now_f + self.post_svc_restart_f = post_svc_restart_f + self.pre_restarts_wait_f = pre_restarts_wait_f + + def __call__(self, f): + """Work like a decorator. + + Returns a wrapped function that performs the restart if triggered. + + :param f: The function that is being wrapped. + :type f: Callable[[Any], Any] + :returns: the wrapped function + :rtype: Callable[[Any], Any] + """ + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + return restart_on_change_helper( + (lambda: f(*args, **kwargs)), + self.restart_map, + stopstart=self.stopstart, + restart_functions=self.restart_functions, + can_restart_now_f=self.can_restart_now_f, + post_svc_restart_f=self.post_svc_restart_f, + pre_restarts_wait_f=self.pre_restarts_wait_f) + return wrapped_f + + def __enter__(self): + """Enter the runtime context related to this object. """ + self.checksums = _pre_restart_on_change_helper(self.restart_map) + + def __exit__(self, exc_type, exc_val, exc_tb): + """Exit the runtime context related to this object. + + The parameters describe the exception that caused the context to be + exited. If the context was exited without an exception, all three + arguments will be None. + """ + if exc_type is None: + _post_restart_on_change_helper( + self.checksums, + self.restart_map, + stopstart=self.stopstart, + restart_functions=self.restart_functions, + can_restart_now_f=self.can_restart_now_f, + post_svc_restart_f=self.post_svc_restart_f, + pre_restarts_wait_f=self.pre_restarts_wait_f) + # All is good, so return False; any exceptions will propagate. + return False + + +def restart_on_change_helper(lambda_f, restart_map, stopstart=False, + restart_functions=None, + can_restart_now_f=None, + post_svc_restart_f=None, + pre_restarts_wait_f=None): + """Helper function to perform the restart_on_change function. + + This is provided for decorators to restart services if files described + in the restart_map have changed after an invocation of lambda_f(). + + This functions allows for a number of helper functions to be passed. + + `restart_functions` is a map with a service as the key and the + corresponding value being the function to call to restart the service. For + example if `restart_functions={'some-service': my_restart_func}` then + `my_restart_func` should a function which takes one argument which is the + service name to be retstarted. + + `can_restart_now_f` is a function which checks that a restart is permitted. + It should return a bool which indicates if a restart is allowed and should + take a service name (str) and a list of changed files (List[str]) as + arguments. + + `post_svc_restart_f` is a function which runs after a service has been + restarted. It takes the service name that was restarted as an argument. + + `pre_restarts_wait_f` is a function which is called before any restarts + occur. The use case for this is an application which wants to try and + stagger restarts between units. + + :param lambda_f: function to call. + :type lambda_f: Callable[[], ANY] + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function called before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + :returns: result of lambda_f() + :rtype: ANY + """ + checksums = _pre_restart_on_change_helper(restart_map) + r = lambda_f() + _post_restart_on_change_helper(checksums, + restart_map, + stopstart, + restart_functions, + can_restart_now_f, + post_svc_restart_f, + pre_restarts_wait_f) + return r + + +def _pre_restart_on_change_helper(restart_map): + """Take a snapshot of file hashes. + + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :returns: Dictionary of file paths and the files checksum. + :rtype: Dict[str, str] + """ + return {path: path_hash(path) for path in restart_map} + + +def _post_restart_on_change_helper(checksums, + restart_map, + stopstart=False, + restart_functions=None, + can_restart_now_f=None, + post_svc_restart_f=None, + pre_restarts_wait_f=None): + """Check whether files have changed. + + :param checksums: Dictionary of file paths and the files checksum. + :type checksums: Dict[str, str] + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function called before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + """ + if restart_functions is None: + restart_functions = {} + changed_files = defaultdict(list) + restarts = [] + # create a list of lists of the services to restart + for path, services in restart_map.items(): + if path_hash(path) != checksums[path]: + restarts.append(services) + for svc in services: + changed_files[svc].append(path) + # create a flat list of ordered services without duplicates from lists + services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) + if services_list: + if pre_restarts_wait_f: + pre_restarts_wait_f() + actions = ('stop', 'start') if stopstart else ('restart',) + for service_name in services_list: + if can_restart_now_f: + if not can_restart_now_f(service_name, + changed_files[service_name]): + continue + if service_name in restart_functions: + restart_functions[service_name](service_name) + else: + for action in actions: + service(action, service_name) + if post_svc_restart_f: + post_svc_restart_f(service_name) + + +def pwgen(length=None): + """Generate a random password.""" + if length is None: + # A random length is ok to use a weak PRNG + length = random.choice(range(35, 45)) + alphanumeric_chars = [ + l for l in (string.ascii_letters + string.digits) + if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() + random_chars = [ + random_generator.choice(alphanumeric_chars) for _ in range(length)] + return(''.join(random_chars)) + + +def is_phy_iface(interface): + """Returns True if interface is not virtual, otherwise False.""" + if interface: + sys_net = '/sys/class/net' + if os.path.isdir(sys_net): + for iface in glob.glob(os.path.join(sys_net, '*')): + if '/virtual/' in os.path.realpath(iface): + continue + + if interface == os.path.basename(iface): + return True + + return False + + +def get_bond_master(interface): + """Returns bond master if interface is bond slave otherwise None. + + NOTE: the provided interface is expected to be physical + """ + if interface: + iface_path = '/sys/class/net/%s' % (interface) + if os.path.exists(iface_path): + if '/virtual/' in os.path.realpath(iface_path): + return None + + master = os.path.join(iface_path, 'master') + if os.path.exists(master): + master = os.path.realpath(master) + # make sure it is a bond master + if os.path.exists(os.path.join(master, 'bonding')): + return os.path.basename(master) + + return None + + +def list_nics(nic_type=None): + """Return a list of nics of given type(s)""" + if isinstance(nic_type, six.string_types): + int_types = [nic_type] + else: + int_types = nic_type + + interfaces = [] + if nic_type: + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace') + ip_output = ip_output.split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + matched = re.search('.*: (' + int_type + + r'[0-9]+\.[0-9]+)@.*', line) + if matched: + iface = matched.groups()[0] + else: + iface = line.split()[1].replace(":", "") + + if iface not in interfaces: + interfaces.append(iface) + else: + cmd = ['ip', 'a'] + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace').split('\n') + ip_output = (line.strip() for line in ip_output if line) + + key = re.compile(r'^[0-9]+:\s+(.+):') + for line in ip_output: + matched = re.search(key, line) + if matched: + iface = matched.group(1) + iface = iface.partition("@")[0] + if iface not in interfaces: + interfaces.append(iface) + + return interfaces + + +def set_nic_mtu(nic, mtu): + """Set the Maximum Transmission Unit (MTU) on a network interface.""" + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + """Return the Maximum Transmission Unit (MTU) for a network interface.""" + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output( + cmd).decode('UTF-8', errors='replace').split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + """Return the Media Access Control (MAC) for a network interface.""" + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).decode('UTF-8', errors='replace') + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr + + +@contextmanager +def chdir(directory): + """Change the current working directory to a different directory for a code + block and return the previous directory after the block exits. Useful to + run commands from a specified directory. + + :param str directory: The directory path to change to for this context. + """ + cur = os.getcwd() + try: + yield os.chdir(directory) + finally: + os.chdir(cur) + + +def chownr(path, owner, group, follow_links=True, chowntopdir=False): + """Recursively change user and group ownership of files and directories + in given path. Doesn't chown path itself by default, only its children. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + :param bool follow_links: Also follow and chown links if True + :param bool chowntopdir: Also chown path itself if True + """ + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + if follow_links: + chown = os.chown + else: + chown = os.lchown + + if chowntopdir: + broken_symlink = os.path.lexists(path) and not os.path.exists(path) + if not broken_symlink: + chown(path, uid, gid) + for root, dirs, files in os.walk(path, followlinks=follow_links): + for name in dirs + files: + full = os.path.join(root, name) + try: + chown(full, uid, gid) + except (IOError, OSError) as e: + # Intended to ignore "file not found". Catching both to be + # compatible with both Python 2.7 and 3.x. + if e.errno == errno.ENOENT: + pass + + +def lchownr(path, owner, group): + """Recursively change user and group ownership of files and directories + in a given path, not following symbolic links. See the documentation for + 'os.lchown' for more information. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + """ + chownr(path, owner, group, follow_links=False) + + +def owner(path): + """Returns a tuple containing the username & groupname owning the path. + + :param str path: the string path to retrieve the ownership + :return tuple(str, str): A (username, groupname) tuple containing the + name of the user and group owning the path. + :raises OSError: if the specified path does not exist + """ + stat = os.stat(path) + username = pwd.getpwuid(stat.st_uid)[0] + groupname = grp.getgrgid(stat.st_gid)[0] + return username, groupname + + +def get_total_ram(): + """The total amount of system RAM in bytes. + + This is what is reported by the OS, and may be overcommitted when + there are multiple containers hosted on the same machine. + """ + with open('/proc/meminfo', 'r') as f: + for line in f.readlines(): + if line: + key, value, unit = line.split() + if key == 'MemTotal:': + assert unit == 'kB', 'Unknown unit' + return int(value) * 1024 # Classic, not KiB. + raise NotImplementedError() + + +UPSTART_CONTAINER_TYPE = '/run/container_type' + + +def is_container(): + """Determine whether unit is running in a container + + @return: boolean indicating if unit is in a container + """ + if init_is_systemd(): + # Detect using systemd-detect-virt + return subprocess.call(['systemd-detect-virt', + '--container']) == 0 + else: + # Detect using upstart container file marker + return os.path.exists(UPSTART_CONTAINER_TYPE) + + +def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): + """Adds the specified path to the mlocate's udpatedb.conf PRUNEPATH list. + + This method has no effect if the path specified by updatedb_path does not + exist or is not a file. + + @param path: string the path to add to the updatedb.conf PRUNEPATHS value + @param updatedb_path: the path the updatedb.conf file + """ + if not os.path.exists(updatedb_path) or os.path.isdir(updatedb_path): + # If the updatedb.conf file doesn't exist then don't attempt to update + # the file as the package providing mlocate may not be installed on + # the local system + return + + with open(updatedb_path, 'r+') as f_id: + updatedb_text = f_id.read() + output = updatedb(updatedb_text, path) + f_id.seek(0) + f_id.write(output) + f_id.truncate() + + +def updatedb(updatedb_text, new_path): + lines = [line for line in updatedb_text.split("\n")] + for i, line in enumerate(lines): + if line.startswith("PRUNEPATHS="): + paths_line = line.split("=")[1].replace('"', '') + paths = paths_line.split(" ") + if new_path not in paths: + paths.append(new_path) + lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) + output = "\n".join(lines) + return output + + +def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): + """ Modulo distribution + + This helper uses the unit number, a modulo value and a constant wait time + to produce a calculated wait time distribution. This is useful in large + scale deployments to distribute load during an expensive operation such as + service restarts. + + If you have 1000 nodes that need to restart 100 at a time 1 minute at a + time: + + time.wait(modulo_distribution(modulo=100, wait=60)) + restart() + + If you need restarts to happen serially set modulo to the exact number of + nodes and set a high constant wait time: + + time.wait(modulo_distribution(modulo=10, wait=120)) + restart() + + @param modulo: int The modulo number creates the group distribution + @param wait: int The constant time wait value + @param non_zero_wait: boolean Override unit % modulo == 0, + return modulo * wait. Used to avoid collisions with + leader nodes which are often given priority. + @return: int Calculated time to wait for unit operation + """ + unit_number = int(local_unit().split('/')[1]) + calculated_wait_time = (unit_number % modulo) * wait + if non_zero_wait and calculated_wait_time == 0: + return modulo * wait + else: + return calculated_wait_time + + +def ca_cert_absolute_path(basename_without_extension): + """Returns absolute path to CA certificate. + + :param basename_without_extension: Filename without extension + :type basename_without_extension: str + :returns: Absolute full path + :rtype: str + """ + return '{}/{}.crt'.format(CA_CERT_DIR, basename_without_extension) + + +def install_ca_cert(ca_cert, name=None): + """ + Install the given cert as a trusted CA. + + The ``name`` is the stem of the filename where the cert is written, and if + not provided, it will default to ``juju-{charm_name}``. + + If the cert is empty or None, or is unchanged, nothing is done. + """ + if not ca_cert: + return + if not isinstance(ca_cert, bytes): + ca_cert = ca_cert.encode('utf8') + if not name: + name = 'juju-{}'.format(charm_name()) + cert_file = ca_cert_absolute_path(name) + new_hash = hashlib.md5(ca_cert).hexdigest() + if file_hash(cert_file) == new_hash: + return + log("Installing new CA cert at: {}".format(cert_file), level=INFO) + write_file(cert_file, ca_cert) + subprocess.check_call(['update-ca-certificates', '--fresh']) + + +def get_system_env(key, default=None): + """Get data from system environment as represented in ``/etc/environment``. + + :param key: Key to look up + :type key: str + :param default: Value to return if key is not found + :type default: any + :returns: Value for key if found or contents of default parameter + :rtype: any + :raises: subprocess.CalledProcessError + """ + env_file = '/etc/environment' + # use the shell and env(1) to parse the global environments file. This is + # done to get the correct result even if the user has shell variable + # substitutions or other shell logic in that file. + output = subprocess.check_output( + ['env', '-i', '/bin/bash', '-c', + 'set -a && source {} && env'.format(env_file)], + universal_newlines=True) + for k, v in (line.split('=', 1) + for line in output.splitlines() if '=' in line): + if k == key: + return v + else: + return default diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/centos.py b/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/centos.py new file mode 100644 index 0000000..7781a39 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/centos.py @@ -0,0 +1,72 @@ +import subprocess +import yum +import os + +from charmhelpers.core.strutils import BasicStringComparator + + +class CompareHostReleases(BasicStringComparator): + """Provide comparisons of Host releases. + + Use in the form of + + if CompareHostReleases(release) > 'trusty': + # do something with mitaka + """ + + def __init__(self, item): + raise NotImplementedError( + "CompareHostReleases() is not implemented for CentOS") + + +def service_available(service_name): + # """Determine whether a system service is available.""" + if os.path.isdir('/run/systemd/system'): + cmd = ['systemctl', 'is-enabled', service_name] + else: + cmd = ['service', service_name, 'is-enabled'] + return subprocess.call(cmd) == 0 + + +def add_new_group(group_name, system_group=False, gid=None): + cmd = ['groupadd'] + if gid: + cmd.extend(['--gid', str(gid)]) + if system_group: + cmd.append('-r') + cmd.append(group_name) + subprocess.check_call(cmd) + + +def lsb_release(): + """Return /etc/os-release in a dict.""" + d = {} + with open('/etc/os-release', 'r') as lsb: + for l in lsb: + s = l.split('=') + if len(s) != 2: + continue + d[s[0].strip()] = s[1].strip() + return d + + +def cmp_pkgrevno(package, revno, pkgcache=None): + """Compare supplied revno with the revno of the installed package. + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + + This function imports YumBase function if the pkgcache argument + is None. + """ + if not pkgcache: + y = yum.YumBase() + packages = y.doPackageLists() + pkgcache = {i.Name: i.version for i in packages['installed']} + pkg = pkgcache[package] + if pkg > revno: + return 1 + if pkg < revno: + return -1 + return 0 diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/ubuntu.py b/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/ubuntu.py new file mode 100644 index 0000000..e710c0e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/host_factory/ubuntu.py @@ -0,0 +1,121 @@ +import subprocess + +from charmhelpers.core.hookenv import cached +from charmhelpers.core.strutils import BasicStringComparator + + +UBUNTU_RELEASES = ( + 'lucid', + 'maverick', + 'natty', + 'oneiric', + 'precise', + 'quantal', + 'raring', + 'saucy', + 'trusty', + 'utopic', + 'vivid', + 'wily', + 'xenial', + 'yakkety', + 'zesty', + 'artful', + 'bionic', + 'cosmic', + 'disco', + 'eoan', + 'focal', + 'groovy', + 'hirsute', + 'impish', +) + + +class CompareHostReleases(BasicStringComparator): + """Provide comparisons of Ubuntu releases. + + Use in the form of + + if CompareHostReleases(release) > 'trusty': + # do something with mitaka + """ + _list = UBUNTU_RELEASES + + +def service_available(service_name): + """Determine whether a system service is available""" + try: + subprocess.check_output( + ['service', service_name, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') + except subprocess.CalledProcessError as e: + return b'unrecognized service' not in e.output + else: + return True + + +def add_new_group(group_name, system_group=False, gid=None): + cmd = ['addgroup'] + if gid: + cmd.extend(['--gid', str(gid)]) + if system_group: + cmd.append('--system') + else: + cmd.extend([ + '--group', + ]) + cmd.append(group_name) + subprocess.check_call(cmd) + + +def lsb_release(): + """Return /etc/lsb-release in a dict""" + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + return d + + +def get_distrib_codename(): + """Return the codename of the distribution + :returns: The codename + :rtype: str + """ + return lsb_release()['DISTRIB_CODENAME'].lower() + + +def cmp_pkgrevno(package, revno, pkgcache=None): + """Compare supplied revno with the revno of the installed package. + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + + This function imports apt_cache function from charmhelpers.fetch if + the pkgcache argument is None. Be sure to add charmhelpers.fetch if + you call this function, or pass an apt_pkg.Cache() instance. + """ + from charmhelpers.fetch import apt_pkg, get_installed_version + if not pkgcache: + current_ver = get_installed_version(package) + else: + pkg = pkgcache[package] + current_ver = pkg.current_ver + + return apt_pkg.version_compare(current_ver.ver_str, revno) + + +@cached +def arch(): + """Return the package architecture as a string. + + :returns: the architecture + :rtype: str + :raises: subprocess.CalledProcessError if dpkg command fails + """ + return subprocess.check_output( + ['dpkg', '--print-architecture'] + ).rstrip().decode('UTF-8') diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/hugepage.py b/nrpe/mod/charmhelpers/charmhelpers/core/hugepage.py new file mode 100644 index 0000000..54b5b5e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/hugepage.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +from charmhelpers.core import fstab +from charmhelpers.core import sysctl +from charmhelpers.core.host import ( + add_group, + add_user_to_group, + fstab_mount, + mkdir, +) +from charmhelpers.core.strutils import bytes_from_string +from subprocess import check_output + + +def hugepage_support(user, group='hugetlb', nr_hugepages=256, + max_map_count=65536, mnt_point='/run/hugepages/kvm', + pagesize='2MB', mount=True, set_shmmax=False): + """Enable hugepages on system. + + Args: + user (str) -- Username to allow access to hugepages to + group (str) -- Group name to own hugepages + nr_hugepages (int) -- Number of pages to reserve + max_map_count (int) -- Number of Virtual Memory Areas a process can own + mnt_point (str) -- Directory to mount hugepages on + pagesize (str) -- Size of hugepages + mount (bool) -- Whether to Mount hugepages + """ + group_info = add_group(group) + gid = group_info.gr_gid + add_user_to_group(user, group) + if max_map_count < 2 * nr_hugepages: + max_map_count = 2 * nr_hugepages + sysctl_settings = { + 'vm.nr_hugepages': nr_hugepages, + 'vm.max_map_count': max_map_count, + 'vm.hugetlb_shm_group': gid, + } + if set_shmmax: + shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) + shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages + if shmmax_minsize > shmmax_current: + sysctl_settings['kernel.shmmax'] = shmmax_minsize + sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') + mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) + lfstab = fstab.Fstab() + fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point) + if fstab_entry: + lfstab.remove_entry(fstab_entry) + entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs', + 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0) + lfstab.add_entry(entry) + if mount: + fstab_mount(mnt_point) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/kernel.py b/nrpe/mod/charmhelpers/charmhelpers/core/kernel.py new file mode 100644 index 0000000..e01f4f8 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/kernel.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + +from charmhelpers.osplatform import get_platform +from charmhelpers.core.hookenv import ( + log, + INFO +) + +__platform__ = get_platform() +if __platform__ == "ubuntu": + from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401 + persistent_modprobe, + update_initramfs, + ) # flake8: noqa -- ignore F401 for this import +elif __platform__ == "centos": + from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401 + persistent_modprobe, + update_initramfs, + ) # flake8: noqa -- ignore F401 for this import + +__author__ = "Jorge Niedbalski " + + +def modprobe(module, persist=True): + """Load a kernel module and configure for auto-load on reboot.""" + cmd = ['modprobe', module] + + log('Loading kernel module %s' % module, level=INFO) + + subprocess.check_call(cmd) + if persist: + persistent_modprobe(module) + + +def rmmod(module, force=False): + """Remove a module from the linux kernel""" + cmd = ['rmmod'] + if force: + cmd.append('-f') + cmd.append(module) + log('Removing kernel module %s' % module, level=INFO) + return subprocess.check_call(cmd) + + +def lsmod(): + """Shows what kernel modules are currently loaded""" + return subprocess.check_output(['lsmod'], + universal_newlines=True) + + +def is_module_loaded(module): + """Checks if a kernel module is already loaded""" + matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) + return len(matches) > 0 diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/centos.py b/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/centos.py new file mode 100644 index 0000000..1c402c1 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/centos.py @@ -0,0 +1,17 @@ +import subprocess +import os + + +def persistent_modprobe(module): + """Load a kernel module and configure for auto-load on reboot.""" + if not os.path.exists('/etc/rc.modules'): + open('/etc/rc.modules', 'a') + os.chmod('/etc/rc.modules', 111) + with open('/etc/rc.modules', 'r+') as modules: + if module not in modules.read(): + modules.write('modprobe %s\n' % module) + + +def update_initramfs(version='all'): + """Updates an initramfs image.""" + return subprocess.check_call(["dracut", "-f", version]) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/ubuntu.py b/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/ubuntu.py new file mode 100644 index 0000000..3de372f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/kernel_factory/ubuntu.py @@ -0,0 +1,13 @@ +import subprocess + + +def persistent_modprobe(module): + """Load a kernel module and configure for auto-load on reboot.""" + with open('/etc/modules', 'r+') as modules: + if module not in modules.read(): + modules.write(module + "\n") + + +def update_initramfs(version='all'): + """Updates an initramfs image.""" + return subprocess.check_call(["update-initramfs", "-k", version, "-u"]) diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/services/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/core/services/__init__.py new file mode 100644 index 0000000..61fd074 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/services/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import * # NOQA +from .helpers import * # NOQA diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/services/base.py b/nrpe/mod/charmhelpers/charmhelpers/core/services/base.py new file mode 100644 index 0000000..9f88029 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/services/base.py @@ -0,0 +1,367 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import inspect +from collections import Iterable, OrderedDict + +import six + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +__all__ = ['ServiceManager', 'ManagerCallback', + 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports', + 'service_restart', 'service_stop'] + + +class ServiceManager(object): + def __init__(self, services=None): + """ + Register a list of services, given their definitions. + + Service definitions are dicts in the following formats (all keys except + 'service' are optional):: + + { + "service": , + "required_data": , + "provided_data": , + "data_ready": , + "data_lost": , + "start": , + "stop": , + "ports": , + } + + The 'required_data' list should contain dicts of required data (or + dependency managers that act like dicts and know how to collect the data). + Only when all items in the 'required_data' list are populated are the list + of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more + information. + + The 'provided_data' list should contain relation data providers, most likely + a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, + that will indicate a set of data to set on a given relation. + + The 'data_ready' value should be either a single callback, or a list of + callbacks, to be called when all items in 'required_data' pass `is_ready()`. + Each callback will be called with the service name as the only parameter. + After all of the 'data_ready' callbacks are called, the 'start' callbacks + are fired. + + The 'data_lost' value should be either a single callback, or a list of + callbacks, to be called when a 'required_data' item no longer passes + `is_ready()`. Each callback will be called with the service name as the + only parameter. After all of the 'data_lost' callbacks are called, + the 'stop' callbacks are fired. + + The 'start' value should be either a single callback, or a list of + callbacks, to be called when starting the service, after the 'data_ready' + callbacks are complete. Each callback will be called with the service + name as the only parameter. This defaults to + `[host.service_start, services.open_ports]`. + + The 'stop' value should be either a single callback, or a list of + callbacks, to be called when stopping the service. If the service is + being stopped because it no longer has all of its 'required_data', this + will be called after all of the 'data_lost' callbacks are complete. + Each callback will be called with the service name as the only parameter. + This defaults to `[services.close_ports, host.service_stop]`. + + The 'ports' value should be a list of ports to manage. The default + 'start' handler will open the ports after the service is started, + and the default 'stop' handler will close the ports prior to stopping + the service. + + + Examples: + + The following registers an Upstart service called bingod that depends on + a mongodb relation and which runs a custom `db_migrate` function prior to + restarting the service, and a Runit service called spadesd:: + + manager = services.ServiceManager([ + { + 'service': 'bingod', + 'ports': [80, 443], + 'required_data': [MongoRelation(), config(), {'my': 'data'}], + 'data_ready': [ + services.template(source='bingod.conf'), + services.template(source='bingod.ini', + target='/etc/bingod.ini', + owner='bingo', perms=0400), + ], + }, + { + 'service': 'spadesd', + 'data_ready': services.template(source='spadesd_run.j2', + target='/etc/sv/spadesd/run', + perms=0555), + 'start': runit_start, + 'stop': runit_stop, + }, + ]) + manager.manage() + """ + self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') + self._ready = None + self.services = OrderedDict() + for service in services or []: + service_name = service['service'] + self.services[service_name] = service + + def manage(self): + """ + Handle the current hook by doing The Right Thing with the registered services. + """ + hookenv._run_atstart() + try: + hook_name = hookenv.hook_name() + if hook_name == 'stop': + self.stop_services() + else: + self.reconfigure_services() + self.provide_data() + except SystemExit as x: + if x.code is None or x.code == 0: + hookenv._run_atexit() + hookenv._run_atexit() + + def provide_data(self): + """ + Set the relation data for each provider in the ``provided_data`` list. + + A provider must have a `name` attribute, which indicates which relation + to set data on, and a `provide_data()` method, which returns a dict of + data to set. + + The `provide_data()` method can optionally accept two parameters: + + * ``remote_service`` The name of the remote service that the data will + be provided to. The `provide_data()` method will be called once + for each connected service (not unit). This allows the method to + tailor its data to the given service. + * ``service_ready`` Whether or not the service definition had all of + its requirements met, and thus the ``data_ready`` callbacks run. + + Note that the ``provided_data`` methods are now called **after** the + ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks + a chance to generate any data necessary for the providing to the remote + services. + """ + for service_name, service in self.services.items(): + service_ready = self.is_ready(service_name) + for provider in service.get('provided_data', []): + for relid in hookenv.relation_ids(provider.name): + units = hookenv.related_units(relid) + if not units: + continue + remote_service = units[0].split('/')[0] + if six.PY2: + argspec = inspect.getargspec(provider.provide_data) + else: + argspec = inspect.getfullargspec(provider.provide_data) + if len(argspec.args) > 1: + data = provider.provide_data(remote_service, service_ready) + else: + data = provider.provide_data() + if data: + hookenv.relation_set(relid, data) + + def reconfigure_services(self, *service_names): + """ + Update all files for one or more registered services, and, + if ready, optionally restart them. + + If no service names are given, reconfigures all registered services. + """ + for service_name in service_names or self.services.keys(): + if self.is_ready(service_name): + self.fire_event('data_ready', service_name) + self.fire_event('start', service_name, default=[ + service_restart, + manage_ports]) + self.save_ready(service_name) + else: + if self.was_ready(service_name): + self.fire_event('data_lost', service_name) + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + self.save_lost(service_name) + + def stop_services(self, *service_names): + """ + Stop one or more registered services, by name. + + If no service names are given, stops all registered services. + """ + for service_name in service_names or self.services.keys(): + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + + def get_service(self, service_name): + """ + Given the name of a registered service, return its service definition. + """ + service = self.services.get(service_name) + if not service: + raise KeyError('Service not registered: %s' % service_name) + return service + + def fire_event(self, event_name, service_name, default=None): + """ + Fire a data_ready, data_lost, start, or stop event on a given service. + """ + service = self.get_service(service_name) + callbacks = service.get(event_name, default) + if not callbacks: + return + if not isinstance(callbacks, Iterable): + callbacks = [callbacks] + for callback in callbacks: + if isinstance(callback, ManagerCallback): + callback(self, service_name, event_name) + else: + callback(service_name) + + def is_ready(self, service_name): + """ + Determine if a registered service is ready, by checking its 'required_data'. + + A 'required_data' item can be any mapping type, and is considered ready + if `bool(item)` evaluates as True. + """ + service = self.get_service(service_name) + reqs = service.get('required_data', []) + return all(bool(req) for req in reqs) + + def _load_ready_file(self): + if self._ready is not None: + return + if os.path.exists(self._ready_file): + with open(self._ready_file) as fp: + self._ready = set(json.load(fp)) + else: + self._ready = set() + + def _save_ready_file(self): + if self._ready is None: + return + with open(self._ready_file, 'w') as fp: + json.dump(list(self._ready), fp) + + def save_ready(self, service_name): + """ + Save an indicator that the given service is now data_ready. + """ + self._load_ready_file() + self._ready.add(service_name) + self._save_ready_file() + + def save_lost(self, service_name): + """ + Save an indicator that the given service is no longer data_ready. + """ + self._load_ready_file() + self._ready.discard(service_name) + self._save_ready_file() + + def was_ready(self, service_name): + """ + Determine if the given service was previously data_ready. + """ + self._load_ready_file() + return service_name in self._ready + + +class ManagerCallback(object): + """ + Special case of a callback that takes the `ServiceManager` instance + in addition to the service name. + + Subclasses should implement `__call__` which should accept three parameters: + + * `manager` The `ServiceManager` instance + * `service_name` The name of the service it's being triggered for + * `event_name` The name of the event that this callback is handling + """ + def __call__(self, manager, service_name, event_name): + raise NotImplementedError() + + +class PortManagerCallback(ManagerCallback): + """ + Callback class that will open or close ports, for use as either + a start or stop action. + """ + def __call__(self, manager, service_name, event_name): + service = manager.get_service(service_name) + # turn this generator into a list, + # as we'll be going over it multiple times + new_ports = list(service.get('ports', [])) + port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name)) + if os.path.exists(port_file): + with open(port_file) as fp: + old_ports = fp.read().split(',') + for old_port in old_ports: + if bool(old_port) and not self.ports_contains(old_port, new_ports): + hookenv.close_port(old_port) + with open(port_file, 'w') as fp: + fp.write(','.join(str(port) for port in new_ports)) + for port in new_ports: + # A port is either a number or 'ICMP' + protocol = 'TCP' + if str(port).upper() == 'ICMP': + protocol = 'ICMP' + if event_name == 'start': + hookenv.open_port(port, protocol) + elif event_name == 'stop': + hookenv.close_port(port, protocol) + + def ports_contains(self, port, ports): + if not bool(port): + return False + if str(port).upper() != 'ICMP': + port = int(port) + return port in ports + + +def service_stop(service_name): + """ + Wrapper around host.service_stop to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_running(service_name): + host.service_stop(service_name) + + +def service_restart(service_name): + """ + Wrapper around host.service_restart to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_available(service_name): + if host.service_running(service_name): + host.service_restart(service_name) + else: + host.service_start(service_name) + + +# Convenience aliases +open_ports = close_ports = manage_ports = PortManagerCallback() diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/services/helpers.py b/nrpe/mod/charmhelpers/charmhelpers/core/services/helpers.py new file mode 100644 index 0000000..3e6e30d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/services/helpers.py @@ -0,0 +1,290 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +from charmhelpers.core import hookenv +from charmhelpers.core import host +from charmhelpers.core import templating + +from charmhelpers.core.services.base import ManagerCallback + + +__all__ = ['RelationContext', 'TemplateCallback', + 'render_template', 'template'] + + +class RelationContext(dict): + """ + Base class for a context generator that gets relation data from juju. + + Subclasses must provide the attributes `name`, which is the name of the + interface of interest, `interface`, which is the type of the interface of + interest, and `required_keys`, which is the set of keys required for the + relation to be considered complete. The data for all interfaces matching + the `name` attribute that are complete will used to populate the dictionary + values (see `get_data`, below). + + The generated context will be namespaced under the relation :attr:`name`, + to prevent potential naming conflicts. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = None + interface = None + + def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + + if name is not None: + self.name = name + if additional_required_keys: + self.required_keys.extend(additional_required_keys) + self.get_data() + + def __bool__(self): + """ + Returns True if all of the required_keys are available. + """ + return self.is_ready() + + __nonzero__ = __bool__ + + def __repr__(self): + return super(RelationContext, self).__repr__() + + def is_ready(self): + """ + Returns True if all of the `required_keys` are available from any units. + """ + ready = len(self.get(self.name, [])) > 0 + if not ready: + hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG) + return ready + + def _is_ready(self, unit_data): + """ + Helper method that tests a set of relation data and returns True if + all of the `required_keys` are present. + """ + return set(unit_data.keys()).issuperset(set(self.required_keys)) + + def get_data(self): + """ + Retrieve the relation data for each unit involved in a relation and, + if complete, store it in a list under `self[self.name]`. This + is automatically called when the RelationContext is instantiated. + + The units are sorted lexographically first by the service ID, then by + the unit ID. Thus, if an interface has two other services, 'db:1' + and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1', + and 'db:2' having one unit, 'mediawiki/0', all of which have a complete + set of data, the relation data for the units will be stored in the + order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'. + + If you only care about a single unit on the relation, you can just + access it as `{{ interface[0]['key'] }}`. However, if you can at all + support multiple units on a relation, you should iterate over the list, + like:: + + {% for unit in interface -%} + {{ unit['key'] }}{% if not loop.last %},{% endif %} + {%- endfor %} + + Note that since all sets of relation data from all related services and + units are in a single list, if you need to know which service or unit a + set of data came from, you'll need to extend this class to preserve + that information. + """ + if not hookenv.relation_ids(self.name): + return + + ns = self.setdefault(self.name, []) + for rid in sorted(hookenv.relation_ids(self.name)): + for unit in sorted(hookenv.related_units(rid)): + reldata = hookenv.relation_get(rid=rid, unit=unit) + if self._is_ready(reldata): + ns.append(reldata) + + def provide_data(self): + """ + Return data to be relation_set for this interface. + """ + return {} + + +class MysqlRelation(RelationContext): + """ + Relation context for the `mysql` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'db' + interface = 'mysql' + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + RelationContext.__init__(self, *args, **kwargs) + + +class HttpRelation(RelationContext): + """ + Relation context for the `http` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'website' + interface = 'http' + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + RelationContext.__init__(self, *args, **kwargs) + + def provide_data(self): + return { + 'host': hookenv.unit_get('private-address'), + 'port': 80, + } + + +class RequiredConfig(dict): + """ + Data context that loads config options with one or more mandatory options. + + Once the required options have been changed from their default values, all + config options will be available, namespaced under `config` to prevent + potential naming conflicts (for example, between a config option and a + relation property). + + :param list *args: List of options that must be changed from their default values. + """ + + def __init__(self, *args): + self.required_options = args + self['config'] = hookenv.config() + with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: + self.config = yaml.load(fp).get('options', {}) + + def __bool__(self): + for option in self.required_options: + if option not in self['config']: + return False + current_value = self['config'][option] + default_value = self.config[option].get('default') + if current_value == default_value: + return False + if current_value in (None, '') and default_value in (None, ''): + return False + return True + + def __nonzero__(self): + return self.__bool__() + + +class StoredContext(dict): + """ + A data context that always returns the data that it was first created with. + + This is useful to do a one-time generation of things like passwords, that + will thereafter use the same value that was originally generated, instead + of generating a new value each time it is run. + """ + def __init__(self, file_name, config_data): + """ + If the file exists, populate `self` with the data from the file. + Otherwise, populate with the given data and persist it to the file. + """ + if os.path.exists(file_name): + self.update(self.read_context(file_name)) + else: + self.store_context(file_name, config_data) + self.update(config_data) + + def store_context(self, file_name, config_data): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'w') as file_stream: + os.fchmod(file_stream.fileno(), 0o600) + yaml.dump(config_data, file_stream) + + def read_context(self, file_name): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'r') as file_stream: + data = yaml.load(file_stream) + if not data: + raise OSError("%s is empty" % file_name) + return data + + +class TemplateCallback(ManagerCallback): + """ + Callback class that will render a Jinja2 template, for use as a ready + action. + + :param str source: The template source file, relative to + `$CHARM_DIR/templates` + + :param str target: The target to write the rendered template to (or None) + :param str owner: The owner of the rendered file + :param str group: The group of the rendered file + :param int perms: The permissions of the rendered file + :param partial on_change_action: functools partial to be executed when + rendered file changes + :param jinja2 loader template_loader: A jinja2 template loader + + :return str: The rendered template + """ + def __init__(self, source, target, + owner='root', group='root', perms=0o444, + on_change_action=None, template_loader=None): + self.source = source + self.target = target + self.owner = owner + self.group = group + self.perms = perms + self.on_change_action = on_change_action + self.template_loader = template_loader + + def __call__(self, manager, service_name, event_name): + pre_checksum = '' + if self.on_change_action and os.path.isfile(self.target): + pre_checksum = host.file_hash(self.target) + service = manager.get_service(service_name) + context = {'ctx': {}} + for ctx in service.get('required_data', []): + context.update(ctx) + context['ctx'].update(ctx) + + result = templating.render(self.source, self.target, context, + self.owner, self.group, self.perms, + template_loader=self.template_loader) + if self.on_change_action: + if pre_checksum == host.file_hash(self.target): + hookenv.log( + 'No change detected: {}'.format(self.target), + hookenv.DEBUG) + else: + self.on_change_action() + + return result + + +# Convenience aliases for templates +render_template = template = TemplateCallback diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/strutils.py b/nrpe/mod/charmhelpers/charmhelpers/core/strutils.py new file mode 100644 index 0000000..28c6b3f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/strutils.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import re + +TRUTHY_STRINGS = {'y', 'yes', 'true', 't', 'on'} +FALSEY_STRINGS = {'n', 'no', 'false', 'f', 'off'} + + +def bool_from_string(value, truthy_strings=TRUTHY_STRINGS, falsey_strings=FALSEY_STRINGS, assume_false=False): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in truthy_strings: + return True + elif value in falsey_strings or assume_false: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) + + +def bytes_from_string(value): + """Interpret human readable string value as bytes. + + Returns int + """ + BYTE_POWER = { + 'K': 1, + 'KB': 1, + 'M': 2, + 'MB': 2, + 'G': 3, + 'GB': 3, + 'T': 4, + 'TB': 4, + 'P': 5, + 'PB': 5, + } + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as bytes" % (value) + raise ValueError(msg) + matches = re.match("([0-9]+)([a-zA-Z]+)", value) + if matches: + size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + else: + # Assume that value passed in is bytes + try: + size = int(value) + except ValueError: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return size + + +class BasicStringComparator(object): + """Provides a class that will compare strings from an iterator type object. + Used to provide > and < comparisons on strings that may not necessarily be + alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the + z-wrap. + """ + + _list = None + + def __init__(self, item): + if self._list is None: + raise Exception("Must define the _list in the class definition!") + try: + self.index = self._list.index(item) + except Exception: + raise KeyError("Item '{}' is not in list '{}'" + .format(item, self._list)) + + def __eq__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index == self._list.index(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index < self._list.index(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __gt__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index > self._list.index(other) + + def __le__(self, other): + return not self.__gt__(other) + + def __str__(self): + """Always give back the item at the index so it can be used in + comparisons like: + + s_mitaka = CompareOpenStack('mitaka') + s_newton = CompareOpenstack('newton') + + assert s_newton > s_mitaka + + @returns: + """ + return self._list[self.index] diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/sysctl.py b/nrpe/mod/charmhelpers/charmhelpers/core/sysctl.py new file mode 100644 index 0000000..386428d --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/sysctl.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from subprocess import check_call, CalledProcessError + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + ERROR, + WARNING, +) + +from charmhelpers.core.host import is_container + +__author__ = 'Jorge Niedbalski R. ' + + +def create(sysctl_dict, sysctl_file, ignore=False): + """Creates a sysctl.conf file from a YAML associative array + + :param sysctl_dict: a dict or YAML-formatted string of sysctl + options eg "{ 'kernel.max_pid': 1337 }" + :type sysctl_dict: str + :param sysctl_file: path to the sysctl file to be saved + :type sysctl_file: str or unicode + :param ignore: If True, ignore "unknown variable" errors. + :type ignore: bool + :returns: None + """ + if type(sysctl_dict) is not dict: + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + else: + sysctl_dict_parsed = sysctl_dict + + with open(sysctl_file, "w") as fd: + for key, value in sysctl_dict_parsed.items(): + fd.write("{}={}\n".format(key, value)) + + log("Updating sysctl_file: {} values: {}".format(sysctl_file, + sysctl_dict_parsed), + level=DEBUG) + + call = ["sysctl", "-p", sysctl_file] + if ignore: + call.append("-e") + + try: + check_call(call) + except CalledProcessError as e: + if is_container(): + log("Error setting some sysctl keys in this container: {}".format(e.output), + level=WARNING) + else: + raise e diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/templating.py b/nrpe/mod/charmhelpers/charmhelpers/core/templating.py new file mode 100644 index 0000000..9014015 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/templating.py @@ -0,0 +1,93 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +def render(source, target, context, owner='root', group='root', + perms=0o444, templates_dir=None, encoding='UTF-8', + template_loader=None, config_template=None): + """ + Render a template. + + The `source` path, if not absolute, is relative to the `templates_dir`. + + The `target` path should be absolute. It can also be `None`, in which + case no file will be written. + + The context should be a dict containing the values to be replaced in the + template. + + config_template may be provided to render from a provided template instead + of loading from a file. + + The `owner`, `group`, and `perms` options will be passed to `write_file`. + + If omitted, `templates_dir` defaults to the `templates` folder in the charm. + + The rendered template will be written to the file as well as being returned + as a string. + + Note: Using this requires python-jinja2 or python3-jinja2; if it is not + installed, calling this will attempt to use charmhelpers.fetch.apt_install + to install it. + """ + try: + from jinja2 import FileSystemLoader, Environment, exceptions + except ImportError: + try: + from charmhelpers.fetch import apt_install + except ImportError: + hookenv.log('Could not import jinja2, and could not import ' + 'charmhelpers.fetch to install it', + level=hookenv.ERROR) + raise + if sys.version_info.major == 2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) + from jinja2 import FileSystemLoader, Environment, exceptions + + if template_loader: + template_env = Environment(loader=template_loader) + else: + if templates_dir is None: + templates_dir = os.path.join(hookenv.charm_dir(), 'templates') + template_env = Environment(loader=FileSystemLoader(templates_dir)) + + # load from a string if provided explicitly + if config_template is not None: + template = template_env.from_string(config_template) + else: + try: + source = source + template = template_env.get_template(source) + except exceptions.TemplateNotFound as e: + hookenv.log('Could not load template %s from %s.' % + (source, templates_dir), + level=hookenv.ERROR) + raise e + content = template.render(context) + if target is not None: + target_dir = os.path.dirname(target) + if not os.path.exists(target_dir): + # This is a terrible default directory permission, as the file + # or its siblings will often contain secrets. + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) + return content diff --git a/nrpe/mod/charmhelpers/charmhelpers/core/unitdata.py b/nrpe/mod/charmhelpers/charmhelpers/core/unitdata.py new file mode 100644 index 0000000..d9b8d0b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/core/unitdata.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# Kapil Thangavelu +# +""" +Intro +----- + +A simple way to store state in units. This provides a key value +storage with support for versioned, transactional operation, +and can calculate deltas from previous values to simplify unit logic +when processing changes. + + +Hook Integration +---------------- + +There are several extant frameworks for hook execution, including + + - charmhelpers.core.hookenv.Hooks + - charmhelpers.core.services.ServiceManager + +The storage classes are framework agnostic, one simple integration is +via the HookData contextmanager. It will record the current hook +execution environment (including relation data, config data, etc.), +setup a transaction and allow easy access to the changes from +previously seen values. One consequence of the integration is the +reservation of particular keys ('rels', 'unit', 'env', 'config', +'charm_revisions') for their respective values. + +Here's a fully worked integration example using hookenv.Hooks:: + + from charmhelper.core import hookenv, unitdata + + hook_data = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # Print all changes to configuration from previously seen + # values. + for changed, (prev, cur) in hook_data.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookkeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + # Directly access all charm config as a mapping. + conf = db.getrange('config', True) + + # Directly access all relation data as a mapping + rels = db.getrange('rels', True) + + if __name__ == '__main__': + with hook_data(): + hook.execute() + + +A more basic integration is via the hook_scope context manager which simply +manages transaction scope (and records hook name, and timestamp):: + + >>> from unitdata import kv + >>> db = kv() + >>> with db.hook_scope('install'): + ... # do work, in transactional scope. + ... db.set('x', 1) + >>> db.get('x') + 1 + + +Usage +----- + +Values are automatically json de/serialized to preserve basic typing +and complex data struct capabilities (dicts, lists, ints, booleans, etc). + +Individual values can be manipulated via get/set:: + + >>> kv.set('y', True) + >>> kv.get('y') + True + + # We can set complex values (dicts, lists) as a single key. + >>> kv.set('config', {'a': 1, 'b': True'}) + + # Also supports returning dictionaries as a record which + # provides attribute access. + >>> config = kv.get('config', record=True) + >>> config.b + True + + +Groups of keys can be manipulated with update/getrange:: + + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") + >>> kv.getrange('gui.', strip=True) + {'z': 1, 'y': 2} + +When updating values, its very helpful to understand which values +have actually changed and how have they changed. The storage +provides a delta method to provide for this:: + + >>> data = {'debug': True, 'option': 2} + >>> delta = kv.delta(data, 'config.') + >>> delta.debug.previous + None + >>> delta.debug.current + True + >>> delta + {'debug': (None, True), 'option': (None, 2)} + +Note the delta method does not persist the actual change, it needs to +be explicitly saved via 'update' method:: + + >>> kv.update(data, 'config.') + +Values modified in the context of a hook scope retain historical values +associated to the hookname. + + >>> with db.hook_scope('config-changed'): + ... db.set('x', 42) + >>> db.gethistory('x') + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] + +""" + +import collections +import contextlib +import datetime +import itertools +import json +import os +import pprint +import sqlite3 +import sys + +__author__ = 'Kapil Thangavelu ' + + +class Storage(object): + """Simple key value database for local unit state within charms. + + Modifications are not persisted unless :meth:`flush` is called. + + To support dicts, lists, integer, floats, and booleans values + are automatically json encoded/decoded. + + Note: to facilitate unit testing, ':memory:' can be passed as the + path parameter which causes sqlite3 to only build the db in memory. + This should only be used for testing purposes. + """ + def __init__(self, path=None): + self.db_path = path + if path is None: + if 'UNIT_STATE_DB' in os.environ: + self.db_path = os.environ['UNIT_STATE_DB'] + else: + self.db_path = os.path.join( + os.environ.get('CHARM_DIR', ''), '.unit-state.db') + if self.db_path != ':memory:': + with open(self.db_path, 'a') as f: + os.fchmod(f.fileno(), 0o600) + self.conn = sqlite3.connect('%s' % self.db_path) + self.cursor = self.conn.cursor() + self.revision = None + self._closed = False + self._init() + + def close(self): + if self._closed: + return + self.flush(False) + self.cursor.close() + self.conn.close() + self._closed = True + + def get(self, key, default=None, record=False): + self.cursor.execute('select data from kv where key=?', [key]) + result = self.cursor.fetchone() + if not result: + return default + if record: + return Record(json.loads(result[0])) + return json.loads(result[0]) + + def getrange(self, key_prefix, strip=False): + """ + Get a range of keys starting with a common prefix as a mapping of + keys to values. + + :param str key_prefix: Common prefix among all keys + :param bool strip: Optionally strip the common prefix from the key + names in the returned dict + :return dict: A (possibly empty) dict of key-value mappings + """ + self.cursor.execute("select key, data from kv where key like ?", + ['%s%%' % key_prefix]) + result = self.cursor.fetchall() + + if not result: + return {} + if not strip: + key_prefix = '' + return dict([ + (k[len(key_prefix):], json.loads(v)) for k, v in result]) + + def update(self, mapping, prefix=""): + """ + Set the values of multiple keys at once. + + :param dict mapping: Mapping of keys to values + :param str prefix: Optional prefix to apply to all keys in `mapping` + before setting + """ + for k, v in mapping.items(): + self.set("%s%s" % (prefix, k), v) + + def unset(self, key): + """ + Remove a key from the database entirely. + """ + self.cursor.execute('delete from kv where key=?', [key]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + [key, self.revision, json.dumps('DELETED')]) + + def unsetrange(self, keys=None, prefix=""): + """ + Remove a range of keys starting with a common prefix, from the database + entirely. + + :param list keys: List of keys to remove. + :param str prefix: Optional prefix to apply to all keys in ``keys`` + before removing. + """ + if keys is not None: + keys = ['%s%s' % (prefix, key) for key in keys] + self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)), + list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys))) + else: + self.cursor.execute('delete from kv where key like ?', + ['%s%%' % prefix]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + ['%s%%' % prefix, self.revision, json.dumps('DELETED')]) + + def set(self, key, value): + """ + Set a value in the database. + + :param str key: Key to set the value for + :param value: Any JSON-serializable value to be set + """ + serialized = json.dumps(value) + + self.cursor.execute('select data from kv where key=?', [key]) + exists = self.cursor.fetchone() + + # Skip mutations to the same value + if exists: + if exists[0] == serialized: + return value + + if not exists: + self.cursor.execute( + 'insert into kv (key, data) values (?, ?)', + (key, serialized)) + else: + self.cursor.execute(''' + update kv + set data = ? + where key = ?''', [serialized, key]) + + # Save + if not self.revision: + return value + + self.cursor.execute( + 'select 1 from kv_revisions where key=? and revision=?', + [key, self.revision]) + exists = self.cursor.fetchone() + + if not exists: + self.cursor.execute( + '''insert into kv_revisions ( + revision, key, data) values (?, ?, ?)''', + (self.revision, key, serialized)) + else: + self.cursor.execute( + ''' + update kv_revisions + set data = ? + where key = ? + and revision = ?''', + [serialized, key, self.revision]) + + return value + + def delta(self, mapping, prefix): + """ + return a delta containing values that have changed. + """ + previous = self.getrange(prefix, strip=True) + if not previous: + pk = set() + else: + pk = set(previous.keys()) + ck = set(mapping.keys()) + delta = DeltaSet() + + # added + for k in ck.difference(pk): + delta[k] = Delta(None, mapping[k]) + + # removed + for k in pk.difference(ck): + delta[k] = Delta(previous[k], None) + + # changed + for k in pk.intersection(ck): + c = mapping[k] + p = previous[k] + if c != p: + delta[k] = Delta(p, c) + + return delta + + @contextlib.contextmanager + def hook_scope(self, name=""): + """Scope all future interactions to the current hook execution + revision.""" + assert not self.revision + self.cursor.execute( + 'insert into hooks (hook, date) values (?, ?)', + (name or sys.argv[0], + datetime.datetime.utcnow().isoformat())) + self.revision = self.cursor.lastrowid + try: + yield self.revision + self.revision = None + except Exception: + self.flush(False) + self.revision = None + raise + else: + self.flush() + + def flush(self, save=True): + if save: + self.conn.commit() + elif self._closed: + return + else: + self.conn.rollback() + + def _init(self): + self.cursor.execute(''' + create table if not exists kv ( + key text, + data text, + primary key (key) + )''') + self.cursor.execute(''' + create table if not exists kv_revisions ( + key text, + revision integer, + data text, + primary key (key, revision) + )''') + self.cursor.execute(''' + create table if not exists hooks ( + version integer primary key autoincrement, + hook text, + date text + )''') + self.conn.commit() + + def gethistory(self, key, deserialize=False): + self.cursor.execute( + ''' + select kv.revision, kv.key, kv.data, h.hook, h.date + from kv_revisions kv, + hooks h + where kv.key=? + and kv.revision = h.version + ''', [key]) + if deserialize is False: + return self.cursor.fetchall() + return map(_parse_history, self.cursor.fetchall()) + + def debug(self, fh=sys.stderr): + self.cursor.execute('select * from kv') + pprint.pprint(self.cursor.fetchall(), stream=fh) + self.cursor.execute('select * from kv_revisions') + pprint.pprint(self.cursor.fetchall(), stream=fh) + + +def _parse_history(d): + return (d[0], d[1], json.loads(d[2]), d[3], + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) + + +class HookData(object): + """Simple integration for existing hook exec frameworks. + + Records all unit information, and stores deltas for processing + by the hook. + + Sample:: + + from charmhelper.core import hookenv, unitdata + + changes = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # View all changes to configuration + for changed, (prev, cur) in changes.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookkeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + if __name__ == '__main__': + with changes(): + hook.execute() + + """ + def __init__(self): + self.kv = kv() + self.conf = None + self.rels = None + + @contextlib.contextmanager + def __call__(self): + from charmhelpers.core import hookenv + hook_name = hookenv.hook_name() + + with self.kv.hook_scope(hook_name): + self._record_charm_version(hookenv.charm_dir()) + delta_config, delta_relation = self._record_hook(hookenv) + yield self.kv, delta_config, delta_relation + + def _record_charm_version(self, charm_dir): + # Record revisions.. charm revisions are meaningless + # to charm authors as they don't control the revision. + # so logic dependnent on revision is not particularly + # useful, however it is useful for debugging analysis. + charm_rev = open( + os.path.join(charm_dir, 'revision')).read().strip() + charm_rev = charm_rev or '0' + revs = self.kv.get('charm_revisions', []) + if charm_rev not in revs: + revs.append(charm_rev.strip() or '0') + self.kv.set('charm_revisions', revs) + + def _record_hook(self, hookenv): + data = hookenv.execution_environment() + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') + self.kv.set('env', dict(data['env'])) + self.kv.set('unit', data['unit']) + self.kv.set('relid', data.get('relid')) + return conf_delta, rels_delta + + +class Record(dict): + + __slots__ = () + + def __getattr__(self, k): + if k in self: + return self[k] + raise AttributeError(k) + + +class DeltaSet(Record): + + __slots__ = () + + +Delta = collections.namedtuple('Delta', ['previous', 'current']) + + +_KV = None + + +def kv(): + global _KV + if _KV is None: + _KV = Storage() + return _KV diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/__init__.py new file mode 100644 index 0000000..9497ee0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/__init__.py @@ -0,0 +1,212 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from charmhelpers.osplatform import get_platform +from yaml import safe_load +from charmhelpers.core.hookenv import ( + config, + log, +) + +import six +if six.PY3: + from urllib.parse import urlparse, urlunparse +else: + from urlparse import urlparse, urlunparse + + +# The order of this list is very important. Handlers should be listed in from +# least- to most-specific URL matching. +FETCH_HANDLERS = ( + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', + 'charmhelpers.fetch.giturl.GitUrlFetchHandler', +) + + +class SourceConfigError(Exception): + pass + + +class UnhandledSource(Exception): + pass + + +class AptLockError(Exception): + pass + + +class GPGKeyError(Exception): + """Exception occurs when a GPG key cannot be fetched or used. The message + indicates what the problem is. + """ + pass + + +class BaseFetchHandler(object): + + """Base class for FetchHandler implementations in fetch plugins""" + + def can_handle(self, source): + """Returns True if the source can be handled. Otherwise returns + a string explaining why it cannot""" + return "Wrong source type" + + def install(self, source): + """Try to download and unpack the source. Return the path to the + unpacked files or raise UnhandledSource.""" + raise UnhandledSource("Wrong source type {}".format(source)) + + def parse_url(self, url): + return urlparse(url) + + def base_url(self, url): + """Return url without querystring or fragment""" + parts = list(self.parse_url(url)) + parts[4:] = ['' for i in parts[4:]] + return urlunparse(parts) + + +__platform__ = get_platform() +module = "charmhelpers.fetch.%s" % __platform__ +fetch = importlib.import_module(module) + +filter_installed_packages = fetch.filter_installed_packages +filter_missing_packages = fetch.filter_missing_packages +install = fetch.apt_install +upgrade = fetch.apt_upgrade +update = _fetch_update = fetch.apt_update +purge = fetch.apt_purge +add_source = fetch.add_source + +if __platform__ == "ubuntu": + apt_cache = fetch.apt_cache + apt_install = fetch.apt_install + apt_update = fetch.apt_update + apt_upgrade = fetch.apt_upgrade + apt_purge = fetch.apt_purge + apt_autoremove = fetch.apt_autoremove + apt_mark = fetch.apt_mark + apt_hold = fetch.apt_hold + apt_unhold = fetch.apt_unhold + import_key = fetch.import_key + get_upstream_version = fetch.get_upstream_version + apt_pkg = fetch.ubuntu_apt_pkg + get_apt_dpkg_env = fetch.get_apt_dpkg_env + get_installed_version = fetch.get_installed_version + OPENSTACK_RELEASES = fetch.OPENSTACK_RELEASES + UBUNTU_OPENSTACK_RELEASE = fetch.UBUNTU_OPENSTACK_RELEASE +elif __platform__ == "centos": + yum_search = fetch.yum_search + + +def configure_sources(update=False, + sources_var='install_sources', + keys_var='install_keys'): + """Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The fragment needs to be included as a string. Sources and their + corresponding keys are of the types supported by add_source(). + + Example config: + install_sources: | + - "ppa:foo" + - "http://example.com/repo precise main" + install_keys: | + - null + - "a1b2c3d4" + + Note that 'null' (a.k.a. None) should not be quoted. + """ + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, six.string_types): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) + else: + if isinstance(keys, six.string_types): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) + if update: + _fetch_update(fatal=True) + + +def install_remote(source, *args, **kwargs): + """Install a file tree from a remote source. + + The specified source should be a url of the form: + scheme://[host]/path[#[option=value][&...]] + + Schemes supported are based on this modules submodules. + Options supported are submodule-specific. + Additional arguments are passed through to the submodule. + + For example:: + + dest = install_remote('http://example.com/archive.tgz', + checksum='deadbeef', + hash_type='sha1') + + This will download `archive.tgz`, validate it using SHA1 and, if + the file is ok, extract it and return the directory in which it + was extracted. If the checksum fails, it will raise + :class:`charmhelpers.core.host.ChecksumError`. + """ + # We ONLY check for True here because can_handle may return a string + # explaining why it can't handle a given source. + handlers = [h for h in plugins() if h.can_handle(source) is True] + for handler in handlers: + try: + return handler.install(source, *args, **kwargs) + except UnhandledSource as e: + log('Install source attempt unsuccessful: {}'.format(e), + level='WARNING') + raise UnhandledSource("No handler found for source {}".format(source)) + + +def install_from_config(config_var_name): + """Install a file from config.""" + charm_config = config() + source = charm_config[config_var_name] + return install_remote(source) + + +def plugins(fetch_handlers=None): + if not fetch_handlers: + fetch_handlers = FETCH_HANDLERS + plugin_list = [] + for handler_name in fetch_handlers: + package, classname = handler_name.rsplit('.', 1) + try: + handler_class = getattr( + importlib.import_module(package), + classname) + plugin_list.append(handler_class()) + except NotImplementedError: + # Skip missing plugins so that they can be omitted from + # installation if desired + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) + return plugin_list diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/archiveurl.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/archiveurl.py new file mode 100644 index 0000000..d25587a --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/archiveurl.py @@ -0,0 +1,165 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import hashlib +import re + +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + +import six +if six.PY3: + from urllib.request import ( + build_opener, install_opener, urlopen, urlretrieve, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ) + from urllib.parse import urlparse, urlunparse, parse_qs + from urllib.error import URLError +else: + from urllib import urlretrieve + from urllib2 import ( + build_opener, install_opener, urlopen, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + URLError + ) + from urlparse import urlparse, urlunparse, parse_qs + + +def splituser(host): + '''urllib.splituser(), but six's support of this seems broken''' + _userprog = re.compile('^(.*)@(.*)$') + match = _userprog.match(host) + if match: + return match.group(1, 2) + return None, host + + +def splitpasswd(user): + '''urllib.splitpasswd(), but six's support of this is missing''' + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) + match = _passwdprog.match(user) + if match: + return match.group(1, 2) + return user, None + + +class ArchiveUrlFetchHandler(BaseFetchHandler): + """ + Handler to download archive files from arbitrary URLs. + + Can fetch from http, https, ftp, and file URLs. + + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. + + Installs the contents of the archive in $CHARM_DIR/fetched/. + """ + def can_handle(self, source): + url_parts = self.parse_url(source) + if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): + # XXX: Why is this returning a boolean and a string? It's + # doomed to fail since "bool(can_handle('foo://'))" will be True. + return "Wrong source type" + if get_archive_handler(self.base_url(source)): + return True + return False + + def download(self, source, dest): + """ + Download an archive file. + + :param str source: URL pointing to an archive file. + :param str dest: Local path location to download archive file to. + """ + # propagate all exceptions + # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse(source) + if proto in ('http', 'https'): + auth, barehost = splituser(netloc) + if auth is not None: + source = urlunparse((proto, barehost, path, params, query, fragment)) + username, password = splitpasswd(auth) + passman = HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = HTTPBasicAuthHandler(passman) + opener = build_opener(authhandler) + install_opener(opener) + response = urlopen(source) + try: + with open(dest, 'wb') as dest_file: + dest_file.write(response.read()) + except Exception as e: + if os.path.isfile(dest): + os.unlink(dest) + raise e + + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + tempfile, headers = urlretrieve(url) + check_hash(tempfile, hashsum, validate) + return tempfile + + def install(self, source, dest=None, checksum=None, hash_type='sha1'): + """ + Download and install an archive file, with optional checksum validation. + + The checksum can also be given on the `source` URL's fragment. + For example:: + + handler.install('http://example.com/file.tgz#sha1=deadbeef') + + :param str source: URL pointing to an archive file. + :param str dest: Local destination path to install to. If not given, + installs to `$CHARM_DIR/archives/archive_file_name`. + :param str checksum: If given, validate the archive file after download. + :param str hash_type: Algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + + """ + url_parts = self.parse_url(source) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') + if not os.path.exists(dest_dir): + mkdir(dest_dir, perms=0o755) + dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) + try: + self.download(source, dld_file) + except URLError as e: + raise UnhandledSource(e.reason) + except OSError as e: + raise UnhandledSource(e.strerror) + options = parse_qs(url_parts.fragment) + for key, value in options.items(): + if not six.PY3: + algorithms = hashlib.algorithms + else: + algorithms = hashlib.algorithms_available + if key in algorithms: + if len(value) != 1: + raise TypeError( + "Expected 1 hash value, not %d" % len(value)) + expected = value[0] + check_hash(dld_file, expected, key) + if checksum: + check_hash(dld_file, checksum, hash_type) + return extract(dld_file, dest) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/bzrurl.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/bzrurl.py new file mode 100644 index 0000000..c4ab3ff --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/bzrurl.py @@ -0,0 +1,76 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from subprocess import STDOUT, check_output +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource, + filter_installed_packages, + install, +) +from charmhelpers.core.host import mkdir + + +if filter_installed_packages(['bzr']) != []: + install(['bzr']) + if filter_installed_packages(['bzr']) != []: + raise NotImplementedError('Unable to install bzr') + + +class BzrUrlFetchHandler(BaseFetchHandler): + """Handler for bazaar branches via generic and lp URLs.""" + + def can_handle(self, source): + url_parts = self.parse_url(source) + if url_parts.scheme not in ('bzr+ssh', 'lp', ''): + return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.bzr')) + else: + return True + + def branch(self, source, dest, revno=None): + if not self.can_handle(source): + raise UnhandledSource("Cannot handle {}".format(source)) + cmd_opts = [] + if revno: + cmd_opts += ['-r', str(revno)] + if os.path.exists(dest): + cmd = ['bzr', 'pull'] + cmd += cmd_opts + cmd += ['--overwrite', '-d', dest, source] + else: + cmd = ['bzr', 'branch'] + cmd += cmd_opts + cmd += [source, dest] + check_output(cmd, stderr=STDOUT) + + def install(self, source, dest=None, revno=None): + url_parts = self.parse_url(source) + branch_name = url_parts.path.strip("/").split("/")[-1] + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + + if dest and not os.path.exists(dest): + mkdir(dest, perms=0o755) + + try: + self.branch(source, dest_dir, revno) + except OSError as e: + raise UnhandledSource(e.strerror) + return dest_dir diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/centos.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/centos.py new file mode 100644 index 0000000..a91dcff --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/centos.py @@ -0,0 +1,171 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import os +import time +import six +import yum + +from tempfile import NamedTemporaryFile +from charmhelpers.core.hookenv import log + +YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM. +YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. +YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +def filter_installed_packages(packages): + """Return a list of packages that require installation.""" + yb = yum.YumBase() + package_list = yb.doPackageLists() + temp_cache = {p.base_package_name: 1 for p in package_list['installed']} + + _pkgs = [p for p in packages if not temp_cache.get(p, False)] + return _pkgs + + +def install(packages, options=None, fatal=False): + """Install one or more packages.""" + cmd = ['yum', '--assumeyes'] + if options is not None: + cmd.extend(options) + cmd.append('install') + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Installing {} with options: {}".format(packages, + options)) + _run_yum_command(cmd, fatal) + + +def upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages.""" + cmd = ['yum', '--assumeyes'] + if options is not None: + cmd.extend(options) + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + _run_yum_command(cmd, fatal) + + +def update(fatal=False): + """Update local yum cache.""" + cmd = ['yum', '--assumeyes', 'update'] + log("Update with fatal: {}".format(fatal)) + _run_yum_command(cmd, fatal) + + +def purge(packages, fatal=False): + """Purge one or more packages.""" + cmd = ['yum', '--assumeyes', 'remove'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Purging {}".format(packages)) + _run_yum_command(cmd, fatal) + + +def yum_search(packages): + """Search for a package.""" + output = {} + cmd = ['yum', 'search'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Searching for {}".format(packages)) + result = subprocess.check_output(cmd) + for package in list(packages): + output[package] = package in result + return output + + +def add_source(source, key=None): + """Add a package source to this system. + + @param source: a URL with a rpm package + + @param key: A key to be added to the system's keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. + """ + if source is None: + log('Source is not present. Skipping') + return + + if source.startswith('http'): + directory = '/etc/yum.repos.d/' + for filename in os.listdir(directory): + with open(directory + filename, 'r') as rpm_file: + if source in rpm_file.read(): + break + else: + log("Add source: {!r}".format(source)) + # write in the charms.repo + with open(directory + 'Charms.repo', 'a') as rpm_file: + rpm_file.write('[%s]\n' % source[7:].replace('/', '_')) + rpm_file.write('name=%s\n' % source[7:]) + rpm_file.write('baseurl=%s\n\n' % source) + else: + log("Unknown source: {!r}".format(source)) + + if key: + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: + with NamedTemporaryFile('w+') as key_file: + key_file.write(key) + key_file.flush() + key_file.seek(0) + subprocess.check_call(['rpm', '--import', key_file.name]) + else: + subprocess.check_call(['rpm', '--import', key]) + + +def _run_yum_command(cmd, fatal=False): + """Run an YUM command. + + Checks the output and retry if the fatal flag is set to True. + + :param: cmd: str: The yum command to run. + :param: fatal: bool: Whether the command's output should be checked and + retried. + """ + env = os.environ.copy() + + if fatal: + retry_count = 0 + result = None + + # If the command is considered "fatal", we need to retry if the yum + # lock was not acquired. + + while result is None or result == YUM_NO_LOCK: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > YUM_NO_LOCK_RETRY_COUNT: + raise + result = e.returncode + log("Couldn't acquire YUM lock. Will retry in {} seconds." + "".format(YUM_NO_LOCK_RETRY_DELAY)) + time.sleep(YUM_NO_LOCK_RETRY_DELAY) + + else: + subprocess.call(cmd, env=env) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/giturl.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/giturl.py new file mode 100644 index 0000000..070ca9b --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/giturl.py @@ -0,0 +1,69 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from subprocess import check_output, CalledProcessError, STDOUT +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource, + filter_installed_packages, + install, +) + +if filter_installed_packages(['git']) != []: + install(['git']) + if filter_installed_packages(['git']) != []: + raise NotImplementedError('Unable to install git') + + +class GitUrlFetchHandler(BaseFetchHandler): + """Handler for git branches via generic and github URLs.""" + + def can_handle(self, source): + url_parts = self.parse_url(source) + # TODO (mattyw) no support for ssh git@ yet + if url_parts.scheme not in ('http', 'https', 'git', ''): + return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.git')) + else: + return True + + def clone(self, source, dest, branch="master", depth=None): + if not self.can_handle(source): + raise UnhandledSource("Cannot handle {}".format(source)) + + if os.path.exists(dest): + cmd = ['git', '-C', dest, 'pull', source, branch] + else: + cmd = ['git', 'clone', source, dest, '--branch', branch] + if depth: + cmd.extend(['--depth', depth]) + check_output(cmd, stderr=STDOUT) + + def install(self, source, branch="master", dest=None, depth=None): + url_parts = self.parse_url(source) + branch_name = url_parts.path.strip("/").split("/")[-1] + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + try: + self.clone(source, dest_dir, branch, depth) + except CalledProcessError as e: + raise UnhandledSource(e) + except OSError as e: + raise UnhandledSource(e.strerror) + return dest_dir diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/python/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/__init__.py new file mode 100644 index 0000000..bff99dc --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2019 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/python/debug.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/debug.py new file mode 100644 index 0000000..757135e --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/debug.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import atexit +import sys + +from charmhelpers.fetch.python.rpdb import Rpdb +from charmhelpers.core.hookenv import ( + open_port, + close_port, + ERROR, + log +) + +__author__ = "Jorge Niedbalski " + +DEFAULT_ADDR = "0.0.0.0" +DEFAULT_PORT = 4444 + + +def _error(message): + log(message, level=ERROR) + + +def set_trace(addr=DEFAULT_ADDR, port=DEFAULT_PORT): + """ + Set a trace point using the remote debugger + """ + atexit.register(close_port, port) + try: + log("Starting a remote python debugger session on %s:%s" % (addr, + port)) + open_port(port) + debugger = Rpdb(addr=addr, port=port) + debugger.set_trace(sys._getframe().f_back) + except Exception: + _error("Cannot start a remote debug session on %s:%s" % (addr, + port)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/python/packages.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/packages.py new file mode 100644 index 0000000..6004835 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/packages.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import six +import subprocess +import sys + +from charmhelpers.fetch import apt_install, apt_update +from charmhelpers.core.hookenv import charm_dir, log + +__author__ = "Jorge Niedbalski " + + +def pip_execute(*args, **kwargs): + """Overridden pip_execute() to stop sys.path being changed. + + The act of importing main from the pip module seems to cause add wheels + from the /usr/share/python-wheels which are installed by various tools. + This function ensures that sys.path remains the same after the call is + executed. + """ + try: + _path = sys.path + try: + from pip import main as _pip_execute + except ImportError: + apt_update() + if six.PY2: + apt_install('python-pip') + else: + apt_install('python3-pip') + from pip import main as _pip_execute + _pip_execute(*args, **kwargs) + finally: + sys.path = _path + + +def parse_options(given, available): + """Given a set of options, check if available""" + for key, value in sorted(given.items()): + if not value: + continue + if key in available: + yield "--{0}={1}".format(key, value) + + +def pip_install_requirements(requirements, constraints=None, **options): + """Install a requirements file. + + :param constraints: Path to pip constraints file. + http://pip.readthedocs.org/en/stable/user_guide/#constraints-files + """ + command = ["install"] + + available_options = ('proxy', 'src', 'log', ) + for option in parse_options(options, available_options): + command.append(option) + + command.append("-r {0}".format(requirements)) + if constraints: + command.append("-c {0}".format(constraints)) + log("Installing from file: {} with constraints {} " + "and options: {}".format(requirements, constraints, command)) + else: + log("Installing from file: {} with options: {}".format(requirements, + command)) + pip_execute(command) + + +def pip_install(package, fatal=False, upgrade=False, venv=None, + constraints=None, **options): + """Install a python package""" + if venv: + venv_python = os.path.join(venv, 'bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] + + available_options = ('proxy', 'src', 'log', 'index-url', ) + for option in parse_options(options, available_options): + command.append(option) + + if upgrade: + command.append('--upgrade') + + if constraints: + command.extend(['-c', constraints]) + + if isinstance(package, list): + command.extend(package) + else: + command.append(package) + + log("Installing {} package with options: {}".format(package, + command)) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) + + +def pip_uninstall(package, **options): + """Uninstall a python package""" + command = ["uninstall", "-q", "-y"] + + available_options = ('proxy', 'log', ) + for option in parse_options(options, available_options): + command.append(option) + + if isinstance(package, list): + command.extend(package) + else: + command.append(package) + + log("Uninstalling {} package with options: {}".format(package, + command)) + pip_execute(command) + + +def pip_list(): + """Returns the list of current python installed packages + """ + return pip_execute(["list"]) + + +def pip_create_virtualenv(path=None): + """Create an isolated Python environment.""" + if six.PY2: + apt_install('python-virtualenv') + extra_flags = [] + else: + apt_install(['python3-virtualenv', 'virtualenv']) + extra_flags = ['--python=python3'] + + if path: + venv_path = path + else: + venv_path = os.path.join(charm_dir(), 'venv') + + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path] + extra_flags) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/python/rpdb.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/rpdb.py new file mode 100644 index 0000000..9b31610 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/rpdb.py @@ -0,0 +1,56 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Remote Python Debugger (pdb wrapper).""" + +import pdb +import socket +import sys + +__author__ = "Bertrand Janin " +__version__ = "0.1.3" + + +class Rpdb(pdb.Pdb): + + def __init__(self, addr="127.0.0.1", port=4444): + """Initialize the socket and initialize pdb.""" + + # Backup stdin and stdout before replacing them by the socket handle + self.old_stdout = sys.stdout + self.old_stdin = sys.stdin + + # Open a 'reusable' socket to let the webapp reload on the same port + self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.skt.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, True) + self.skt.bind((addr, port)) + self.skt.listen(1) + (clientsocket, address) = self.skt.accept() + handle = clientsocket.makefile('rw') + pdb.Pdb.__init__(self, completekey='tab', stdin=handle, stdout=handle) + sys.stdout = sys.stdin = handle + + def shutdown(self): + """Revert stdin and stdout, close the socket.""" + sys.stdout = self.old_stdout + sys.stdin = self.old_stdin + self.skt.close() + self.set_continue() + + def do_continue(self, arg): + """Stop all operation on ``continue``.""" + self.shutdown() + return 1 + + do_EOF = do_quit = do_exit = do_c = do_cont = do_continue diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/python/version.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/version.py new file mode 100644 index 0000000..3eb4210 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/python/version.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# coding: utf-8 + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +__author__ = "Jorge Niedbalski " + + +def current_version(): + """Current system python version""" + return sys.version_info + + +def current_version_string(): + """Current system python version as string major.minor.micro""" + return "{0}.{1}.{2}".format(sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/snap.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/snap.py new file mode 100644 index 0000000..36d6bce --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/snap.py @@ -0,0 +1,150 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Charm helpers snap for classic charms. + +If writing reactive charms, use the snap layer: +https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html +""" +import subprocess +import os +from time import sleep +from charmhelpers.core.hookenv import log + +__author__ = 'Joseph Borg ' + +# The return code for "couldn't acquire lock" in Snap +# (hopefully this will be improved). +SNAP_NO_LOCK = 1 +SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. +SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. +SNAP_CHANNELS = [ + 'edge', + 'beta', + 'candidate', + 'stable', +] + + +class CouldNotAcquireLockException(Exception): + pass + + +class InvalidSnapChannel(Exception): + pass + + +def _snap_exec(commands): + """ + Execute snap commands. + + :param commands: List commands + :return: Integer exit code + """ + assert type(commands) == list + + retry_count = 0 + return_code = None + + while return_code is None or return_code == SNAP_NO_LOCK: + try: + return_code = subprocess.check_call(['snap'] + commands, + env=os.environ) + except subprocess.CalledProcessError as e: + retry_count += + 1 + if retry_count > SNAP_NO_LOCK_RETRY_COUNT: + raise CouldNotAcquireLockException( + 'Could not acquire lock after {} attempts' + .format(SNAP_NO_LOCK_RETRY_COUNT)) + return_code = e.returncode + log('Snap failed to acquire lock, trying again in {} seconds.' + .format(SNAP_NO_LOCK_RETRY_DELAY), level='WARN') + sleep(SNAP_NO_LOCK_RETRY_DELAY) + + return return_code + + +def snap_install(packages, *flags): + """ + Install a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to install command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Installing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with option(s) "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['install'] + flags + packages) + + +def snap_remove(packages, *flags): + """ + Remove a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to remove command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Removing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['remove'] + flags + packages) + + +def snap_refresh(packages, *flags): + """ + Refresh / Update snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to refresh command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Refreshing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['refresh'] + flags + packages) + + +def valid_snap_channel(channel): + """ Validate snap channel exists + + :raises InvalidSnapChannel: When channel does not exist + :return: Boolean + """ + if channel.lower() in SNAP_CHANNELS: + return True + else: + raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu.py new file mode 100644 index 0000000..6c7cf6f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu.py @@ -0,0 +1,1011 @@ +# Copyright 2014-2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +import os +import platform +import re +import six +import subprocess +import sys +import time + +from charmhelpers import deprecate +from charmhelpers.core.host import get_distrib_codename, get_system_env + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + WARNING, + env_proxy_settings, +) +from charmhelpers.fetch import SourceConfigError, GPGKeyError +from charmhelpers.fetch import ubuntu_apt_pkg + +PROPOSED_POCKET = ( + "# Proposed\n" + "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe " + "multiverse restricted\n") +PROPOSED_PORTS_POCKET = ( + "# Proposed\n" + "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe " + "multiverse restricted\n") +# Only supports 64bit and ppc64 at the moment. +ARCH_TO_PROPOSED_POCKET = { + 'x86_64': PROPOSED_POCKET, + 'ppc64le': PROPOSED_PORTS_POCKET, + 'aarch64': PROPOSED_PORTS_POCKET, + 's390x': PROPOSED_PORTS_POCKET, +} +CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" +CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' +CLOUD_ARCHIVE = """# Ubuntu Cloud Archive +deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main +""" +CLOUD_ARCHIVE_POCKETS = { + # Folsom + 'folsom': 'precise-updates/folsom', + 'folsom/updates': 'precise-updates/folsom', + 'precise-folsom': 'precise-updates/folsom', + 'precise-folsom/updates': 'precise-updates/folsom', + 'precise-updates/folsom': 'precise-updates/folsom', + 'folsom/proposed': 'precise-proposed/folsom', + 'precise-folsom/proposed': 'precise-proposed/folsom', + 'precise-proposed/folsom': 'precise-proposed/folsom', + # Grizzly + 'grizzly': 'precise-updates/grizzly', + 'grizzly/updates': 'precise-updates/grizzly', + 'precise-grizzly': 'precise-updates/grizzly', + 'precise-grizzly/updates': 'precise-updates/grizzly', + 'precise-updates/grizzly': 'precise-updates/grizzly', + 'grizzly/proposed': 'precise-proposed/grizzly', + 'precise-grizzly/proposed': 'precise-proposed/grizzly', + 'precise-proposed/grizzly': 'precise-proposed/grizzly', + # Havana + 'havana': 'precise-updates/havana', + 'havana/updates': 'precise-updates/havana', + 'precise-havana': 'precise-updates/havana', + 'precise-havana/updates': 'precise-updates/havana', + 'precise-updates/havana': 'precise-updates/havana', + 'havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', + 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', + # Juno + 'juno': 'trusty-updates/juno', + 'juno/updates': 'trusty-updates/juno', + 'trusty-juno': 'trusty-updates/juno', + 'trusty-juno/updates': 'trusty-updates/juno', + 'trusty-updates/juno': 'trusty-updates/juno', + 'juno/proposed': 'trusty-proposed/juno', + 'trusty-juno/proposed': 'trusty-proposed/juno', + 'trusty-proposed/juno': 'trusty-proposed/juno', + # Kilo + 'kilo': 'trusty-updates/kilo', + 'kilo/updates': 'trusty-updates/kilo', + 'trusty-kilo': 'trusty-updates/kilo', + 'trusty-kilo/updates': 'trusty-updates/kilo', + 'trusty-updates/kilo': 'trusty-updates/kilo', + 'kilo/proposed': 'trusty-proposed/kilo', + 'trusty-kilo/proposed': 'trusty-proposed/kilo', + 'trusty-proposed/kilo': 'trusty-proposed/kilo', + # Liberty + 'liberty': 'trusty-updates/liberty', + 'liberty/updates': 'trusty-updates/liberty', + 'trusty-liberty': 'trusty-updates/liberty', + 'trusty-liberty/updates': 'trusty-updates/liberty', + 'trusty-updates/liberty': 'trusty-updates/liberty', + 'liberty/proposed': 'trusty-proposed/liberty', + 'trusty-liberty/proposed': 'trusty-proposed/liberty', + 'trusty-proposed/liberty': 'trusty-proposed/liberty', + # Mitaka + 'mitaka': 'trusty-updates/mitaka', + 'mitaka/updates': 'trusty-updates/mitaka', + 'trusty-mitaka': 'trusty-updates/mitaka', + 'trusty-mitaka/updates': 'trusty-updates/mitaka', + 'trusty-updates/mitaka': 'trusty-updates/mitaka', + 'mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', + # Newton + 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', + 'xenial-newton': 'xenial-updates/newton', + 'xenial-newton/updates': 'xenial-updates/newton', + 'xenial-updates/newton': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', + 'xenial-newton/proposed': 'xenial-proposed/newton', + 'xenial-proposed/newton': 'xenial-proposed/newton', + # Ocata + 'ocata': 'xenial-updates/ocata', + 'ocata/updates': 'xenial-updates/ocata', + 'xenial-ocata': 'xenial-updates/ocata', + 'xenial-ocata/updates': 'xenial-updates/ocata', + 'xenial-updates/ocata': 'xenial-updates/ocata', + 'ocata/proposed': 'xenial-proposed/ocata', + 'xenial-ocata/proposed': 'xenial-proposed/ocata', + 'xenial-proposed/ocata': 'xenial-proposed/ocata', + # Pike + 'pike': 'xenial-updates/pike', + 'xenial-pike': 'xenial-updates/pike', + 'xenial-pike/updates': 'xenial-updates/pike', + 'xenial-updates/pike': 'xenial-updates/pike', + 'pike/proposed': 'xenial-proposed/pike', + 'xenial-pike/proposed': 'xenial-proposed/pike', + 'xenial-proposed/pike': 'xenial-proposed/pike', + # Queens + 'queens': 'xenial-updates/queens', + 'xenial-queens': 'xenial-updates/queens', + 'xenial-queens/updates': 'xenial-updates/queens', + 'xenial-updates/queens': 'xenial-updates/queens', + 'queens/proposed': 'xenial-proposed/queens', + 'xenial-queens/proposed': 'xenial-proposed/queens', + 'xenial-proposed/queens': 'xenial-proposed/queens', + # Rocky + 'rocky': 'bionic-updates/rocky', + 'bionic-rocky': 'bionic-updates/rocky', + 'bionic-rocky/updates': 'bionic-updates/rocky', + 'bionic-updates/rocky': 'bionic-updates/rocky', + 'rocky/proposed': 'bionic-proposed/rocky', + 'bionic-rocky/proposed': 'bionic-proposed/rocky', + 'bionic-proposed/rocky': 'bionic-proposed/rocky', + # Stein + 'stein': 'bionic-updates/stein', + 'bionic-stein': 'bionic-updates/stein', + 'bionic-stein/updates': 'bionic-updates/stein', + 'bionic-updates/stein': 'bionic-updates/stein', + 'stein/proposed': 'bionic-proposed/stein', + 'bionic-stein/proposed': 'bionic-proposed/stein', + 'bionic-proposed/stein': 'bionic-proposed/stein', + # Train + 'train': 'bionic-updates/train', + 'bionic-train': 'bionic-updates/train', + 'bionic-train/updates': 'bionic-updates/train', + 'bionic-updates/train': 'bionic-updates/train', + 'train/proposed': 'bionic-proposed/train', + 'bionic-train/proposed': 'bionic-proposed/train', + 'bionic-proposed/train': 'bionic-proposed/train', + # Ussuri + 'ussuri': 'bionic-updates/ussuri', + 'bionic-ussuri': 'bionic-updates/ussuri', + 'bionic-ussuri/updates': 'bionic-updates/ussuri', + 'bionic-updates/ussuri': 'bionic-updates/ussuri', + 'ussuri/proposed': 'bionic-proposed/ussuri', + 'bionic-ussuri/proposed': 'bionic-proposed/ussuri', + 'bionic-proposed/ussuri': 'bionic-proposed/ussuri', + # Victoria + 'victoria': 'focal-updates/victoria', + 'focal-victoria': 'focal-updates/victoria', + 'focal-victoria/updates': 'focal-updates/victoria', + 'focal-updates/victoria': 'focal-updates/victoria', + 'victoria/proposed': 'focal-proposed/victoria', + 'focal-victoria/proposed': 'focal-proposed/victoria', + 'focal-proposed/victoria': 'focal-proposed/victoria', + # Wallaby + 'wallaby': 'focal-updates/wallaby', + 'focal-wallaby': 'focal-updates/wallaby', + 'focal-wallaby/updates': 'focal-updates/wallaby', + 'focal-updates/wallaby': 'focal-updates/wallaby', + 'wallaby/proposed': 'focal-proposed/wallaby', + 'focal-wallaby/proposed': 'focal-proposed/wallaby', + 'focal-proposed/wallaby': 'focal-proposed/wallaby', + # Xena + 'xena': 'focal-updates/xena', + 'focal-xena': 'focal-updates/xena', + 'focal-xena/updates': 'focal-updates/xena', + 'focal-updates/xena': 'focal-updates/xena', + 'xena/proposed': 'focal-proposed/xena', + 'focal-xena/proposed': 'focal-proposed/xena', + 'focal-proposed/xena': 'focal-proposed/xena', + # Yoga + 'yoga': 'focal-updates/yoga', + 'focal-yoga': 'focal-updates/yoga', + 'focal-yoga/updates': 'focal-updates/yoga', + 'focal-updates/yoga': 'focal-updates/yoga', + 'yoga/proposed': 'focal-proposed/yoga', + 'focal-yoga/proposed': 'focal-proposed/yoga', + 'focal-proposed/yoga': 'focal-proposed/yoga', +} + + +OPENSTACK_RELEASES = ( + 'diablo', + 'essex', + 'folsom', + 'grizzly', + 'havana', + 'icehouse', + 'juno', + 'kilo', + 'liberty', + 'mitaka', + 'newton', + 'ocata', + 'pike', + 'queens', + 'rocky', + 'stein', + 'train', + 'ussuri', + 'victoria', + 'wallaby', + 'xena', + 'yoga', +) + + +UBUNTU_OPENSTACK_RELEASE = OrderedDict([ + ('oneiric', 'diablo'), + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ('wily', 'liberty'), + ('xenial', 'mitaka'), + ('yakkety', 'newton'), + ('zesty', 'ocata'), + ('artful', 'pike'), + ('bionic', 'queens'), + ('cosmic', 'rocky'), + ('disco', 'stein'), + ('eoan', 'train'), + ('focal', 'ussuri'), + ('groovy', 'victoria'), + ('hirsute', 'wallaby'), + ('impish', 'xena'), +]) + + +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. +CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. +CMD_RETRY_COUNT = 10 # Retry a failing fatal command X times. + + +def filter_installed_packages(packages): + """Return a list of packages that require installation.""" + cache = apt_cache() + _pkgs = [] + for package in packages: + try: + p = cache[package] + p.current_ver or _pkgs.append(package) + except KeyError: + log('Package {} has no installation candidate.'.format(package), + level='WARNING') + _pkgs.append(package) + return _pkgs + + +def filter_missing_packages(packages): + """Return a list of packages that are installed. + + :param packages: list of packages to evaluate. + :returns list: Packages that are installed. + """ + return list( + set(packages) - + set(filter_installed_packages(packages)) + ) + + +def apt_cache(*_, **__): + """Shim returning an object simulating the apt_pkg Cache. + + :param _: Accept arguments for compatibility, not used. + :type _: any + :param __: Accept keyword arguments for compatibility, not used. + :type __: any + :returns:Object used to interrogate the system apt and dpkg databases. + :rtype:ubuntu_apt_pkg.Cache + """ + if 'apt_pkg' in sys.modules: + # NOTE(fnordahl): When our consumer use the upstream ``apt_pkg`` module + # in conjunction with the apt_cache helper function, they may expect us + # to call ``apt_pkg.init()`` for them. + # + # Detect this situation, log a warning and make the call to + # ``apt_pkg.init()`` to avoid the consumer Python interpreter from + # crashing with a segmentation fault. + @deprecate( + 'Support for use of upstream ``apt_pkg`` module in conjunction' + 'with charm-helpers is deprecated since 2019-06-25', + date=None, log=lambda x: log(x, level=WARNING)) + def one_shot_log(): + pass + + one_shot_log() + sys.modules['apt_pkg'].init() + return ubuntu_apt_pkg.Cache() + + +def apt_install(packages, options=None, fatal=False, quiet=False): + """Install one or more packages. + + :param packages: Package(s) to install + :type packages: Option[str, List[str]] + :param options: Options to pass on to apt-get + :type options: Option[None, List[str]] + :param fatal: Whether the command's output should be checked and + retried. + :type fatal: bool + :param quiet: if True (default), suppress log message to stdout/stderr + :type quiet: bool + :raises: subprocess.CalledProcessError + """ + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + cmd.append('install') + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + if not quiet: + log("Installing {} with options: {}" + .format(packages, options)) + _run_apt_command(cmd, fatal, quiet=quiet) + + +def apt_upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages. + + :param options: Options to pass on to apt-get + :type options: Option[None, List[str]] + :param fatal: Whether the command's output should be checked and + retried. + :type fatal: bool + :param dist: Whether ``dist-upgrade`` should be used over ``upgrade`` + :type dist: bool + :raises: subprocess.CalledProcessError + """ + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + if dist: + cmd.append('dist-upgrade') + else: + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + _run_apt_command(cmd, fatal) + + +def apt_update(fatal=False): + """Update local apt cache.""" + cmd = ['apt-get', 'update'] + _run_apt_command(cmd, fatal) + + +def apt_purge(packages, fatal=False): + """Purge one or more packages. + + :param packages: Package(s) to install + :type packages: Option[str, List[str]] + :param fatal: Whether the command's output should be checked and + retried. + :type fatal: bool + :raises: subprocess.CalledProcessError + """ + cmd = ['apt-get', '--assume-yes', 'purge'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Purging {}".format(packages)) + _run_apt_command(cmd, fatal) + + +def apt_autoremove(purge=True, fatal=False): + """Purge one or more packages. + :param purge: Whether the ``--purge`` option should be passed on or not. + :type purge: bool + :param fatal: Whether the command's output should be checked and + retried. + :type fatal: bool + :raises: subprocess.CalledProcessError + """ + cmd = ['apt-get', '--assume-yes', 'autoremove'] + if purge: + cmd.append('--purge') + _run_apt_command(cmd, fatal) + + +def apt_mark(packages, mark, fatal=False): + """Flag one or more packages using apt-mark.""" + log("Marking {} as {}".format(packages, mark)) + cmd = ['apt-mark', mark] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + + if fatal: + subprocess.check_call(cmd, universal_newlines=True) + else: + subprocess.call(cmd, universal_newlines=True) + + +def apt_hold(packages, fatal=False): + return apt_mark(packages, 'hold', fatal=fatal) + + +def apt_unhold(packages, fatal=False): + return apt_mark(packages, 'unhold', fatal=fatal) + + +def import_key(key): + """Import an ASCII Armor key. + + A Radix64 format keyid is also supported for backwards + compatibility. In this case Ubuntu keyserver will be + queried for a key via HTTPS by its keyid. This method + is less preferable because https proxy servers may + require traffic decryption which is equivalent to a + man-in-the-middle attack (a proxy server impersonates + keyserver TLS certificates and has to be explicitly + trusted by the system). + + :param key: A GPG key in ASCII armor format, + including BEGIN and END markers or a keyid. + :type key: (bytes, str) + :raises: GPGKeyError if the key could not be imported + """ + key = key.strip() + if '-' in key or '\n' in key: + # Send everything not obviously a keyid to GPG to import, as + # we trust its validation better than our own. eg. handling + # comments before the key. + log("PGP key found (looks like ASCII Armor format)", level=DEBUG) + if ('-----BEGIN PGP PUBLIC KEY BLOCK-----' in key and + '-----END PGP PUBLIC KEY BLOCK-----' in key): + log("Writing provided PGP key in the binary format", level=DEBUG) + if six.PY3: + key_bytes = key.encode('utf-8') + else: + key_bytes = key + key_name = _get_keyid_by_gpg_key(key_bytes) + key_gpg = _dearmor_gpg_key(key_bytes) + _write_apt_gpg_keyfile(key_name=key_name, key_material=key_gpg) + else: + raise GPGKeyError("ASCII armor markers missing from GPG key") + else: + log("PGP key found (looks like Radix64 format)", level=WARNING) + log("SECURELY importing PGP key from keyserver; " + "full key not provided.", level=WARNING) + # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL + # to retrieve GPG keys. `apt-key adv` command is deprecated as is + # apt-key in general as noted in its manpage. See lp:1433761 for more + # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop + # gpg + key_asc = _get_key_by_keyid(key) + # write the key in GPG format so that apt-key list shows it + key_gpg = _dearmor_gpg_key(key_asc) + _write_apt_gpg_keyfile(key_name=key, key_material=key_gpg) + + +def _get_keyid_by_gpg_key(key_material): + """Get a GPG key fingerprint by GPG key material. + Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded + or binary GPG key material. Can be used, for example, to generate file + names for keys passed via charm options. + + :param key_material: ASCII armor-encoded or binary GPG key material + :type key_material: bytes + :raises: GPGKeyError if invalid key material has been provided + :returns: A GPG key fingerprint + :rtype: str + """ + # Use the same gpg command for both Xenial and Bionic + cmd = 'gpg --with-colons --with-fingerprint' + ps = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + out, err = ps.communicate(input=key_material) + if six.PY3: + out = out.decode('utf-8') + err = err.decode('utf-8') + if 'gpg: no valid OpenPGP data found.' in err: + raise GPGKeyError('Invalid GPG key material provided') + # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) + return re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE).group(1) + + +def _get_key_by_keyid(keyid): + """Get a key via HTTPS from the Ubuntu keyserver. + Different key ID formats are supported by SKS keyservers (the longer ones + are more secure, see "dead beef attack" and https://evil32.com/). Since + HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will + impersonate keyserver.ubuntu.com and generate a certificate with + keyserver.ubuntu.com in the CN field or in SubjAltName fields of a + certificate. If such proxy behavior is expected it is necessary to add the + CA certificate chain containing the intermediate CA of the SSLBump proxy to + every machine that this code runs on via ca-certs cloud-init directive (via + cloudinit-userdata model-config) or via other means (such as through a + custom charm option). Also note that DNS resolution for the hostname in a + URL is done at a proxy server - not at the client side. + + 8-digit (32 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 + 16-digit (64 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 + 40-digit key ID: + https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 + + :param keyid: An 8, 16 or 40 hex digit keyid to find a key for + :type keyid: (bytes, str) + :returns: A key material for the specified GPG key id + :rtype: (str, bytes) + :raises: subprocess.CalledProcessError + """ + # options=mr - machine-readable output (disables html wrappers) + keyserver_url = ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr&exact=on&search=0x{}') + curl_cmd = ['curl', keyserver_url.format(keyid)] + # use proxy server settings in order to retrieve the key + return subprocess.check_output(curl_cmd, + env=env_proxy_settings(['https'])) + + +def _dearmor_gpg_key(key_asc): + """Converts a GPG key in the ASCII armor format to the binary format. + + :param key_asc: A GPG key in ASCII armor format. + :type key_asc: (str, bytes) + :returns: A GPG key in binary format + :rtype: (str, bytes) + :raises: GPGKeyError + """ + ps = subprocess.Popen(['gpg', '--dearmor'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + out, err = ps.communicate(input=key_asc) + # no need to decode output as it is binary (invalid utf-8), only error + if six.PY3: + err = err.decode('utf-8') + if 'gpg: no valid OpenPGP data found.' in err: + raise GPGKeyError('Invalid GPG key material. Check your network setup' + ' (MTU, routing, DNS) and/or proxy server settings' + ' as well as destination keyserver status.') + else: + return out + + +def _write_apt_gpg_keyfile(key_name, key_material): + """Writes GPG key material into a file at a provided path. + + :param key_name: A key name to use for a key file (could be a fingerprint) + :type key_name: str + :param key_material: A GPG key material (binary) + :type key_material: (str, bytes) + """ + with open('/etc/apt/trusted.gpg.d/{}.gpg'.format(key_name), + 'wb') as keyf: + keyf.write(key_material) + + +def add_source(source, key=None, fail_invalid=False): + """Add a package source to this system. + + @param source: a URL or sources.list entry, as supported by + add-apt-repository(1). Examples:: + + ppa:charmers/example + deb https://stub:key@private.example.com/ubuntu trusty main + + In addition: + 'proposed:' may be used to enable the standard 'proposed' + pocket for the release. + 'cloud:' may be used to activate official cloud archive pockets, + such as 'cloud:icehouse' + 'distro' may be used as a noop + + Full list of source specifications supported by the function are: + + 'distro': A NOP; i.e. it has no effect. + 'proposed': the proposed deb spec [2] is wrtten to + /etc/apt/sources.list/proposed + 'distro-proposed': adds -proposed to the debs [2] + 'ppa:': add-apt-repository --yes + 'deb ': add-apt-repository --yes deb + 'http://....': add-apt-repository --yes http://... + 'cloud-archive:': add-apt-repository -yes cloud-archive: + 'cloud:[-staging]': specify a Cloud Archive pocket with + optional staging version. If staging is used then the staging PPA [2] + with be used. If staging is NOT used then the cloud archive [3] will be + added, and the 'ubuntu-cloud-keyring' package will be added for the + current distro. + '': translate to cloud: based on the current + distro version (i.e. for 'ussuri' this will either be 'bionic-ussuri' or + 'distro'. + '/proposed': as above, but for proposed. + + Otherwise the source is not recognised and this is logged to the juju log. + However, no error is raised, unless sys_error_on_exit is True. + + [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main + where {} is replaced with the derived pocket name. + [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \ + main universe multiverse restricted + where {} is replaced with the lsb_release codename (e.g. xenial) + [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu + to /etc/apt/sources.list.d/cloud-archive-list + + @param key: A key to be added to the system's APT keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. ppa and cloud archive keys + are securely added automatically, so should not be provided. + + @param fail_invalid: (boolean) if True, then the function raises a + SourceConfigError is there is no matching installation source. + + @raises SourceConfigError() if for cloud:, the is not a + valid pocket in CLOUD_ARCHIVE_POCKETS + """ + # extract the OpenStack versions from the CLOUD_ARCHIVE_POCKETS; can't use + # the list in contrib.openstack.utils as it might not be included in + # classic charms and would break everything. Having OpenStack specific + # code in this file is a bit of an antipattern, anyway. + os_versions_regex = "({})".format("|".join(OPENSTACK_RELEASES)) + + _mapping = OrderedDict([ + (r"^distro$", lambda: None), # This is a NOP + (r"^(?:proposed|distro-proposed)$", _add_proposed), + (r"^cloud-archive:(.*)$", _add_apt_repository), + (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), + (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), + (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), + (r"^cloud:(.*)$", _add_cloud_pocket), + (r"^snap:.*-(.*)-(.*)$", _add_cloud_distro_check), + (r"^{}\/proposed$".format(os_versions_regex), + _add_bare_openstack_proposed), + (r"^{}$".format(os_versions_regex), _add_bare_openstack), + ]) + if source is None: + source = '' + for r, fn in six.iteritems(_mapping): + m = re.match(r, source) + if m: + if key: + # Import key before adding the source which depends on it, + # as refreshing packages could fail otherwise. + try: + import_key(key) + except GPGKeyError as e: + raise SourceConfigError(str(e)) + # call the associated function with the captured groups + # raises SourceConfigError on error. + fn(*m.groups()) + break + else: + # nothing matched. log an error and maybe sys.exit + err = "Unknown source: {!r}".format(source) + log(err) + if fail_invalid: + raise SourceConfigError(err) + + +def _add_proposed(): + """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list + + Uses get_distrib_codename to determine the correct stanza for + the deb line. + + For Intel architectures PROPOSED_POCKET is used for the release, but for + other architectures PROPOSED_PORTS_POCKET is used for the release. + """ + release = get_distrib_codename() + arch = platform.machine() + if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): + raise SourceConfigError("Arch {} not supported for (distro-)proposed" + .format(arch)) + with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: + apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release)) + + +def _add_apt_repository(spec): + """Add the spec using add_apt_repository + + :param spec: the parameter to pass to add_apt_repository + :type spec: str + """ + if '{series}' in spec: + series = get_distrib_codename() + spec = spec.replace('{series}', series) + _run_with_retries(['add-apt-repository', '--yes', spec], + cmd_env=env_proxy_settings(['https', 'http', 'no_proxy']) + ) + + +def _add_cloud_pocket(pocket): + """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list + + Note that this overwrites the existing file if there is one. + + This function also converts the simple pocket in to the actual pocket using + the CLOUD_ARCHIVE_POCKETS mapping. + + :param pocket: string representing the pocket to add a deb spec for. + :raises: SourceConfigError if the cloud pocket doesn't exist or the + requested release doesn't match the current distro version. + """ + apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), + fatal=True) + if pocket not in CLOUD_ARCHIVE_POCKETS: + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] + with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + + +def _add_cloud_staging(cloud_archive_release, openstack_release): + """Add the cloud staging repository which is in + ppa:ubuntu-cloud-archive/-staging + + This function checks that the cloud_archive_release matches the current + codename for the distro that charm is being installed on. + + :param cloud_archive_release: string, codename for the release. + :param openstack_release: String, codename for the openstack release. + :raises: SourceConfigError if the cloud_archive_release doesn't match the + current version of the os. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release) + cmd = 'add-apt-repository -y {}'.format(ppa) + _run_with_retries(cmd.split(' ')) + + +def _add_cloud_distro_check(cloud_archive_release, openstack_release): + """Add the cloud pocket, but also check the cloud_archive_release against + the current distro, and use the openstack_release as the full lookup. + + This just calls _add_cloud_pocket() with the openstack_release as pocket + to get the correct cloud-archive.list for dpkg to work with. + + :param cloud_archive_release:String, codename for the distro release. + :param openstack_release: String, spec for the release to look up in the + CLOUD_ARCHIVE_POCKETS + :raises: SourceConfigError if this is the wrong distro, or the pocket spec + doesn't exist. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release)) + + +def _verify_is_ubuntu_rel(release, os_release): + """Verify that the release is in the same as the current ubuntu release. + + :param release: String, lowercase for the release. + :param os_release: String, the os_release being asked for + :raises: SourceConfigError if the release is not the same as the ubuntu + release. + """ + ubuntu_rel = get_distrib_codename() + if release != ubuntu_rel: + raise SourceConfigError( + 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu' + 'version ({})'.format(release, os_release, ubuntu_rel)) + + +def _add_bare_openstack(openstack_release): + """Add cloud or distro based on the release given. + + The spec given is, say, 'ussuri', but this could apply cloud:bionic-ussuri + or 'distro' depending on whether the ubuntu release is bionic or focal. + + :param openstack_release: the OpenStack codename to determine the release + for. + :type openstack_release: str + :raises: SourceConfigError + """ + # TODO(ajkavanagh) - surely this means we should be removing cloud archives + # if they exist? + __add_bare_helper(openstack_release, "{}-{}", lambda: None) + + +def _add_bare_openstack_proposed(openstack_release): + """Add cloud of distro but with proposed. + + The spec given is, say, 'ussuri' but this could apply + cloud:bionic-ussuri/proposed or 'distro/proposed' depending on whether the + ubuntu release is bionic or focal. + + :param openstack_release: the OpenStack codename to determine the release + for. + :type openstack_release: str + :raises: SourceConfigError + """ + __add_bare_helper(openstack_release, "{}-{}/proposed", _add_proposed) + + +def __add_bare_helper(openstack_release, pocket_format, final_function): + """Helper for _add_bare_openstack[_proposed] + + The bulk of the work between the two functions is exactly the same except + for the pocket format and the function that is run if it's the distro + version. + + :param openstack_release: the OpenStack codename. e.g. ussuri + :type openstack_release: str + :param pocket_format: the pocket formatter string to construct a pocket str + from the openstack_release and the current ubuntu version. + :type pocket_format: str + :param final_function: the function to call if it is the distro version. + :type final_function: Callable + :raises SourceConfigError on error + """ + ubuntu_version = get_distrib_codename() + possible_pocket = pocket_format.format(ubuntu_version, openstack_release) + if possible_pocket in CLOUD_ARCHIVE_POCKETS: + _add_cloud_pocket(possible_pocket) + return + # Otherwise it's almost certainly the distro version; verify that it + # exists. + try: + assert UBUNTU_OPENSTACK_RELEASE[ubuntu_version] == openstack_release + except KeyError: + raise SourceConfigError( + "Invalid ubuntu version {} isn't known to this library" + .format(ubuntu_version)) + except AssertionError: + raise SourceConfigError( + 'Invalid OpenStack release specified: {} for Ubuntu version {}' + .format(openstack_release, ubuntu_version)) + final_function() + + +def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), + retry_message="", cmd_env=None, quiet=False): + """Run a command and retry until success or max_retries is reached. + + :param cmd: The apt command to run. + :type cmd: str + :param max_retries: The number of retries to attempt on a fatal + command. Defaults to CMD_RETRY_COUNT. + :type max_retries: int + :param retry_exitcodes: Optional additional exit codes to retry. + Defaults to retry on exit code 1. + :type retry_exitcodes: tuple + :param retry_message: Optional log prefix emitted during retries. + :type retry_message: str + :param: cmd_env: Environment variables to add to the command run. + :type cmd_env: Option[None, Dict[str, str]] + :param quiet: if True, silence the output of the command from stdout and + stderr + :type quiet: bool + """ + env = get_apt_dpkg_env() + if cmd_env: + env.update(cmd_env) + + kwargs = {} + if quiet: + devnull = os.devnull if six.PY2 else subprocess.DEVNULL + kwargs['stdout'] = devnull + kwargs['stderr'] = devnull + + if not retry_message: + retry_message = "Failed executing '{}'".format(" ".join(cmd)) + retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) + + retry_count = 0 + result = None + + retry_results = (None,) + retry_exitcodes + while result in retry_results: + try: + result = subprocess.check_call(cmd, env=env, **kwargs) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > max_retries: + raise + result = e.returncode + log(retry_message) + time.sleep(CMD_RETRY_DELAY) + + +def _run_apt_command(cmd, fatal=False, quiet=False): + """Run an apt command with optional retries. + + :param cmd: The apt command to run. + :type cmd: str + :param fatal: Whether the command's output should be checked and + retried. + :type fatal: bool + :param quiet: if True, silence the output of the command from stdout and + stderr + :type quiet: bool + """ + if fatal: + _run_with_retries( + cmd, retry_exitcodes=(1, APT_NO_LOCK,), + retry_message="Couldn't acquire DPKG lock", + quiet=quiet) + else: + kwargs = {} + if quiet: + devnull = os.devnull if six.PY2 else subprocess.DEVNULL + kwargs['stdout'] = devnull + kwargs['stderr'] = devnull + subprocess.call(cmd, env=get_apt_dpkg_env(), **kwargs) + + +def get_upstream_version(package): + """Determine upstream version based on installed package + + @returns None (if not installed) or the upstream version + """ + cache = apt_cache() + try: + pkg = cache[package] + except Exception: + # the package is unknown to the current apt cache. + return None + + if not pkg.current_ver: + # package is known, but no version is currently installed. + return None + + return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str) + + +def get_installed_version(package): + """Determine installed version of a package + + @returns None (if not installed) or the installed version as + Version object + """ + cache = apt_cache() + dpkg_result = cache._dpkg_list([package]).get(package, {}) + current_ver = None + installed_version = dpkg_result.get('version') + + if installed_version: + current_ver = ubuntu_apt_pkg.Version({'ver_str': installed_version}) + return current_ver + + +def get_apt_dpkg_env(): + """Get environment suitable for execution of APT and DPKG tools. + + We keep this in a helper function instead of in a global constant to + avoid execution on import of the library. + :returns: Environment suitable for execution of APT and DPKG tools. + :rtype: Dict[str, str] + """ + # The fallback is used in the event of ``/etc/environment`` not containing + # avalid PATH variable. + return {'DEBIAN_FRONTEND': 'noninteractive', + 'PATH': get_system_env('PATH', '/usr/sbin:/usr/bin:/sbin:/bin')} diff --git a/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu_apt_pkg.py b/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu_apt_pkg.py new file mode 100644 index 0000000..436e177 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -0,0 +1,312 @@ +# Copyright 2019-2021 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provide a subset of the ``python-apt`` module API. + +Data collection is done through subprocess calls to ``apt-cache`` and +``dpkg-query`` commands. + +The main purpose for this module is to avoid dependency on the +``python-apt`` python module. + +The indicated python module is a wrapper around the ``apt`` C++ library +which is tightly connected to the version of the distribution it was +shipped on. It is not developed in a backward/forward compatible manner. + +This in turn makes it incredibly hard to distribute as a wheel for a piece +of python software that supports a span of distro releases [0][1]. + +Upstream feedback like [2] does not give confidence in this ever changing, +so with this we get rid of the dependency. + +0: https://github.com/juju-solutions/layer-basic/pull/135 +1: https://bugs.launchpad.net/charm-octavia/+bug/1824112 +2: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=845330#10 +""" + +import locale +import os +import subprocess +import sys + + +class _container(dict): + """Simple container for attributes.""" + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + + +class Package(_container): + """Simple container for package attributes.""" + + +class Version(_container): + """Simple container for version attributes.""" + + +class Cache(object): + """Simulation of ``apt_pkg`` Cache object.""" + def __init__(self, progress=None): + pass + + def __contains__(self, package): + try: + pkg = self.__getitem__(package) + return pkg is not None + except KeyError: + return False + + def __getitem__(self, package): + """Get information about a package from apt and dpkg databases. + + :param package: Name of package + :type package: str + :returns: Package object + :rtype: object + :raises: KeyError, subprocess.CalledProcessError + """ + apt_result = self._apt_cache_show([package])[package] + apt_result['name'] = apt_result.pop('package') + pkg = Package(apt_result) + dpkg_result = self._dpkg_list([package]).get(package, {}) + current_ver = None + installed_version = dpkg_result.get('version') + if installed_version: + current_ver = Version({'ver_str': installed_version}) + pkg.current_ver = current_ver + pkg.architecture = dpkg_result.get('architecture') + return pkg + + def _dpkg_list(self, packages): + """Get data from system dpkg database for package. + + :param packages: Packages to get data from + :type packages: List[str] + :returns: Structured data about installed packages, keys like + ``dpkg-query --list`` + :rtype: dict + :raises: subprocess.CalledProcessError + """ + pkgs = {} + cmd = ['dpkg-query', '--list'] + cmd.extend(packages) + if locale.getlocale() == (None, None): + # subprocess calls out to locale.getpreferredencoding(False) to + # determine encoding. Workaround for Trusty where the + # environment appears to not be set up correctly. + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + try: + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True) + except subprocess.CalledProcessError as cp: + # ``dpkg-query`` may return error and at the same time have + # produced useful output, for example when asked for multiple + # packages where some are not installed + if cp.returncode != 1: + raise + output = cp.output + headings = [] + for line in output.splitlines(): + if line.startswith('||/'): + headings = line.split() + headings.pop(0) + continue + elif (line.startswith('|') or line.startswith('+') or + line.startswith('dpkg-query:')): + continue + else: + data = line.split(None, 4) + status = data.pop(0) + if status not in ('ii', 'hi'): + continue + pkg = {} + pkg.update({k.lower(): v for k, v in zip(headings, data)}) + if 'name' in pkg: + pkgs.update({pkg['name']: pkg}) + return pkgs + + def _apt_cache_show(self, packages): + """Get data from system apt cache for package. + + :param packages: Packages to get data from + :type packages: List[str] + :returns: Structured data about package, keys like + ``apt-cache show`` + :rtype: dict + :raises: subprocess.CalledProcessError + """ + pkgs = {} + cmd = ['apt-cache', 'show', '--no-all-versions'] + cmd.extend(packages) + if locale.getlocale() == (None, None): + # subprocess calls out to locale.getpreferredencoding(False) to + # determine encoding. Workaround for Trusty where the + # environment appears to not be set up correctly. + locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') + try: + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True) + previous = None + pkg = {} + for line in output.splitlines(): + if not line: + if 'package' in pkg: + pkgs.update({pkg['package']: pkg}) + pkg = {} + continue + if line.startswith(' '): + if previous and previous in pkg: + pkg[previous] += os.linesep + line.lstrip() + continue + if ':' in line: + kv = line.split(':', 1) + key = kv[0].lower() + if key == 'n': + continue + previous = key + pkg.update({key: kv[1].lstrip()}) + except subprocess.CalledProcessError as cp: + # ``apt-cache`` returns 100 if none of the packages asked for + # exist in the apt cache. + if cp.returncode != 100: + raise + return pkgs + + +class Config(_container): + def __init__(self): + super(Config, self).__init__(self._populate()) + + def _populate(self): + cfgs = {} + cmd = ['apt-config', 'dump'] + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT, + universal_newlines=True) + for line in output.splitlines(): + if not line.startswith("CommandLine"): + k, v = line.split(" ", 1) + cfgs[k] = v.strip(";").strip("\"") + + return cfgs + + +# Backwards compatibility with old apt_pkg module +sys.modules[__name__].config = Config() + + +def init(): + """Compatibility shim that does nothing.""" + pass + + +def upstream_version(version): + """Extracts upstream version from a version string. + + Upstream reference: https://salsa.debian.org/apt-team/apt/blob/master/ + apt-pkg/deb/debversion.cc#L259 + + :param version: Version string + :type version: str + :returns: Upstream version + :rtype: str + """ + if version: + version = version.split(':')[-1] + version = version.split('-')[0] + return version + + +def version_compare(a, b): + """Compare the given versions. + + Call out to ``dpkg`` to make sure the code doing the comparison is + compatible with what the ``apt`` library would do. Mimic the return + values. + + Upstream reference: + https://apt-team.pages.debian.net/python-apt/library/apt_pkg.html + ?highlight=version_compare#apt_pkg.version_compare + + :param a: version string + :type a: str + :param b: version string + :type b: str + :returns: >0 if ``a`` is greater than ``b``, 0 if a equals b, + <0 if ``a`` is smaller than ``b`` + :rtype: int + :raises: subprocess.CalledProcessError, RuntimeError + """ + for op in ('gt', 1), ('eq', 0), ('lt', -1): + try: + subprocess.check_call(['dpkg', '--compare-versions', + a, op[0], b], + stderr=subprocess.STDOUT, + universal_newlines=True) + return op[1] + except subprocess.CalledProcessError as cp: + if cp.returncode == 1: + continue + raise + else: + raise RuntimeError('Unable to compare "{}" and "{}", according to ' + 'our logic they are neither greater, equal nor ' + 'less than each other.'.format(a, b)) + + +class PkgVersion(): + """Allow package versions to be compared. + + For example:: + + >>> import charmhelpers.fetch as fetch + >>> (fetch.apt_pkg.PkgVersion('2:20.4.0') < + ... fetch.apt_pkg.PkgVersion('2:20.5.0')) + True + >>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'), + ... fetch.apt_pkg.PkgVersion('2:21.4.0'), + ... fetch.apt_pkg.PkgVersion('2:17.4.0')] + >>> pkgs.sort() + >>> pkgs + [2:17.4.0, 2:20.4.0, 2:21.4.0] + """ + + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return version_compare(self.version, other.version) == -1 + + def __le__(self, other): + return self.__lt__(other) or self.__eq__(other) + + def __gt__(self, other): + return version_compare(self.version, other.version) == 1 + + def __ge__(self, other): + return self.__gt__(other) or self.__eq__(other) + + def __eq__(self, other): + return version_compare(self.version, other.version) == 0 + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return self.version + + def __hash__(self): + return hash(repr(self)) diff --git a/nrpe/mod/charmhelpers/charmhelpers/osplatform.py b/nrpe/mod/charmhelpers/charmhelpers/osplatform.py new file mode 100644 index 0000000..1ace468 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/osplatform.py @@ -0,0 +1,49 @@ +import platform +import os + + +def get_platform(): + """Return the current OS platform. + + For example: if current os platform is Ubuntu then a string "ubuntu" + will be returned (which is the name of the module). + This string is used to decide which platform module should be imported. + """ + # linux_distribution is deprecated and will be removed in Python 3.7 + # Warnings *not* disabled, as we certainly need to fix this. + if hasattr(platform, 'linux_distribution'): + tuple_platform = platform.linux_distribution() + current_platform = tuple_platform[0] + else: + current_platform = _get_platform_from_fs() + + if "Ubuntu" in current_platform: + return "ubuntu" + elif "CentOS" in current_platform: + return "centos" + elif "debian" in current_platform: + # Stock Python does not detect Ubuntu and instead returns debian. + # Or at least it does in some build environments like Travis CI + return "ubuntu" + elif "elementary" in current_platform: + # ElementaryOS fails to run tests locally without this. + return "ubuntu" + elif "Pop!_OS" in current_platform: + # Pop!_OS also fails to run tests locally without this. + return "ubuntu" + else: + raise RuntimeError("This module is not supported on {}." + .format(current_platform)) + + +def _get_platform_from_fs(): + """Get Platform from /etc/os-release.""" + with open(os.path.join(os.sep, 'etc', 'os-release')) as fin: + content = dict( + line.split('=', 1) + for line in fin.read().splitlines() + if '=' in line + ) + for k, v in content.items(): + content[k] = v.strip('"') + return content["NAME"] diff --git a/nrpe/mod/charmhelpers/charmhelpers/payload/__init__.py b/nrpe/mod/charmhelpers/charmhelpers/payload/__init__.py new file mode 100644 index 0000000..ee55cb3 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/payload/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Tools for working with files injected into a charm just before deployment." diff --git a/nrpe/mod/charmhelpers/charmhelpers/payload/archive.py b/nrpe/mod/charmhelpers/charmhelpers/payload/archive.py new file mode 100644 index 0000000..7fc453f --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/payload/archive.py @@ -0,0 +1,71 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tarfile +import zipfile +from charmhelpers.core import ( + host, + hookenv, +) + + +class ArchiveError(Exception): + pass + + +def get_archive_handler(archive_name): + if os.path.isfile(archive_name): + if tarfile.is_tarfile(archive_name): + return extract_tarfile + elif zipfile.is_zipfile(archive_name): + return extract_zipfile + else: + # look at the file name + for ext in ('.tar', '.tar.gz', '.tgz', 'tar.bz2', '.tbz2', '.tbz'): + if archive_name.endswith(ext): + return extract_tarfile + for ext in ('.zip', '.jar'): + if archive_name.endswith(ext): + return extract_zipfile + + +def archive_dest_default(archive_name): + archive_file = os.path.basename(archive_name) + return os.path.join(hookenv.charm_dir(), "archives", archive_file) + + +def extract(archive_name, destpath=None): + handler = get_archive_handler(archive_name) + if handler: + if not destpath: + destpath = archive_dest_default(archive_name) + if not os.path.isdir(destpath): + host.mkdir(destpath) + handler(archive_name, destpath) + return destpath + else: + raise ArchiveError("No handler for archive") + + +def extract_tarfile(archive_name, destpath): + "Unpack a tar archive, optionally compressed" + archive = tarfile.open(archive_name) + archive.extractall(destpath) + + +def extract_zipfile(archive_name, destpath): + "Unpack a zip file" + archive = zipfile.ZipFile(archive_name) + archive.extractall(destpath) diff --git a/nrpe/mod/charmhelpers/charmhelpers/payload/execd.py b/nrpe/mod/charmhelpers/charmhelpers/payload/execd.py new file mode 100644 index 0000000..1502aa0 --- /dev/null +++ b/nrpe/mod/charmhelpers/charmhelpers/payload/execd.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import subprocess +from charmhelpers.core import hookenv + + +def default_execd_dir(): + return os.path.join(os.environ['CHARM_DIR'], 'exec.d') + + +def execd_module_paths(execd_dir=None): + """Generate a list of full paths to modules within execd_dir.""" + if not execd_dir: + execd_dir = default_execd_dir() + + if not os.path.exists(execd_dir): + return + + for subpath in os.listdir(execd_dir): + module = os.path.join(execd_dir, subpath) + if os.path.isdir(module): + yield module + + +def execd_submodule_paths(command, execd_dir=None): + """Generate a list of full paths to the specified command within exec_dir. + """ + for module_path in execd_module_paths(execd_dir): + path = os.path.join(module_path, command) + if os.access(path, os.X_OK) and os.path.isfile(path): + yield path + + +def execd_run(command, execd_dir=None, die_on_error=True, stderr=subprocess.STDOUT): + """Run command for each module within execd_dir which defines it.""" + for submodule_path in execd_submodule_paths(command, execd_dir): + try: + subprocess.check_output(submodule_path, stderr=stderr, + universal_newlines=True) + except subprocess.CalledProcessError as e: + hookenv.log("Error ({}) running {}. Output: {}".format( + e.returncode, e.cmd, e.output)) + if die_on_error: + sys.exit(e.returncode) + + +def execd_preinstall(execd_dir=None): + """Run charm-pre-install for each module within execd_dir.""" + execd_run('charm-pre-install', execd_dir=execd_dir) diff --git a/nrpe/mod/charmhelpers/debian/compat b/nrpe/mod/charmhelpers/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/nrpe/mod/charmhelpers/debian/compat @@ -0,0 +1 @@ +7 diff --git a/nrpe/mod/charmhelpers/debian/control b/nrpe/mod/charmhelpers/debian/control new file mode 100644 index 0000000..c4992af --- /dev/null +++ b/nrpe/mod/charmhelpers/debian/control @@ -0,0 +1,20 @@ +Source: charmhelpers +Maintainer: Matthew Wedgwood +Section: python +Priority: optional +Build-Depends: python-all (>= 2.6.6-3), debhelper (>= 7) +Standards-Version: 3.9.1 + +Package: python-charmhelpers +Architecture: all +Depends: ${misc:Depends}, ${python:Depends} +Description: UNKNOWN + ============ + CharmHelpers + ============ + . + CharmHelpers provides an opinionated set of tools for building Juju + charms that work together. In addition to basic tasks like interact- + ing with the charm environment and the machine it runs on, it also + helps keep you build hooks and establish relations effortlessly. + . diff --git a/nrpe/mod/charmhelpers/debian/rules b/nrpe/mod/charmhelpers/debian/rules new file mode 100755 index 0000000..7f2101b --- /dev/null +++ b/nrpe/mod/charmhelpers/debian/rules @@ -0,0 +1,9 @@ +#!/usr/bin/make -f + +# This file was automatically generated by stdeb 0.6.0+git at +# Fri, 04 Jan 2013 15:14:11 -0600 + +%: + dh $@ --with python2 --buildsystem=python_distutils + + diff --git a/nrpe/mod/charmhelpers/debian/source/format b/nrpe/mod/charmhelpers/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/nrpe/mod/charmhelpers/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/nrpe/mod/charmhelpers/docs/Makefile b/nrpe/mod/charmhelpers/docs/Makefile new file mode 100644 index 0000000..17396a1 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/CharmHelpers.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CharmHelpers.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/CharmHelpers" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CharmHelpers" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/nrpe/mod/charmhelpers/docs/_extensions/automembersummary.py b/nrpe/mod/charmhelpers/docs/_extensions/automembersummary.py new file mode 100644 index 0000000..6a1e707 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/_extensions/automembersummary.py @@ -0,0 +1,84 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect + +from docutils.parsers.rst import directives +from sphinx.ext.autosummary import Autosummary +from sphinx.ext.autosummary import get_import_prefixes_from_env +from sphinx.ext.autosummary import import_by_name + + +class AutoMemberSummary(Autosummary): + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + has_content = True + option_spec = { + 'toctree': directives.unchanged, + 'nosignatures': directives.flag, + 'template': directives.unchanged, + } + + def get_items(self, names): + env = self.state.document.settings.env + prefixes = get_import_prefixes_from_env(env) + + items = [] + prefix = '' + shorten = '' + + def _get_items(name): + _items = super(AutoMemberSummary, self).get_items([shorten + name]) + for dn, sig, summary, rn in _items: + items.append(('%s%s' % (prefix, dn), sig, summary, rn)) + + for name in names: + if '~' in name: + prefix, name = name.split('~') + shorten = '~' + else: + prefix = '' + shorten = '' + + try: + real_name, obj, parent, _ = import_by_name(name, prefixes=prefixes) + except ImportError: + self.warn('failed to import %s' % name) + continue + + if not inspect.ismodule(obj): + _get_items(name) + continue + + for member in dir(obj): + if member.startswith('_'): + continue + mobj = getattr(obj, member) + if hasattr(mobj, '__module__'): + if not mobj.__module__.startswith(real_name): + continue # skip imported classes & functions + elif hasattr(mobj, '__name__'): + if not mobj.__name__.startswith(real_name): + continue # skip imported modules + else: + continue # skip instances + _get_items('%s.%s' % (name, member)) + + return items + + +def setup(app): + app.add_directive('automembersummary', AutoMemberSummary) diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.cli.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.cli.rst new file mode 100644 index 0000000..749ad24 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.cli.rst @@ -0,0 +1,24 @@ +charmhelpers.cli package +======================== + +charmhelpers.cli.commands module +-------------------------------- + +.. automodule:: charmhelpers.cli.commands + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.cli.host module +---------------------------- + +.. automodule:: charmhelpers.cli.host + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.cli + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ansible.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ansible.rst new file mode 100644 index 0000000..a664493 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ansible.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.ansible package +==================================== + +.. automodule:: charmhelpers.contrib.ansible + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmhelpers.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmhelpers.rst new file mode 100644 index 0000000..5d5222d --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmhelpers.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.charmhelpers package +========================================= + +.. automodule:: charmhelpers.contrib.charmhelpers + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmsupport.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmsupport.rst new file mode 100644 index 0000000..5f33aeb --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.charmsupport.rst @@ -0,0 +1,24 @@ +charmhelpers.contrib.charmsupport package +========================================= + +charmhelpers.contrib.charmsupport.nrpe module +--------------------------------------------- + +.. automodule:: charmhelpers.contrib.charmsupport.nrpe + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.charmsupport.volumes module +------------------------------------------------ + +.. automodule:: charmhelpers.contrib.charmsupport.volumes + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.charmsupport + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.hahelpers.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.hahelpers.rst new file mode 100644 index 0000000..129db24 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.hahelpers.rst @@ -0,0 +1,24 @@ +charmhelpers.contrib.hahelpers package +====================================== + +charmhelpers.contrib.hahelpers.apache module +-------------------------------------------- + +.. automodule:: charmhelpers.contrib.hahelpers.apache + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.hahelpers.cluster module +--------------------------------------------- + +.. automodule:: charmhelpers.contrib.hahelpers.cluster + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.hahelpers + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.ovs.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.ovs.rst new file mode 100644 index 0000000..98ab8cb --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.ovs.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.network.ovs package +======================================== + +.. automodule:: charmhelpers.contrib.network.ovs + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.rst new file mode 100644 index 0000000..c7b89b2 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.network.rst @@ -0,0 +1,20 @@ +charmhelpers.contrib.network package +==================================== + +.. toctree:: + + charmhelpers.contrib.network.ovs + +charmhelpers.contrib.network.ip module +-------------------------------------- + +.. automodule:: charmhelpers.contrib.network.ip + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.network + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.rst new file mode 100644 index 0000000..d969ed6 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.rst @@ -0,0 +1,52 @@ +charmhelpers.contrib.openstack package +====================================== + +.. toctree:: + + charmhelpers.contrib.openstack.templates + +charmhelpers.contrib.openstack.alternatives module +-------------------------------------------------- + +.. automodule:: charmhelpers.contrib.openstack.alternatives + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.openstack.context module +--------------------------------------------- + +.. automodule:: charmhelpers.contrib.openstack.context + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.openstack.neutron module +--------------------------------------------- + +.. automodule:: charmhelpers.contrib.openstack.neutron + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.openstack.templating module +------------------------------------------------ + +.. automodule:: charmhelpers.contrib.openstack.templating + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.openstack.utils module +------------------------------------------- + +.. automodule:: charmhelpers.contrib.openstack.utils + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.openstack + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.templates.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.templates.rst new file mode 100644 index 0000000..f9eaa2f --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.openstack.templates.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.openstack.templates package +================================================ + +.. automodule:: charmhelpers.contrib.openstack.templates + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.peerstorage.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.peerstorage.rst new file mode 100644 index 0000000..e33e82f --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.peerstorage.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.peerstorage package +======================================== + +.. automodule:: charmhelpers.contrib.peerstorage + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.python.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.python.rst new file mode 100644 index 0000000..98d6a62 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.python.rst @@ -0,0 +1,40 @@ +charmhelpers.contrib.python package +=================================== + +charmhelpers.contrib.python.debug module +---------------------------------------- + +.. automodule:: charmhelpers.contrib.python.debug + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.python.packages module +------------------------------------------- + +.. automodule:: charmhelpers.contrib.python.packages + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.python.rpdb module +--------------------------------------- + +.. automodule:: charmhelpers.contrib.python.rpdb + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.python.version module +------------------------------------------ + +.. automodule:: charmhelpers.contrib.python.version + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.python + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.rst new file mode 100644 index 0000000..d365543 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.rst @@ -0,0 +1,23 @@ +charmhelpers.contrib package +============================ + +.. toctree:: + + charmhelpers.contrib.ansible + charmhelpers.contrib.charmhelpers + charmhelpers.contrib.charmsupport + charmhelpers.contrib.hahelpers + charmhelpers.contrib.network + charmhelpers.contrib.openstack + charmhelpers.contrib.peerstorage + charmhelpers.contrib.python + charmhelpers.contrib.saltstack + charmhelpers.contrib.ssl + charmhelpers.contrib.storage + charmhelpers.contrib.templating + charmhelpers.contrib.unison + +.. automodule:: charmhelpers.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.saltstack.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.saltstack.rst new file mode 100644 index 0000000..ca14b85 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.saltstack.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.saltstack package +====================================== + +.. automodule:: charmhelpers.contrib.saltstack + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ssl.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ssl.rst new file mode 100644 index 0000000..8b7cde8 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.ssl.rst @@ -0,0 +1,16 @@ +charmhelpers.contrib.ssl package +================================ + +charmhelpers.contrib.ssl.service module +--------------------------------------- + +.. automodule:: charmhelpers.contrib.ssl.service + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.ssl + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.linux.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.linux.rst new file mode 100644 index 0000000..5acb073 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.linux.rst @@ -0,0 +1,40 @@ +charmhelpers.contrib.storage.linux package +========================================== + +charmhelpers.contrib.storage.linux.ceph module +---------------------------------------------- + +.. automodule:: charmhelpers.contrib.storage.linux.ceph + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.storage.linux.loopback module +-------------------------------------------------- + +.. automodule:: charmhelpers.contrib.storage.linux.loopback + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.storage.linux.lvm module +--------------------------------------------- + +.. automodule:: charmhelpers.contrib.storage.linux.lvm + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.storage.linux.utils module +----------------------------------------------- + +.. automodule:: charmhelpers.contrib.storage.linux.utils + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.storage.linux + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.rst new file mode 100644 index 0000000..a3adfdf --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.storage.rst @@ -0,0 +1,11 @@ +charmhelpers.contrib.storage package +==================================== + +.. toctree:: + + charmhelpers.contrib.storage.linux + +.. automodule:: charmhelpers.contrib.storage + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.templating.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.templating.rst new file mode 100644 index 0000000..c3eb2a2 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.templating.rst @@ -0,0 +1,24 @@ +charmhelpers.contrib.templating package +======================================= + +charmhelpers.contrib.templating.contexts module +----------------------------------------------- + +.. automodule:: charmhelpers.contrib.templating.contexts + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.contrib.templating.pyformat module +----------------------------------------------- + +.. automodule:: charmhelpers.contrib.templating.pyformat + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.contrib.templating + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.unison.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.unison.rst new file mode 100644 index 0000000..af3a525 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.contrib.unison.rst @@ -0,0 +1,7 @@ +charmhelpers.contrib.unison package +=================================== + +.. automodule:: charmhelpers.contrib.unison + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.coordinator.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.coordinator.rst new file mode 100644 index 0000000..13eeb67 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.coordinator.rst @@ -0,0 +1,10 @@ +charmhelpers.coordinator package +================================ + +charmhelpers.coordinator module +------------------------------- + +.. automodule:: charmhelpers.coordinator + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.decorators.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.decorators.rst new file mode 100644 index 0000000..5b4fb40 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.decorators.rst @@ -0,0 +1,7 @@ +charmhelpers.core.decorators +============================ + +.. automodule:: charmhelpers.core.decorators + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.fstab.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.fstab.rst new file mode 100644 index 0000000..a4c9f94 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.fstab.rst @@ -0,0 +1,7 @@ +charmhelpers.core.fstab +======================= + +.. automodule:: charmhelpers.core.fstab + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.hookenv.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.hookenv.rst new file mode 100644 index 0000000..70a0df8 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.hookenv.rst @@ -0,0 +1,12 @@ +charmhelpers.core.hookenv +========================= + +.. automembersummary:: + :nosignatures: + + ~charmhelpers.core.hookenv + +.. automodule:: charmhelpers.core.hookenv + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.host.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.host.rst new file mode 100644 index 0000000..7b6b9f0 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.host.rst @@ -0,0 +1,12 @@ +charmhelpers.core.host +====================== + +.. automembersummary:: + :nosignatures: + + ~charmhelpers.core.host + +.. automodule:: charmhelpers.core.host + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.rst new file mode 100644 index 0000000..0aec7b7 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.rst @@ -0,0 +1,19 @@ +charmhelpers.core package +========================= + +.. toctree:: + + charmhelpers.core.decorators + charmhelpers.core.fstab + charmhelpers.core.hookenv + charmhelpers.core.host + charmhelpers.core.strutils + charmhelpers.core.sysctl + charmhelpers.core.templating + charmhelpers.core.unitdata + charmhelpers.core.services + +.. automodule:: charmhelpers.core + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.base.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.base.rst new file mode 100644 index 0000000..79ef6cb --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.base.rst @@ -0,0 +1,12 @@ +charmhelpers.core.services.base +=============================== + +.. automembersummary:: + :nosignatures: + + ~charmhelpers.core.services.base + +.. automodule:: charmhelpers.core.services.base + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.helpers.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.helpers.rst new file mode 100644 index 0000000..ccba56c --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.helpers.rst @@ -0,0 +1,12 @@ +charmhelpers.core.services.helpers +================================== + +.. automembersummary:: + :nosignatures: + + ~charmhelpers.core.services.helpers + +.. automodule:: charmhelpers.core.services.helpers + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.rst new file mode 100644 index 0000000..515103c --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.services.rst @@ -0,0 +1,12 @@ +charmhelpers.core.services +========================== + +.. toctree:: + + charmhelpers.core.services.base + charmhelpers.core.services.helpers + +.. automodule:: charmhelpers.core.services + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.strutils.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.strutils.rst new file mode 100644 index 0000000..3b96809 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.strutils.rst @@ -0,0 +1,7 @@ +charmhelpers.core.strutils +============================ + +.. automodule:: charmhelpers.core.strutils + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.sysctl.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.sysctl.rst new file mode 100644 index 0000000..45b960f --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.sysctl.rst @@ -0,0 +1,7 @@ +charmhelpers.core.sysctl +============================ + +.. automodule:: charmhelpers.core.sysctl + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.templating.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.templating.rst new file mode 100644 index 0000000..c131d97 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.templating.rst @@ -0,0 +1,7 @@ +charmhelpers.core.templating +============================ + +.. automodule:: charmhelpers.core.templating + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.unitdata.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.unitdata.rst new file mode 100644 index 0000000..2b50978 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.core.unitdata.rst @@ -0,0 +1,7 @@ +charmhelpers.core.unitdata +========================== + +.. automodule:: charmhelpers.core.unitdata + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.archiveurl.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.archiveurl.rst new file mode 100644 index 0000000..b9d0944 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.archiveurl.rst @@ -0,0 +1,7 @@ +charmhelpers.fetch.archiveurl module +==================================== + +.. automodule:: charmhelpers.fetch.archiveurl + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.bzrurl.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.bzrurl.rst new file mode 100644 index 0000000..1be5910 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.bzrurl.rst @@ -0,0 +1,7 @@ +charmhelpers.fetch.bzrurl module +================================ + +.. automodule:: charmhelpers.fetch.bzrurl + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.python.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.python.rst new file mode 100644 index 0000000..ed89d42 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.python.rst @@ -0,0 +1,7 @@ +charmhelpers.fetch.python module +==================================== + +.. automodule:: charmhelpers.fetch.python + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.rst new file mode 100644 index 0000000..6fb42ef --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.rst @@ -0,0 +1,37 @@ +charmhelpers.fetch package +========================== + +.. automodule:: charmhelpers.fetch + :members: + :undoc-members: + :show-inheritance: + + +charmhelpers.fetch.archiveurl module +------------------------------------ + +.. toctree:: + + charmhelpers.fetch.archiveurl + +charmhelpers.fetch.bzrurl module +-------------------------------- + +.. toctree:: + + charmhelpers.fetch.bzrurl + +charmhelpers.fetch.snap module +------------------------------ + +.. toctree:: + + charmhelpers.fetch.snap + +charmhelpers.fetch.python module +------------------------------ + +.. toctree:: + + charmhelpers.fetch.python + diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.snap.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.snap.rst new file mode 100644 index 0000000..882a88e --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.fetch.snap.rst @@ -0,0 +1,26 @@ +charmhelpers.fetch.snap package +=============================== + +.. automodule:: charmhelpers.fetch.snap + :members: + :undoc-members: + :show-inheritance: + +Examples +-------- + +.. code-block:: python + + snap_install('hello-world', '--classic', '--stable') + snap_install(['hello-world', 'htop']) + +.. code-block:: python + + snap_refresh('hello-world', '--classic', '--stable') + snap_refresh(['hello-world', 'htop']) + +.. code-block:: python + + snap_remove('hello-world') + snap_remove(['hello-world', 'htop']) + diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.payload.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.payload.rst new file mode 100644 index 0000000..b1d9607 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.payload.rst @@ -0,0 +1,24 @@ +charmhelpers.payload package +============================ + +charmhelpers.payload.archive module +----------------------------------- + +.. automodule:: charmhelpers.payload.archive + :members: + :undoc-members: + :show-inheritance: + +charmhelpers.payload.execd module +--------------------------------- + +.. automodule:: charmhelpers.payload.execd + :members: + :undoc-members: + :show-inheritance: + + +.. automodule:: charmhelpers.payload + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/api/charmhelpers.rst b/nrpe/mod/charmhelpers/docs/api/charmhelpers.rst new file mode 100644 index 0000000..14266fb --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/api/charmhelpers.rst @@ -0,0 +1,17 @@ +API Documentation +================= + +.. toctree:: + :maxdepth: 3 + + charmhelpers.core + charmhelpers.contrib + charmhelpers.fetch + charmhelpers.payload + charmhelpers.cli + charmhelpers.coordinator + +.. automodule:: charmhelpers + :members: + :undoc-members: + :show-inheritance: diff --git a/nrpe/mod/charmhelpers/docs/conf.py b/nrpe/mod/charmhelpers/docs/conf.py new file mode 100644 index 0000000..f495d55 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/conf.py @@ -0,0 +1,266 @@ +# -*- coding: utf-8 -*- +# +# Charm Helpers documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 6 10:34:44 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('../')) +sys.path.append(os.path.abspath('_extensions/')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'automembersummary', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Charm Helpers' +copyright = u'2014-2018, Canonical Ltd.' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +version_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../', 'VERSION')) +VERSION = open(version_file).read().strip() +# The short X.Y version. +version = VERSION +# The full version, including alpha/beta/rc tags. +release = VERSION + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build', '_extensions'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +html_theme_path = ['_themes'] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'CharmHelpersdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'CharmHelpers.tex', u'Charm Helpers Documentation', + u'Charm Helpers Developers', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'charmhelpers', u'Charm Helpers Documentation', + [u'Charm Helpers Developers'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'CharmHelpers', u'Charm Helpers Documentation', + u'Charm Helpers Developers', 'CharmHelpers', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/nrpe/mod/charmhelpers/docs/contributing.rst b/nrpe/mod/charmhelpers/docs/contributing.rst new file mode 100644 index 0000000..ed4728b --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/contributing.rst @@ -0,0 +1,49 @@ +Contributing +============ + +All contributions, both code and documentation, are welcome! + +Source +------ + +The source code is located at https://github.com/juju/charm-helpers. To +submit contributions you'll need to create a GitHub account if you do not +already have one. + +To get the code:: + + $ git clone https://github.com/juju/charm-helpers + +To build and run tests:: + + $ cd charm-helpers + $ make + +Submitting a Merge Proposal +--------------------------- + +Run ``make test`` and ensure all tests pass. Then commit your changes to a +`fork `_ and create a +`pull request `_. + +Open Bugs +--------- + +If you're looking for something to work on, the open bug/feature list can be +found at https://bugs.launchpad.net/charm-helpers. + +Documentation +------------- + +If you'd like to contribute to the documentation, please refer to the ``HACKING.md`` +document in the root of the source tree for instructions on building the documentation. + +Contributions to the :doc:`example-index` section of the documentation are +especially welcome, and are easy to add. Simply add a new ``.rst`` file under +``charmhelpers/docs/examples``. + +Getting Help +------------ + +If you need help you can find it in ``#juju`` on the Freenode IRC network. Come +talk to us - we're a friendly bunch! diff --git a/nrpe/mod/charmhelpers/docs/example-index.rst b/nrpe/mod/charmhelpers/docs/example-index.rst new file mode 100644 index 0000000..e0251c0 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/example-index.rst @@ -0,0 +1,11 @@ +Examples +======== + +If you'd like to contribute an example (please do!), please refer to the +:doc:`contributing` page for instructions on how to do so. + +.. toctree:: + :maxdepth: 1 + :glob: + + examples/* diff --git a/nrpe/mod/charmhelpers/docs/examples/config.rst b/nrpe/mod/charmhelpers/docs/examples/config.rst new file mode 100644 index 0000000..d6d6089 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/examples/config.rst @@ -0,0 +1,88 @@ +Interacting with Charm Configuration +==================================== + +The :func:`charmhelpers.core.hookenv.config`, when called with no arguments, +returns a :class:`charmhelpers.core.hookenv.Config` instance - a dictionary +representation of a charm's ``config.yaml`` file. This object can +be used to: + +* get a charm's current config values +* check if a config value has changed since the last hook invocation +* view the previous value of a changed config item +* save arbitrary key/value data for use in a later hook + +For the following examples we'll assume our charm has a config.yaml file that +looks like this:: + + options: + app-name: + type: string + default: "My App" + description: "Name of your app." + + +Getting charm config values +--------------------------- + +:: + + # hooks/hooks.py + + from charmhelpers.core import hookenv + + hooks = hookenv.Hooks() + + @hooks.hook('install') + def install(): + config = hookenv.config() + + assert config['app-name'] == 'My App' + +Checking if a config value has changed +-------------------------------------- + +Let's say the user changes the ``app-name`` config value at runtime by +executing the following juju command:: + + juju set mycharm app-name="My New App" + +which triggers a ``config-changed`` hook:: + + # hooks/hooks.py + + from charmhelpers.core import hookenv + + hooks = hookenv.Hooks() + + @hooks.hook('config-changed') + def config_changed(): + config = hookenv.config() + + assert config.changed('app-name') + assert config['app-name'] == 'My New App' + assert config.previous('app-name') == 'My App' + +Saving arbitrary key/value data +------------------------------- + +The :class:`Config ` object maybe also be +used to store arbitrary data that you want to persist across hook +invocations:: + + # hooks/hooks.py + + from charmhelpers.core import hookenv + + hooks = hookenv.Hooks() + + @hooks.hook('install') + def install(): + config = hookenv.config() + + config['mykey'] = 'myval' + + @hooks.hook('config-changed') + def config_changed(): + config = hookenv.config() + + assert config['mykey'] == 'myval' diff --git a/nrpe/mod/charmhelpers/docs/examples/services.rst b/nrpe/mod/charmhelpers/docs/examples/services.rst new file mode 100644 index 0000000..db350d2 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/examples/services.rst @@ -0,0 +1,160 @@ +Managing Charms with the Services Framework +=========================================== + +Traditional charm authoring is focused on implementing hooks. That is, +the charm author is thinking in terms of "What hook am I handling; what +does this hook need to do?" However, in most cases, the real question +should be "Do I have the information I need to configure and start this +piece of software and, if so, what are the steps for doing so?" The +services framework tries to bring the focus to the data and the +setup tasks, in the most declarative way possible. + + +Hooks as Data Sources for Service Definitions +--------------------------------------------- + +While the ``install``, ``start``, and ``stop`` hooks clearly represent +state transitions, all of the other hooks are really notifications of +changes in data from external sources, such as config data values in +the case of ``config-changed`` or relation data for any of the +``*-relation-*`` hooks. Moreover, many charms that rely on external +data from config options or relations find themselves needing some +piece of external data before they can even configure and start anything, +and so the ``start`` hook loses its semantic usefulness. + +If data is required from multiple sources, it even becomes impossible to +know which hook will be executing when all required data is available. +(E.g., which relation will be the last to execute; will the required +config option be set before or after all of the relations are available?) +One common solution to this problem is to create "flag files" to track +whether a given bit of data has been observed, but this can get cluttered +quickly and is difficult to understand what conditions lead to which actions. + +When using the services framework, all hooks other than ``install`` +are handled by a single call to :meth:`manager.manage() `. +This can be done with symlinks, or by having a ``definitions.py`` file +containing the service definitions, and every hook can be reduced to:: + + #!/bin/env python + from charmhelpers.core.services import ServiceManager + from definitions import service_definitions + ServiceManager(service_definitions).manage() + +So, what magic goes into ``definitions.py``? + + +Service Definitions Overview +---------------------------- + +The format of service definitions are fully documented in +:class:`~charmhelpers.core.services.base.ServiceManager`, but most commonly +will consist of one or more dictionaries containing four items: the name of +a service being managed, the list of data contexts required before the service +can be configured and started, the list of actions to take when the data +requirements are satisfied, and list of ports to open. The service name +generally maps to an Upstart job, the required data contexts are ``dict`` +or ``dict``-like structures that contain the data once available (usually +subclasses of :class:`~charmhelpers.core.services.helpers.RelationContext` +or wrappers around :func:`charmhelpers.core.hookenv.config`), and the actions +are just callbacks that are passed the service name for which they are executing +(or a subclass of :class:`~charmhelpers.core.services.base.ManagerCallback` +for more complex cases). + +An example service definition might be:: + + service_definitions = [ + { + 'service': 'wordpress', + 'ports': [80], + 'required_data': [config(), MySQLRelation()], + 'data_ready': [ + actions.install_frontend, + services.render_template(source='wp-config.php.j2', + target=os.path.join(WP_INSTALL_DIR, 'wp-config.php')) + services.render_template(source='wordpress.upstart.j2', + target='/etc/init/wordpress'), + ], + }, + ] + +Each time a hook is fired, the conditions will be checked (in this case, just +that MySQL is available) and, if met, the appropriate actions taken (correct +front-end installed, config files written / updated, and the Upstart job +(re)started, implicitly). + + +Required Data Contexts +---------------------- + +Required data contexts are, at the most basic level, are just dictionaries, +and if they evaluate as True (e.g., if the contain data), their condition is +considered to be met. A simple sentinel could just be a function that returns +data if available or an empty ``dict`` otherwise. + +For the common case of gathering data from relations, the +:class:`~charmhelpers.core.services.helpers.RelationContext` base class gathers +data from a named relation and checks for a set of required keys to be present +and set on the relation before considering that relation complete. For example, +a basic MySQL context might be:: + + class MySQLRelation(RelationContext): + name = 'db' + interface = 'mysql' + required_keys = ['host', 'user', 'password', 'database'] + +Because there could potentially be multiple units on a given relation, and +to prevent conflicts when the data contexts are merged to be sent to templates +(see below), the data for a ``RelationContext`` is nested in the following way:: + + relation[relation.name][unit_number][relation_key] + +For example, to get the host of the first MySQL unit (``mysql/0``):: + + mysql = MySQLRelation() + unit_0_host = mysql[mysql.name][0]['host'] + +Note that only units that have set values for all of the required keys are +included in the list, and if no units have set all of the required keys, +instantiating the ``RelationContext`` will result in an empty list. + + +Data-Ready Actions +------------------ + +When a hook is triggered and all of the ``required_data`` contexts are complete, +the list of "data ready" actions are executed. These callbacks are passed +the service name from the ``service`` key of the service definition for which +they are running, and are responsible for (re)configuring the service +according to the required data. + +The most common action should be to render a config file from a template. +The :class:`render_template ` +helper will merge all of the ``required_data`` contexts and render a +`Jinja2 `_ template with the combined data. For +example, to render a list of DSNs for units on the db relation, the +template should include:: + + databases: [ + {% for unit in db %} + "mysql://{{unit['user']}}:{{unit['password']}}@{{unit['host']}}/{{unit['database']}}", + {% endfor %} + ] + +Note that the actions need to be idempotent, since they will all be re-run +if something about the charm changes (that is, if a hook is triggered). That +is why rendering a template is preferred to editing a file via regular expression +substitutions. + +Also note that the actions are not responsible for starting the service; there +are separate ``start`` and ``stop`` options that default to starting and stopping +an Upstart service with the name given by the ``service`` value. + + +Conclusion +---------- + +By using this framework, it is easy to see what the preconditions for the charm +are, and there is never a concern about things being in a partially configured +state. As a charm author, you can focus on what is important to you: what +data is mandatory, what is optional, and what actions should be taken once +the requirements are met. diff --git a/nrpe/mod/charmhelpers/docs/getting-started.rst b/nrpe/mod/charmhelpers/docs/getting-started.rst new file mode 100644 index 0000000..a20b5e5 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/getting-started.rst @@ -0,0 +1,152 @@ +Getting Started +=============== + +For a video introduction to ``charmhelpers``, check out this +`Charm School session `_. To start +using ``charmhelpers``, proceed with the instructions on the remainder of this +page. + +Installing Charm Tools +---------------------- + +First, follow `these instructions `_ +to install the ``charm-tools`` package for your platform. + +Creating a New Charm +-------------------- + +:: + + $ cd ~ + $ mkdirs -p charms/precise + $ cd charms/precise + $ charm create -t python mycharm + INFO: Generating template for mycharm in ./mycharm + INFO: No mycharm in apt cache; creating an empty charm instead. + Symlink all hooks to one python source file? [yN] y + INFO:root:Loading charm helper config from charm-helpers.yaml. + INFO:root:Checking out lp:charm-helpers to /tmp/tmpPAqUyN/charm-helpers. + Branched 160 revisions. + INFO:root:Syncing directory: /tmp/tmpPAqUyN/charm-helpers/charmhelpers/core -> lib/charmhelpers/core. + INFO:root:Adding missing __init__.py: lib/charmhelpers/__init__.py + +Let's see what our new charm looks like:: + + $ tree mycharm/ + mycharm/ + ├── charm-helpers.yaml + ├── config.yaml + ├── hooks + │   ├── config-changed -> hooks.py + │   ├── hooks.py + │   ├── install -> hooks.py + │   ├── start -> hooks.py + │   ├── stop -> hooks.py + │   └── upgrade-charm -> hooks.py + ├── icon.svg + ├── lib + │   └── charmhelpers + │   ├── core + │   │   ├── fstab.py + │   │   ├── hookenv.py + │   │   ├── host.py + │   │   └── __init__.py + │   └── __init__.py + ├── metadata.yaml + ├── README.ex + ├── revision + ├── scripts + │   └── charm_helpers_sync.py + └── tests + ├── 00-setup + └── 10-deploy + + 6 directories, 20 files + +The ``charmhelpers`` code is bundled in our charm in the ``lib/`` directory. +All of our python code will go in ``hooks/hook.py``. A look at that file reveals +that ``charmhelpers`` has been added to the python path and imported for us:: + + $ head mycharm/hooks/hooks.py -n11 + #!/usr/bin/python + + import os + import sys + + sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib')) + + from charmhelpers.core import ( + hookenv, + host, + ) + +Updating Charmhelpers Packages +------------------------------ + +By default, a new charm installs only the ``charmhelpers.core`` package, but +other packages are available (for a complete list, see the :doc:`api/charmhelpers`). +The installed packages are controlled by the ``charm-helpers.yaml`` file in our charm:: + + $ cd mycharm + $ cat charm-helpers.yaml + destination: lib/charmhelpers + branch: lp:charm-helpers + include: + - core + +Let's update this file to include some more packages:: + + $ vim charm-helpers.yaml + $ cat charm-helpers.yaml + destination: lib/charmhelpers + branch: lp:charm-helpers + include: + - core + - contrib.storage + - fetch + +Now we need to download the new packages into our charm:: + + $ ./scripts/charm_helpers_sync.py -c charm-helpers.yaml + INFO:root:Loading charm helper config from charm-helpers.yaml. + INFO:root:Checking out lp:charm-helpers to /tmp/tmpT38Y87/charm-helpers. + Branched 160 revisions. + INFO:root:Syncing directory: /tmp/tmpT38Y87/charm-helpers/charmhelpers/core -> lib/charmhelpers/core. + INFO:root:Syncing directory: /tmp/tmpT38Y87/charm-helpers/charmhelpers/contrib/storage -> lib/charmhelpers/contrib/storage. + INFO:root:Adding missing __init__.py: lib/charmhelpers/contrib/__init__.py + INFO:root:Syncing directory: /tmp/tmpT38Y87/charm-helpers/charmhelpers/fetch -> lib/charmhelpers/fetch. + +A look at our charmhelpers directory reveals that the new packages have indeed +been added. We are now free to import and use them in our charm:: + + $ tree lib/charmhelpers/ + lib/charmhelpers/ + ├── contrib + │   ├── __init__.py + │   └── storage + │   ├── __init__.py + │   └── linux + │   ├── ceph.py + │   ├── __init__.py + │   ├── loopback.py + │   ├── lvm.py + │   └── utils.py + ├── core + │   ├── fstab.py + │   ├── hookenv.py + │   ├── host.py + │   └── __init__.py + ├── fetch + │   ├── archiveurl.py + │   ├── bzrurl.py + │   └── __init__.py + └── __init__.py + + 5 directories, 15 files + +Next Steps +---------- + +Now that you have access to ``charmhelpers`` in your charm, check out the +:doc:`example-index` or :doc:`api/charmhelpers` to learn about all the great +functionality that ``charmhelpers`` provides. diff --git a/nrpe/mod/charmhelpers/docs/index.rst b/nrpe/mod/charmhelpers/docs/index.rst new file mode 100644 index 0000000..7a75555 --- /dev/null +++ b/nrpe/mod/charmhelpers/docs/index.rst @@ -0,0 +1,41 @@ +.. Charm Helpers documentation master file, created by + sphinx-quickstart on Fri Jun 6 10:34:44 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Charm Helpers Documentation +=========================== + +The ``charmhelpers`` Python library is an extensive collection of functions and classes +for simplifying the development of `Juju Charms`_. It includes utilities for: + +* Interacting with the host environment +* Managing hook events +* Reading and writing charm configuration +* Installing dependencies +* Much, much more! + +.. toctree:: + :maxdepth: 2 + + getting-started + example-index + api/charmhelpers + +.. toctree:: + :caption: Project + :glob: + :maxdepth: 3 + + contributing + changelog + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` + + +.. _Juju Charms: https://juju.ubuntu.com/docs/ diff --git a/nrpe/mod/charmhelpers/pip.sh b/nrpe/mod/charmhelpers/pip.sh new file mode 100755 index 0000000..8a71ce4 --- /dev/null +++ b/nrpe/mod/charmhelpers/pip.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +pip install pip==20.2.3 +pip "$@" diff --git a/nrpe/mod/charmhelpers/requirements.txt b/nrpe/mod/charmhelpers/requirements.txt new file mode 100644 index 0000000..97dbeba --- /dev/null +++ b/nrpe/mod/charmhelpers/requirements.txt @@ -0,0 +1,16 @@ +# Runtime Dependencies + +# https://pyyaml.org/wiki/PyYAML#history +# PyYAML==5.2 is last supported for py34 +PyYAML + +# https://jinja.palletsprojects.com/en/2.11.x/changelog/ +# Jinja2==2.10 is last supported for py34 (trusty) +# Jinja2==2.11 is last supported for py27 & py35 (xenial) +Jinja2 + +six +netaddr +Tempita + +pbr!=2.1.0,>=2.0.0 # Apache-2.0 diff --git a/nrpe/mod/charmhelpers/scripts/README b/nrpe/mod/charmhelpers/scripts/README new file mode 100644 index 0000000..baaf919 --- /dev/null +++ b/nrpe/mod/charmhelpers/scripts/README @@ -0,0 +1 @@ +This directory contains scripts for managing the charmhelpers project diff --git a/nrpe/mod/charmhelpers/scripts/update-revno b/nrpe/mod/charmhelpers/scripts/update-revno new file mode 100755 index 0000000..48e6bba --- /dev/null +++ b/nrpe/mod/charmhelpers/scripts/update-revno @@ -0,0 +1,11 @@ +#!/bin/bash +VERSION=$(cat VERSION) +REVNO=$(bzr revno) +bzr di &>/dev/null +if [ $? ]; then + REVNO="${REVNO}+" +fi +cat << EOF > charmhelpers/version.py +CHARMHELPERS_VERSION = '${VERSION}' +CHARMHELPERS_BZRREVNO = '${REVNO}' +EOF diff --git a/nrpe/mod/charmhelpers/setup.cfg b/nrpe/mod/charmhelpers/setup.cfg new file mode 100644 index 0000000..af820fb --- /dev/null +++ b/nrpe/mod/charmhelpers/setup.cfg @@ -0,0 +1,36 @@ +[metadata] +name = charmhelpers +summary = Helpers for Juju Charm development +description-file = + README.rst +author = Charmers +author-email = juju@lists.ubuntu.com +home-page = https://github.com/juju/charm-helpers +classifier = + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + +[files] +packages = + charmhelpers +scripts = + bin/chlp + bin/contrib/charmsupport/charmsupport + bin/contrib/saltstack/salt-call + +[nosetests] +with-coverage=1 +cover-erase=1 +cover-package=charmhelpers,tools + +[upload_sphinx] +upload-dir = docs/_build/html diff --git a/nrpe/mod/charmhelpers/setup.py b/nrpe/mod/charmhelpers/setup.py new file mode 100644 index 0000000..9d8594c --- /dev/null +++ b/nrpe/mod/charmhelpers/setup.py @@ -0,0 +1,27 @@ +# Copyright 2016 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr>=2.0.0'], + pbr=True) diff --git a/nrpe/mod/charmhelpers/tarmac_tests.sh b/nrpe/mod/charmhelpers/tarmac_tests.sh new file mode 100755 index 0000000..28f7e5a --- /dev/null +++ b/nrpe/mod/charmhelpers/tarmac_tests.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# How the tests are run in Jenkins by Tarmac + +set -e + +pkgs='python-flake8 python-shelltoolbox python-tempita python-nose python-mock python-testtools python-jinja2 python-coverage python-git python-netifaces python-netaddr python-pip zip' +if ! dpkg -s $pkgs 2>/dev/null >/dev/null ; then + echo "Required packages are missing. Please ensure that the missing packages are installed." + echo "Run: sudo apt-get install $pkgs" + exit 1 +fi + +make build diff --git a/nrpe/mod/charmhelpers/test-requirements.txt b/nrpe/mod/charmhelpers/test-requirements.txt new file mode 100644 index 0000000..b6c5871 --- /dev/null +++ b/nrpe/mod/charmhelpers/test-requirements.txt @@ -0,0 +1,35 @@ +# Test-only dependencies are unpinned. +# +git+https://git.launchpad.net/ubuntu/+source/python-distutils-extra +pip +coverage>=3.6 +mock>=1.0.1,<1.1.0 +nose>=1.3.1 +flake8 +testtools==0.9.14 # Before dependent on modern 'six' +sphinx_rtd_theme +ipaddress;python_version<'3.0' # Py27 unit test requirement + + +########################################################## +# Specify versions of runtime dependencies where possible. +# The requirements.txt file cannot be so specific + +# https://pyyaml.org/wiki/PyYAML#history +# PyYAML==5.2 is last supported for py34 +PyYAML==5.2;python_version >= '3.0' and python_version <= '3.4' # py3 trusty +PyYAML; python_version == '2.7' or python_version >= '3.5' # all else + +# https://jinja.palletsprojects.com/en/2.11.x/changelog/ +# Jinja2==2.10 is last supported for py34 +# Jinja2==2.11.3 is last supported for py27 & py35 +Jinja2==2.10;python_version >= '3.0' and python_version <= '3.4' # py3 trusty +Jinja2==2.11.3;python_version == '2.7' or python_version == '3.5' # py27, py35 +Jinja2; python_version >= '3.6' # py36 and on + +############################################################## + +netifaces==0.10 # trusty is 0.8, but using py3 compatible version for tests. +psutil==1.2.1 # trusty +python-keystoneclient==2.3.2 # xenial +dnspython==1.11.1 # trusty diff --git a/nrpe/mod/charmhelpers/tests/__init__.py b/nrpe/mod/charmhelpers/tests/__init__.py new file mode 100644 index 0000000..5707deb --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/__init__.py @@ -0,0 +1,16 @@ +import sys +import mock + + +sys.modules['yum'] = mock.MagicMock() +sys.modules['sriov_netplan_shim'] = mock.MagicMock() +sys.modules['sriov_netplan_shim.pci'] = mock.MagicMock() +with mock.patch('charmhelpers.deprecate') as ch_deprecate: + def mock_deprecate(warning, date=None, log=None): + def mock_wrap(f): + def wrapped_f(*args, **kwargs): + return f(*args, **kwargs) + return wrapped_f + return mock_wrap + ch_deprecate.side_effect = mock_deprecate + import charmhelpers.contrib.openstack.utils as openstack # noqa: F401 diff --git a/nrpe/mod/charmhelpers/tests/cli/__init__.py b/nrpe/mod/charmhelpers/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/cli/test_cmdline.py b/nrpe/mod/charmhelpers/tests/cli/test_cmdline.py new file mode 100644 index 0000000..549e70c --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/cli/test_cmdline.py @@ -0,0 +1,236 @@ +"""Tests for the commandant code that analyzes a function signature to +determine the parameters to argparse.""" + +from unittest import TestCase +from mock import ( + patch, + MagicMock, + ANY, +) +import json +from pprint import pformat +import yaml +import csv + +from six import StringIO + +from charmhelpers import cli + + +class SubCommandTest(TestCase): + """Test creation of subcommands""" + + def setUp(self): + super(SubCommandTest, self).setUp() + self.cl = cli.CommandLine() + + @patch('sys.exit') + def test_subcommand_wrapper(self, _sys_exit): + """Test function name detection""" + @self.cl.subcommand() + def payload(): + "A function that does work." + pass + args = self.cl.argument_parser.parse_args(['payload']) + self.assertEqual(args.func, payload) + self.assertEqual(_sys_exit.mock_calls, []) + + @patch('sys.exit') + def test_subcommand_wrapper_bogus_arguments(self, _sys_exit): + """Test function name detection""" + @self.cl.subcommand() + def payload(): + "A function that does work." + pass + with self.assertRaises(TypeError): + with patch("sys.argv", "tests deliberately bad input".split()): + with patch("sys.stderr"): + self.cl.argument_parser.parse_args() + _sys_exit.assert_called_once_with(2) + + @patch('sys.exit') + def test_subcommand_wrapper_cmdline_options(self, _sys_exit): + """Test detection of positional arguments and optional parameters.""" + @self.cl.subcommand() + def payload(x, y=None): + "A function that does work." + return x + args = self.cl.argument_parser.parse_args(['payload', 'positional', '--y=optional']) + self.assertEqual(args.func, payload) + self.assertEqual(args.x, 'positional') + self.assertEqual(args.y, 'optional') + self.assertEqual(_sys_exit.mock_calls, []) + + @patch('sys.exit') + def test_subcommand_builder(self, _sys_exit): + def noop(z): + pass + + @self.cl.subcommand_builder('payload', description="A subcommand") + def payload_command(subparser): + subparser.add_argument('-z', action='store_true') + return noop + + args = self.cl.argument_parser.parse_args(['payload', '-z']) + self.assertEqual(args.func, noop) + self.assertTrue(args.z) + self.assertFalse(_sys_exit.called) + + def test_subcommand_builder_bogus_wrapped_args(self): + with self.assertRaises(TypeError): + @self.cl.subcommand_builder('payload', description="A subcommand") + def payload_command(subparser, otherarg): + pass + + def test_run(self): + self.bar_called = False + + @self.cl.subcommand() + def bar(x, y=None, *vargs): + "A function that does work." + self.assertEqual(x, 'baz') + self.assertEqual(y, 'why') + self.assertEqual(vargs, ('mux', 'zob')) + self.bar_called = True + return "qux" + + args = ['chlp', 'bar', '--y', 'why', 'baz', 'mux', 'zob'] + self.cl.formatter = MagicMock() + with patch("sys.argv", args): + with patch("charmhelpers.core.unitdata._KV") as _KV: + self.cl.run() + assert _KV.flush.called + self.assertTrue(self.bar_called) + self.cl.formatter.format_output.assert_called_once_with('qux', ANY) + + def test_no_output(self): + self.bar_called = False + + @self.cl.subcommand() + @self.cl.no_output + def bar(x, y=None, *vargs): + "A function that does work." + self.bar_called = True + return "qux" + + args = ['foo', 'bar', 'baz'] + self.cl.formatter = MagicMock() + with patch("sys.argv", args): + self.cl.run() + self.assertTrue(self.bar_called) + self.cl.formatter.format_output.assert_called_once_with('', ANY) + + def test_test_command(self): + self.bar_called = False + self.bar_result = True + + @self.cl.subcommand() + @self.cl.test_command + def bar(x, y=None, *vargs): + "A function that does work." + self.bar_called = True + return self.bar_result + + args = ['foo', 'bar', 'baz'] + self.cl.formatter = MagicMock() + with patch("sys.argv", args): + self.cl.run() + self.assertTrue(self.bar_called) + self.assertEqual(self.cl.exit_code, 0) + self.cl.formatter.format_output.assert_called_once_with('', ANY) + + self.bar_result = False + with patch("sys.argv", args): + self.cl.run() + self.assertEqual(self.cl.exit_code, 1) + + +class OutputFormatterTest(TestCase): + def setUp(self): + super(OutputFormatterTest, self).setUp() + self.expected_formats = ( + "raw", + "json", + "py", + "yaml", + "csv", + "tab", + ) + self.outfile = StringIO() + self.of = cli.OutputFormatter(outfile=self.outfile) + self.output_data = {"this": "is", "some": 1, "data": dict()} + + def test_supports_formats(self): + self.assertEqual(sorted(self.expected_formats), + sorted(self.of.supported_formats)) + + def test_adds_arguments(self): + ap = MagicMock() + arg_group = MagicMock() + add_arg = MagicMock() + arg_group.add_argument = add_arg + ap.add_mutually_exclusive_group.return_value = arg_group + self.of.add_arguments(ap) + + self.assertTrue(add_arg.called) + + for call_args in add_arg.call_args_list: + if "--format" in call_args[0]: + self.assertEqual(sorted(call_args[1]['choices']), + sorted(self.expected_formats)) + self.assertEqual(call_args[1]['default'], 'raw') + break + else: + print(arg_group.call_args_list) + self.fail("No --format argument was created") + + all_args = [c[0][0] for c in add_arg.call_args_list] + all_args.extend([c[0][1] for c in add_arg.call_args_list if len(c[0]) > 1]) + for fmt in self.expected_formats: + self.assertIn("-{}".format(fmt[0]), all_args) + self.assertIn("--{}".format(fmt), all_args) + + def test_outputs_raw(self): + self.of.raw(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), str(self.output_data)) + + def test_outputs_json(self): + self.of.json(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), json.dumps(self.output_data)) + + def test_outputs_py(self): + self.of.py(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), pformat(self.output_data) + "\n") + + def test_outputs_yaml(self): + self.of.yaml(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), yaml.dump(self.output_data)) + + def test_outputs_csv(self): + sample = StringIO() + writer = csv.writer(sample) + writer.writerows(self.output_data) + sample.seek(0) + self.of.csv(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), sample.read()) + + def test_outputs_tab(self): + sample = StringIO() + writer = csv.writer(sample, dialect=csv.excel_tab) + writer.writerows(self.output_data) + sample.seek(0) + self.of.tab(self.output_data) + self.outfile.seek(0) + self.assertEqual(self.outfile.read(), sample.read()) + + def test_formats_output(self): + for format in self.expected_formats: + mock_f = MagicMock() + setattr(self.of, format, mock_f) + self.of.format_output(self.output_data, format) + mock_f.assert_called_with(self.output_data) diff --git a/nrpe/mod/charmhelpers/tests/cli/test_function_signature_analysis.py b/nrpe/mod/charmhelpers/tests/cli/test_function_signature_analysis.py new file mode 100644 index 0000000..515d9ce --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/cli/test_function_signature_analysis.py @@ -0,0 +1,46 @@ +"""Tests for the commandant code that analyzes a function signature to +determine the parameters to argparse.""" + +from testtools import TestCase + +from charmhelpers import cli + + +class FunctionSignatureTest(TestCase): + """Test a variety of function signatures.""" + + def test_positional_arguments(self): + """Finite number of order-dependent required arguments.""" + argparams = tuple(cli.describe_arguments(lambda x, y, z: False)) + self.assertEqual(3, len(argparams)) + for argspec in ((('x',), {}), (('y',), {}), (('z',), {})): + self.assertIn(argspec, argparams) + + def test_keyword_arguments(self): + """Function has optional parameters with default values.""" + argparams = tuple(cli.describe_arguments(lambda x, y=3, z="bar": False)) + self.assertEqual(3, len(argparams)) + for argspec in ((('x',), {}), + (('--y',), {"default": 3}), + (('--z',), {"default": "bar"})): + self.assertIn(argspec, argparams) + + def test_varargs(self): + """Function has a splat-operator parameter to catch an arbitrary number + of positional parameters.""" + argparams = tuple(cli.describe_arguments( + lambda x, y=3, *z: False)) + self.assertEqual(3, len(argparams)) + for argspec in ((('x',), {}), + (('--y',), {"default": 3}), + (('z',), {"nargs": "*"})): + self.assertIn(argspec, argparams) + + def test_keyword_splat_missing(self): + """Double-splat arguments can't be represented in the current version + of commandant.""" + args = cli.describe_arguments(lambda x, y=3, *z, **missing: False) + for opts, _ in args: + # opts should be ('varname',) at this point + self.assertTrue(len(opts) == 1) + self.assertNotIn('missing', opts) diff --git a/nrpe/mod/charmhelpers/tests/context/__init__.py b/nrpe/mod/charmhelpers/tests/context/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/context/test_context.py b/nrpe/mod/charmhelpers/tests/context/test_context.py new file mode 100644 index 0000000..3554301 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/context/test_context.py @@ -0,0 +1,199 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from mock import patch, sentinel +import six + +from charmhelpers import context +from charmhelpers.core import hookenv + + +class TestRelations(unittest.TestCase): + def setUp(self): + def install(*args, **kw): + p = patch.object(*args, **kw) + p.start() + self.addCleanup(p.stop) + + install(hookenv, 'relation_types', return_value=['rel', 'pear']) + install(hookenv, 'peer_relation_id', return_value='pear:9') + install(hookenv, 'relation_ids', + side_effect=lambda x: ['{}:{}'.format(x, i) + for i in range(9, 11)]) + install(hookenv, 'related_units', + side_effect=lambda x: ['svc_' + x.replace(':', '/')]) + install(hookenv, 'local_unit', return_value='foo/1') + install(hookenv, 'relation_get') + install(hookenv, 'relation_set') + # install(hookenv, 'is_leader', return_value=False) + + def test_relations(self): + rels = context.Relations() + self.assertListEqual(list(rels.keys()), + ['pear', 'rel']) # Ordered alphabetically + self.assertListEqual(list(rels['rel'].keys()), + ['rel:9', 'rel:10']) # Ordered numerically + + # Relation data is loaded on demand, not on instantiation. + self.assertFalse(hookenv.relation_get.called) + + # But we did have to retrieve some lists of units etc. + self.assertGreaterEqual(hookenv.relation_ids.call_count, 2) + self.assertGreaterEqual(hookenv.related_units.call_count, 2) + + def test_relations_peer(self): + # The Relations instance has a short cut to the peer relation. + # If the charm has managed to get multiple peer relations, + # it returns the 'primary' one used here and returned by + # hookenv.peer_relation_id() + rels = context.Relations() + self.assertIs(rels.peer, rels['pear']['pear:9']) + + def test_relation(self): + rel = context.Relations()['rel']['rel:9'] + self.assertEqual(rel.relid, 'rel:9') + self.assertEqual(rel.relname, 'rel') + self.assertEqual(rel.service, 'svc_rel') + self.assertTrue(isinstance(rel.local, context.RelationInfo)) + self.assertEqual(rel.local.unit, hookenv.local_unit()) + self.assertTrue(isinstance(rel.peers, context.OrderedDict)) + self.assertTrue(len(rel.peers), 2) + self.assertTrue(isinstance(rel.peers['svc_pear/9'], + context.RelationInfo)) + + # I use this in my log messages. Relation id for identity + # plus service name for ease of reference. + self.assertEqual(str(rel), 'rel:9 (svc_rel)') + + def test_relation_no_peer_relation(self): + hookenv.peer_relation_id.return_value = None + rel = context.Relation('rel:10') + self.assertTrue(rel.peers is None) + + def test_relation_no_peers(self): + hookenv.related_units.side_effect = None + hookenv.related_units.return_value = [] + rel = context.Relation('rel:10') + self.assertDictEqual(rel.peers, {}) + + def test_peer_relation(self): + peer_rel = context.Relations().peer + # The peer relation does not have a 'peers' properly. We + # could give it one for symmetry, but it seems somewhat silly. + self.assertTrue(peer_rel.peers is None) + + def test_relationinfo(self): + hookenv.relation_get.return_value = {sentinel.key: 'value'} + r = context.RelationInfo('rel:10', 'svc_rel/9') + + self.assertEqual(r.relname, 'rel') + self.assertEqual(r.relid, 'rel:10') + self.assertEqual(r.unit, 'svc_rel/9') + self.assertEqual(r.service, 'svc_rel') + self.assertEqual(r.number, 9) + + self.assertFalse(hookenv.relation_get.called) + self.assertEqual(r[sentinel.key], 'value') + hookenv.relation_get.assert_called_with(unit='svc_rel/9', rid='rel:10') + + # Updates fail + with self.assertRaises(TypeError): + r['newkey'] = 'foo' + + # Deletes fail + with self.assertRaises(TypeError): + del r[sentinel.key] + + # I use this for logging. + self.assertEqual(str(r), 'rel:10 (svc_rel/9)') + + def test_relationinfo_local(self): + r = context.RelationInfo('rel:10', hookenv.local_unit()) + + # Updates work, with standard strings. + r[sentinel.key] = 'value' + hookenv.relation_set.assert_called_once_with( + 'rel:10', {sentinel.key: 'value'}) + + # Python 2 unicode strings work too. + hookenv.relation_set.reset_mock() + r[sentinel.key] = six.u('value') + hookenv.relation_set.assert_called_once_with( + 'rel:10', {sentinel.key: six.u('value')}) + + # Byte strings fail under Python 3. + if six.PY3: + with self.assertRaises(ValueError): + r[sentinel.key] = six.b('value') + + # Deletes work + del r[sentinel.key] + hookenv.relation_set.assert_called_with('rel:10', {sentinel.key: None}) + + # Attempting to write a non-string fails + with self.assertRaises(ValueError): + r[sentinel.key] = 42 + + +class TestLeader(unittest.TestCase): + @patch.object(hookenv, 'leader_get') + def test_get(self, leader_get): + leader_get.return_value = {'a_key': 'a_value'} + + leader = context.Leader() + self.assertEqual(leader['a_key'], 'a_value') + leader_get.assert_called_with() + + with self.assertRaises(KeyError): + leader['missing'] + + @patch.object(hookenv, 'leader_set') + @patch.object(hookenv, 'leader_get') + @patch.object(hookenv, 'is_leader') + def test_set(self, is_leader, leader_get, leader_set): + is_leader.return_value = True + leader = context.Leader() + + # Updates work + leader[sentinel.key] = 'foo' + leader_set.assert_called_with({sentinel.key: 'foo'}) + del leader[sentinel.key] + leader_set.assert_called_with({sentinel.key: None}) + + # Python 2 unicode string values work too + leader[sentinel.key] = six.u('bar') + leader_set.assert_called_with({sentinel.key: 'bar'}) + + # Byte strings fail under Python 3 + if six.PY3: + with self.assertRaises(ValueError): + leader[sentinel.key] = six.b('baz') + + # Non strings fail, as implicit casting causes more trouble + # than it solves. Simple types like integers would round trip + # back as strings. + with self.assertRaises(ValueError): + leader[sentinel.key] = 42 + + @patch.object(hookenv, 'leader_set') + @patch.object(hookenv, 'leader_get') + @patch.object(hookenv, 'is_leader') + def test_set_not_leader(self, is_leader, leader_get, leader_set): + is_leader.return_value = False + leader_get.return_value = {'a_key': 'a_value'} + leader = context.Leader() + with self.assertRaises(TypeError): + leader['a_key'] = 'foo' + with self.assertRaises(TypeError): + del leader['a_key'] diff --git a/nrpe/mod/charmhelpers/tests/contrib/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/ansible/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/ansible/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/ansible/test_ansible.py b/nrpe/mod/charmhelpers/tests/contrib/ansible/test_ansible.py new file mode 100644 index 0000000..5261da2 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/ansible/test_ansible.py @@ -0,0 +1,339 @@ +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +import mock +import os +import shutil +import stat +import tempfile +import unittest +import yaml + + +import charmhelpers.contrib.ansible +from charmhelpers.core import hookenv + + +class InstallAnsibleSupportTestCase(unittest.TestCase): + + def setUp(self): + super(InstallAnsibleSupportTestCase, self).setUp() + + patcher = mock.patch('charmhelpers.fetch') + self.mock_fetch = patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch('charmhelpers.core') + self.mock_core = patcher.start() + self.addCleanup(patcher.stop) + + hosts_file = tempfile.NamedTemporaryFile() + self.ansible_hosts_path = hosts_file.name + self.addCleanup(hosts_file.close) + patcher = mock.patch.object(charmhelpers.contrib.ansible, + 'ansible_hosts_path', + self.ansible_hosts_path) + patcher.start() + self.addCleanup(patcher.stop) + + def test_adds_ppa_by_default(self): + charmhelpers.contrib.ansible.install_ansible_support() + + self.mock_fetch.add_source.assert_called_once_with( + 'ppa:ansible/ansible') + self.mock_fetch.apt_update.assert_called_once_with(fatal=True) + self.mock_fetch.apt_install.assert_called_once_with( + 'ansible') + + def test_no_ppa(self): + charmhelpers.contrib.ansible.install_ansible_support( + from_ppa=False) + + self.assertEqual(self.mock_fetch.add_source.call_count, 0) + self.mock_fetch.apt_install.assert_called_once_with( + 'ansible') + + def test_writes_ansible_hosts(self): + with open(self.ansible_hosts_path) as hosts_file: + self.assertEqual(hosts_file.read(), '') + + charmhelpers.contrib.ansible.install_ansible_support() + + with open(self.ansible_hosts_path) as hosts_file: + self.assertEqual(hosts_file.read(), + 'localhost ansible_connection=local ' + 'ansible_remote_tmp=/root/.ansible/tmp') + + +class ApplyPlaybookTestCases(unittest.TestCase): + + unit_data = { + 'private-address': '10.0.3.2', + 'public-address': '123.123.123.123', + } + + def setUp(self): + super(ApplyPlaybookTestCases, self).setUp() + + # Hookenv patches (a single patch to hookenv doesn't work): + patcher = mock.patch('charmhelpers.core.hookenv.config') + self.mock_config = patcher.start() + self.addCleanup(patcher.stop) + Serializable = charmhelpers.core.hookenv.Serializable + self.mock_config.return_value = Serializable({}) + patcher = mock.patch('charmhelpers.core.hookenv.relation_get') + self.mock_relation_get = patcher.start() + self.mock_relation_get.return_value = {} + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relations') + self.mock_relations = patcher.start() + self.mock_relations.return_value = { + 'wsgi-file': {}, + 'website': {}, + 'nrpe-external-master': {}, + } + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relations_of_type') + self.mock_relations_of_type = patcher.start() + self.mock_relations_of_type.return_value = [] + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relation_type') + self.mock_relation_type = patcher.start() + self.mock_relation_type.return_value = None + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.local_unit') + self.mock_local_unit = patcher.start() + self.addCleanup(patcher.stop) + self.mock_local_unit.return_value = {} + + def unit_get_data(argument): + "dummy unit_get that accesses dummy unit data" + return self.unit_data[argument] + + patcher = mock.patch( + 'charmhelpers.core.hookenv.unit_get', unit_get_data) + self.mock_unit_get = patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch('charmhelpers.contrib.ansible.subprocess') + self.mock_subprocess = patcher.start() + self.addCleanup(patcher.stop) + + etc_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, etc_dir) + self.vars_path = os.path.join(etc_dir, 'ansible', 'vars.yaml') + patcher = mock.patch.object(charmhelpers.contrib.ansible, + 'ansible_vars_path', self.vars_path) + patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch.object(charmhelpers.contrib.ansible.os, + 'environ', {}) + patcher.start() + self.addCleanup(patcher.stop) + + def test_calls_ansible_playbook(self): + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/dependencies.yaml') + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'playbooks/dependencies.yaml'], + env={'PYTHONUNBUFFERED': '1'}) + + def test_writes_vars_file(self): + self.assertFalse(os.path.exists(self.vars_path)) + self.mock_config.return_value = charmhelpers.core.hookenv.Serializable({ + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + 'private-address': '10.10.10.10', + }) + self.mock_relation_type.return_value = 'wsgi-file' + self.mock_relation_get.return_value = { + 'relation_key1': 'relation_value1', + 'relation-key2': 'relation_value2', + } + + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/dependencies.yaml') + + self.assertTrue(os.path.exists(self.vars_path)) + stats = os.stat(self.vars_path) + self.assertEqual( + stats.st_mode & stat.S_IRWXU, + stat.S_IRUSR | stat.S_IWUSR) + self.assertEqual(stats.st_mode & stat.S_IRWXG, 0) + self.assertEqual(stats.st_mode & stat.S_IRWXO, 0) + with open(self.vars_path, 'r') as vars_file: + result = yaml.safe_load(vars_file.read()) + self.assertEqual({ + "group_code_owner": "webops_deploy", + "user_code_runner": "ubunet", + "private_address": "10.10.10.10", + "charm_dir": "", + "local_unit": {}, + 'current_relation': { + 'relation_key1': 'relation_value1', + 'relation-key2': 'relation_value2', + }, + 'relations_full': { + 'nrpe-external-master': {}, + 'website': {}, + 'wsgi-file': {}, + }, + 'relations': { + 'nrpe-external-master': [], + 'website': [], + 'wsgi-file': [], + }, + "wsgi_file__relation_key1": "relation_value1", + "wsgi_file__relation_key2": "relation_value2", + "unit_private_address": "10.0.3.2", + "unit_public_address": "123.123.123.123", + }, result) + + def test_calls_with_tags(self): + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/complete-state.yaml', tags=['install', 'somethingelse']) + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'playbooks/complete-state.yaml', + '--tags', 'install,somethingelse'], env={'PYTHONUNBUFFERED': '1'}) + + @mock.patch.object(hookenv, 'config') + def test_calls_with_extra_vars(self, config): + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/complete-state.yaml', tags=['install', 'somethingelse'], + extra_vars={'a': 'b'}) + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'playbooks/complete-state.yaml', + '--tags', 'install,somethingelse', '--extra-vars', '{"a": "b"}'], + env={'PYTHONUNBUFFERED': '1'}) + + @mock.patch.object(hookenv, 'config') + def test_calls_with_extra_vars_path(self, config): + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/complete-state.yaml', tags=['install', 'somethingelse'], + extra_vars='@myvars.json') + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'playbooks/complete-state.yaml', + '--tags', 'install,somethingelse', '--extra-vars', '"@myvars.json"'], + env={'PYTHONUNBUFFERED': '1'}) + + @mock.patch.object(hookenv, 'config') + def test_calls_with_extra_vars_dict(self, config): + charmhelpers.contrib.ansible.apply_playbook( + 'playbooks/complete-state.yaml', tags=['install', 'somethingelse'], + extra_vars={'pkg': {'a': 'present', 'b': 'absent'}}) + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'playbooks/complete-state.yaml', + '--tags', 'install,somethingelse', '--extra-vars', + '{"pkg": {"a": "present", "b": "absent"}}'], + env={'PYTHONUNBUFFERED': '1'}) + + @mock.patch.object(hookenv, 'config') + def test_hooks_executes_playbook_with_tag(self, config): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('my/playbook.yaml') + foo = mock.MagicMock() + hooks.register('foo', foo) + + hooks.execute(['foo']) + + self.assertEqual(foo.call_count, 1) + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'my/playbook.yaml', + '--tags', 'foo'], env={'PYTHONUNBUFFERED': '1'}) + + @mock.patch.object(hookenv, 'config') + def test_specifying_ansible_handled_hooks(self, config): + hooks = charmhelpers.contrib.ansible.AnsibleHooks( + 'my/playbook.yaml', default_hooks=['start', 'stop']) + + hooks.execute(['start']) + + self.mock_subprocess.check_call.assert_called_once_with([ + 'ansible-playbook', '-c', 'local', 'my/playbook.yaml', + '--tags', 'start'], env={'PYTHONUNBUFFERED': '1'}) + + +class TestActionDecorator(unittest.TestCase): + + def setUp(self): + p = mock.patch('charmhelpers.contrib.ansible.apply_playbook') + self.apply_playbook = p.start() + self.addCleanup(p.stop) + + def test_action_no_args(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(): + return {} + + hooks.execute(['test']) + self.apply_playbook.assert_called_once_with( + 'playbook.yaml', tags=['test'], extra_vars={}) + + def test_action_required_arg_keyword(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(x): + return locals() + + hooks.execute(['test', 'x=a']) + self.apply_playbook.assert_called_once_with( + 'playbook.yaml', tags=['test'], extra_vars={'x': 'a'}) + + def test_action_required_arg_missing(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(x): + """Requires x""" + return locals() + + try: + hooks.execute(['test']) + self.fail("should have thrown TypeError") + except TypeError as e: + self.assertEqual(e.args[1], "Requires x") + + def test_action_required_unknown_arg(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(x='a'): + """Requires x""" + return locals() + + try: + hooks.execute(['test', 'z=c']) + self.fail("should have thrown TypeError") + except TypeError as e: + self.assertEqual(e.args[1], "Requires x") + + def test_action_default_arg(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(x='b'): + return locals() + + hooks.execute(['test']) + self.apply_playbook.assert_called_once_with( + 'playbook.yaml', tags=['test'], extra_vars={'x': 'b'}) + + def test_action_mutliple(self): + hooks = charmhelpers.contrib.ansible.AnsibleHooks('playbook.yaml') + + @hooks.action() + def test(x, y='b'): + return locals() + + hooks.execute(['test', 'x=a', 'y=b']) + self.apply_playbook.assert_called_once_with( + 'playbook.yaml', tags=['test'], extra_vars={'x': 'a', 'y': 'b'}) diff --git a/nrpe/mod/charmhelpers/tests/contrib/benchmark/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/benchmark/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/benchmark/test_benchmark.py b/nrpe/mod/charmhelpers/tests/contrib/benchmark/test_benchmark.py new file mode 100644 index 0000000..52ec086 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/benchmark/test_benchmark.py @@ -0,0 +1,124 @@ +from functools import partial +from os.path import join +from tempfile import mkdtemp +from shutil import rmtree + +import mock +from testtools import TestCase +# import unittest +from charmhelpers.contrib.benchmark import Benchmark, action_set # noqa +from tests.helpers import patch_open, FakeRelation + +TO_PATCH = [ + 'in_relation_hook', + 'relation_ids', + 'relation_set', + 'relation_get', +] + +FAKE_RELATION = { + 'benchmark:0': { + 'benchmark/0': { + 'hostname': '127.0.0.1', + 'port': '1111', + 'graphite_port': '2222', + 'graphite_endpoint': 'http://localhost:3333', + 'api_port': '4444' + } + } +} + + +class TestBenchmark(TestCase): + + def setUp(self): + super(TestBenchmark, self).setUp() + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + self.fake_relation = FakeRelation(FAKE_RELATION) + # self.hook_name.return_value = 'benchmark-relation-changed' + + self.relation_get.side_effect = partial( + self.fake_relation.get, rid="benchmark:0", unit="benchmark/0") + self.relation_ids.side_effect = self.fake_relation.relation_ids + + def _patch(self, method): + _m = mock.patch('charmhelpers.contrib.benchmark.' + method) + m = _m.start() + self.addCleanup(_m.stop) + return m + + @mock.patch('os.path.exists') + @mock.patch('subprocess.check_output') + def test_benchmark_start(self, check_output, exists): + + exists.return_value = True + check_output.return_value = "data" + + with patch_open() as (_open, _file): + self.assertIsNone(Benchmark.start()) + # _open.assert_called_with('/etc/benchmark.conf', 'w') + + COLLECT_PROFILE_DATA = '/usr/local/bin/collect-profile-data' + exists.assert_any_call(COLLECT_PROFILE_DATA) + check_output.assert_any_call([COLLECT_PROFILE_DATA]) + + def test_benchmark_finish(self): + with patch_open() as (_open, _file): + self.assertIsNone(Benchmark.finish()) + # _open.assert_called_with('/etc/benchmark.conf', 'w') + + @mock.patch('charmhelpers.contrib.benchmark.action_set') + def test_benchmark_set_composite_score(self, action_set): + self.assertTrue(Benchmark.set_composite_score(15.7, 'hits/sec', 'desc')) + action_set.assert_called_once_with('meta.composite', {'value': 15.7, 'units': 'hits/sec', 'direction': 'desc'}) + + @mock.patch('charmhelpers.contrib.benchmark.find_executable') + @mock.patch('charmhelpers.contrib.benchmark.subprocess.check_call') + def test_benchmark_action_set(self, check_call, find_executable): + find_executable.return_value = "/usr/bin/action-set" + self.assertTrue(action_set('foo', 'bar')) + + find_executable.assert_called_once_with('action-set') + check_call.assert_called_once_with(['action-set', 'foo=bar']) + + @mock.patch('charmhelpers.contrib.benchmark.find_executable') + @mock.patch('charmhelpers.contrib.benchmark.subprocess.check_call') + def test_benchmark_action_set_dict(self, check_call, find_executable): + find_executable.return_value = "/usr/bin/action-set" + self.assertTrue(action_set('baz', {'foo': 1, 'bar': 2})) + + find_executable.assert_called_with('action-set') + + check_call.assert_any_call(['action-set', 'baz.foo=1']) + check_call.assert_any_call(['action-set', 'baz.bar=2']) + + @mock.patch('charmhelpers.contrib.benchmark.relation_ids') + @mock.patch('charmhelpers.contrib.benchmark.in_relation_hook') + def test_benchmark_init(self, in_relation_hook, relation_ids): + + in_relation_hook.return_value = True + relation_ids.return_value = ['benchmark:0'] + actions = ['asdf', 'foobar'] + + tempdir = mkdtemp(prefix=self.__class__.__name__) + self.addCleanup(rmtree, tempdir) + conf_path = join(tempdir, "benchmark.conf") + with mock.patch.object(Benchmark, "BENCHMARK_CONF", conf_path): + b = Benchmark(actions) + + self.assertIsInstance(b, Benchmark) + + self.assertTrue(self.relation_get.called) + self.assertTrue(self.relation_set.called) + + relation_ids.assert_called_once_with('benchmark') + + self.relation_set.assert_called_once_with( + relation_id='benchmark:0', + relation_settings={'benchmarks': ",".join(actions)} + ) + + conf_contents = open(conf_path).readlines() + for key, val in iter(FAKE_RELATION['benchmark:0']['benchmark/0'].items()): + self.assertIn("%s=%s\n" % (key, val), conf_contents) diff --git a/nrpe/mod/charmhelpers/tests/contrib/charmhelpers/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/charmhelpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py b/nrpe/mod/charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py new file mode 100644 index 0000000..2837d2b --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/charmhelpers/test_charmhelpers.py @@ -0,0 +1,274 @@ +# Tests for Python charm helpers. + +import unittest +import yaml +from testtools import TestCase + +from six import StringIO + +import sys +# Path hack to ensure we test the local code, not a version installed in +# /usr/local/lib. This is necessary since /usr/local/lib is prepended before +# what is specified in PYTHONPATH. +sys.path.insert(0, 'helpers/python') +from charmhelpers.contrib import charmhelpers # noqa + + +class CharmHelpersTestCase(TestCase): + """A basic test case for Python charm helpers.""" + + def _patch_command(self, replacement_command): + """Monkeypatch charmhelpers.command for testing purposes. + + :param replacement_command: The replacement Callable for + command(). + """ + self.patch(charmhelpers, 'command', lambda *args: replacement_command) + + def _make_juju_status_dict(self, num_units=1, + service_name='test-service', + unit_state='pending', + machine_state='not-started'): + """Generate valid juju status dict and return it.""" + machine_data = {} + # The 0th machine is the Zookeeper. + machine_data[0] = {'dns-name': 'zookeeper.example.com', + 'instance-id': 'machine0', + 'state': 'not-started'} + service_data = {'charm': 'local:precise/{}-1'.format(service_name), + 'relations': {}, + 'units': {}} + for i in range(num_units): + # The machine is always going to be i+1 because there + # will always be num_units+1 machines. + machine_number = i + 1 + unit_machine_data = { + 'dns-name': 'machine{}.example.com'.format(machine_number), + 'instance-id': 'machine{}'.format(machine_number), + 'state': machine_state, + 'instance-state': machine_state} + machine_data[machine_number] = unit_machine_data + unit_data = { + 'machine': machine_number, + 'public-address': + '{}-{}.example.com'.format(service_name, i), + 'relations': {'db': {'state': 'up'}}, + 'agent-state': unit_state} + service_data['units']['{}/{}'.format(service_name, i)] = ( + unit_data) + juju_status_data = {'machines': machine_data, + 'services': {service_name: service_data}} + return juju_status_data + + def _make_juju_status_yaml(self, num_units=1, + service_name='test-service', + unit_state='pending', + machine_state='not-started'): + """Convert the dict returned by `_make_juju_status_dict` to YAML.""" + return yaml.dump( + self._make_juju_status_dict( + num_units, service_name, unit_state, machine_state)) + + def test_make_charm_config_file(self): + # make_charm_config_file() writes the passed configuration to a + # temporary file as YAML. + charm_config = {'foo': 'bar', + 'spam': 'eggs', + 'ham': 'jam'} + # make_charm_config_file() returns the file object so that it + # can be garbage collected properly. + charm_config_file = charmhelpers.make_charm_config_file(charm_config) + with open(charm_config_file.name) as config_in: + written_config = config_in.read() + self.assertEqual(yaml.dump(charm_config), written_config) + + def test_unit_info(self): + # unit_info returns requested data about a given service. + juju_yaml = self._make_juju_status_yaml() + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertEqual( + 'pending', + charmhelpers.unit_info('test-service', 'agent-state')) + + def test_unit_info_returns_empty_for_nonexistent_service(self): + # If the service passed to unit_info() has not yet started (or + # otherwise doesn't exist), unit_info() will return an empty + # string. + juju_yaml = "services: {}" + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertEqual( + '', charmhelpers.unit_info('test-service', 'state')) + + def test_unit_info_accepts_data(self): + # It's possible to pass a `data` dict, containing the parsed + # result of juju status, to unit_info(). + juju_status_data = yaml.safe_load( + self._make_juju_status_yaml()) + self.patch(charmhelpers, 'juju_status', lambda: None) + service_data = juju_status_data['services']['test-service'] + unit_info_dict = service_data['units']['test-service/0'] + for key, value in unit_info_dict.items(): + item_info = charmhelpers.unit_info( + 'test-service', key, data=juju_status_data) + self.assertEqual(value, item_info) + + def test_unit_info_returns_first_unit_by_default(self): + # By default, unit_info() just returns the value of the + # requested item for the first unit in a service. + juju_yaml = self._make_juju_status_yaml(num_units=2) + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + unit_address = charmhelpers.unit_info( + 'test-service', 'public-address') + self.assertEqual('test-service-0.example.com', unit_address) + + def test_unit_info_accepts_unit_name(self): + # By default, unit_info() just returns the value of the + # requested item for the first unit in a service. However, it's + # possible to pass a unit name to it, too. + juju_yaml = self._make_juju_status_yaml(num_units=2) + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + unit_address = charmhelpers.unit_info( + 'test-service', 'public-address', unit='test-service/1') + self.assertEqual('test-service-1.example.com', unit_address) + + def test_get_machine_data(self): + # get_machine_data() returns a dict containing the machine data + # parsed from juju status. + juju_yaml = self._make_juju_status_yaml() + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + machine_0_data = charmhelpers.get_machine_data()[0] + self.assertEqual('zookeeper.example.com', machine_0_data['dns-name']) + + def test_wait_for_machine_returns_if_machine_up(self): + # If wait_for_machine() is called and the machine(s) it is + # waiting for are already up, it will return. + juju_yaml = self._make_juju_status_yaml(machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + machines, time_taken = charmhelpers.wait_for_machine(timeout=1) + self.assertEqual(1, machines) + + def test_wait_for_machine_times_out(self): + # If the machine that wait_for_machine is waiting for isn't + # 'running' before the passed timeout is reached, + # wait_for_machine will raise an error. + juju_yaml = self._make_juju_status_yaml() + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertRaises( + RuntimeError, charmhelpers.wait_for_machine, timeout=0) + + def test_wait_for_machine_always_returns_if_running_locally(self): + # If juju is actually running against a local LXC container, + # wait_for_machine will always return. + juju_status_dict = self._make_juju_status_dict() + # We'll update the 0th machine to make it look like it's an LXC + # container. + juju_status_dict['machines'][0]['dns-name'] = 'localhost' + juju_yaml = yaml.dump(juju_status_dict) + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + machines, time_taken = charmhelpers.wait_for_machine(timeout=1) + # wait_for_machine will always return 1 machine started here, + # since there's only one machine to start. + self.assertEqual(1, machines) + # time_taken will be 0, since no actual waiting happened. + self.assertEqual(0, time_taken) + + def test_wait_for_machine_waits_for_multiple_machines(self): + # wait_for_machine can be told to wait for multiple machines. + juju_yaml = self._make_juju_status_yaml( + num_units=2, machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + machines, time_taken = charmhelpers.wait_for_machine(num_machines=2) + self.assertEqual(2, machines) + + def test_wait_for_unit_returns_if_unit_started(self): + # wait_for_unit() will return if the service it's waiting for is + # already up. + juju_yaml = self._make_juju_status_yaml( + unit_state='started', machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + charmhelpers.wait_for_unit('test-service', timeout=0) + + def test_wait_for_unit_raises_error_on_error_state(self): + # If the unit is in some kind of error state, wait_for_unit will + # raise a RuntimeError. + juju_yaml = self._make_juju_status_yaml( + unit_state='start-error', machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertRaises(RuntimeError, charmhelpers.wait_for_unit, + 'test-service', timeout=0) + + def test_wait_for_unit_raises_error_on_timeout(self): + # If the unit does not start before the timeout is reached, + # wait_for_unit will raise a RuntimeError. + juju_yaml = self._make_juju_status_yaml( + unit_state='pending', machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertRaises(RuntimeError, charmhelpers.wait_for_unit, + 'test-service', timeout=0) + + def test_wait_for_relation_returns_if_relation_up(self): + # wait_for_relation() waits for relations to come up. If a + # relation is already 'up', wait_for_relation() will return + # immediately. + juju_yaml = self._make_juju_status_yaml( + unit_state='started', machine_state='running') + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + charmhelpers.wait_for_relation('test-service', 'db', timeout=0) + + def test_wait_for_relation_times_out_if_relation_not_present(self): + # If a relation does not exist at all before a timeout is + # reached, wait_for_relation() will raise a RuntimeError. + juju_dict = self._make_juju_status_dict( + unit_state='started', machine_state='running') + units = juju_dict['services']['test-service']['units'] + # We'll remove all the relations for test-service for this test. + units['test-service/0']['relations'] = {} + juju_dict['services']['test-service']['units'] = units + juju_yaml = yaml.dump(juju_dict) + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertRaises( + RuntimeError, charmhelpers.wait_for_relation, 'test-service', + 'db', timeout=0) + + def test_wait_for_relation_times_out_if_relation_not_up(self): + # If a relation does not transition to an 'up' state, before a + # timeout is reached, wait_for_relation() will raise a + # RuntimeError. + juju_dict = self._make_juju_status_dict( + unit_state='started', machine_state='running') + units = juju_dict['services']['test-service']['units'] + units['test-service/0']['relations']['db']['state'] = 'down' + juju_dict['services']['test-service']['units'] = units + juju_yaml = yaml.dump(juju_dict) + self.patch(charmhelpers, 'juju_status', lambda: juju_yaml) + self.assertRaises( + RuntimeError, charmhelpers.wait_for_relation, 'test-service', + 'db', timeout=0) + + def test_wait_for_page_contents_returns_if_contents_available(self): + # wait_for_page_contents() will wait until a given string is + # contained within the results of a given url and will return + # once it does. + # We need to patch the charmhelpers instance of urlopen so that + # it doesn't try to connect out. + test_content = "Hello, world." + self.patch(charmhelpers, 'urlopen', + lambda *args: StringIO(test_content)) + charmhelpers.wait_for_page_contents( + 'http://example.com', test_content, timeout=0) + + def test_wait_for_page_contents_times_out(self): + # If the desired contents do not appear within the page before + # the specified timeout, wait_for_page_contents() will raise a + # RuntimeError. + # We need to patch the charmhelpers instance of urlopen so that + # it doesn't try to connect out. + self.patch(charmhelpers, 'urlopen', + lambda *args: StringIO("This won't work.")) + self.assertRaises( + RuntimeError, charmhelpers.wait_for_page_contents, + 'http://example.com', "This will error", timeout=0) + + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/mod/charmhelpers/tests/contrib/charmsupport/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/charmsupport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/charmsupport/test_nrpe.py b/nrpe/mod/charmhelpers/tests/contrib/charmsupport/test_nrpe.py new file mode 100644 index 0000000..334cf18 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/charmsupport/test_nrpe.py @@ -0,0 +1,496 @@ +import os +import yaml +import subprocess + +from testtools import TestCase +from mock import patch, call, MagicMock + +from charmhelpers.contrib.charmsupport import nrpe +from charmhelpers.core import host + + +class NRPEBaseTestCase(TestCase): + patches = { + 'config': {'object': nrpe}, + 'copy2': {'object': nrpe.shutil}, + 'log': {'object': nrpe}, + 'getpwnam': {'object': nrpe.pwd}, + 'getgrnam': {'object': nrpe.grp}, + 'glob': {'object': nrpe.glob}, + 'mkdir': {'object': os}, + 'chown': {'object': os}, + 'chmod': {'object': os}, + 'exists': {'object': os.path}, + 'listdir': {'object': os}, + 'remove': {'object': os}, + 'open': {'object': nrpe, 'create': True}, + 'isfile': {'object': os.path}, + 'isdir': {'object': os.path}, + 'call': {'object': subprocess}, + 'relation_get': {'object': nrpe}, + 'relation_ids': {'object': nrpe}, + 'relation_set': {'object': nrpe}, + 'relations_of_type': {'object': nrpe}, + 'service': {'object': nrpe}, + 'init_is_systemd': {'object': host}, + } + + def setUp(self): + super(NRPEBaseTestCase, self).setUp() + self.patched = {} + # Mock the universe. + for attr, data in self.patches.items(): + create = data.get('create', False) + patcher = patch.object(data['object'], attr, create=create) + self.patched[attr] = patcher.start() + self.addCleanup(patcher.stop) + env_patcher = patch.dict('os.environ', + {'JUJU_UNIT_NAME': 'testunit', + 'CHARM_DIR': '/usr/lib/test_charm_dir'}) + env_patcher.start() + self.addCleanup(env_patcher.stop) + + def check_call_counts(self, **kwargs): + for attr, expected in kwargs.items(): + patcher = self.patched[attr] + self.assertEqual(expected, patcher.call_count, attr) + + +class NRPETestCase(NRPEBaseTestCase): + + def test_init_gets_config(self): + self.patched['config'].return_value = {'nagios_context': 'testctx', + 'nagios_servicegroups': 'testsgrps'} + + checker = nrpe.NRPE() + + self.assertEqual('testctx', checker.nagios_context) + self.assertEqual('testsgrps', checker.nagios_servicegroups) + self.assertEqual('testunit', checker.unit_name) + self.assertEqual('testctx-testunit', checker.hostname) + self.check_call_counts(config=1) + + def test_init_hostname(self): + """Test that the hostname parameter is correctly set""" + checker = nrpe.NRPE() + self.assertEqual(checker.hostname, + "{}-{}".format(checker.nagios_context, + checker.unit_name)) + hostname = "test.host" + checker = nrpe.NRPE(hostname=hostname) + self.assertEqual(checker.hostname, hostname) + + def test_default_servicegroup(self): + """Test that nagios_servicegroups gets set to the default if omitted""" + self.patched['config'].return_value = {'nagios_context': 'testctx'} + checker = nrpe.NRPE() + self.assertEqual(checker.nagios_servicegroups, 'testctx') + + def test_no_nagios_installed_bails(self): + self.patched['config'].return_value = {'nagios_context': 'test', + 'nagios_servicegroups': ''} + self.patched['getgrnam'].side_effect = KeyError + checker = nrpe.NRPE() + + self.assertEqual(None, checker.write()) + + expected = 'Nagios user not set up, nrpe checks not updated' + self.patched['log'].assert_called_with(expected) + self.check_call_counts(log=2, config=1, getpwnam=1, getgrnam=1) + + def test_write_no_checker(self): + self.patched['config'].return_value = {'nagios_context': 'test', + 'nagios_servicegroups': ''} + self.patched['exists'].return_value = True + checker = nrpe.NRPE() + + self.assertEqual(None, checker.write()) + + self.check_call_counts(config=1, getpwnam=1, getgrnam=1, exists=1) + + def test_write_restarts_service(self): + self.patched['config'].return_value = {'nagios_context': 'test', + 'nagios_servicegroups': ''} + self.patched['exists'].return_value = True + checker = nrpe.NRPE() + + self.assertEqual(None, checker.write()) + + self.patched['service'].assert_called_with('restart', 'nagios-nrpe-server') + self.check_call_counts(config=1, getpwnam=1, getgrnam=1, + exists=1, service=1) + + def test_update_nrpe(self): + self.patched['config'].return_value = {'nagios_context': 'a', + 'nagios_servicegroups': ''} + self.patched['exists'].return_value = True + self.patched['relation_get'].return_value = { + 'egress-subnets': '10.66.111.24/32', + 'ingress-address': '10.66.111.24', + 'private-address': '10.66.111.24' + } + + def _rels(rname): + relations = { + 'local-monitors': 'local-monitors:1', + 'nrpe-external-master': 'nrpe-external-master:2', + } + return [relations[rname]] + self.patched['relation_ids'].side_effect = _rels + + checker = nrpe.NRPE() + checker.add_check(shortname="myservice", + description="Check MyService", + check_cmd="check_http http://localhost") + + self.assertEqual(None, checker.write()) + + self.assertEqual(2, self.patched['open'].call_count) + filename = 'check_myservice.cfg' + expected = [ + ('/etc/nagios/nrpe.d/%s' % filename, 'w'), + ('/var/lib/nagios/export/service__a-testunit_%s' % filename, 'w'), + ] + actual = [x[0] for x in self.patched['open'].call_args_list] + self.assertEqual(expected, actual) + outfile = self.patched['open'].return_value.__enter__.return_value + service_file_contents = """ +#--------------------------------------------------- +# This file is Juju managed +#--------------------------------------------------- +define service { + use active-service + host_name a-testunit + service_description a-testunit[myservice] Check MyService + check_command check_nrpe!check_myservice + servicegroups a + +} +""" + expected = [ + '# check myservice\n', + '# The following header was added automatically by juju\n', + '# Modifying it will affect nagios monitoring and alerting\n', + '# servicegroups: a\n', + 'command[check_myservice]=/usr/lib/nagios/plugins/check_http http://localhost\n', + service_file_contents, + ] + actual = [x[0][0] for x in outfile.write.call_args_list] + self.assertEqual(expected, actual) + + nrpe_monitors = {'myservice': + {'command': 'check_myservice', + }} + monitors = yaml.dump( + {"monitors": {"remote": {"nrpe": nrpe_monitors}}}) + relation_set_calls = [ + call(monitors=monitors, relation_id="local-monitors:1"), + call(monitors=monitors, relation_id="nrpe-external-master:2"), + ] + self.patched['relation_set'].assert_has_calls(relation_set_calls, any_order=True) + self.check_call_counts(config=1, getpwnam=1, getgrnam=1, + exists=4, open=2, listdir=1, relation_get=2, + relation_ids=3, relation_set=3) + + def test_max_check_attmpts(self): + self.patched['config'].return_value = {'nagios_context': 'a', + 'nagios_servicegroups': ''} + self.patched['exists'].return_value = True + self.patched['relation_get'].return_value = { + 'egress-subnets': '10.66.111.24/32', + 'ingress-address': '10.66.111.24', + 'private-address': '10.66.111.24' + } + + def _rels(rname): + relations = { + 'local-monitors': 'local-monitors:1', + 'nrpe-external-master': 'nrpe-external-master:2', + } + return [relations[rname]] + self.patched['relation_ids'].side_effect = _rels + + checker = nrpe.NRPE() + checker.add_check(shortname="myservice", + description="Check MyService", + check_cmd="check_http http://localhost", + max_check_attempts=8, + ) + + self.assertEqual(None, checker.write()) + + self.assertEqual(2, self.patched['open'].call_count) + filename = 'check_myservice.cfg' + expected = [ + ('/etc/nagios/nrpe.d/%s' % filename, 'w'), + ('/var/lib/nagios/export/service__a-testunit_%s' % filename, 'w'), + ] + actual = [x[0] for x in self.patched['open'].call_args_list] + self.assertEqual(expected, actual) + outfile = self.patched['open'].return_value.__enter__.return_value + service_file_contents = """ +#--------------------------------------------------- +# This file is Juju managed +#--------------------------------------------------- +define service { + use active-service + host_name a-testunit + service_description a-testunit[myservice] Check MyService + check_command check_nrpe!check_myservice + servicegroups a + max_check_attempts 8 +} +""" + expected = [ + '# check myservice\n', + '# The following header was added automatically by juju\n', + '# Modifying it will affect nagios monitoring and alerting\n', + '# servicegroups: a\n', + 'command[check_myservice]=/usr/lib/nagios/plugins/check_http http://localhost\n', + service_file_contents, + ] + actual = [x[0][0] for x in outfile.write.call_args_list] + self.assertEqual(expected, actual) + + nrpe_monitors = {'myservice': + {'command': 'check_myservice', + 'max_check_attempts': 8, + }} + monitors = yaml.dump( + {"monitors": {"remote": {"nrpe": nrpe_monitors}}}) + relation_set_calls = [ + call(monitors=monitors, relation_id="local-monitors:1"), + call(monitors=monitors, relation_id="nrpe-external-master:2"), + ] + self.patched['relation_set'].assert_has_calls(relation_set_calls, any_order=True) + self.check_call_counts(config=1, getpwnam=1, getgrnam=1, + exists=4, open=2, listdir=1, relation_get=2, + relation_ids=3, relation_set=3) + + +class NRPECheckTestCase(NRPEBaseTestCase): + + def test_invalid_shortname(self): + cases = [ + 'invalid:name', + '', + ] + for shortname in cases: + self.assertRaises(nrpe.CheckException, nrpe.Check, shortname, + 'description', '/some/command') + + def test_valid_shortname(self): + cases = [ + '1_number_is_fine', + 'dots.are.good', + 'dashes-ok', + 'UPPER_case_allowed', + '5', + '@valid', + ] + for shortname in cases: + check = nrpe.Check(shortname, 'description', '/some/command') + self.assertEqual(shortname, check.shortname) + + def test_write_removes_existing_config(self): + self.patched['listdir'].return_value = [ + 'foo', 'bar.cfg', '_check_shortname.cfg'] + check = nrpe.Check('shortname', 'description', '/some/command') + + self.assertEqual(None, check.write('testctx', 'hostname', 'testsgrp')) + + expected = '/var/lib/nagios/export/_check_shortname.cfg' + self.patched['remove'].assert_called_once_with(expected) + self.check_call_counts(exists=3, remove=1, open=2, listdir=1) + + def test_check_write_nrpe_exportdir_not_accessible(self): + self.patched['exists'].return_value = False + check = nrpe.Check('shortname', 'description', '/some/command') + + self.assertEqual(None, check.write('testctx', 'hostname', 'testsgrps')) + expected = ('Not writing service config as ' + '/var/lib/nagios/export is not accessible') + self.patched['log'].assert_has_calls( + [call(expected)], any_order=True) + self.check_call_counts(log=2, open=1) + + def test_locate_cmd_no_args(self): + self.patched['exists'].return_value = True + + check = nrpe.Check('shortname', 'description', '/bin/ls') + + self.assertEqual('/bin/ls', check.check_cmd) + + def test_locate_cmd_not_found(self): + self.patched['exists'].return_value = False + check = nrpe.Check('shortname', 'description', 'check_http -x -y -z') + + self.assertEqual('', check.check_cmd) + self.assertEqual(2, self.patched['exists'].call_count) + expected = [ + '/usr/lib/nagios/plugins/check_http', + '/usr/local/lib/nagios/plugins/check_http', + ] + actual = [x[0][0] for x in self.patched['exists'].call_args_list] + self.assertEqual(expected, actual) + self.check_call_counts(exists=2, log=1) + expected = 'Check command not found: check_http' + self.assertEqual(expected, self.patched['log'].call_args[0][0]) + + def test_run(self): + self.patched['exists'].return_value = True + command = '/usr/bin/wget foo' + check = nrpe.Check('shortname', 'description', command) + + self.assertEqual(None, check.run()) + + self.check_call_counts(exists=1, call=1) + self.assertEqual(command, self.patched['call'].call_args[0][0]) + + +class NRPEMiscTestCase(NRPEBaseTestCase): + def test_get_nagios_hostcontext(self): + rel_info = { + 'nagios_hostname': 'bob-openstack-dashboard-0', + 'private-address': '10.5.3.103', + '__unit__': u'dashboard-nrpe/1', + '__relid__': u'nrpe-external-master:2', + 'nagios_host_context': u'bob', + } + self.patched['relations_of_type'].return_value = [rel_info] + self.assertEqual(nrpe.get_nagios_hostcontext(), 'bob') + + def test_get_nagios_hostname(self): + rel_info = { + 'nagios_hostname': 'bob-openstack-dashboard-0', + 'private-address': '10.5.3.103', + '__unit__': u'dashboard-nrpe/1', + '__relid__': u'nrpe-external-master:2', + 'nagios_host_context': u'bob', + } + self.patched['relations_of_type'].return_value = [rel_info] + self.assertEqual(nrpe.get_nagios_hostname(), 'bob-openstack-dashboard-0') + + def test_get_nagios_unit_name(self): + rel_info = { + 'nagios_hostname': 'bob-openstack-dashboard-0', + 'private-address': '10.5.3.103', + '__unit__': u'dashboard-nrpe/1', + '__relid__': u'nrpe-external-master:2', + 'nagios_host_context': u'bob', + } + self.patched['relations_of_type'].return_value = [rel_info] + self.assertEqual(nrpe.get_nagios_unit_name(), 'bob:testunit') + + def test_get_nagios_unit_name_no_hc(self): + self.patched['relations_of_type'].return_value = [] + self.assertEqual(nrpe.get_nagios_unit_name(), 'testunit') + + @patch.object(os.path, 'isdir') + def test_add_init_service_checks(self, mock_isdir): + def _exists(init_file): + files = ['/etc/init/apache2.conf', + '/usr/lib/nagios/plugins/check_upstart_job', + '/etc/init.d/haproxy', + '/usr/lib/nagios/plugins/check_status_file.py', + '/etc/cron.d/nagios-service-check-haproxy', + '/var/lib/nagios/service-check-haproxy.txt', + '/usr/lib/nagios/plugins/check_systemd.py' + ] + return init_file in files + + self.patched['exists'].side_effect = _exists + + # Test without systemd and /var/lib/nagios does not exist + self.patched['init_is_systemd'].return_value = False + mock_isdir.return_value = False + bill = nrpe.NRPE() + services = ['apache2', 'haproxy'] + nrpe.add_init_service_checks(bill, services, 'testunit') + mock_isdir.assert_called_with('/var/lib/nagios') + self.patched['call'].assert_not_called() + expect_cmds = { + 'apache2': '/usr/lib/nagios/plugins/check_upstart_job apache2', + 'haproxy': '/usr/lib/nagios/plugins/check_status_file.py -f ' + '/var/lib/nagios/service-check-haproxy.txt', + } + self.assertEqual(bill.checks[0].shortname, 'apache2') + self.assertEqual(bill.checks[0].check_cmd, expect_cmds['apache2']) + self.assertEqual(bill.checks[1].shortname, 'haproxy') + self.assertEqual(bill.checks[1].check_cmd, expect_cmds['haproxy']) + + # without systemd and /var/lib/nagios does exist + mock_isdir.return_value = True + f = MagicMock() + self.patched['open'].return_value = f + bill = nrpe.NRPE() + services = ['apache2', 'haproxy'] + nrpe.add_init_service_checks(bill, services, 'testunit') + mock_isdir.assert_called_with('/var/lib/nagios') + self.patched['call'].assert_called_with( + ['/usr/local/lib/nagios/plugins/check_exit_status.pl', '-e', '-s', + '/etc/init.d/haproxy', 'status'], stdout=f, + stderr=subprocess.STDOUT) + + # Test regular services and snap services with systemd + services = ['apache2', 'haproxy', 'snap.test.test', + 'ceph-radosgw@hostname'] + self.patched['init_is_systemd'].return_value = True + nrpe.add_init_service_checks(bill, services, 'testunit') + expect_cmds = { + 'apache2': '/usr/lib/nagios/plugins/check_systemd.py apache2', + 'haproxy': '/usr/lib/nagios/plugins/check_systemd.py haproxy', + 'snap.test.test': '/usr/lib/nagios/plugins/check_systemd.py snap.test.test', + } + self.assertEqual(bill.checks[2].shortname, 'apache2') + self.assertEqual(bill.checks[2].check_cmd, expect_cmds['apache2']) + self.assertEqual(bill.checks[3].shortname, 'haproxy') + self.assertEqual(bill.checks[3].check_cmd, expect_cmds['haproxy']) + self.assertEqual(bill.checks[4].shortname, 'snap.test.test') + self.assertEqual(bill.checks[4].check_cmd, expect_cmds['snap.test.test']) + + def test_copy_nrpe_checks(self): + file_presence = { + 'filea': True, + 'fileb': False} + self.patched['exists'].return_value = True + self.patched['glob'].return_value = ['filea', 'fileb'] + self.patched['isdir'].side_effect = [False, True] + self.patched['isfile'].side_effect = lambda x: file_presence[x] + nrpe.copy_nrpe_checks() + self.patched['glob'].assert_called_once_with( + ('/usr/lib/test_charm_dir/hooks/charmhelpers/contrib/openstack/' + 'files/check_*')) + self.patched['copy2'].assert_called_once_with( + 'filea', + '/usr/local/lib/nagios/plugins/filea') + + def test_copy_nrpe_checks_other_root(self): + file_presence = { + 'filea': True, + 'fileb': False} + self.patched['exists'].return_value = True + self.patched['glob'].return_value = ['filea', 'fileb'] + self.patched['isdir'].side_effect = [True, False] + self.patched['isfile'].side_effect = lambda x: file_presence[x] + nrpe.copy_nrpe_checks() + self.patched['glob'].assert_called_once_with( + ('/usr/lib/test_charm_dir/charmhelpers/contrib/openstack/' + 'files/check_*')) + self.patched['copy2'].assert_called_once_with( + 'filea', + '/usr/local/lib/nagios/plugins/filea') + + def test_copy_nrpe_checks_nrpe_files_dir(self): + file_presence = { + 'filea': True, + 'fileb': False} + self.patched['exists'].return_value = True + self.patched['glob'].return_value = ['filea', 'fileb'] + self.patched['isfile'].side_effect = lambda x: file_presence[x] + nrpe.copy_nrpe_checks(nrpe_files_dir='/other/dir') + self.patched['glob'].assert_called_once_with( + '/other/dir/check_*') + self.patched['copy2'].assert_called_once_with( + 'filea', + '/usr/local/lib/nagios/plugins/filea') diff --git a/nrpe/mod/charmhelpers/tests/contrib/database/README b/nrpe/mod/charmhelpers/tests/contrib/database/README new file mode 100644 index 0000000..c582403 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/database/README @@ -0,0 +1,7 @@ +The Mysql test is not enabled. In order to enable it, touch __init__.py. + +As it currently stands, the test has poor coverage and fails under python3. +These things should be addressed before re-enabling the test. + +Adam Israel +March 17, 2015 diff --git a/nrpe/mod/charmhelpers/tests/contrib/database/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/database/test_mysql.py b/nrpe/mod/charmhelpers/tests/contrib/database/test_mysql.py new file mode 100644 index 0000000..b99b783 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/database/test_mysql.py @@ -0,0 +1,949 @@ +import os +import mock +import json +import unittest +import sys +import shutil +import tempfile + +from collections import OrderedDict + +sys.modules['MySQLdb'] = mock.Mock() +from charmhelpers.contrib.database import mysql # noqa + + +class MysqlTests(unittest.TestCase): + def setUp(self): + super(MysqlTests, self).setUp() + + def test_connect_host_defined(self): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + with mock.patch.object(mysql, 'log'): + helper.connect(user='user', password='password', host='1.1.1.1') + mysql.MySQLdb.connect.assert_called_with( + passwd='password', host='1.1.1.1', user='user', connect_timeout=30) + + def test_connect_host_not_defined(self): + helper = mysql.MySQLHelper('foo', 'bar') + with mock.patch.object(mysql, 'log'): + helper.connect(user='user', password='password') + mysql.MySQLdb.connect.assert_called_with( + passwd='password', host='localhost', user='user', + connect_timeout=30) + + def test_connect_port_defined(self): + helper = mysql.MySQLHelper('foo', 'bar') + with mock.patch.object(mysql, 'log'): + helper.connect(user='user', password='password', port=3316) + mysql.MySQLdb.connect.assert_called_with( + passwd='password', host='localhost', user='user', port=3316, + connect_timeout=30) + + def test_connect_new_default_timeout(self): + helper = mysql.MySQLHelper('foo', 'bar', connect_timeout=10) + with mock.patch.object(mysql, 'log'): + helper.connect(user='user', password='password', port=3316) + mysql.MySQLdb.connect.assert_called_with( + passwd='password', host='localhost', user='user', port=3316, + connect_timeout=10) + + def test_connect_new_default_override(self): + helper = mysql.MySQLHelper('foo', 'bar', connect_timeout=10) + with mock.patch.object(mysql, 'log'): + helper.connect(user='user', password='password', port=3316, + connect_timeout=20) + mysql.MySQLdb.connect.assert_called_with( + passwd='password', host='localhost', user='user', port=3316, + connect_timeout=20) + + @mock.patch.object(mysql.MySQLHelper, 'normalize_address') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password') + @mock.patch.object(mysql.MySQLHelper, 'grant_exists') + @mock.patch.object(mysql, 'relation_get') + @mock.patch.object(mysql, 'related_units') + @mock.patch.object(mysql, 'log') + def test_get_allowed_units(self, mock_log, mock_related_units, + mock_relation_get, + mock_grant_exists, + mock_get_password, + mock_normalize_address): + + # echo + mock_normalize_address.side_effect = lambda addr: addr + + def mock_rel_get(unit, rid): + if unit == 'unit/0': + # Non-prefixed settings + d = {'private-address': '10.0.0.1', + 'hostname': 'hostA'} + elif unit == 'unit/1': + # Containing prefixed settings + d = {'private-address': '10.0.0.2', + 'dbA_hostname': json.dumps(['10.0.0.2', '2001:db8:1::2'])} + elif unit == 'unit/2': + # No hostname + d = {'private-address': '10.0.0.3'} + elif unit == 'unit/3': + # Prefixed hostname + d = {'private-address': '10.0.0.4', + 'PRE_hostname': json.dumps(['10.0.0.4', '2001:db8:1::4'])} + return d + + mock_relation_get.side_effect = mock_rel_get + mock_related_units.return_value = ['unit/0', 'unit/1', 'unit/2'] + + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + units = helper.get_allowed_units('dbA', 'userA') + + calls = [mock.call('dbA', 'userA', 'hostA'), + mock.call('dbA', 'userA', '10.0.0.2'), + mock.call('dbA', 'userA', '2001:db8:1::2'), + mock.call('dbA', 'userA', '10.0.0.3')] + + helper.grant_exists.assert_has_calls(calls, any_order=True) + self.assertEqual(units, set(['unit/0', 'unit/1', 'unit/2'])) + + # With prefix + calls = [mock.call('dbB', 'userB', 'hostA'), + mock.call('dbB', 'userB', '10.0.0.2'), + mock.call('dbB', 'userB', '10.0.0.3'), + mock.call('dbB', 'userB', '2001:db8:1::4'), + mock.call('dbB', 'userB', '10.0.0.4')] + + mock_related_units.return_value = [ + 'unit/0', 'unit/1', 'unit/2', 'unit/3'] + units = helper.get_allowed_units('dbB', 'userB', prefix="PRE") + helper.grant_exists.assert_has_calls(calls, any_order=True) + self.assertEqual(units, set(['unit/0', 'unit/1', 'unit/2', 'unit/3'])) + + @mock.patch('charmhelpers.contrib.network.ip.log', + lambda *args, **kwargs: None) + @mock.patch('charmhelpers.contrib.network.ip.ns_query') + @mock.patch('charmhelpers.contrib.network.ip.socket') + @mock.patch.object(mysql, 'unit_get') + @mock.patch.object(mysql, 'config_get') + @mock.patch.object(mysql, 'log') + def test_normalize_address(self, mock_log, mock_config_get, mock_unit_get, + mock_socket, mock_ns_query): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + # prefer-ipv6 + mock_config_get.return_value = False + # echo + mock_socket.gethostbyname.side_effect = lambda addr: addr + + mock_unit_get.return_value = '10.0.0.1' + out = helper.normalize_address('10.0.0.1') + self.assertEqual('127.0.0.1', out) + mock_config_get.assert_called_with('prefer-ipv6') + + mock_unit_get.return_value = '10.0.0.1' + out = helper.normalize_address('10.0.0.2') + self.assertEqual('10.0.0.2', out) + mock_config_get.assert_called_with('prefer-ipv6') + + out = helper.normalize_address('2001:db8:1::1') + self.assertEqual('2001:db8:1::1', out) + mock_config_get.assert_called_with('prefer-ipv6') + + mock_socket.gethostbyname.side_effect = Exception + mock_ns_query.return_value = None + out = helper.normalize_address('unresolvable') + self.assertEqual('unresolvable', out) + mock_config_get.assert_called_with('prefer-ipv6') + + # prefer-ipv6 + mock_config_get.return_value = True + mock_socket.gethostbyname.side_effect = 'other' + out = helper.normalize_address('unresolvable') + self.assertEqual('unresolvable', out) + mock_config_get.assert_called_with('prefer-ipv6') + + def test_passwd_keys(self): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + self.assertEqual(list(helper.passwd_keys(None)), ['mysql.passwd']) + self.assertEqual(list(helper.passwd_keys('auser')), + ['mysql-auser.passwd', 'auser.passwd']) + + @mock.patch.object(mysql.MySQLHelper, 'migrate_passwords_to_leader_storage') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password_on_disk') + @mock.patch.object(mysql, 'leader_get') + def test_get_mysql_password_no_peer_passwd(self, mock_leader_get, + mock_get_disk_pw, + mock_migrate_pw): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + store = {} + mock_leader_get.side_effect = lambda key: store.get(key) + mock_get_disk_pw.return_value = "disk-passwd" + self.assertEqual(helper.get_mysql_password(), "disk-passwd") + self.assertTrue(mock_migrate_pw.called) + + @mock.patch.object(mysql.MySQLHelper, 'migrate_passwords_to_leader_storage') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password_on_disk') + @mock.patch.object(mysql, 'leader_get') + def test_get_mysql_password_peer_passwd(self, mock_leader_get, + mock_get_disk_pw, mock_migrate_pw): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + store = {'mysql-userA.passwd': 'passwdA'} + mock_leader_get.side_effect = lambda key: store.get(key) + mock_get_disk_pw.return_value = "disk-passwd" + self.assertEqual(helper.get_mysql_password(username='userA'), + "passwdA") + self.assertTrue(mock_migrate_pw.called) + + @mock.patch.object(mysql.MySQLHelper, 'migrate_passwords_to_leader_storage') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password_on_disk') + @mock.patch.object(mysql, 'leader_get') + def test_get_mysql_password_peer_passwd_legacy(self, mock_leader_get, + mock_get_disk_pw, + mock_migrate_pw): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + store = {'userA.passwd': 'passwdA'} + mock_leader_get.side_effect = lambda key: store.get(key) + mock_get_disk_pw.return_value = "disk-passwd" + self.assertEqual(helper.get_mysql_password(username='userA'), + "passwdA") + self.assertTrue(mock_migrate_pw.called) + + @mock.patch.object(mysql.MySQLHelper, 'migrate_passwords_to_leader_storage') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password_on_disk') + @mock.patch.object(mysql, 'leader_get') + def test_get_mysql_password_peer_passwd_all(self, mock_leader_get, + mock_get_disk_pw, + mock_migrate_pw): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + # Add * so we can identify that the new format key takes precedence + # if found. + store = {'mysql-userA.passwd': 'passwdA', + 'userA.passwd': 'passwdA*'} + mock_leader_get.side_effect = lambda key: store.get(key) + mock_get_disk_pw.return_value = "disk-passwd" + self.assertEqual(helper.get_mysql_password(username='userA'), + "passwdA") + self.assertTrue(mock_migrate_pw.called) + + @mock.patch.object(mysql.MySQLHelper, 'set_mysql_password') + def test_set_mysql_root_password(self, mock_set_passwd): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + helper.set_mysql_root_password(password='1234') + mock_set_passwd.assert_called_with( + 'root', + '1234', + current_password=None) + + @mock.patch.object(mysql.MySQLHelper, 'set_mysql_password') + def test_set_mysql_root_password_cur_passwd(self, mock_set_passwd): + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + helper.set_mysql_root_password(password='1234', current_password='abc') + mock_set_passwd.assert_called_with( + 'root', + '1234', + current_password='abc') + + @mock.patch.object(mysql, 'log', lambda *args, **kwargs: None) + @mock.patch.object(mysql, 'is_leader') + @mock.patch.object(mysql, 'leader_get') + @mock.patch.object(mysql, 'leader_set') + @mock.patch.object(mysql, 'CompareHostReleases') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password') + @mock.patch.object(mysql.MySQLHelper, 'connect') + def test_set_mysql_password(self, mock_connect, mock_get_passwd, + mock_compare_releases, mock_leader_set, + mock_leader_get, mock_is_leader): + mock_connection = mock.MagicMock() + mock_cursor = mock.MagicMock() + mock_connection.cursor.return_value = mock_cursor + mock_get_passwd.return_value = 'asdf' + mock_is_leader.return_value = True + mock_leader_get.return_value = '1234' + mock_compare_releases.return_value = 'artful' + + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + helper.connection = mock_connection + + helper.set_mysql_password(username='root', password='1234') + + mock_connect.assert_has_calls( + [mock.call(user='root', password='asdf'), # original password + mock.call(user='root', password='1234')]) # new password + mock_leader_get.assert_has_calls([mock.call('mysql.passwd')]) + mock_leader_set.assert_has_calls( + [mock.call(settings={'mysql.passwd': '1234'})] + ) + SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET password = " + "PASSWORD( %s ) WHERE user = %s;") + mock_cursor.assert_has_calls( + [mock.call.execute(SQL_UPDATE_PASSWD, ('1234', 'root')), + mock.call.execute('FLUSH PRIVILEGES;'), + mock.call.close(), + mock.call.execute('select 1;'), + mock.call.close()] + ) + mock_get_passwd.assert_called_once_with(None) + + # make sure for the non-leader leader-set is not called + mock_is_leader.return_value = False + mock_leader_set.reset_mock() + mock_get_passwd.reset_mock() + helper.set_mysql_password(username='root', password='1234') + mock_leader_set.assert_not_called() + mock_get_passwd.assert_called_once_with(None) + + mock_get_passwd.reset_mock() + mock_compare_releases.return_value = 'bionic' + helper.set_mysql_password(username='root', password='1234') + SQL_UPDATE_PASSWD = ("UPDATE mysql.user SET " + "authentication_string = " + "PASSWORD( %s ) WHERE user = %s;") + mock_cursor.assert_has_calls( + [mock.call.execute(SQL_UPDATE_PASSWD, ('1234', 'root')), + mock.call.execute('FLUSH PRIVILEGES;'), + mock.call.close(), + mock.call.execute('select 1;'), + mock.call.close()] + ) + mock_get_passwd.assert_called_once_with(None) + + # Test supplying the current password + mock_is_leader.return_value = False + mock_connect.reset_mock() + mock_get_passwd.reset_mock() + helper.set_mysql_password( + username='root', + password='1234', + current_password='currpass') + self.assertFalse(mock_get_passwd.called) + mock_connect.assert_has_calls( + [mock.call(user='root', password='currpass'), # original password + mock.call(user='root', password='1234')]) # new password + + @mock.patch.object(mysql, 'leader_get') + @mock.patch.object(mysql, 'leader_set') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password') + @mock.patch.object(mysql.MySQLHelper, 'connect') + def test_set_mysql_password_fail_to_connect(self, mock_connect, + mock_get_passwd, + mock_leader_set, + mock_leader_get): + + class FakeOperationalError(Exception): + pass + + def fake_connect(*args, **kwargs): + raise FakeOperationalError('foobar') + + mysql.MySQLdb.OperationalError = FakeOperationalError + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + mock_connect.side_effect = fake_connect + self.assertRaises(mysql.MySQLSetPasswordError, + helper.set_mysql_password, + username='root', password='1234') + + @mock.patch.object(mysql, 'lsb_release') + @mock.patch.object(mysql, 'leader_get') + @mock.patch.object(mysql, 'leader_set') + @mock.patch.object(mysql.MySQLHelper, 'get_mysql_password') + @mock.patch.object(mysql.MySQLHelper, 'connect') + def test_set_mysql_password_fail_to_connect2(self, mock_connect, + mock_get_passwd, + mock_leader_set, + mock_leader_get, + mock_lsb_release): + + class FakeOperationalError(Exception): + def __str__(self): + return 'some-error' + + operational_error = FakeOperationalError('foobar') + + def fake_connect(user, password): + # fail for the new password + if user == 'root' and password == '1234': + raise operational_error + else: + return mock.MagicMock() + + mysql.MySQLdb.OperationalError = FakeOperationalError + helper = mysql.MySQLHelper('foo', 'bar', host='hostA') + helper.connection = mock.MagicMock() + mock_connect.side_effect = fake_connect + mock_lsb_release.return_value = { + 'DISTRIB_CODENAME': 'bionic', + } + with self.assertRaises(mysql.MySQLSetPasswordError) as cm: + helper.set_mysql_password(username='root', password='1234') + + ex = cm.exception + self.assertEqual(ex.args[0], 'Cannot connect using new password: some-error') + self.assertEqual(ex.args[1], operational_error) + + @mock.patch.object(mysql, 'is_leader') + @mock.patch.object(mysql, 'leader_set') + def test_migrate_passwords_to_leader_storage(self, mock_leader_set, + mock_is_leader): + files = {'mysql.passwd': '1', + 'userA.passwd': '2', + 'mysql-userA.passwd': '3'} + store = {} + + def _store(settings): + store.update(settings) + + mock_is_leader.return_value = True + + tmpdir = tempfile.mkdtemp('charm-helpers-unit-tests') + try: + root_tmplt = "%s/mysql.passwd" % (tmpdir) + helper = mysql.MySQLHelper(root_tmplt, None, host='hostA') + for f in files: + with open(os.path.join(tmpdir, f), 'w') as fd: + fd.write(files[f]) + + mock_leader_set.side_effect = _store + helper.migrate_passwords_to_leader_storage() + + calls = [mock.call(settings={'mysql.passwd': '1'}), + mock.call(settings={'userA.passwd': '2'}), + mock.call(settings={'mysql-userA.passwd': '3'})] + + mock_leader_set.assert_has_calls(calls, + any_order=True) + finally: + shutil.rmtree(tmpdir) + + # Note that legacy key/val is NOT overwritten + self.assertEqual(store, {'mysql.passwd': '1', + 'userA.passwd': '2', + 'mysql-userA.passwd': '3'}) + + @mock.patch.object(mysql, 'log', lambda *args, **kwargs: None) + @mock.patch.object(mysql, 'is_leader') + @mock.patch.object(mysql, 'leader_set') + def test_migrate_passwords_to_leader_storage_not_leader(self, mock_leader_set, + mock_is_leader): + mock_is_leader.return_value = False + tmpdir = tempfile.mkdtemp('charm-helpers-unit-tests') + try: + root_tmplt = "%s/mysql.passwd" % (tmpdir) + helper = mysql.MySQLHelper(root_tmplt, None, host='hostA') + helper.migrate_passwords_to_leader_storage() + finally: + shutil.rmtree(tmpdir) + mock_leader_set.assert_not_called() + + +class MySQLConfigHelperTests(unittest.TestCase): + + def setUp(self): + super(MySQLConfigHelperTests, self).setUp() + self.config_data = {} + self.config = mock.MagicMock() + mysql.config_get = self.config + self.config.side_effect = self._fake_config + + def _fake_config(self, key=None): + if key: + try: + return self.config_data[key] + except KeyError: + return None + else: + return self.config_data + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_pool_fixed(self, log, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': "50%", + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_buffer_pool_size(), + helper.human_to_bytes("50G")) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_pool_not_set(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': '', + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_buffer_pool_size(), + helper.DEFAULT_INNODB_BUFFER_SIZE_MAX) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_buffer_unset(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': None, + 'dataset-size': None, + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_buffer_pool_size(), + helper.DEFAULT_INNODB_BUFFER_SIZE_MAX) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_buffer_unset_small(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-buffer-pool-size': None, + 'dataset-size': None, + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_buffer_pool_size(), + int(helper.human_to_bytes(mem.return_value) * + helper.DEFAULT_INNODB_BUFFER_FACTOR)) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_dataset_size(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'dataset-size': "10G", + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_buffer_pool_size(), + int(helper.human_to_bytes("10G"))) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_tuning_level(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'safest', + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_flush_log_at_trx_commit(), + 1 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_tuning_level_fast(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'fast', + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_flush_log_at_trx_commit(), + 2 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_tuning_level_unsafe(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'unsafe', + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_flush_log_at_trx_commit(), + 0 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_valid_values(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-change-buffering': 'all', + } + + helper = mysql.MySQLConfigHelper() + + self.assertEqual( + helper.get_innodb_change_buffering(), + 'all' + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_get_innodb_invalid_values(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-change-buffering': 'invalid', + } + + helper = mysql.MySQLConfigHelper() + + self.assertTrue(helper.get_innodb_change_buffering() is None) + + +class PerconaTests(unittest.TestCase): + + def setUp(self): + super(PerconaTests, self).setUp() + self.config_data = {} + self.config = mock.MagicMock() + mysql.config_get = self.config + self.config.side_effect = self._fake_config + + def _fake_config(self, key=None): + if key: + try: + return self.config_data[key] + except KeyError: + return None + else: + return self.config_data + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_pool_fixed(self, log, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': "50%", + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual(mysql_config.get('innodb_buffer_pool_size'), + helper.human_to_bytes("50G")) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_pool_not_set(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': '', + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_buffer_pool_size'), + helper.DEFAULT_INNODB_BUFFER_SIZE_MAX) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_buffer_unset(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'innodb-buffer-pool-size': None, + 'dataset-size': None, + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_buffer_pool_size'), + helper.DEFAULT_INNODB_BUFFER_SIZE_MAX) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_buffer_unset_small(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-buffer-pool-size': None, + 'dataset-size': None, + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_buffer_pool_size'), + int(helper.human_to_bytes(mem.return_value) * + helper.DEFAULT_INNODB_BUFFER_FACTOR)) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_dataset_size(self, mog, mem): + mem.return_value = "100G" + self.config_data = { + 'dataset-size': "10G", + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_buffer_pool_size'), + int(helper.human_to_bytes("10G"))) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_wait_timeout(self, mog, mem): + mem.return_value = "100G" + + timeout = 314 + self.config_data = { + 'wait-timeout': timeout, + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('wait_timeout'), + timeout) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_tuning_level(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'safest', + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_flush_log_at_trx_commit'), + 1 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_tuning_level_fast(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'fast', + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_flush_log_at_trx_commit'), + 2 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_tuning_level_unsafe(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'tuning-level': 'unsafe', + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_flush_log_at_trx_commit'), + 0 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_valid_values(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-change-buffering': 'all', + 'innodb-io-capacity': 100, + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertEqual( + mysql_config.get('innodb_change_buffering'), + 'all' + ) + + self.assertEqual( + mysql_config.get('innodb_io_capacity'), + 100 + ) + + @mock.patch.object(mysql.MySQLConfigHelper, 'get_mem_total') + @mock.patch.object(mysql, 'log') + def test_parse_config_innodb_invalid_values(self, mog, mem): + mem.return_value = "512M" + self.config_data = { + 'innodb-change-buffering': 'invalid', + } + + helper = mysql.PerconaClusterHelper() + mysql_config = helper.parse_config() + + self.assertTrue('innodb_change_buffering' not in mysql_config) + self.assertTrue('innodb_io_capacity' not in mysql_config) + + +class Mysql8Tests(unittest.TestCase): + + def setUp(self): + super(Mysql8Tests, self).setUp() + self.template = "/tmp/mysql-passwd.txt" + self.connection = mock.MagicMock() + self.cursor = mock.MagicMock() + self.connection.cursor.return_value = self.cursor + self.helper = mysql.MySQL8Helper( + rpasswdf_template=self.template, + upasswdf_template=self.template) + self.helper.connection = self.connection + self.user = "user" + self.host = "10.5.0.21" + self.password = "passwd" + self.db = "mydb" + + def test_grant_exists(self): + # With backticks + self.cursor.fetchall.return_value = ( + ("GRANT USAGE ON *.* TO `{}`@`{}`".format(self.user, self.host),), + ("GRANT ALL PRIVILEGES ON `{}`.* TO `{}`@`{}`" + .format(self.db, self.user, self.host),)) + self.assertTrue(self.helper.grant_exists(self.db, self.user, self.host)) + + self.cursor.execute.assert_called_with( + "SHOW GRANTS FOR '{}'@'{}'".format(self.user, self.host)) + + # With single quotes + self.cursor.fetchall.return_value = ( + ("GRANT USAGE ON *.* TO '{}'@'{}'".format(self.user, self.host),), + ("GRANT ALL PRIVILEGES ON '{}'.* TO '{}'@'{}'" + .format(self.db, self.user, self.host),)) + self.assertTrue(self.helper.grant_exists(self.db, self.user, self.host)) + + # Grant not there + self.cursor.fetchall.return_value = ( + ("GRANT USAGE ON *.* TO '{}'@'{}'".format("someuser", "notmyhost"),), + ("GRANT ALL PRIVILEGES ON '{}'.* TO '{}'@'{}'" + .format("somedb", "someuser", "notmyhost"),)) + self.assertFalse(self.helper.grant_exists(self.db, self.user, self.host)) + + def test_create_grant(self): + self.helper.grant_exists = mock.MagicMock(return_value=False) + self.helper.create_user = mock.MagicMock() + + self.helper.create_grant(self.db, self.user, self.host, self.password) + self.cursor.execute.assert_called_with( + "GRANT ALL PRIVILEGES ON `{}`.* TO '{}'@'{}'" + .format(self.db, self.user, self.host)) + self.helper.create_user.assert_called_with(self.user, self.host, self.password) + + def test_create_user(self): + self.helper.create_user(self.user, self.host, self.password) + self.cursor.execute.assert_called_with( + "CREATE USER '{}'@'{}' IDENTIFIED BY '{}'". + format(self.user, self.host, self.password)) + + def test_create_router_grant(self): + self.helper.create_user = mock.MagicMock() + + self.helper.create_router_grant(self.user, self.host, self.password) + _calls = [ + mock.call("GRANT CREATE USER ON *.* TO '{}'@'{}' WITH GRANT OPTION" + .format(self.user, self.host)), + mock.call("GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON " + "mysql_innodb_cluster_metadata.* TO '{}'@'{}'" + .format(self.user, self.host)), + mock.call("GRANT SELECT ON mysql.user TO '{}'@'{}'" + .format(self.user, self.host)), + mock.call("GRANT SELECT ON " + "performance_schema.replication_group_members TO " + "'{}'@'{}'".format(self.user, self.host)), + mock.call("GRANT SELECT ON " + "performance_schema.replication_group_member_stats TO " + "'{}'@'{}'".format(self.user, self.host)), + mock.call("GRANT SELECT ON " + "performance_schema.global_variables TO " + "'{}'@'{}'".format(self.user, self.host))] + + self.cursor.execute.assert_has_calls(_calls) + self.helper.create_user.assert_called_with(self.user, self.host, self.password) + + def test_configure_router(self): + self.helper.create_user = mock.MagicMock() + self.helper.create_router_grant = mock.MagicMock() + self.helper.normalize_address = mock.MagicMock(return_value=self.host) + self.helper.get_mysql_password = mock.MagicMock(return_value=self.password) + + self.assertEqual(self.password, self.helper.configure_router(self.host, self.user)) + self.helper.create_user.assert_called_with(self.user, self.host, self.password) + self.helper.create_router_grant.assert_called_with(self.user, self.host, self.password) + + +class MysqlHelperTests(unittest.TestCase): + + def setUp(self): + super(MysqlHelperTests, self).setUp() + + def test_get_prefix(self): + _tests = { + "prefix1": "prefix1_username", + "prefix2": "prefix2_database", + "prefix3": "prefix3_hostname"} + + for key in _tests.keys(): + self.assertEqual( + key, + mysql.get_prefix(_tests[key])) + + def test_get_db_data(self): + _unprefixed = "myprefix" + # Test relation data has every variation of shared-db/db-router data + _relation_data = { + "egress-subnets": "10.5.0.43/32", + "ingress-address": "10.5.0.43", + "nova_database": "nova", + "nova_hostname": "10.5.0.43", + "nova_username": "nova", + "novaapi_database": "nova_api", + "novaapi_hostname": "10.5.0.43", + "novaapi_username": "nova", + "novacell0_database": "nova_cell0", + "novacell0_hostname": "10.5.0.43", + "novacell0_username": "nova", + "private-address": "10.5.0.43", + "database": "keystone", + "username": "keystone", + "hostname": "10.5.0.43", + "mysqlrouter_username": + "mysqlrouteruser", + "mysqlrouter_hostname": "10.5.0.43"} + + _expected_data = OrderedDict([ + ('nova', OrderedDict([('database', 'nova'), + ('hostname', '10.5.0.43'), + ('username', 'nova')])), + ('novaapi', OrderedDict([('database', 'nova_api'), + ('hostname', '10.5.0.43'), + ('username', 'nova')])), + ('novacell0', OrderedDict([('database', 'nova_cell0'), + ('hostname', '10.5.0.43'), + ('username', 'nova')])), + ('mysqlrouter', OrderedDict([('username', 'mysqlrouteruser'), + ('hostname', '10.5.0.43')])), + ('myprefix', OrderedDict([('hostname', '10.5.0.43'), + ('database', 'keystone'), + ('username', 'keystone')]))]) + + _results = mysql.get_db_data(_relation_data, unprefixed=_unprefixed) + + for prefix in _expected_data.keys(): + for key in _expected_data[prefix].keys(): + self.assertEqual( + _results[prefix][key], _expected_data[prefix][key]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hahelpers/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hahelpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_apache_utils.py b/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_apache_utils.py new file mode 100644 index 0000000..7b75569 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_apache_utils.py @@ -0,0 +1,133 @@ +from mock import patch, call + +from testtools import TestCase +from tests.helpers import patch_open, FakeRelation + +import charmhelpers.contrib.hahelpers.apache as apache_utils + +cert = ''' + -----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAMO1fWOu8ntUMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTQwNDIyMTUzNDA0WhcNMjQwNDE5MTUzNDA0WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAuk6dmZnMvVxykNidNjbIwXM3ShhMpwCvUmWwpybFAIqhtNTuGJF9Ikp5 +kzB+ThQV1onK8O8YarNGQx+MOISEnlJ5npj3Atp33pKGHRn69lHKGVfJvRbN4A90 +1hTueYsELzfPV2YWm4c6nQiXRT6Cy0yaw/DE8fBTHzAiE9+/XGPsjn5VPv8H6Wa1 +f/d5FblE+RtHP6YpRo9Jh3XAn3iC9fVr8rblS4rk7ev8LfH/yIG2wRVOEPC6lYfu +MEIwPpxKV0c3Z6lqtMOgC5dgzWjrbItnQfB0JaIzSFMMxDhNCJocQRJDQ+0jmj+K +rMGB1QRZlVLZxx0xnv38G0GyfFMv8QIDAQABo1AwTjAdBgNVHQ4EFgQUcxEj7X26 +poFDa0lw40aAKIqyNp0wHwYDVR0jBBgwFoAUcxEj7X26poFDa0lw40aAKIqyNp0w +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAQe6RUCqTYf0Ns8fKfAEb +QSxZKqCst02oC0F3Gm0opWiUetxZqmAYTAjztmlRFIw7hgF/P95SY1ujGLZmiAlU +poOTjQ/i7MvjkXPVCo92izwXi65qRmJGbjduIirOAYtBmBmm3qS9BmoDlLQMVNYn +bwFImc9ar0h+o3/VH1hry+2vEVikXiKK5uKZI6B7ejNYfAWydq6ilzfKIh75W852 +OSbKt3NB/BTZZUdCvK6+B+MoeuzQHDO0/QKBEBfaKFeJki3mdyzFlNbYio1z00rM +E2zl3kh9gkZnMuV1uzHdfKJbtTcNn4hCls5x7T21jn4joADHaVez8FloykBUABu3 +qw== +-----END CERTIFICATE----- +''' + +IDENTITY_NEW_STYLE_CERTS = { + 'identity-service:0': { + 'keystone/0': { + 'ssl_cert_test-cn': 'keystone_provided_cert', + 'ssl_key_test-cn': 'keystone_provided_key', + } + } +} + +IDENTITY_OLD_STYLE_CERTS = { + 'identity-service:0': { + 'keystone/0': { + 'ssl_cert': 'keystone_provided_cert', + 'ssl_key': 'keystone_provided_key', + } + } +} + + +class ApacheUtilsTests(TestCase): + def setUp(self): + super(ApacheUtilsTests, self).setUp() + [self._patch(m) for m in [ + 'log', + 'config_get', + 'relation_get', + 'relation_ids', + 'relation_list', + 'host', + ]] + + def _patch(self, method): + _m = patch.object(apache_utils, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def test_get_cert_from_config(self): + '''Ensure cert and key from charm config override relation''' + self.config_get.side_effect = [ + 'some_ca_cert', # config_get('ssl_cert') + 'some_ca_key', # config_Get('ssl_key') + ] + result = apache_utils.get_cert('test-cn') + self.assertEquals(('some_ca_cert', 'some_ca_key'), result) + + def test_get_ca_cert_from_config(self): + self.config_get.return_value = "some_ca_cert" + self.assertEquals('some_ca_cert', apache_utils.get_ca_cert()) + + def test_get_cert_from_relation(self): + self.config_get.return_value = None + rel = FakeRelation(IDENTITY_NEW_STYLE_CERTS) + self.relation_ids.side_effect = rel.relation_ids + self.relation_list.side_effect = rel.relation_units + self.relation_get.side_effect = rel.get + result = apache_utils.get_cert('test-cn') + self.assertEquals(('keystone_provided_cert', 'keystone_provided_key'), + result) + + def test_get_cert_from_relation_deprecated(self): + self.config_get.return_value = None + rel = FakeRelation(IDENTITY_OLD_STYLE_CERTS) + self.relation_ids.side_effect = rel.relation_ids + self.relation_list.side_effect = rel.relation_units + self.relation_get.side_effect = rel.get + result = apache_utils.get_cert() + self.assertEquals(('keystone_provided_cert', 'keystone_provided_key'), + result) + + def test_get_ca_cert_from_relation(self): + self.config_get.return_value = None + self.relation_ids.side_effect = [['identity-service:0'], + ['identity-credentials:1']] + self.relation_list.return_value = 'keystone/0' + self.relation_get.side_effect = [ + 'keystone_provided_ca', + ] + result = apache_utils.get_ca_cert() + self.relation_ids.assert_has_calls([call('identity-service'), + call('identity-credentials')]) + self.assertEquals('keystone_provided_ca', + result) + + @patch.object(apache_utils.os.path, 'isfile') + def test_retrieve_ca_cert(self, _isfile): + _isfile.return_value = True + with patch_open() as (_open, _file): + _file.read.return_value = cert + self.assertEqual( + apache_utils.retrieve_ca_cert('mycertfile'), + cert) + _open.assert_called_once_with('mycertfile', 'rb') + + @patch.object(apache_utils.os.path, 'isfile') + def test_retrieve_ca_cert_no_file(self, _isfile): + _isfile.return_value = False + with patch_open() as (_open, _file): + self.assertEqual( + apache_utils.retrieve_ca_cert('mycertfile'), + None) + self.assertFalse(_open.called) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_cluster_utils.py b/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_cluster_utils.py new file mode 100644 index 0000000..990f1dc --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hahelpers/test_cluster_utils.py @@ -0,0 +1,590 @@ +from mock import patch, MagicMock, call + +from subprocess import CalledProcessError +from testtools import TestCase + +import charmhelpers.contrib.hahelpers.cluster as cluster_utils + +CRM_STATUS = b''' +Last updated: Thu May 14 14:46:35 2015 +Last change: Thu May 14 14:43:51 2015 via crmd on juju-trusty-machine-1 +Stack: corosync +Current DC: juju-trusty-machine-2 (168108171) - partition with quorum +Version: 1.1.10-42f2063 +3 Nodes configured +4 Resources configured + + +Online: [ juju-trusty-machine-1 juju-trusty-machine-2 juju-trusty-machine-3 ] + + Resource Group: grp_percona_cluster + res_mysql_vip (ocf::heartbeat:IPaddr2): Started juju-trusty-machine-1 + Clone Set: cl_mysql_monitor [res_mysql_monitor] + Started: [ juju-trusty-machine-1 juju-trusty-machine-2 juju-trusty-machine-3 ] +''' + +CRM_DC_NONE = b''' +Last updated: Thu May 14 14:46:35 2015 +Last change: Thu May 14 14:43:51 2015 via crmd on juju-trusty-machine-1 +Stack: corosync +Current DC: NONE +1 Nodes configured, 2 expected votes +0 Resources configured + + +Node node1: UNCLEAN (offline) +''' + + +class ClusterUtilsTests(TestCase): + def setUp(self): + super(ClusterUtilsTests, self).setUp() + [self._patch(m) for m in [ + 'log', + 'relation_ids', + 'relation_list', + 'relation_get', + 'get_unit_hostname', + 'config_get', + 'unit_get', + ]] + + def _patch(self, method): + _m = patch.object(cluster_utils, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def test_is_clustered(self): + '''It determines whether or not a unit is clustered''' + self.relation_ids.return_value = ['ha:0'] + self.relation_list.return_value = ['ha/0'] + self.relation_get.return_value = 'yes' + self.assertTrue(cluster_utils.is_clustered()) + + def test_is_not_clustered(self): + '''It determines whether or not a unit is clustered''' + self.relation_ids.return_value = ['ha:0'] + self.relation_list.return_value = ['ha/0'] + self.relation_get.return_value = None + self.assertFalse(cluster_utils.is_clustered()) + + @patch('subprocess.check_output') + def test_is_crm_dc(self, check_output): + '''It determines its unit is leader''' + self.get_unit_hostname.return_value = 'juju-trusty-machine-2' + check_output.return_value = CRM_STATUS + self.assertTrue(cluster_utils.is_crm_dc()) + + @patch('subprocess.check_output') + def test_is_crm_dc_no_cluster(self, check_output): + '''It is not leader if there is no cluster up''' + def r(*args, **kwargs): + raise CalledProcessError(1, 'crm') + + check_output.side_effect = r + self.assertRaises(cluster_utils.CRMDCNotFound, cluster_utils.is_crm_dc) + + @patch('subprocess.check_output') + def test_is_crm_dc_false(self, check_output): + '''It determines its unit is leader''' + self.get_unit_hostname.return_value = 'juju-trusty-machine-1' + check_output.return_value = CRM_STATUS + self.assertFalse(cluster_utils.is_crm_dc()) + + @patch('subprocess.check_output') + def test_is_crm_dc_current_none(self, check_output): + '''It determines its unit is leader''' + self.get_unit_hostname.return_value = 'juju-trusty-machine-1' + check_output.return_value = CRM_DC_NONE + self.assertRaises(cluster_utils.CRMDCNotFound, cluster_utils.is_crm_dc) + + @patch('subprocess.check_output') + def test_is_crm_leader(self, check_output): + '''It determines its unit is leader''' + self.get_unit_hostname.return_value = 'node1' + crm = b'resource vip is running on: node1' + check_output.return_value = crm + self.assertTrue(cluster_utils.is_crm_leader('vip')) + + @patch('charmhelpers.core.decorators.time') + @patch('subprocess.check_output') + def test_is_not_leader(self, check_output, mock_time): + '''It determines its unit is not leader''' + self.get_unit_hostname.return_value = 'node1' + crm = b'resource vip is running on: node2' + check_output.return_value = crm + self.assertFalse(cluster_utils.is_crm_leader('some_resource')) + self.assertFalse(mock_time.called) + + @patch('charmhelpers.core.decorators.log') + @patch('charmhelpers.core.decorators.time') + @patch('subprocess.check_output') + def test_is_not_leader_resource_not_exists(self, check_output, mock_time, + mock_log): + '''It determines its unit is not leader''' + self.get_unit_hostname.return_value = 'node1' + check_output.return_value = "resource vip is NOT running" + self.assertRaises(cluster_utils.CRMResourceNotFound, + cluster_utils.is_crm_leader, 'vip') + mock_time.assert_has_calls([call.sleep(2), call.sleep(4), + call.sleep(6)]) + + @patch('charmhelpers.core.decorators.time') + @patch('subprocess.check_output') + def test_is_crm_leader_no_cluster(self, check_output, mock_time): + '''It is not leader if there is no cluster up''' + check_output.side_effect = CalledProcessError(1, 'crm') + self.assertFalse(cluster_utils.is_crm_leader('vip')) + self.assertFalse(mock_time.called) + + @patch.object(cluster_utils, 'is_crm_dc') + def test_is_crm_leader_dc_resource(self, _is_crm_dc): + '''Call out to is_crm_dc''' + cluster_utils.is_crm_leader(cluster_utils.DC_RESOURCE_NAME) + _is_crm_dc.assert_called_with() + + def test_peer_units(self): + '''It lists all peer units for cluster relation''' + peers = ['peer_node/1', 'peer_node/2'] + self.relation_ids.return_value = ['cluster:0'] + self.relation_list.return_value = peers + self.assertEquals(peers, cluster_utils.peer_units()) + + def test_peer_ips(self): + '''Get a dict of peers and their ips''' + peers = { + 'peer_node/1': '10.0.0.1', + 'peer_node/2': '10.0.0.2', + } + + def _relation_get(attr, rid, unit): + return peers[unit] + self.relation_ids.return_value = ['cluster:0'] + self.relation_list.return_value = peers.keys() + self.relation_get.side_effect = _relation_get + self.assertEquals(peers, cluster_utils.peer_ips()) + + @patch('os.getenv') + def test_is_oldest_peer(self, getenv): + '''It detects local unit is the oldest of all peers''' + peers = ['peer_node/1', 'peer_node/2', 'peer_node/3'] + getenv.return_value = 'peer_node/1' + self.assertTrue(cluster_utils.oldest_peer(peers)) + + @patch('os.getenv') + def test_is_not_oldest_peer(self, getenv): + '''It detects local unit is not the oldest of all peers''' + peers = ['peer_node/1', 'peer_node/2', 'peer_node/3'] + getenv.return_value = 'peer_node/2' + self.assertFalse(cluster_utils.oldest_peer(peers)) + + @patch.object(cluster_utils, 'is_crm_leader') + @patch.object(cluster_utils, 'is_clustered') + def test_is_elected_leader_clustered(self, is_clustered, is_crm_leader): + '''It detects it is the eligible leader in a hacluster of units''' + is_clustered.return_value = True + is_crm_leader.return_value = True + self.assertTrue(cluster_utils.is_elected_leader('vip')) + + @patch.object(cluster_utils, 'is_crm_leader') + @patch.object(cluster_utils, 'is_clustered') + def test_not_is_elected_leader_clustered(self, is_clustered, is_crm_leader): + '''It detects it is not the eligible leader in a hacluster of units''' + is_clustered.return_value = True + is_crm_leader.return_value = False + self.assertFalse(cluster_utils.is_elected_leader('vip')) + + @patch.object(cluster_utils, 'oldest_peer') + @patch.object(cluster_utils, 'peer_units') + @patch.object(cluster_utils, 'is_clustered') + def test_is_is_elected_leader_unclustered(self, is_clustered, + peer_units, oldest_peer): + '''It detects it is the eligible leader in non-clustered peer group''' + is_clustered.return_value = False + oldest_peer.return_value = True + self.assertTrue(cluster_utils.is_elected_leader('vip')) + + @patch.object(cluster_utils, 'oldest_peer') + @patch.object(cluster_utils, 'peer_units') + @patch.object(cluster_utils, 'is_clustered') + def test_not_is_elected_leader_unclustered(self, is_clustered, + peer_units, oldest_peer): + '''It detects it is not the eligible leader in non-clustered group''' + is_clustered.return_value = False + oldest_peer.return_value = False + self.assertFalse(cluster_utils.is_elected_leader('vip')) + + def test_https_explict(self): + '''It determines https is available if configured explicitly''' + # config_get('use-https') + self.config_get.return_value = 'yes' + self.assertTrue(cluster_utils.https()) + + def test_https_cert_key_in_config(self): + '''It determines https is available if cert + key in charm config''' + # config_get('use-https') + self.config_get.side_effect = [ + 'no', # config_get('use-https') + 'cert', # config_get('ssl_cert') + 'key', # config_get('ssl_key') + ] + self.assertTrue(cluster_utils.https()) + + def test_https_cert_key_in_identity_relation(self): + '''It determines https is available if cert in identity-service''' + self.config_get.return_value = False + self.relation_ids.return_value = 'identity-service:0' + self.relation_list.return_value = 'keystone/0' + self.relation_get.side_effect = [ + 'yes', # relation_get('https_keystone') + 'cert', # relation_get('ssl_cert') + 'key', # relation_get('ssl_key') + 'ca_cert', # relation_get('ca_cert') + ] + self.assertTrue(cluster_utils.https()) + + def test_https_cert_key_incomplete_identity_relation(self): + '''It determines https unavailable if cert not in identity-service''' + self.config_get.return_value = False + self.relation_ids.return_value = 'identity-service:0' + self.relation_list.return_value = 'keystone/0' + self.relation_get.return_value = None + self.assertFalse(cluster_utils.https()) + + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'peer_units') + def test_determine_api_port_with_peers(self, peer_units, https): + '''It determines API port in presence of peers''' + peer_units.return_value = ['peer1'] + https.return_value = False + self.assertEquals(9686, cluster_utils.determine_api_port(9696)) + + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'peer_units') + def test_determine_api_port_nopeers_singlemode(self, peer_units, https): + '''It determines API port with a single unit in singlemode''' + peer_units.return_value = [] + https.return_value = False + port = cluster_utils.determine_api_port(9696, singlenode_mode=True) + self.assertEquals(9686, port) + + @patch.object(cluster_utils, 'is_clustered') + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'peer_units') + def test_determine_api_port_clustered(self, peer_units, https, + is_clustered): + '''It determines API port in presence of an hacluster''' + peer_units.return_value = [] + is_clustered.return_value = True + https.return_value = False + self.assertEquals(9686, cluster_utils.determine_api_port(9696)) + + @patch.object(cluster_utils, 'is_clustered') + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'peer_units') + def test_determine_api_port_clustered_https(self, peer_units, https, + is_clustered): + '''It determines API port in presence of hacluster + https''' + peer_units.return_value = [] + is_clustered.return_value = True + https.return_value = True + self.assertEquals(9676, cluster_utils.determine_api_port(9696)) + + @patch.object(cluster_utils, 'https') + def test_determine_apache_port_https(self, https): + '''It determines haproxy port with https enabled''' + https.return_value = True + self.assertEquals(9696, cluster_utils.determine_apache_port(9696)) + + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'is_clustered') + def test_determine_apache_port_clustered(self, https, is_clustered): + '''It determines haproxy port with https disabled''' + https.return_value = True + is_clustered.return_value = True + self.assertEquals(9686, cluster_utils.determine_apache_port(9696)) + + @patch.object(cluster_utils, 'peer_units') + @patch.object(cluster_utils, 'https') + @patch.object(cluster_utils, 'is_clustered') + def test_determine_apache_port_nopeers_singlemode(self, https, + is_clustered, + peer_units): + '''It determines haproxy port with a single unit in singlemode''' + peer_units.return_value = [] + https.return_value = False + is_clustered.return_value = False + port = cluster_utils.determine_apache_port(9696, singlenode_mode=True) + self.assertEquals(9686, port) + + @patch.object(cluster_utils, 'valid_hacluster_config') + def test_get_hacluster_config_complete(self, valid_hacluster_config): + '''It fetches all hacluster charm config''' + conf = { + 'ha-bindiface': 'eth1', + 'ha-mcastport': '3333', + 'vip': '10.0.0.1', + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + } + + valid_hacluster_config.return_value = True + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertEquals(conf, cluster_utils.get_hacluster_config()) + + @patch.object(cluster_utils, 'valid_hacluster_config') + def test_get_hacluster_config_incomplete(self, valid_hacluster_config): + '''It raises exception if some hacluster charm config missing''' + conf = { + 'ha-bindiface': 'eth1', + 'ha-mcastport': '3333', + 'vip': None, + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + } + + valid_hacluster_config.return_value = False + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertRaises(cluster_utils.HAIncorrectConfig, + cluster_utils.get_hacluster_config) + + @patch.object(cluster_utils, 'valid_hacluster_config') + def test_get_hacluster_config_with_excludes(self, valid_hacluster_config): + '''It fetches all hacluster charm config''' + conf = { + 'ha-bindiface': 'eth1', + 'ha-mcastport': '3333', + } + valid_hacluster_config.return_value = True + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + exclude_keys = ['vip', 'os-admin-hostname', 'os-internal-hostname', + 'os-public-hostname', 'os-access-hostname'] + result = cluster_utils.get_hacluster_config(exclude_keys) + self.assertEquals(conf, result) + + @patch.object(cluster_utils, 'is_clustered') + def test_canonical_url_bare(self, is_clustered): + '''It constructs a URL to host with no https or cluster present''' + self.unit_get.return_value = 'foohost1' + is_clustered.return_value = False + configs = MagicMock() + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + url = cluster_utils.canonical_url(configs) + self.assertEquals('http://foohost1', url) + + @patch.object(cluster_utils, 'is_clustered') + def test_canonical_url_https_no_cluster(self, is_clustered): + '''It constructs a URL to host with https and no cluster present''' + self.unit_get.return_value = 'foohost1' + is_clustered.return_value = False + configs = MagicMock() + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['https'] + url = cluster_utils.canonical_url(configs) + self.assertEquals('https://foohost1', url) + + @patch.object(cluster_utils, 'is_clustered') + def test_canonical_url_https_cluster(self, is_clustered): + '''It constructs a URL to host with https and cluster present''' + self.config_get.return_value = '10.0.0.1' + is_clustered.return_value = True + configs = MagicMock() + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = ['https'] + url = cluster_utils.canonical_url(configs) + self.assertEquals('https://10.0.0.1', url) + + @patch.object(cluster_utils, 'is_clustered') + def test_canonical_url_cluster_no_https(self, is_clustered): + '''It constructs a URL to host with no https and cluster present''' + self.config_get.return_value = '10.0.0.1' + self.unit_get.return_value = 'foohost1' + is_clustered.return_value = True + configs = MagicMock() + configs.complete_contexts = MagicMock() + configs.complete_contexts.return_value = [] + url = cluster_utils.canonical_url(configs) + self.assertEquals('http://10.0.0.1', url) + + @patch.object(cluster_utils, 'status_set') + def test_valid_hacluster_config_incomplete(self, status_set): + '''Returns False with incomplete HA config''' + conf = { + 'vip': None, + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + 'dns-ha': False, + } + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertRaises(cluster_utils.HAIncorrectConfig, + cluster_utils.valid_hacluster_config) + + @patch.object(cluster_utils, 'status_set') + def test_valid_hacluster_config_both(self, status_set): + '''Returns False when both VIP and DNS HA are set''' + conf = { + 'vip': '10.0.0.1', + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + 'dns-ha': True, + } + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertRaises(cluster_utils.HAIncorrectConfig, + cluster_utils.valid_hacluster_config) + + @patch.object(cluster_utils, 'status_set') + def test_valid_hacluster_config_vip_ha(self, status_set): + '''Returns True with complete VIP HA config''' + conf = { + 'vip': '10.0.0.1', + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + 'dns-ha': False, + } + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertTrue(cluster_utils.valid_hacluster_config()) + self.assertFalse(status_set.called) + + @patch.object(cluster_utils, 'status_set') + def test_valid_hacluster_config_dns_incomplete(self, status_set): + '''Returns False with incomplete DNS HA config''' + conf = { + 'vip': None, + 'os-admin-hostname': None, + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + 'dns-ha': True, + } + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertRaises(cluster_utils.HAIncompleteConfig, + cluster_utils.valid_hacluster_config) + + @patch.object(cluster_utils, 'status_set') + def test_valid_hacluster_config_dns_ha(self, status_set): + '''Returns True with complete DNS HA config''' + conf = { + 'vip': None, + 'os-admin-hostname': 'somehostname', + 'os-public-hostname': None, + 'os-internal-hostname': None, + 'os-access-hostname': None, + 'dns-ha': True, + } + + def _fake_config_get(setting): + return conf[setting] + + self.config_get.side_effect = _fake_config_get + self.assertTrue(cluster_utils.valid_hacluster_config()) + self.assertFalse(status_set.called) + + @patch.object(cluster_utils, 'juju_is_leader') + @patch.object(cluster_utils, 'status_set') + @patch.object(cluster_utils.time, 'sleep') + @patch.object(cluster_utils, 'modulo_distribution') + @patch.object(cluster_utils, 'log') + def test_distributed_wait(self, log, modulo_distribution, sleep, + status_set, is_leader): + + # Leader regardless of modulo should not wait + is_leader.return_value = True + cluster_utils.distributed_wait(modulo=9, wait=23) + modulo_distribution.assert_not_called() + sleep.assert_called_with(0) + + # The rest of the tests are non-leader units + is_leader.return_value = False + + def _fake_config_get(setting): + return conf[setting] + + # Uses fallback defaults + conf = { + 'modulo-nodes': None, + 'known-wait': None, + } + self.config_get.side_effect = _fake_config_get + cluster_utils.distributed_wait() + modulo_distribution.assert_called_with(modulo=3, wait=30, + non_zero_wait=True) + + # Uses config values + conf = { + 'modulo-nodes': 7, + 'known-wait': 10, + } + self.config_get.side_effect = _fake_config_get + cluster_utils.distributed_wait() + modulo_distribution.assert_called_with(modulo=7, wait=10, + non_zero_wait=True) + + # Uses passed values + cluster_utils.distributed_wait(modulo=5, wait=45) + modulo_distribution.assert_called_with(modulo=5, wait=45, + non_zero_wait=True) + + @patch.object(cluster_utils, 'relation_ids') + def test_get_managed_services_and_ports(self, relation_ids): + relation_ids.return_value = ['rel:2'] + self.assertEqual( + cluster_utils.get_managed_services_and_ports( + ['apache2', 'haproxy'], + [8067, 4545, 6732]), + (['apache2'], [8057, 4535, 6722])) + self.assertEqual( + cluster_utils.get_managed_services_and_ports( + ['apache2', 'haproxy'], + [8067, 4545, 6732], + external_services=['apache2']), + (['haproxy'], [8057, 4535, 6722])) + + def add_ten(x): + return x + 10 + + self.assertEqual( + cluster_utils.get_managed_services_and_ports( + ['apache2', 'haproxy'], + [8067, 4545, 6732], + port_conv_f=add_ten), + (['apache2'], [8077, 4555, 6742])) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/checks/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/checks/test_config.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/checks/test_config.py new file mode 100644 index 0000000..da080da --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/apache/checks/test_config.py @@ -0,0 +1,110 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +from unittest import TestCase +from mock import patch + +from charmhelpers.contrib.hardening.apache.checks import config + +TEST_TMPDIR = None +APACHE_VERSION_STR = b"""Server version: Apache/2.4.7 (Ubuntu) +Server built: Jan 14 2016 17:45:23 +""" + + +class ApacheConfigTestCase(TestCase): + + def setUp(self): + global TEST_TMPDIR + TEST_TMPDIR = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(TEST_TMPDIR) + + @patch.object(config.subprocess, 'call', lambda *args, **kwargs: 1) + def test_get_audits_apache_not_installed(self): + audits = config.get_audits() + self.assertEqual([], audits) + + @patch.object(config.utils, 'get_settings', lambda x: { + 'common': {'apache_dir': TEST_TMPDIR, + 'traceenable': 'Off'}, + 'hardening': { + 'allowed_http_methods': {'GOGETEM'}, + 'modules_to_disable': {'modfoo'} + } + }) + @patch.object(config.subprocess, 'call', lambda *args, **kwargs: 0) + def test_get_audits_apache_is_installed(self): + audits = config.get_audits() + self.assertEqual(7, len(audits)) + + @patch.object(config.utils, 'get_settings', lambda x: { + 'common': {'apache_dir': TEST_TMPDIR}, + 'hardening': { + 'allowed_http_methods': {'GOGETEM'}, + 'modules_to_disable': {'modfoo'}, + 'traceenable': 'off', + 'servertokens': 'Prod', + 'honor_cipher_order': 'on', + 'cipher_suite': 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' + } + }) + @patch.object(config, 'subprocess') + def test_ApacheConfContext(self, mock_subprocess): + mock_subprocess.call.return_value = 0 + + with tempfile.NamedTemporaryFile() as ftmp: # noqa + def fake_check_output(cmd, *args, **kwargs): + if cmd[0] == 'apache2': + return APACHE_VERSION_STR + + mock_subprocess.check_output.side_effect = fake_check_output + ctxt = config.ApacheConfContext() + self.assertEqual(ctxt(), { + 'allowed_http_methods': set(['GOGETEM']), + 'apache_icondir': + '/usr/share/apache2/icons/', + 'apache_version': '2.4.7', + 'modules_to_disable': set(['modfoo']), + 'servertokens': 'Prod', + 'traceenable': 'off', + 'honor_cipher_order': 'on', + 'cipher_suite': 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' + }) + + @patch.object(config.utils, 'get_settings', lambda x: { + 'common': {'apache_dir': TEST_TMPDIR}, + 'hardening': { + 'allowed_http_methods': {'GOGETEM'}, + 'modules_to_disable': {'modfoo'}, + 'traceenable': 'off', + 'servertokens': 'Prod', + 'honor_cipher_order': 'on', + 'cipher_suite': 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' + } + }) + @patch.object(config.subprocess, 'call', lambda *args, **kwargs: 0) + def test_file_permission_audit(self): + audits = config.get_audits() + settings = config.utils.get_settings('apache') + conf_file_name = 'apache2.conf' + conf_file_path = os.path.join( + settings['common']['apache_dir'], conf_file_name + ) + self.assertEqual(audits[0].paths[0], conf_file_path) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/__init__.py new file mode 100644 index 0000000..30a3e94 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apache_audits.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apache_audits.py new file mode 100644 index 0000000..c62b1a5 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apache_audits.py @@ -0,0 +1,97 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mock import call +from mock import patch + +from unittest import TestCase +from charmhelpers.contrib.hardening.audits import apache +from subprocess import CalledProcessError + + +class DisabledModuleAuditsTest(TestCase): + + def setup(self): + super(DisabledModuleAuditsTest, self).setUp() + self._patch_obj(apache, 'log') + + def _patch_obj(self, obj, method): + _m = patch.object(obj, method) + mock = _m.start() + self.addCleanup(mock.stop) + setattr(self, method, mock) + + def test_init_string(self): + audit = apache.DisabledModuleAudit('foo') + self.assertEqual(['foo'], audit.modules) + + def test_init_list(self): + audit = apache.DisabledModuleAudit(['foo', 'bar']) + self.assertEqual(['foo', 'bar'], audit.modules) + + @patch.object(apache.DisabledModuleAudit, '_get_loaded_modules') + def test_ensure_compliance_no_modules(self, mock_loaded_modules): + audit = apache.DisabledModuleAudit(None) + audit.ensure_compliance() + self.assertFalse(mock_loaded_modules.called) + + @patch.object(apache.DisabledModuleAudit, '_get_loaded_modules') + @patch.object(apache, 'log', lambda *args, **kwargs: None) + def test_ensure_compliance_loaded_modules_raises_ex(self, + mock_loaded_modules): + mock_loaded_modules.side_effect = CalledProcessError(1, 'test', 'err') + audit = apache.DisabledModuleAudit('foo') + audit.ensure_compliance() + + @patch.object(apache.DisabledModuleAudit, '_get_loaded_modules') + @patch.object(apache.DisabledModuleAudit, '_disable_module') + @patch.object(apache, 'log', lambda *args, **kwargs: None) + def test_disabled_modules_not_loaded(self, mock_disable_module, + mock_loaded_modules): + mock_loaded_modules.return_value = ['foo'] + audit = apache.DisabledModuleAudit('bar') + audit.ensure_compliance() + self.assertFalse(mock_disable_module.called) + + @patch.object(apache.DisabledModuleAudit, '_get_loaded_modules') + @patch.object(apache.DisabledModuleAudit, '_disable_module') + @patch.object(apache.DisabledModuleAudit, '_restart_apache') + @patch.object(apache, 'log', lambda *args, **kwargs: None) + def test_disabled_modules_loaded(self, mock_restart_apache, + mock_disable_module, mock_loaded_modules): + mock_loaded_modules.return_value = ['foo', 'bar'] + audit = apache.DisabledModuleAudit('bar') + audit.ensure_compliance() + mock_disable_module.assert_has_calls([call('bar')]) + mock_restart_apache.assert_has_calls([call()]) + + @patch('subprocess.check_output') + def test_get_loaded_modules(self, mock_check_output): + mock_check_output.return_value = (b'Loaded Modules:\n' + b' foo_module (static)\n' + b' bar_module (shared)\n') + audit = apache.DisabledModuleAudit('bar') + result = audit._get_loaded_modules() + self.assertEqual(['foo', 'bar'], result) + + @patch('subprocess.check_output') + def test_is_ssl_enabled(self, mock_check_output): + mock_check_output.return_value = (b'Loaded Modules:\n' + b' foo_module (static)\n' + b' bar_module (shared)\n' + b' ssl_module (shared)\n') + audit = apache.DisabledModuleAudit('bar') + result = audit._get_loaded_modules() + self.assertEqual(['foo', 'bar', 'ssl'], result) + self.assertTrue(audit.is_ssl_enabled()) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apt_audits.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apt_audits.py new file mode 100644 index 0000000..fc14b2a --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_apt_audits.py @@ -0,0 +1,87 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import call +from mock import MagicMock +from mock import patch + +from charmhelpers.contrib.hardening.audits import apt +from charmhelpers.fetch import ubuntu_apt_pkg as apt_pkg +from charmhelpers.core import hookenv + + +class RestrictedPackagesTestCase(TestCase): + def setUp(self): + super(RestrictedPackagesTestCase, self).setUp() + + def create_package(self, name, virtual=False): + pkg = MagicMock() + pkg.name = name + pkg.current_ver = '2.0' + if virtual: + pkgver = MagicMock() + pkgver.parent_pkg = self.create_package('foo') + pkg.provides_list = [('virtualfoo', None, pkgver)] + resp = { + 'has_provides': True, + 'has_versions': False, + } + pkg.get.side_effect = resp.get + + return pkg + + @patch.object(apt, 'apt_cache') + @patch.object(apt, 'apt_purge') + @patch.object(apt, 'log', lambda *args, **kwargs: None) + def test_ensure_compliance(self, mock_purge, mock_apt_cache): + pkg = self.create_package('bar') + mock_apt_cache.return_value = {'bar': pkg} + + audit = apt.RestrictedPackages(pkgs=['bar']) + audit.ensure_compliance() + mock_purge.assert_has_calls([call(pkg.name)]) + + @patch.object(apt, 'apt_purge') + @patch.object(apt, 'apt_cache') + @patch.object(apt, 'log', lambda *args, **kwargs: None) + def test_apt_harden_virtual_package(self, mock_apt_cache, mock_apt_purge): + vpkg = self.create_package('virtualfoo', virtual=True) + mock_apt_cache.return_value = {'foo': vpkg} + audit = apt.RestrictedPackages(pkgs=['foo']) + audit.ensure_compliance() + self.assertTrue(mock_apt_cache.called) + mock_apt_purge.assert_has_calls([call('foo')]) + + +class AptConfigTestCase(TestCase): + + @patch.object(apt, 'apt_pkg') + def test_ensure_compliance(self, mock_apt_pkg): + mock_apt_pkg.init.return_value = None + mock_apt_pkg.config.side_effect = {} + mock_apt_pkg.config.get.return_value = None + audit = apt.AptConfig([{'key': 'APT::Get::AllowUnauthenticated', + 'expected': 'false'}]) + audit.ensure_compliance() + self.assertTrue(mock_apt_pkg.config.get.called) + + @patch.object(hookenv, 'log') + def test_verify_config(self, mock_log): + cfg = apt_pkg.config + key, value = list(cfg.items())[0] + audit = apt.AptConfig([{"key": key, "expected": value}]) + audit.ensure_compliance() + self.assertFalse(mock_log.called) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_base_audits.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_base_audits.py new file mode 100644 index 0000000..cf9f236 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_base_audits.py @@ -0,0 +1,52 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from charmhelpers.contrib.hardening.audits import BaseAudit + + +class BaseAuditTestCase(TestCase): + + def setUp(self): + super(BaseAuditTestCase, self).setUp() + + def test_take_action_default(self): + check = BaseAudit() + take_action = check._take_action() + self.assertTrue(take_action) + + def test_take_action_unless_true(self): + check = BaseAudit(unless=True) + take_action = check._take_action() + self.assertFalse(take_action) + + def test_take_action_unless_false(self): + check = BaseAudit(unless=False) + take_action = check._take_action() + self.assertTrue(take_action) + + def test_take_action_unless_callback_false(self): + def callback(): + return False + check = BaseAudit(unless=callback) + take_action = check._take_action() + self.assertTrue(take_action) + + def test_take_action_unless_callback_true(self): + def callback(): + return True + check = BaseAudit(unless=callback) + take_action = check._take_action() + self.assertFalse(take_action) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_file_audits.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_file_audits.py new file mode 100644 index 0000000..8718f61 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/audits/test_file_audits.py @@ -0,0 +1,334 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile + +from mock import call, patch + +from unittest import TestCase + +from charmhelpers.core import unitdata +from charmhelpers.contrib.hardening.audits import file + + +@patch('os.path.exists') +class BaseFileAuditTestCase(TestCase): + + def setUp(self): + super(BaseFileAuditTestCase, self).setUp() + self._patch_obj(file.BaseFileAudit, 'is_compliant') + self._patch_obj(file.BaseFileAudit, 'comply') + self._patch_obj(file, 'log') + + def _patch_obj(self, obj, method): + _m = patch.object(obj, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def test_ensure_compliance(self, mock_exists): + mock_exists.return_value = False + check = file.BaseFileAudit(paths='/tmp/foo') + check.ensure_compliance() + self.assertFalse(self.comply.called) + + def test_ensure_compliance_in_compliance(self, mock_exists): + mock_exists.return_value = True + self.is_compliant.return_value = True + check = file.BaseFileAudit(paths=['/tmp/foo']) + check.ensure_compliance() + mock_exists.assert_has_calls([call('/tmp/foo')]) + self.is_compliant.assert_has_calls([call('/tmp/foo')]) + self.assertFalse(self.log.called) + self.assertFalse(self.comply.called) + + def test_ensure_compliance_out_of_compliance(self, mock_exists): + mock_exists.return_value = True + self.is_compliant.return_value = False + check = file.BaseFileAudit(paths=['/tmp/foo']) + check.ensure_compliance() + mock_exists.assert_has_calls([call('/tmp/foo')]) + self.is_compliant.assert_has_calls([call('/tmp/foo')]) + self.assertTrue(self.log.called) + self.comply.assert_has_calls([call('/tmp/foo')]) + + +class EasyMock(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + + +class FilePermissionAuditTestCase(TestCase): + def setUp(self): + super(FilePermissionAuditTestCase, self).setUp() + self._patch_obj(file.grp, 'getgrnam') + self._patch_obj(file.pwd, 'getpwnam') + self._patch_obj(file.FilePermissionAudit, '_get_stat') + self.getpwnam.return_value = EasyMock({'pw_name': 'testuser', + 'pw_uid': 1000}) + self.getgrnam.return_value = EasyMock({'gr_name': 'testgroup', + 'gr_gid': 1000}) + self._get_stat.return_value = EasyMock({'st_mode': 0o644, + 'st_uid': 1000, + 'st_gid': 1000}) + + def _patch_obj(self, obj, method): + _m = patch.object(obj, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def test_is_compliant(self): + check = file.FilePermissionAudit(paths=['/foo/bar'], + user='testuser', + group='testgroup', mode=0o644) + compliant = check.is_compliant('/foo/bar') + self.assertTrue(compliant) + + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_not_compliant_wrong_group(self): + self.getgrnam.return_value = EasyMock({'gr_name': 'testgroup', + 'gr_gid': 222}) + check = file.FilePermissionAudit(paths=['/foo/bar'], user='testuser', + group='testgroup', mode=0o644) + compliant = check.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_not_compliant_wrong_user(self): + self.getpwnam.return_value = EasyMock({'pw_name': 'fred', + 'pw_uid': 123}) + check = file.FilePermissionAudit(paths=['/foo/bar'], user='testuser', + group='testgroup', mode=0o644) + compliant = check.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_not_compliant_wrong_permissions(self): + self._get_stat.return_value = EasyMock({'st_mode': 0o777, + 'st_uid': 1000, + 'st_gid': 1000}) + check = file.FilePermissionAudit(paths=['/foo/bar'], user='testuser', + group='testgroup', mode=0o644) + compliant = check.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch('charmhelpers.contrib.hardening.utils.ensure_permissions') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_comply(self, _ensure_permissions): + check = file.FilePermissionAudit(paths=['/foo/bar'], user='testuser', + group='testgroup', mode=0o644) + check.comply('/foo/bar') + c = call('/foo/bar', 'testuser', 'testgroup', 0o644) + _ensure_permissions.assert_has_calls([c]) + + +class DirectoryPermissionAuditTestCase(TestCase): + def setUp(self): + super(DirectoryPermissionAuditTestCase, self).setUp() + + @patch('charmhelpers.contrib.hardening.audits.file.os.path.isdir') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_compliant_not_directory(self, mock_isdir): + mock_isdir.return_value = False + check = file.DirectoryPermissionAudit(paths=['/foo/bar'], + user='testuser', + group='testgroup', mode=0o0700) + self.assertRaises(ValueError, check.is_compliant, '/foo/bar') + + @patch.object(file.FilePermissionAudit, 'is_compliant') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_compliant_file_not_compliant(self, mock_is_compliant): + mock_is_compliant.return_value = False + tmpdir = tempfile.mkdtemp() + try: + check = file.DirectoryPermissionAudit(paths=[tmpdir], + user='testuser', + group='testgroup', + mode=0o0700) + compliant = check.is_compliant(tmpdir) + self.assertFalse(compliant) + finally: + shutil.rmtree(tmpdir) + + +class NoSUIDGUIDAuditTestCase(TestCase): + def setUp(self): + super(NoSUIDGUIDAuditTestCase, self).setUp() + + @patch.object(file.NoSUIDSGIDAudit, '_get_stat') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_compliant(self, mock_get_stat): + mock_get_stat.return_value = EasyMock({'st_mode': 0o0644, + 'st_uid': 0, + 'st_gid': 0}) + audit = file.NoSUIDSGIDAudit('/foo/bar') + compliant = audit.is_compliant('/foo/bar') + self.assertTrue(compliant) + + @patch.object(file.NoSUIDSGIDAudit, '_get_stat') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_noncompliant(self, mock_get_stat): + mock_get_stat.return_value = EasyMock({'st_mode': 0o6644, + 'st_uid': 0, + 'st_gid': 0}) + audit = file.NoSUIDSGIDAudit('/foo/bar') + compliant = audit.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch.object(file, 'log') + @patch.object(file, 'check_output') + def test_comply(self, mock_check_output, mock_log): + audit = file.NoSUIDSGIDAudit('/foo/bar') + audit.comply('/foo/bar') + mock_check_output.assert_has_calls([call(['chmod', '-s', '/foo/bar'])]) + self.assertTrue(mock_log.called) + + +class TemplatedFileTestCase(TestCase): + def setUp(self): + super(TemplatedFileTestCase, self).setUp() + self.kv = patch.object(unitdata, 'kv') + self.kv.start() + self.addCleanup(self.kv.stop) + + @patch.object(file.TemplatedFile, 'templates_match') + @patch.object(file.TemplatedFile, 'contents_match') + @patch.object(file.TemplatedFile, 'permissions_match') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_not_compliant(self, contents_match_, permissions_match_, + templates_match_): + contents_match_.return_value = False + permissions_match_.return_value = False + templates_match_.return_value = False + + f = file.TemplatedFile('/foo/bar', None, '/tmp', 0o0644) + compliant = f.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch.object(file.TemplatedFile, 'templates_match') + @patch.object(file.TemplatedFile, 'contents_match') + @patch.object(file.TemplatedFile, 'permissions_match') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_is_compliant(self, contents_match_, permissions_match_, + templates_match_): + contents_match_.return_value = True + permissions_match_.return_value = True + templates_match_.return_value = True + + f = file.TemplatedFile('/foo/bar', None, '/tmp', 0o0644) + compliant = f.is_compliant('/foo/bar') + self.assertTrue(compliant) + + @patch.object(file.TemplatedFile, 'templates_match') + @patch.object(file.TemplatedFile, 'contents_match') + @patch.object(file.TemplatedFile, 'permissions_match') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_template_changes(self, contents_match_, permissions_match_, + templates_match_): + contents_match_.return_value = True + permissions_match_.return_value = True + templates_match_.return_value = False + + f = file.TemplatedFile('/foo/bar', None, '/tmp', 0o0644) + compliant = f.is_compliant('/foo/bar') + self.assertFalse(compliant) + + @patch.object(file, 'render_and_write') + @patch.object(file.utils, 'ensure_permissions') + @patch.object(file, 'log', lambda *args, **kwargs: None) + def test_comply(self, mock_ensure_permissions, mock_render_and_write): + class Context(object): + def __call__(self): + return {} + with tempfile.NamedTemporaryFile() as ftmp: + f = file.TemplatedFile(ftmp.name, Context(), + os.path.dirname(ftmp.name), 0o0644) + f.comply(ftmp.name) + calls = [call(os.path.dirname(ftmp.name), ftmp.name, {})] + mock_render_and_write.assert_has_calls(calls) + mock_ensure_permissions.assert_has_calls([call(ftmp.name, 'root', + 'root', 0o0644)]) + + +CONTENTS_PASS = """Ciphers aes256-ctr,aes192-ctr,aes128-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160 +KexAlgorithms diffie-hellman-group-exchange-sha256 +""" + + +CONTENTS_FAIL = """Ciphers aes256-ctr,aes192-ctr,aes128-ctr +MACs hmac-sha2-512,hmac-sha2-256,hmac-ripemd160 +KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1 +""" + + +class FileContentAuditTestCase(TestCase): + + @patch.object(file, 'log') + def test_audit_contents_pass(self, mock_log): + conditions = {'pass': + [r'^KexAlgorithms\s+diffie-hellman-group-exchange-' + 'sha256$'], + 'fail': [r'^KexAlgorithms\s+diffie-hellman-group-' + 'exchange-sha256.+$']} + with tempfile.NamedTemporaryFile() as ftmp: + filename = ftmp.name + with open(filename, 'w') as fd: + fd.write(CONTENTS_FAIL) + + audit = file.FileContentAudit(filename, conditions) + self.assertFalse(audit.is_compliant(filename)) + + calls = [call("Auditing contents of file '%s'" % filename, + level='DEBUG'), + call("Pattern '^KexAlgorithms\\s+diffie-hellman-group-" + "exchange-sha256$' was expected to pass but instead it " + "failed", level='WARNING'), + call("Pattern '^KexAlgorithms\\s+diffie-hellman-group-" + "exchange-sha256.+$' was expected to fail but instead " + "it passed", level='WARNING'), + call('Checked 2 cases and 0 passed', level='DEBUG')] + mock_log.assert_has_calls(calls) + + @patch.object(file, 'log') + def test_audit_contents_fail(self, mock_log): + conditions = {'pass': + [r'^KexAlgorithms\s+diffie-hellman-group-exchange-' + 'sha256$'], + 'fail': + [r'^KexAlgorithms\s+diffie-hellman-group-exchange-' + 'sha256.+$']} + with tempfile.NamedTemporaryFile() as ftmp: + filename = ftmp.name + with open(filename, 'w') as fd: + fd.write(CONTENTS_FAIL) + + audit = file.FileContentAudit(filename, conditions) + self.assertFalse(audit.is_compliant(filename)) + + calls = [call("Auditing contents of file '%s'" % filename, + level='DEBUG'), + call("Pattern '^KexAlgorithms\\s+diffie-hellman-group-" + "exchange-sha256$' was expected to pass but instead " + "it failed", + level='WARNING'), + call("Pattern '^KexAlgorithms\\s+diffie-hellman-group-" + "exchange-sha256.+$' was expected to fail but instead " + "it passed", + level='WARNING'), + call('Checked 2 cases and 0 passed', level='DEBUG')] + mock_log.assert_has_calls(calls) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/defaults b/nrpe/mod/charmhelpers/tests/contrib/hardening/defaults new file mode 120000 index 0000000..e7fff54 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/defaults @@ -0,0 +1 @@ +../../../charmhelpers/contrib/hardening/defaults \ No newline at end of file diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_apt.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_apt.py new file mode 100644 index 0000000..ea7ed8e --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_apt.py @@ -0,0 +1,46 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import apt + + +class AptHardeningTestCase(TestCase): + + @patch.object(apt, 'get_settings', lambda x: { + 'security': {'packages_clean': False} + }) + def test_dont_clean_packages(self): + audits = apt.get_audits() + self.assertEqual(1, len(audits)) + + @patch.object(apt, 'get_settings', lambda x: { + 'security': {'packages_clean': True, + 'packages_list': []} + }) + def test_no_security_packages(self): + audits = apt.get_audits() + self.assertEqual(1, len(audits)) + + @patch.object(apt, 'get_settings', lambda x: { + 'security': {'packages_clean': True, + 'packages_list': ['foo', 'bar']} + }) + def test_restricted_packages(self): + audits = apt.get_audits() + self.assertEqual(2, len(audits)) + self.assertTrue(isinstance(audits[1], apt.RestrictedPackages)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_limits.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_limits.py new file mode 100644 index 0000000..0750fde --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_limits.py @@ -0,0 +1,43 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import limits + + +class LimitsTestCase(TestCase): + + @patch.object(limits.utils, 'get_settings', + lambda x: {'security': {'kernel_enable_core_dump': False}}) + def test_core_dump_disabled(self): + audits = limits.get_audits() + self.assertEqual(2, len(audits)) + audit = audits[0] + self.assertTrue(isinstance(audit, limits.DirectoryPermissionAudit)) + self.assertEqual('/etc/security/limits.d', audit.paths[0]) + audit = audits[1] + self.assertTrue(isinstance(audit, limits.TemplatedFile)) + self.assertEqual('/etc/security/limits.d/10.hardcore.conf', + audit.paths[0]) + + @patch.object(limits.utils, 'get_settings', lambda x: { + 'security': {'kernel_enable_core_dump': True} + }) + def test_core_dump_enabled(self): + audits = limits.get_audits() + self.assertEqual(1, len(audits)) + self.assertTrue(isinstance(audits[0], limits.DirectoryPermissionAudit)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_login.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_login.py new file mode 100644 index 0000000..e3c6fc8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_login.py @@ -0,0 +1,27 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from charmhelpers.contrib.hardening.host.checks import login + + +class LoginTestCase(TestCase): + + def test_login(self): + audits = login.get_audits() + self.assertEqual(1, len(audits)) + audit = audits[0] + self.assertTrue(isinstance(audit, login.TemplatedFile)) + self.assertEqual('/etc/login.defs', audit.paths[0]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_minimize_access.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_minimize_access.py new file mode 100644 index 0000000..2e38387 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_minimize_access.py @@ -0,0 +1,68 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import minimize_access + + +class MinimizeAccessTestCase(TestCase): + + @patch.object(minimize_access.utils, 'get_settings', lambda x: + {'environment': {'extra_user_paths': []}, + 'security': {'users_allow': []}}) + def test_default_options(self): + audits = minimize_access.get_audits() + self.assertEqual(3, len(audits)) + + # First audit is to ensure that all folders in the $PATH variable + # are read-only. + self.assertTrue(isinstance(audits[0], minimize_access.ReadOnly)) + self.assertEqual({'/usr/local/sbin', '/usr/local/bin', + '/usr/sbin', '/usr/bin', '/bin'}, audits[0].paths) + + # Second audit is to ensure that the /etc/shadow is only readable + # by the root user. + self.assertTrue(isinstance(audits[1], + minimize_access.FilePermissionAudit)) + self.assertEqual(audits[1].paths[0], '/etc/shadow') + self.assertEqual(audits[1].mode, 0o0600) + + # Last audit is to ensure that only root has access to the su + self.assertTrue(isinstance(audits[2], + minimize_access.FilePermissionAudit)) + self.assertEqual(audits[2].paths[0], '/bin/su') + self.assertEqual(audits[2].mode, 0o0750) + + @patch.object(minimize_access.utils, 'get_settings', lambda x: + {'environment': {'extra_user_paths': []}, + 'security': {'users_allow': ['change_user']}}) + def test_allow_change_user(self): + audits = minimize_access.get_audits() + self.assertEqual(2, len(audits)) + + # First audit is to ensure that all folders in the $PATH variable + # are read-only. + self.assertTrue(isinstance(audits[0], minimize_access.ReadOnly)) + self.assertEqual({'/usr/local/sbin', '/usr/local/bin', + '/usr/sbin', '/usr/bin', '/bin'}, audits[0].paths) + + # Second audit is to ensure that the /etc/shadow is only readable + # by the root user. + self.assertTrue(isinstance(audits[1], + minimize_access.FilePermissionAudit)) + self.assertEqual(audits[1].paths[0], '/etc/shadow') + self.assertEqual(audits[1].mode, 0o0600) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_pam.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_pam.py new file mode 100644 index 0000000..2879bc8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_pam.py @@ -0,0 +1,53 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import pam + + +class PAMTestCase(TestCase): + + @patch.object(pam.utils, 'get_settings', lambda x: { + 'auth': {'pam_passwdqc_enable': True, + 'retries': False} + }) + def test_enable_passwdqc(self): + audits = pam.get_audits() + self.assertEqual(2, len(audits)) + audit = audits[0] + self.assertTrue(isinstance(audit, pam.PasswdqcPAM)) + audit = audits[1] + self.assertTrue(isinstance(audit, pam.DeletedFile)) + self.assertEqual('/usr/share/pam-configs/tally2', audit.paths[0]) + + @patch.object(pam.utils, 'get_settings', lambda x: { + 'auth': {'pam_passwdqc_enable': False, + 'retries': True} + }) + def test_disable_passwdqc(self): + audits = pam.get_audits() + self.assertEqual(1, len(audits)) + self.assertFalse(isinstance(audits[0], pam.PasswdqcPAM)) + + @patch.object(pam.utils, 'get_settings', lambda x: { + 'auth': {'pam_passwdqc_enable': False, + 'retries': True} + }) + def test_auth_retries(self): + audits = pam.get_audits() + self.assertEqual(1, len(audits)) + self.assertTrue(isinstance(audits[0], pam.Tally2PAM)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_profile.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_profile.py new file mode 100644 index 0000000..1d7fc99 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_profile.py @@ -0,0 +1,65 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import profile + + +class ProfileTestCase(TestCase): + + def setUp(self): + super(ProfileTestCase, self).setUp() + + os.environ['JUJU_CHARM_DIR'] = '/tmp' + self.addCleanup(lambda: os.environ.pop('JUJU_CHARM_DIR')) + + @patch.object(profile.utils, 'get_settings', lambda x: + {'security': {'kernel_enable_core_dump': False, 'ssh_tmout': False}}) + def test_core_dump_disabled(self): + audits = profile.get_audits() + self.assertEqual(1, len(audits)) + self.assertTrue(isinstance(audits[0], profile.TemplatedFile)) + + @patch.object(profile.utils, 'get_settings', lambda x: { + 'security': {'kernel_enable_core_dump': True, 'ssh_tmout': False} + }) + def test_core_dump_enabled(self): + audits = profile.get_audits() + self.assertEqual(0, len(audits)) + + @patch.object(profile.utils, 'get_settings', lambda x: + {'security': {'kernel_enable_core_dump': True, 'ssh_tmout': False}}) + def test_ssh_tmout_disabled(self): + audits = profile.get_audits() + self.assertEqual(0, len(audits)) + + @patch.object(profile.utils, 'get_settings', lambda x: { + 'security': {'kernel_enable_core_dump': True, 'ssh_tmout': 300} + }) + def test_ssh_tmout_enabled(self): + audits = profile.get_audits() + self.assertEqual(1, len(audits)) + self.assertTrue(isinstance(audits[0], profile.TemplatedFile)) + + @patch.object(profile.utils, 'log', lambda *args, **kwargs: None) + def test_ProfileContext(self): + ctxt = profile.ProfileContext() + self.assertEqual(ctxt(), { + 'ssh_tmout': 300 + }) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_securetty.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_securetty.py new file mode 100644 index 0000000..4c5d7ac --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_securetty.py @@ -0,0 +1,27 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from charmhelpers.contrib.hardening.host.checks import securetty + + +class SecureTTYTestCase(TestCase): + + def test_securetty(self): + audits = securetty.get_audits() + self.assertEqual(1, len(audits)) + audit = audits[0] + self.assertTrue(isinstance(audit, securetty.TemplatedFile)) + self.assertEqual('/etc/securetty', audit.paths[0]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_suid_guid.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_suid_guid.py new file mode 100644 index 0000000..96f78b2 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/host/checks/test_suid_guid.py @@ -0,0 +1,55 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile + +from unittest import TestCase + +from mock import call +from mock import patch + +from charmhelpers.contrib.hardening.host.checks import suid_sgid + + +@patch.object(suid_sgid, 'log', lambda *args, **kwargs: None) +class SUIDSGIDTestCase(TestCase): + + @patch.object(suid_sgid.utils, 'get_settings', lambda x: { + 'security': {'suid_sgid_enforce': False} + }) + def test_no_enforcement(self): + audits = suid_sgid.get_audits() + self.assertEqual(0, len(audits)) + + @patch.object(suid_sgid, 'subprocess') + @patch.object(suid_sgid.utils, 'get_settings', lambda x: { + 'security': {'suid_sgid_enforce': True, + 'suid_sgid_remove_from_unknown': True, + 'suid_sgid_blacklist': [], + 'suid_sgid_whitelist': [], + 'suid_sgid_dry_run_on_unknown': True}, + 'environment': {'root_path': '/'} + }) + def test_suid_guid_harden(self, mock_subprocess): + p = mock_subprocess.Popen.return_value + with tempfile.NamedTemporaryFile() as tmp: + p.communicate.return_value = (tmp.name, "stderr") + + audits = suid_sgid.get_audits() + self.assertEqual(2, len(audits)) + cmd = ['find', '/', '-perm', '-4000', '-o', '-perm', '-2000', '-type', + 'f', '!', '-path', '/proc/*', '-print'] + calls = [call(cmd, stderr=mock_subprocess.PIPE, + stdout=mock_subprocess.PIPE)] + mock_subprocess.Popen.assert_has_calls(calls) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/checks/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/checks/test_config.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/checks/test_config.py new file mode 100644 index 0000000..af015f6 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/mysql/checks/test_config.py @@ -0,0 +1,33 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.mysql.checks import config + + +class MySQLConfigTestCase(TestCase): + + @patch.object(config.subprocess, 'call', lambda *args, **kwargs: 0) + @patch.object(config.utils, 'get_settings', lambda x: { + 'hardening': { + 'mysql-conf': {}, + 'hardening-conf': {} + } + }) + def test_get_audits(self): + audits = config.get_audits() + self.assertEqual(4, len(audits)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/checks/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/checks/test_config.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/checks/test_config.py new file mode 100644 index 0000000..49cea4e --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/ssh/checks/test_config.py @@ -0,0 +1,27 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testtools import TestCase + +from mock import patch + +from charmhelpers.contrib.hardening.ssh.checks import config + + +class SSHConfigTestCase(TestCase): + + @patch.object(config.utils, 'get_settings', lambda x: {}) + def test_dont_clean_packages(self): + audits = config.get_audits() + self.assertEqual(4, len(audits)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/test_defaults.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_defaults.py new file mode 100644 index 0000000..35baa2b --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_defaults.py @@ -0,0 +1,58 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import glob +import yaml + +from unittest import TestCase + +TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), 'defaults') + + +class DefaultsTestCase(TestCase): + + def setUp(self): + super(DefaultsTestCase, self).setUp() + + def get_keys(self, dicto, keys=None): + if keys is None: + keys = [] + + if dicto: + if type(dicto) is not dict: + raise Exception("Unexpected entry: %s" % dicto) + + for key in dicto.keys(): + keys.append(key) + if type(dicto[key]) is dict: + self.get_keys(dicto[key], keys) + + return keys + + def test_defaults(self): + defaults_paths = glob.glob('%s/*.yaml' % TEMPLATES_DIR) + for defaults in defaults_paths: + schema = "%s.schema" % defaults + self.assertTrue(os.path.exists(schema)) + a = yaml.safe_load(open(schema)) + b = yaml.safe_load(open(defaults)) + if not a and not b: + continue + + # Test that all keys in default are present in their associated + # schema. + skeys = self.get_keys(a) + dkeys = self.get_keys(b) + self.assertEqual(set(dkeys).symmetric_difference(skeys), set([])) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/test_harden.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_harden.py new file mode 100644 index 0000000..2536435 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_harden.py @@ -0,0 +1,81 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mock import patch, call +from unittest import TestCase + +from charmhelpers.contrib.hardening import harden + + +class HardenTestCase(TestCase): + + def setUp(self): + super(HardenTestCase, self).setUp() + + @patch.object(harden, 'log', lambda *args, **kwargs: None) + @patch.object(harden, 'run_apache_checks') + @patch.object(harden, 'run_mysql_checks') + @patch.object(harden, 'run_ssh_checks') + @patch.object(harden, 'run_os_checks') + def test_harden(self, mock_host, mock_ssh, mock_mysql, mock_apache): + mock_host.__name__ = 'host' + mock_ssh.__name__ = 'ssh' + mock_mysql.__name__ = 'mysql' + mock_apache.__name__ = 'apache' + + @harden.harden(overrides=['ssh', 'mysql']) + def foo(arg1, kwarg1=None): + return "done." + + self.assertEqual(foo('anarg', kwarg1='akwarg'), "done.") + self.assertTrue(mock_ssh.called) + self.assertTrue(mock_mysql.called) + self.assertFalse(mock_apache.called) + self.assertFalse(mock_host.called) + + @patch.object(harden, 'log') + @patch.object(harden, 'run_apache_checks') + @patch.object(harden, 'run_mysql_checks') + @patch.object(harden, 'run_ssh_checks') + @patch.object(harden, 'run_os_checks') + def test_harden_logs_work(self, mock_host, mock_ssh, mock_mysql, + mock_apache, mock_log): + mock_host.__name__ = 'host' + mock_ssh.__name__ = 'ssh' + mock_mysql.__name__ = 'mysql' + mock_apache.__name__ = 'apache' + + @harden.harden(overrides=['ssh', 'mysql']) + def foo(arg1, kwarg1=None): + return arg1 + kwarg1 + + mock_log.assert_not_called() + self.assertEqual(foo('anarg', kwarg1='akwarg'), "anargakwarg") + mock_log.assert_any_call("Hardening function 'foo'", level="DEBUG") + + @harden.harden(overrides=['ssh', 'mysql']) + def bar(arg1, kwarg1=None): + return arg1 + kwarg1 + + mock_log.reset_mock() + self.assertEqual(bar("a", kwarg1="b"), "ab") + mock_log.assert_any_call("Hardening function 'bar'", level="DEBUG") + + # check it only logs the function name once + mock_log.reset_mock() + self.assertEqual(bar("a", kwarg1="b"), "ab") + self.assertEqual( + mock_log.call_args_list, + [call("Executing hardening module 'ssh'", level="DEBUG"), + call("Executing hardening module 'mysql'", level="DEBUG")]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/test_templating.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_templating.py new file mode 100644 index 0000000..6a2f26d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_templating.py @@ -0,0 +1,310 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import os +import six + +from mock import call, patch +from unittest import TestCase + +from charmhelpers.contrib.hardening import templating +from charmhelpers.contrib.hardening import utils +from charmhelpers.contrib.hardening.audits.file import ( + TemplatedFile, + FileContentAudit, +) +from charmhelpers.contrib.hardening.ssh.checks import ( + config as ssh_config_check +) +from charmhelpers.contrib.hardening.host.checks import ( + sysctl, + securetty, +) +from charmhelpers.contrib.hardening.apache.checks import ( + config as apache_config_check +) +from charmhelpers.contrib.hardening.mysql.checks import ( + config as mysql_config_check +) + + +class TemplatingTestCase(TestCase): + + def setUp(self): + super(TemplatingTestCase, self).setUp() + + os.environ['JUJU_CHARM_DIR'] = '/tmp' + self.pathindex = {} + self.addCleanup(lambda: os.environ.pop('JUJU_CHARM_DIR')) + + def get_renderers(self, audits): + renderers = [] + for a in audits: + if issubclass(a.__class__, TemplatedFile): + renderers.append(a) + + return renderers + + def get_contentcheckers(self, audits): + contentcheckers = [] + for a in audits: + if issubclass(a.__class__, FileContentAudit): + contentcheckers.append(a) + + return contentcheckers + + def render(self, renderers): + for template in renderers: + with patch.object(template, 'pre_write', lambda: None): + with patch.object(template, 'post_write', lambda: None): + with patch.object(template, 'run_service_actions'): + with patch.object(template, 'save_checksum'): + for p in template.paths: + template.comply(p) + + def checkcontents(self, contentcheckers): + for check in contentcheckers: + if check.path not in self.pathindex: + continue + + self.assertTrue(check.is_compliant(self.pathindex[check.path])) + + @patch.object(ssh_config_check, 'lsb_release', + lambda: {'DISTRIB_CODENAME': 'precise'}) + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch('charmhelpers.contrib.hardening.audits.file.log') + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + @patch.object(ssh_config_check, 'log', lambda *args, **kwargs: None) + def test_ssh_config_render_and_check_lt_trusty(self, mock_log, mock_write, + mock_ensure_permissions): + audits = ssh_config_check.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + configs = {} + + def write(path, data): + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + if os.path.basename(path) == "ssh_config": + configs['ssh'] = ftmp.name + elif os.path.basename(path) == "sshd_config": + configs['sshd'] = ftmp.name + + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/etc/ssh/ssh_config', args_list[0][0][0]) + self.assertEqual('/etc/ssh/sshd_config', args_list[1][0][0]) + self.assertEqual(mock_write.call_count, 2) + + calls = [call("Auditing contents of file '%s'" % configs['ssh'], + level='DEBUG'), + call('Checked 10 cases and 10 passed', level='DEBUG'), + call("Auditing contents of file '%s'" % configs['sshd'], + level='DEBUG'), + call('Checked 10 cases and 10 passed', level='DEBUG')] + mock_log.assert_has_calls(calls) + + @patch.object(ssh_config_check, 'lsb_release', + lambda: {'DISTRIB_CODENAME': 'trusty'}) + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch('charmhelpers.contrib.hardening.audits.file.log') + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + @patch.object(ssh_config_check, 'log', lambda *args, **kwargs: None) + def test_ssh_config_render_and_check_gte_trusty(self, mock_log, mock_write, + mock_ensure_permissions): + audits = ssh_config_check.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + + def write(path, data): + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/etc/ssh/ssh_config', args_list[0][0][0]) + self.assertEqual('/etc/ssh/sshd_config', args_list[1][0][0]) + self.assertEqual(mock_write.call_count, 2) + + mock_log.assert_has_calls([call('Checked 9 cases and 9 passed', + level='DEBUG')]) + + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch.object(sysctl, 'log', lambda *args, **kwargs: None) + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + def test_os_sysctl_and_check(self, mock_write, mock_ensure_permissions): + audits = sysctl.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + + def write(path, data): + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/etc/sysctl.d/99-juju-hardening.conf', + args_list[0][0][0]) + self.assertEqual(mock_write.call_count, 1) + + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch.object(sysctl, 'log', lambda *args, **kwargs: None) + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + def test_os_securetty_and_check(self, mock_write, mock_ensure_permissions): + audits = securetty.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + + def write(path, data): + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/etc/securetty', args_list[0][0][0]) + self.assertEqual(mock_write.call_count, 1) + + @patch.object(apache_config_check.utils, 'get_settings', lambda x: { + 'common': {'apache_dir': '/tmp/foo'}, + 'hardening': { + 'allowed_http_methods': {'GOGETEM'}, + 'modules_to_disable': {'modfoo'}, + 'traceenable': 'off', + 'servertokens': 'Prod', + 'honor_cipher_order': 'on', + 'cipher_suite': 'ALL:+MEDIUM:+HIGH:!LOW:!MD5:!RC4:!eNULL:!aNULL:!3DES' + } + }) + @patch('charmhelpers.contrib.hardening.audits.file.os.path.exists', + lambda *a, **kwa: True) + @patch.object(apache_config_check, 'subprocess') + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + def test_apache_conf_and_check(self, mock_write, mock_ensure_permissions, + mock_subprocess): + mock_subprocess.call.return_value = 0 + apache_version = b"""Server version: Apache/2.4.7 (Ubuntu) + Server built: Jan 14 2016 17:45:23 + """ + mock_subprocess.check_output.return_value = apache_version + audits = apache_config_check.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + + def write(path, data): + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/tmp/foo/mods-available/alias.conf', + args_list[0][0][0]) + self.assertEqual(mock_write.call_count, 2) + + @patch.object(apache_config_check.utils, 'get_settings', lambda x: { + 'security': {}, + 'hardening': { + 'mysql-conf': '/tmp/foo/mysql.cnf', + 'hardening-conf': '/tmp/foo/conf.d/hardening.cnf' + } + }) + @patch('charmhelpers.contrib.hardening.audits.file.os.path.exists', + lambda *a, **kwa: True) + @patch.object(utils, 'ensure_permissions') + @patch.object(templating, 'write') + @patch.object(mysql_config_check.subprocess, 'call', + lambda *args, **kwargs: 0) + @patch.object(templating, 'log', lambda *args, **kwargs: None) + @patch.object(utils, 'log', lambda *args, **kwargs: None) + def test_mysql_conf_and_check(self, mock_write, mock_ensure_permissions): + audits = mysql_config_check.get_audits() + contentcheckers = self.get_contentcheckers(audits) + renderers = self.get_renderers(audits) + + def write(path, data): + if path in self.pathindex: + raise Exception("File already rendered '%s'" % path) + + with tempfile.NamedTemporaryFile(delete=False) as ftmp: + self.pathindex[path] = ftmp.name + with open(ftmp.name, 'wb') as fd: + fd.write(data) + + mock_write.side_effect = write + self.render(renderers) + self.checkcontents(contentcheckers) + self.assertTrue(mock_write.called) + args_list = mock_write.call_args_list + self.assertEqual('/tmp/foo/conf.d/hardening.cnf', + args_list[0][0][0]) + self.assertEqual(mock_write.call_count, 1) + + def tearDown(self): + # Cleanup + for path in six.itervalues(self.pathindex): + os.remove(path) + + super(TemplatingTestCase, self).tearDown() diff --git a/nrpe/mod/charmhelpers/tests/contrib/hardening/test_utils.py b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_utils.py new file mode 100644 index 0000000..2b83895 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/hardening/test_utils.py @@ -0,0 +1,63 @@ +# Copyright 2016 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import tempfile + +from mock import ( + MagicMock, + call, + patch +) +from unittest import TestCase + +from charmhelpers.contrib.hardening import utils + + +class UtilsTestCase(TestCase): + + def setUp(self): + super(UtilsTestCase, self).setUp() + utils.__SETTINGS__ = {} + + @patch.object(utils.grp, 'getgrnam') + @patch.object(utils.pwd, 'getpwnam') + @patch.object(utils, 'os') + @patch.object(utils, 'log', lambda *args, **kwargs: None) + def test_ensure_permissions(self, mock_os, mock_getpwnam, mock_getgrnam): + user = MagicMock() + user.pw_uid = '12' + mock_getpwnam.return_value = user + group = MagicMock() + group.gr_gid = '23' + mock_getgrnam.return_value = group + + with tempfile.NamedTemporaryFile() as tmp: + utils.ensure_permissions(tmp.name, 'testuser', 'testgroup', 0o0440) + + mock_getpwnam.assert_has_calls([call('testuser')]) + mock_getgrnam.assert_has_calls([call('testgroup')]) + mock_os.chown.assert_has_calls([call(tmp.name, '12', '23')]) + mock_os.chmod.assert_has_calls([call(tmp.name, 0o0440)]) + + @patch.object(utils, '_get_user_provided_overrides') + def test_settings_cache(self, mock_get_user_provided_overrides): + mock_get_user_provided_overrides.return_value = {} + self.assertEqual(utils.__SETTINGS__, {}) + self.assertTrue('sysctl' in utils.get_settings('os')) + self.assertEqual(sorted(list(six.iterkeys(utils.__SETTINGS__))), + ['os']) + self.assertTrue('server' in utils.get_settings('ssh')) + self.assertEqual(sorted(list(six.iterkeys(utils.__SETTINGS__))), + ['os', 'ssh']) diff --git a/nrpe/mod/charmhelpers/tests/contrib/mellanox/test_infiniband.py b/nrpe/mod/charmhelpers/tests/contrib/mellanox/test_infiniband.py new file mode 100644 index 0000000..bc60be7 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/mellanox/test_infiniband.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +from charmhelpers.contrib.mellanox import infiniband + +from mock import patch, call +import unittest + +TO_PATCH = [ + "log", + "INFO", + "apt_install", + "apt_update", + "modprobe", + "network_interfaces" +] + +NETWORK_INTERFACES = [ + 'lo', + 'eth0', + 'eth1', + 'eth2', + 'eth3', + 'eth4', + 'juju-br0', + 'ib0', + 'virbr0', + 'ovs-system', + 'br-int', + 'br-ex', + 'br-data', + 'phy-br-data', + 'int-br-data', + 'br-tun' +] + + +IBSTAT_OUTPUT = """ +CA 'mlx4_0' + CA type: MT4103 + Number of ports: 2 + Firmware version: 2.33.5000 + Hardware version: 0 + Node GUID: 0xe41d2d03000a1120 + System image GUID: 0xe41d2d03000a1123 +""" + + +class InfinibandTest(unittest.TestCase): + + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.contrib.mellanox.infiniband.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_load_modules(self): + infiniband.load_modules() + + self.modprobe.assert_has_calls(map(lambda x: call(x, persist=True), + infiniband.REQUIRED_MODULES)) + + def test_install_packages(self): + infiniband.install_packages() + + self.apt_update.assert_is_called_once() + self.apt_install.assert_is_called_once() + + @patch("os.path.exists") + def test_is_enabled(self, exists): + exists.return_value = True + self.assertTrue(infiniband.is_enabled()) + + @patch("subprocess.check_output") + def test_stat(self, check_output): + infiniband.stat() + + check_output.assert_called_with(["ibstat"]) + + @patch("subprocess.check_output") + def test_devices(self, check_output): + infiniband.devices() + + check_output.assert_called_with(["ibstat", "-l"]) + + @patch("subprocess.check_output") + def test_device_info(self, check_output): + check_output.return_value = IBSTAT_OUTPUT + + info = infiniband.device_info("mlx4_0") + + self.assertEquals(info.num_ports, "2") + self.assertEquals(info.device_type, "MT4103") + self.assertEquals(info.fw_ver, "2.33.5000") + self.assertEquals(info.hw_ver, "0") + self.assertEquals(info.node_guid, "0xe41d2d03000a1120") + self.assertEquals(info.sys_guid, "0xe41d2d03000a1123") + + @patch("subprocess.check_output") + def test_ipoib_interfaces(self, check_output): + self.network_interfaces.return_value = NETWORK_INTERFACES + + ipoib_nic = "ib0" + + def c(*args, **kwargs): + if ipoib_nic in args[0]: + return "driver: ib_ipoib" + else: + return "driver: mock" + + check_output.side_effect = c + self.assertEquals(infiniband.ipoib_interfaces(), [ipoib_nic]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/ovs/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovn.py b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovn.py new file mode 100644 index 0000000..c039693 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovn.py @@ -0,0 +1,122 @@ +import textwrap +import uuid + +import charmhelpers.contrib.network.ovs.ovn as ovn + +import tests.utils as test_utils + + +CLUSTER_STATUS = textwrap.dedent(""" + 0ea6 + Name: OVN_Northbound + Cluster ID: f6a3 (f6a36e77-97bf-4740-b46a-705cbe4fef45) + Server ID: 0ea6 (0ea6e785-c2bb-4640-b7a2-85104c11a2c1) + Address: ssl:10.219.3.174:6643 + Status: cluster member + Role: follower + Term: 3 + Leader: 22dd + Vote: unknown + + Election timer: 1000 + Log: [2, 10] + Entries not yet committed: 0 + Entries not yet applied: 0 + Connections: ->f6cf ->22dd <-22dd <-f6cf + Servers: + 0ea6 (0ea6 at ssl:10.219.3.174:6643) (self) + f6cf (f6cf at ssl:10.219.3.64:6643) + 22dd (22dd at ssl:10.219.3.137:6643) + """) + +NORTHD_STATUS_ACTIVE = textwrap.dedent(""" + Status: active + """) + +NORTHD_STATUS_STANDBY = textwrap.dedent(""" + Status: standby + """) + + +class TestOVN(test_utils.BaseTestCase): + + def test_ovn_appctl(self): + self.patch_object(ovn.utils, '_run') + ovn.ovn_appctl('ovn-northd', ('is-paused',)) + self._run.assert_called_once_with('ovn-appctl', '-t', 'ovn-northd', + 'is-paused') + self._run.reset_mock() + ovn.ovn_appctl('ovnnb_db', ('cluster/status',)) + self._run.assert_called_once_with('ovn-appctl', '-t', + '/var/run/ovn/ovnnb_db.ctl', + 'cluster/status') + self._run.reset_mock() + ovn.ovn_appctl('ovnnb_db', ('cluster/status',), use_ovs_appctl=True) + self._run.assert_called_once_with('ovs-appctl', '-t', + '/var/run/ovn/ovnnb_db.ctl', + 'cluster/status') + self._run.reset_mock() + ovn.ovn_appctl('ovnsb_db', ('cluster/status',), + rundir='/var/run/openvswitch') + self._run.assert_called_once_with('ovn-appctl', '-t', + '/var/run/openvswitch/ovnsb_db.ctl', + 'cluster/status') + + def test_cluster_status(self): + self.patch_object(ovn, 'ovn_appctl') + self.ovn_appctl.return_value = CLUSTER_STATUS + expect = ovn.OVNClusterStatus( + 'OVN_Northbound', + uuid.UUID('f6a36e77-97bf-4740-b46a-705cbe4fef45'), + uuid.UUID('0ea6e785-c2bb-4640-b7a2-85104c11a2c1'), + 'ssl:10.219.3.174:6643', + 'cluster member', + 'follower', + 3, + '22dd', + 'unknown', + 1000, + '[2, 10]', + 0, + 0, + '->f6cf ->22dd <-22dd <-f6cf', + [ + ('0ea6', 'ssl:10.219.3.174:6643'), + ('f6cf', 'ssl:10.219.3.64:6643'), + ('22dd', 'ssl:10.219.3.137:6643'), + ]) + self.assertEquals(ovn.cluster_status('ovnnb_db'), expect) + self.ovn_appctl.assert_called_once_with('ovnnb_db', ('cluster/status', + 'OVN_Northbound'), + rundir=None, + use_ovs_appctl=False) + self.assertFalse(expect.is_cluster_leader) + expect = ovn.OVNClusterStatus( + 'OVN_Northbound', + uuid.UUID('f6a36e77-97bf-4740-b46a-705cbe4fef45'), + uuid.UUID('0ea6e785-c2bb-4640-b7a2-85104c11a2c1'), + 'ssl:10.219.3.174:6643', + 'cluster member', + 'leader', + 3, + 'self', + 'unknown', + 1000, + '[2, 10]', + 0, + 0, + '->f6cf ->22dd <-22dd <-f6cf', + [ + ('0ea6', 'ssl:10.219.3.174:6643'), + ('f6cf', 'ssl:10.219.3.64:6643'), + ('22dd', 'ssl:10.219.3.137:6643'), + ]) + self.assertTrue(expect.is_cluster_leader) + + def test_is_northd_active(self): + self.patch_object(ovn, 'ovn_appctl') + self.ovn_appctl.return_value = NORTHD_STATUS_ACTIVE + self.assertTrue(ovn.is_northd_active()) + self.ovn_appctl.assert_called_once_with('ovn-northd', ('status',)) + self.ovn_appctl.return_value = NORTHD_STATUS_STANDBY + self.assertFalse(ovn.is_northd_active()) diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovs.py b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovs.py new file mode 100644 index 0000000..ecdf2ec --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovs.py @@ -0,0 +1,283 @@ +import mock +import types +import uuid + +import charmhelpers.contrib.network.ovs as ovs + +import tests.utils as test_utils + + +# NOTE(fnordahl): some functions drectly under the ``contrib.network.ovs`` +# module have their unit tests in the ``test_ovs.py`` module in the +# ``tests.contrib.network`` package. + + +class TestOVS(test_utils.BaseTestCase): + + def test__dict_to_vsctl_set(self): + indata = { + 'key': 'value', + 'otherkey': { + 'nestedkey': 'nestedvalue', + }, + } + # due to varying Dict ordering depending on Python version we need + # to be a bit elaborate rather than comparing result directly + result1 = ('--', 'set', 'aTable', 'anItem', 'key=value') + result2 = ('--', 'set', 'aTable', 'anItem', + 'otherkey:nestedkey=nestedvalue') + for setcmd in ovs._dict_to_vsctl_set(indata, 'aTable', 'anItem'): + self.assertTrue(setcmd == result1 or setcmd == result2) + + def test_add_bridge(self): + self.patch_object(ovs.subprocess, 'check_call') + self.patch_object(ovs, 'log') + ovs.add_bridge('test') + self.check_call.assert_called_once_with([ + "ovs-vsctl", "--", "--may-exist", + "add-br", 'test']) + self.assertTrue(self.log.call_count == 1) + + self.check_call.reset_mock() + self.log.reset_mock() + ovs.add_bridge('test', datapath_type='netdev') + self.check_call.assert_called_with([ + "ovs-vsctl", "--", "--may-exist", + "add-br", 'test', "--", "set", + "bridge", "test", "datapath_type=netdev", + ]) + self.assertTrue(self.log.call_count == 2) + + self.check_call.reset_mock() + ovs.add_bridge('test', exclusive=True) + self.check_call.assert_called_once_with([ + "ovs-vsctl", "--", "add-br", 'test']) + + self.check_call.reset_mock() + self.patch_object(ovs, '_dict_to_vsctl_set') + self._dict_to_vsctl_set.return_value = [['--', 'fakeextradata']] + ovs.add_bridge('test', brdata={'fakeinput': None}) + self._dict_to_vsctl_set.assert_called_once_with( + {'fakeinput': None}, 'bridge', 'test') + self.check_call.assert_called_once_with([ + 'ovs-vsctl', '--', '--may-exist', 'add-br', 'test', + '--', 'fakeextradata']) + + def test_add_bridge_port(self): + self.patch_object(ovs.subprocess, 'check_call') + self.patch_object(ovs, 'log') + ovs.add_bridge_port('test', 'eth1') + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--may-exist', 'add-port', + 'test', 'eth1']), + mock.call(['ip', 'link', 'set', 'eth1', 'up']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'off']) + ]) + self.assertTrue(self.log.call_count == 1) + + self.check_call.reset_mock() + self.log.reset_mock() + ovs.add_bridge_port('test', 'eth1', promisc=True) + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--may-exist', 'add-port', + 'test', 'eth1']), + mock.call(['ip', 'link', 'set', 'eth1', 'up']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'on']) + ]) + self.assertTrue(self.log.call_count == 1) + + self.check_call.reset_mock() + self.log.reset_mock() + ovs.add_bridge_port('test', 'eth1', promisc=None) + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--may-exist', 'add-port', + 'test', 'eth1']), + mock.call(['ip', 'link', 'set', 'eth1', 'up']), + ]) + self.assertTrue(self.log.call_count == 1) + + self.check_call.reset_mock() + ovs.add_bridge_port('test', 'eth1', exclusive=True, linkup=False) + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', 'add-port', 'test', 'eth1']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'off']) + ]) + + self.check_call.reset_mock() + self.patch_object(ovs, '_dict_to_vsctl_set') + self._dict_to_vsctl_set.return_value = [['--', 'fakeextradata']] + ovs.add_bridge_port('test', 'eth1', ifdata={'fakeinput': None}) + self._dict_to_vsctl_set.assert_called_once_with( + {'fakeinput': None}, 'Interface', 'eth1') + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--may-exist', 'add-port', + 'test', 'eth1', '--', 'fakeextradata']), + mock.call(['ip', 'link', 'set', 'eth1', 'up']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'off']) + ]) + self._dict_to_vsctl_set.reset_mock() + self.check_call.reset_mock() + ovs.add_bridge_port('test', 'eth1', portdata={'fakeportinput': None}) + self._dict_to_vsctl_set.assert_called_once_with( + {'fakeportinput': None}, 'Port', 'eth1') + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--may-exist', 'add-port', + 'test', 'eth1', '--', 'fakeextradata']), + mock.call(['ip', 'link', 'set', 'eth1', 'up']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'off']) + ]) + + def test_del_bridge_port(self): + self.patch_object(ovs.subprocess, 'check_call') + self.patch_object(ovs, 'log') + ovs.del_bridge_port('test', 'eth1') + self.check_call.assert_has_calls([ + mock.call(['ovs-vsctl', '--', '--if-exists', 'del-port', + 'test', 'eth1']), + mock.call(['ip', 'link', 'set', 'eth1', 'down']), + mock.call(['ip', 'link', 'set', 'eth1', 'promisc', 'off']) + ]) + self.assertTrue(self.log.call_count == 1) + self.assertTrue(self.check_call.call_count == 3) + self.check_call.reset_mock() + ovs.del_bridge_port('test', 'eth1', linkdown=False) + self.check_call.assert_called_once_with( + ['ovs-vsctl', '--', '--if-exists', 'del-port', 'test', 'eth1']) + + def test_ovs_appctl(self): + self.patch_object(ovs.subprocess, 'check_output') + ovs.ovs_appctl('ovs-vswitchd', ('ofproto/list',)) + self.check_output.assert_called_once_with( + ['ovs-appctl', '-t', 'ovs-vswitchd', 'ofproto/list'], + universal_newlines=True) + + def test_add_bridge_bond(self): + self.patch_object(ovs.subprocess, 'check_call') + self.patch_object(ovs, '_dict_to_vsctl_set') + self._dict_to_vsctl_set.return_value = [['--', 'fakekey=fakevalue']] + portdata = { + 'bond-mode': 'balance-tcp', + 'lacp': 'active', + 'other-config': { + 'lacp-time': 'fast', + }, + } + ifdatamap = { + 'eth0': { + 'type': 'dpdk', + 'mtu-request': '9000', + 'options': { + 'dpdk-devargs': '0000:01:00.0', + }, + }, + 'eth1': { + 'type': 'dpdk', + 'mtu-request': '9000', + 'options': { + 'dpdk-devargs': '0000:02:00.0', + }, + }, + } + ovs.add_bridge_bond('br-ex', 'bond42', ['eth0', 'eth1'], + portdata, ifdatamap) + self._dict_to_vsctl_set.assert_has_calls([ + mock.call(portdata, 'port', 'bond42'), + mock.call(ifdatamap['eth0'], 'Interface', 'eth0'), + mock.call(ifdatamap['eth1'], 'Interface', 'eth1'), + ], any_order=True) + self.check_call.assert_called_once_with([ + 'ovs-vsctl', + '--', '--may-exist', 'add-bond', 'br-ex', 'bond42', 'eth0', 'eth1', + '--', 'fakekey=fakevalue', + '--', 'fakekey=fakevalue', + '--', 'fakekey=fakevalue']) + + def test_uuid_for_port(self): + self.patch_object(ovs.ch_ovsdb, 'SimpleOVSDB') + fake_uuid = uuid.UUID('efdce2cf-cd66-4060-a9f8-1db0e9a06216') + ovsdb = mock.MagicMock() + ovsdb.port.find.return_value = [ + {'_uuid': fake_uuid}, + ] + self.SimpleOVSDB.return_value = ovsdb + self.assertEquals(ovs.uuid_for_port('fake-port'), fake_uuid) + ovsdb.port.find.assert_called_once_with('name=fake-port') + + def test_bridge_for_port(self): + self.patch_object(ovs.ch_ovsdb, 'SimpleOVSDB') + fake_uuid = uuid.UUID('818d03dd-efb8-44be-aba3-bde423bf1cc9') + ovsdb = mock.MagicMock() + ovsdb.bridge.__iter__.return_value = [ + { + 'name': 'fake-bridge', + 'ports': [fake_uuid], + }, + ] + self.SimpleOVSDB.return_value = ovsdb + self.assertEquals(ovs.bridge_for_port(fake_uuid), 'fake-bridge') + # If there is a single port on a bridge the ports property will not be + # a list. ref: juju/charm-helpers#510 + ovsdb.bridge.__iter__.return_value = [ + { + 'name': 'fake-bridge', + 'ports': fake_uuid, + }, + ] + self.assertEquals(ovs.bridge_for_port(fake_uuid), 'fake-bridge') + + def test_patch_ports_on_bridge(self): + self.patch_object(ovs.ch_ovsdb, 'SimpleOVSDB') + self.patch_object(ovs, 'bridge_for_port') + self.patch_object(ovs, 'uuid_for_port') + ovsdb = mock.MagicMock() + ovsdb.interface.find.return_value = [ + { + 'name': 'fake-interface-with-port-for-other-bridge', + 'options': { + 'peer': 'fake-peer' + }, + }, + { + 'name': 'fake-interface', + 'options': { + 'peer': 'fake-peer' + }, + }, + ] + port_uuid = uuid.UUID('0d43905b-f80e-4eaa-9feb-a9017da8c6bc') + ovsdb.port.find.side_effect = [ + [{ + '_uuid': port_uuid, + 'name': 'port-on-other-bridge', + }], + [{ + '_uuid': port_uuid, + 'name': 'fake-port', + }], + ] + self.SimpleOVSDB.return_value = ovsdb + self.bridge_for_port.side_effect = ['some-other-bridge', 'fake-bridge', 'fake-peer-bridge'] + for patch in ovs.patch_ports_on_bridge('fake-bridge'): + self.assertEquals( + patch, + ovs.Patch( + this_end=ovs.PatchPort( + bridge='fake-bridge', + port='fake-port'), + other_end=ovs.PatchPort( + bridge='fake-peer-bridge', + port='fake-peer')) + ) + break + else: + assert 0, 'Expected generator to provide output' + ovsdb.port.find.side_effect = None + ovsdb.port.find.return_value = [] + with self.assertRaises(ValueError): + for patch in ovs.patch_ports_on_bridge('fake-bridge'): + pass + ovsdb.interface.find.return_value = [] + for patch in ovs.patch_ports_on_bridge('fake-bridge'): + assert 0, 'Expected generator to provide empty iterator' + self.assertTrue(isinstance( + ovs.patch_ports_on_bridge('fake-bridge'), types.GeneratorType)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovsdb.py b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovsdb.py new file mode 100644 index 0000000..9d6af9b --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_ovsdb.py @@ -0,0 +1,124 @@ +import mock +import textwrap +import uuid + +import charmhelpers.contrib.network.ovs.ovsdb as ovsdb + +import tests.utils as test_utils + + +VSCTL_BRIDGE_TBL = textwrap.dedent(""" + {"data":[[["uuid","1e21ba48-61ff-4b32-b35e-cb80411da351"], + ["set",[]],["set",[]],"0000a0369fdd3890","","", + ["map",[["charm-ovn-chassis","managed"],["other","value"]]], + ["set",[]],["set",[]],["map",[]],["set",[]],false,["set",[]], + "br-test",["set",[]],["map",[]],["set", + [["uuid","617f9359-77e2-41be-8af6-4c44e7a6bcc3"], + ["uuid","da840476-8809-4107-8733-591f4696f056"]]], + ["set",["OpenFlow10","OpenFlow13","OpenFlow14"]],false,["map",[]], + ["set",[]],["map",[]],false], + [["uuid","bb685b0f-a383-40a1-b7a5-b5c2066bfa42"], + ["set",[]],["set",[]],"00000e5b68bba140","","", + ["map",[]],"secure",["set",[]],["map",[]],["set",[]],false, + ["set",[]],"br-int",["set",[]],["map",[["disable-in-band","true"]]], + ["set",[["uuid","07f4c231-9fd2-49b0-a558-5b69d657fdb0"], + ["uuid","8bbd2441-866f-4317-a284-09491702776c"], + ["uuid","d9e9c081-6482-4006-b7d6-239182b56c2e"]]], + ["set",[]],false,["map",[]],["set",[]],["map",[]],false]], + "headings":["_uuid","auto_attach","controller","datapath_id", + "datapath_type","datapath_version","external_ids","fail_mode", + "flood_vlans","flow_tables","ipfix","mcast_snooping_enable", + "mirrors","name","netflow","other_config","ports","protocols", + "rstp_enable","rstp_status","sflow","status","stp_enable"]} + """) + + +class TestSimpleOVSDB(test_utils.BaseTestCase): + + def patch_target(self, attr, return_value=None): + mocked = mock.patch.object(self.target, attr) + self._patches[attr] = mocked + started = mocked.start() + started.return_value = return_value + self._patches_start[attr] = started + setattr(self, attr, started) + + def test___init__(self): + with self.assertRaises(RuntimeError): + self.target = ovsdb.SimpleOVSDB('atool') + with self.assertRaises(AttributeError): + self.target = ovsdb.SimpleOVSDB('ovs-vsctl') + self.target.unknown_table.find() + + def test__find_tbl(self): + self.target = ovsdb.SimpleOVSDB('ovs-vsctl') + self.patch_object(ovsdb.utils, '_run') + self._run.return_value = VSCTL_BRIDGE_TBL + self.maxDiff = None + expect = { + '_uuid': uuid.UUID('1e21ba48-61ff-4b32-b35e-cb80411da351'), + 'auto_attach': [], + 'controller': [], + 'datapath_id': '0000a0369fdd3890', + 'datapath_type': '', + 'datapath_version': '', + 'external_ids': { + 'charm-ovn-chassis': 'managed', + 'other': 'value', + }, + 'fail_mode': [], + 'flood_vlans': [], + 'flow_tables': {}, + 'ipfix': [], + 'mcast_snooping_enable': False, + 'mirrors': [], + 'name': 'br-test', + 'netflow': [], + 'other_config': {}, + 'ports': [uuid.UUID('617f9359-77e2-41be-8af6-4c44e7a6bcc3'), + uuid.UUID('da840476-8809-4107-8733-591f4696f056')], + 'protocols': ['OpenFlow10', 'OpenFlow13', 'OpenFlow14'], + 'rstp_enable': False, + 'rstp_status': {}, + 'sflow': [], + 'status': {}, + 'stp_enable': False} + # this in effect also tests the __iter__ front end method + for el in self.target.bridge: + self.assertDictEqual(el, expect) + break + self._run.assert_called_once_with( + 'ovs-vsctl', '-f', 'json', 'find', 'bridge') + self._run.reset_mock() + # this in effect also tests the find front end method + for el in self.target.bridge.find(condition='name=br-test'): + break + self._run.assert_called_once_with( + 'ovs-vsctl', '-f', 'json', 'find', 'bridge', 'name=br-test') + + def test_clear(self): + self.target = ovsdb.SimpleOVSDB('ovs-vsctl') + self.patch_object(ovsdb.utils, '_run') + self.target.interface.clear('1e21ba48-61ff-4b32-b35e-cb80411da351', + 'external_ids') + self._run.assert_called_once_with( + 'ovs-vsctl', 'clear', 'interface', + '1e21ba48-61ff-4b32-b35e-cb80411da351', 'external_ids') + + def test_remove(self): + self.target = ovsdb.SimpleOVSDB('ovs-vsctl') + self.patch_object(ovsdb.utils, '_run') + self.target.interface.remove('1e21ba48-61ff-4b32-b35e-cb80411da351', + 'external_ids', 'other') + self._run.assert_called_once_with( + 'ovs-vsctl', 'remove', 'interface', + '1e21ba48-61ff-4b32-b35e-cb80411da351', 'external_ids', 'other') + + def test_set(self): + self.target = ovsdb.SimpleOVSDB('ovs-vsctl') + self.patch_object(ovsdb.utils, '_run') + self.target.interface.set('1e21ba48-61ff-4b32-b35e-cb80411da351', + 'external_ids:other', 'value') + self._run.assert_called_once_with( + 'ovs-vsctl', 'set', 'interface', + '1e21ba48-61ff-4b32-b35e-cb80411da351', 'external_ids:other=value') diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_utils.py b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_utils.py new file mode 100644 index 0000000..8b7e4b1 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/ovs/test_utils.py @@ -0,0 +1,13 @@ +import charmhelpers.contrib.network.ovs.utils as utils + +import tests.utils as test_utils + + +class TestUtils(test_utils.BaseTestCase): + + def test__run(self): + self.patch_object(utils.subprocess, 'check_output') + self.check_output.return_value = 'aReturn' + self.assertEquals(utils._run('aArg'), 'aReturn') + self.check_output.assert_called_once_with( + ('aArg',), universal_newlines=True) diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/test_ip.py b/nrpe/mod/charmhelpers/tests/contrib/network/test_ip.py new file mode 100644 index 0000000..c253b1d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/test_ip.py @@ -0,0 +1,848 @@ +import subprocess +import unittest + +import mock +import netifaces + +import charmhelpers.contrib.network.ip as net_ip +from mock import patch, MagicMock + +import nose.tools +import six + +if not six.PY3: + builtin_open = '__builtin__.open' + builtin_import = '__builtin__.__import__' +else: + builtin_open = 'builtins.open' + builtin_import = 'builtins.__import__' + +DUMMY_ADDRESSES = { + 'lo': { + 17: [{'peer': '00:00:00:00:00:00', + 'addr': '00:00:00:00:00:00'}], + 2: [{'peer': '127.0.0.1', 'netmask': + '255.0.0.0', 'addr': '127.0.0.1'}], + 10: [{'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff', + 'addr': '::1'}] + }, + 'eth0': { + 2: [{'addr': '192.168.1.55', + 'broadcast': '192.168.1.255', + 'netmask': '255.255.255.0'}], + 10: [{'addr': '2a01:348:2f4:0:685e:5748:ae62:209f', + 'netmask': 'ffff:ffff:ffff:ffff::'}, + {'addr': 'fe80::3e97:eff:fe8b:1cf7%eth0', + 'netmask': 'ffff:ffff:ffff:ffff::'}, + {'netmask': 'ffff:ffff:ffff:ffff::/64', + 'addr': 'fd2d:dec4:cf59:3c16::1'}, + {'addr': '2001:db8:1::', + 'netmask': 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'}], + 17: [{'addr': '3c:97:0e:8b:1c:f7', + 'broadcast': 'ff:ff:ff:ff:ff:ff'}] + }, + 'eth0:1': { + 2: [{'addr': '192.168.1.56', + 'broadcast': '192.168.1.255', + 'netmask': '255.255.255.0'}], + }, + 'eth1': { + 2: [{'addr': '10.5.0.1', + 'broadcast': '10.5.255.255', + 'netmask': '255.255.0.0'}, + {'addr': '10.6.0.2', + 'broadcast': '10.6.0.255', + 'netmask': '255.255.255.0'}], + 3: [{'addr': 'fe80::3e97:eff:fe8b:1cf7%eth1', + 'netmask': 'ffff:ffff:ffff:ffff::'}], + 17: [{'addr': '3c:97:0e:8b:1c:f7', + 'broadcast': 'ff:ff:ff:ff:ff:ff'}] + }, + 'eth2': { + 10: [{'addr': '3a01:348:2f4:0:685e:5748:ae62:209f', + 'netmask': 'ffff:ffff:ffff:ffff::'}, + {'addr': 'fe80::3e97:edd:fe8b:1cf7%eth0', + 'netmask': 'ffff:ffff:ffff:ffff::'}], + 17: [{'addr': '3c:97:0e:8b:1c:f7', + 'broadcast': 'ff:ff:ff:ff:ff:ff'}] + }, + 'eth2:1': { + 2: [{'addr': '192.168.10.58', + 'broadcast': '192.168.1.255', + 'netmask': '255.255.255.0'}], + }, +} + +IP_OUTPUT = b"""link/ether fa:16:3e:2a:cc:ce brd ff:ff:ff:ff:ff:ff + inet 10.5.16.93/16 brd 10.5.255.255 scope global eth0 + valid_lft forever preferred_lft forever + inet6 2001:db8:1:0:d0cf:528c:23eb:6000/64 scope global + valid_lft forever preferred_lft forever + inet6 2001:db8:1:0:2918:3444:852:5b8a/64 scope global temporary dynamic + valid_lft 86400sec preferred_lft 14400sec + inet6 2001:db8:1:0:f816:3eff:fe2a:ccce/64 scope global dynamic + valid_lft 86400sec preferred_lft 14400sec + inet6 fe80::f816:3eff:fe2a:ccce/64 scope link + valid_lft forever preferred_lft forever +""" + +IP2_OUTPUT = b"""link/ether fa:16:3e:2a:cc:ce brd ff:ff:ff:ff:ff:ff + inet 10.5.16.93/16 brd 10.5.255.255 scope global eth0 + valid_lft forever preferred_lft forever + inet6 2001:db8:1:0:d0cf:528c:23eb:6000/64 scope global + valid_lft forever preferred_lft forever + inet6 2001:db8:1:0:2918:3444:852:5b8a/64 scope global temporary dynamic + valid_lft 86400sec preferred_lft 14400sec + inet6 2001:db8:1:0:f816:3eff:fe2a:ccce/64 scope global mngtmpaddr dynamic + valid_lft 86400sec preferred_lft 14400sec + inet6 fe80::f816:3eff:fe2a:ccce/64 scope link + valid_lft forever preferred_lft forever +""" + +IP_OUTPUT_NO_VALID = b"""link/ether fa:16:3e:2a:cc:ce brd ff:ff:ff:ff:ff:ff + inet 10.5.16.93/16 brd 10.5.255.255 scope global eth0 + valid_lft forever preferred_lft forever + inet6 2001:db8:1:0:2918:3444:852:5b8a/64 scope global temporary dynamic + valid_lft 86400sec preferred_lft 14400sec + inet6 fe80::f816:3eff:fe2a:ccce/64 scope link + valid_lft forever preferred_lft forever +""" + + +class FakeAnswer(object): + def __init__(self, ip): + self.ip = ip + + def __str__(self): + return self.ip + + +class FakeResolver(object): + def __init__(self, ip): + self.ip = ip + + def query(self, hostname, query_type): + if self.ip == '': + return [] + else: + return [FakeAnswer(self.ip)] + + +class FakeReverse(object): + def from_address(self, address): + return '156.94.189.91.in-addr.arpa' + + +class FakeDNSName(object): + def __init__(self, dnsname): + pass + + +class FakeDNS(object): + def __init__(self, ip): + self.resolver = FakeResolver(ip) + self.reversename = FakeReverse() + self.name = MagicMock() + self.name.Name = FakeDNSName + + +class IPTest(unittest.TestCase): + + def mock_ifaddresses(self, iface): + return DUMMY_ADDRESSES[iface] + + def test_get_address_in_network_with_invalid_net(self): + for net in ['192.168.300/22', '192.168.1.0/2a', '2.a']: + self.assertRaises(ValueError, + net_ip.get_address_in_network, + net) + + def _test_get_address_in_network(self, expect_ip_addr, + network, fallback=None, fatal=False): + + def side_effect(iface): + return DUMMY_ADDRESSES[iface] + + with mock.patch.object(netifaces, 'interfaces') as interfaces: + interfaces.return_value = sorted(DUMMY_ADDRESSES.keys()) + with mock.patch.object(netifaces, 'ifaddresses') as ifaddresses: + ifaddresses.side_effect = side_effect + if not fatal: + self.assertEqual(expect_ip_addr, + net_ip.get_address_in_network(network, + fallback, + fatal)) + else: + net_ip.get_address_in_network(network, fallback, fatal) + + @mock.patch.object(subprocess, 'call') + def test_get_address_in_network_with_none(self, popen): + fallback = '10.10.10.10' + self.assertEqual(fallback, + net_ip.get_address_in_network(None, fallback)) + self.assertEqual(None, + net_ip.get_address_in_network(None)) + + self.assertRaises(ValueError, self._test_get_address_in_network, + None, None, fatal=True) + + def test_get_address_in_network_ipv4(self): + self._test_get_address_in_network('192.168.1.55', '192.168.1.0/24') + + def test_get_address_in_network_ipv4_multi(self): + # Assumes that there is an address configured on both but the first + # one is picked# + self._test_get_address_in_network('192.168.1.55', + '192.168.1.0/24 192.168.10.0/24') + + def test_get_address_in_network_ipv4_multi2(self): + # Assumes that there is nothing configured on 192.168.11.0/24 + self._test_get_address_in_network('192.168.10.58', + '192.168.11.0/24 192.168.10.0/24') + + def test_get_address_in_network_ipv4_secondary(self): + self._test_get_address_in_network('10.6.0.2', + '10.6.0.0/24') + + def test_get_address_in_network_ipv6(self): + self._test_get_address_in_network('2a01:348:2f4:0:685e:5748:ae62:209f', + '2a01:348:2f4::/64') + + def test_get_address_in_network_with_non_existent_net(self): + self._test_get_address_in_network(None, '172.16.0.0/16') + + def test_get_address_in_network_fallback_works(self): + fallback = '10.10.0.0' + self._test_get_address_in_network(fallback, '172.16.0.0/16', fallback) + + @mock.patch.object(subprocess, 'call') + def test_get_address_in_network_not_found_fatal(self, popen): + self.assertRaises(ValueError, self._test_get_address_in_network, + None, '172.16.0.0/16', fatal=True) + + def test_get_address_in_network_not_found_not_fatal(self): + self._test_get_address_in_network(None, '172.16.0.0/16', fatal=False) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_address_in_network_netmask(self, _interfaces, _ifaddresses): + """ + Validates that get_address_in_network works with a netmask + that uses the format 'ffff:ffff:ffff::/prefixlen' + """ + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + self._test_get_address_in_network('fd2d:dec4:cf59:3c16::1', + 'fd2d:dec4:cf59:3c16::/64', + fatal=False) + + def test_is_address_in_network(self): + self.assertTrue( + net_ip.is_address_in_network( + '192.168.1.0/24', + '192.168.1.1')) + self.assertFalse( + net_ip.is_address_in_network( + '192.168.1.0/24', + '10.5.1.1')) + self.assertRaises(ValueError, net_ip.is_address_in_network, + 'broken', '192.168.1.1') + self.assertRaises(ValueError, net_ip.is_address_in_network, + '192.168.1.0/24', 'hostname') + self.assertTrue( + net_ip.is_address_in_network( + '2a01:348:2f4::/64', + '2a01:348:2f4:0:685e:5748:ae62:209f') + ) + self.assertFalse( + net_ip.is_address_in_network( + '2a01:348:2f4::/64', + 'fdfc:3bd5:210b:cc8d:8c80:9e10:3f07:371') + ) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_for_address(self, _interfaces, _ifaddresses): + def mock_ifaddresses(iface): + return DUMMY_ADDRESSES[iface] + _interfaces.return_value = ['eth0', 'eth1'] + _ifaddresses.side_effect = mock_ifaddresses + self.assertEquals( + net_ip.get_iface_for_address('192.168.1.220'), + 'eth0') + self.assertEquals(net_ip.get_iface_for_address('10.5.20.4'), 'eth1') + self.assertEquals( + net_ip.get_iface_for_address('2a01:348:2f4:0:685e:5748:ae62:210f'), + 'eth0' + ) + self.assertEquals(net_ip.get_iface_for_address('172.4.5.5'), None) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_netmask_for_address(self, _interfaces, _ifaddresses): + def mock_ifaddresses(iface): + return DUMMY_ADDRESSES[iface] + _interfaces.return_value = ['eth0', 'eth1'] + _ifaddresses.side_effect = mock_ifaddresses + self.assertEquals( + net_ip.get_netmask_for_address('192.168.1.220'), + '255.255.255.0') + self.assertEquals( + net_ip.get_netmask_for_address('10.5.20.4'), + '255.255.0.0') + self.assertEquals(net_ip.get_netmask_for_address('172.4.5.5'), None) + self.assertEquals( + net_ip.get_netmask_for_address( + '2a01:348:2f4:0:685e:5748:ae62:210f'), + '64' + ) + self.assertEquals( + net_ip.get_netmask_for_address('2001:db8:1::'), + '128' + ) + + def test_is_ipv6(self): + self.assertFalse(net_ip.is_ipv6('myhost')) + self.assertFalse(net_ip.is_ipv6('172.4.5.5')) + self.assertTrue(net_ip.is_ipv6('2a01:348:2f4:0:685e:5748:ae62:209f')) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr_no_ipv6(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + with nose.tools.assert_raises(Exception): + net_ip.get_ipv6_addr('eth0:1') + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr_no_global_ipv6(self, _interfaces, + _ifaddresses): + DUMMY_ADDRESSES = { + 'eth0': { + 10: [{'addr': 'fe80::3e97:eff:fe8b:1cf7%eth0', + 'netmask': 'ffff:ffff:ffff:ffff::'}], + } + } + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + self.assertRaises(Exception, net_ip.get_ipv6_addr) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr_exc_list(self, _interfaces, _ifaddresses, + mock_get_iface_from_addr): + def mock_ifaddresses(iface): + return DUMMY_ADDRESSES[iface] + + _interfaces.return_value = ['eth0', 'eth1'] + _ifaddresses.side_effect = mock_ifaddresses + + result = net_ip.get_ipv6_addr( + exc_list='2a01:348:2f4:0:685e:5748:ae62:209f', + inc_aliases=True, + fatal=False + ) + self.assertEqual([], result) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr(self, _interfaces, _ifaddresses, mock_check_out, + mock_get_iface_from_addr): + mock_get_iface_from_addr.return_value = 'eth0' + mock_check_out.return_value = \ + b"inet6 2a01:348:2f4:0:685e:5748:ae62:209f/64 scope global dynamic" + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_ipv6_addr(dynamic_only=False) + self.assertEqual(['2a01:348:2f4:0:685e:5748:ae62:209f'], result) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr_global_dynamic(self, _interfaces, _ifaddresses, + mock_check_out, + mock_get_iface_from_addr): + mock_get_iface_from_addr.return_value = 'eth0' + mock_check_out.return_value = \ + b"inet6 2a01:348:2f4:0:685e:5748:ae62:209f/64 scope global dynamic" + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_ipv6_addr(dynamic_only=False) + self.assertEqual(['2a01:348:2f4:0:685e:5748:ae62:209f'], result) + + @patch.object(netifaces, 'interfaces') + def test_get_ipv6_addr_invalid_nic(self, _interfaces): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + self.assertRaises(Exception, net_ip.get_ipv6_addr, 'eth1') + + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + def test_is_ipv6_disabled(self, mock_check_output): + # verify that the function does look for the right thing + mock_check_output.return_value = """ + Some lines before + net.ipv6.conf.all.disable_ipv6 = 1 + Some lines afterward + """ + self.assertTrue(net_ip.is_ipv6_disabled()) + mock_check_output.assert_called_once_with( + ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], + stderr=subprocess.STDOUT, universal_newlines=True) + # if it isn't there, it must return false + mock_check_output.return_value = "" + self.assertFalse(net_ip.is_ipv6_disabled()) + # If the syscall returns an error, then return True + + def fake_check_call(*args, **kwargs): + raise subprocess.CalledProcessError(['called'], 1) + mock_check_output.side_effect = fake_check_call + self.assertTrue(net_ip.is_ipv6_disabled()) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("eth0") + self.assertEqual(["192.168.1.55"], result) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_excaliases(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("eth0") + self.assertEqual(['192.168.1.55'], result) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_incaliases(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("eth0", inc_aliases=True) + self.assertEqual(['192.168.1.55', '192.168.1.56'], result) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_exclist(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("eth0", inc_aliases=True, + exc_list=['192.168.1.55']) + self.assertEqual(['192.168.1.56'], result) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_mixedaddr(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("eth2", inc_aliases=True) + self.assertEqual(["192.168.10.58"], result) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_full_interface_path(self, _interfaces, + _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + result = net_ip.get_iface_addr("/dev/eth0") + self.assertEqual(["192.168.1.55"], result) + + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_invalid_type(self, _interfaces): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + with nose.tools.assert_raises(Exception): + net_ip.get_iface_addr(iface='eth0', inet_type='AF_BOB') + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_invalid_interface(self, _interfaces, _ifaddresses): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + result = net_ip.get_ipv4_addr("eth3", fatal=False) + self.assertEqual([], result) + + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_invalid_interface_fatal(self, _interfaces): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + with nose.tools.assert_raises(Exception): + net_ip.get_ipv4_addr("eth3", fatal=True) + + @patch.object(netifaces, 'interfaces') + def test_get_iface_addr_invalid_interface_fatal_incaliases(self, + _interfaces): + _interfaces.return_value = DUMMY_ADDRESSES.keys() + with nose.tools.assert_raises(Exception): + net_ip.get_ipv4_addr("eth3", fatal=True, inc_aliases=True) + + @patch.object(netifaces, 'ifaddresses') + @patch.object(netifaces, 'interfaces') + def test_get_get_iface_addr_interface_has_no_ipv4(self, _interfaces, + _ifaddresses): + + # This will raise a KeyError since we are looking for "2" + # (actually, netiface.AF_INET). + DUMMY_ADDRESSES = { + 'eth0': { + 10: [{'addr': 'fe80::3e97:eff:fe8b:1cf7%eth0', + 'netmask': 'ffff:ffff:ffff:ffff::'}], + } + } + + _interfaces.return_value = DUMMY_ADDRESSES.keys() + _ifaddresses.side_effect = DUMMY_ADDRESSES.__getitem__ + + result = net_ip.get_ipv4_addr("eth0", fatal=False) + self.assertEqual([], result) + + @patch('glob.glob') + def test_get_bridges(self, _glob): + _glob.return_value = ['/sys/devices/virtual/net/br0/bridge'] + self.assertEqual(['br0'], net_ip.get_bridges()) + + @patch.object(net_ip, 'get_bridges') + @patch('glob.glob') + def test_get_bridge_nics(self, _glob, _get_bridges): + _glob.return_value = ['/sys/devices/virtual/net/br0/brif/eth4', + '/sys/devices/virtual/net/br0/brif/eth5'] + self.assertEqual(['eth4', 'eth5'], net_ip.get_bridge_nics('br0')) + + @patch.object(net_ip, 'get_bridges') + @patch('glob.glob') + def test_get_bridge_nics_invalid_br(self, _glob, _get_bridges): + _glob.return_value = [] + self.assertEqual([], net_ip.get_bridge_nics('br1')) + + @patch.object(net_ip, 'get_bridges') + @patch.object(net_ip, 'get_bridge_nics') + def test_is_bridge_member(self, _get_bridge_nics, _get_bridges): + _get_bridges.return_value = ['br0'] + _get_bridge_nics.return_value = ['eth4', 'eth5'] + self.assertTrue(net_ip.is_bridge_member('eth4')) + self.assertFalse(net_ip.is_bridge_member('eth6')) + + def test_format_ipv6_addr(self): + DUMMY_ADDRESS = '2001:db8:1:0:f131:fc84:ea37:7d4' + self.assertEquals(net_ip.format_ipv6_addr(DUMMY_ADDRESS), + '[2001:db8:1:0:f131:fc84:ea37:7d4]') + + def test_format_invalid_ipv6_addr(self): + INVALID_IPV6_ADDR = 'myhost' + self.assertEquals(net_ip.format_ipv6_addr(INVALID_IPV6_ADDR), + None) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_global_address(self, mock_get_iface_addr, mock_check_out, + mock_get_iface_from_addr): + mock_get_iface_from_addr.return_value = 'eth0' + mock_check_out.return_value = IP_OUTPUT + scope_global_addr = '2001:db8:1:0:d0cf:528c:23eb:6000' + scope_global_dyn_addr = '2001:db8:1:0:f816:3eff:fe2a:ccce' + mock_get_iface_addr.return_value = [scope_global_addr, + scope_global_dyn_addr, + '2001:db8:1:0:2918:3444:852:5b8a', + 'fe80::f816:3eff:fe2a:ccce%eth0'] + self.assertEqual([scope_global_addr, scope_global_dyn_addr], + net_ip.get_ipv6_addr(dynamic_only=False)) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_global_dynamic_address(self, mock_get_iface_addr, + mock_check_out, + mock_get_iface_from_addr): + mock_get_iface_from_addr.return_value = 'eth0' + mock_check_out.return_value = IP_OUTPUT + scope_global_addr = '2001:db8:1:0:d0cf:528c:23eb:6000' + scope_global_dyn_addr = '2001:db8:1:0:f816:3eff:fe2a:ccce' + mock_get_iface_addr.return_value = [scope_global_addr, + scope_global_dyn_addr, + '2001:db8:1:0:2918:3444:852:5b8a', + 'fe80::f816:3eff:fe2a:ccce%eth0'] + self.assertEqual([scope_global_dyn_addr], net_ip.get_ipv6_addr()) + + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_global_dynamic_address_ip2(self, mock_get_iface_addr, + mock_check_out, + mock_get_iface_from_addr): + mock_get_iface_from_addr.return_value = 'eth0' + mock_check_out.return_value = IP2_OUTPUT + scope_global_addr = '2001:db8:1:0:d0cf:528c:23eb:6000' + scope_global_dyn_addr = '2001:db8:1:0:f816:3eff:fe2a:ccce' + mock_get_iface_addr.return_value = [scope_global_addr, + scope_global_dyn_addr, + '2001:db8:1:0:2918:3444:852:5b8a', + 'fe80::f816:3eff:fe2a:ccce%eth0'] + self.assertEqual([scope_global_dyn_addr], net_ip.get_ipv6_addr()) + + @patch('charmhelpers.contrib.network.ip.subprocess.check_output') + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_global_dynamic_address_invalid_address( + self, mock_get_iface_addr, mock_check_out): + mock_get_iface_addr.return_value = [] + with nose.tools.assert_raises(Exception): + net_ip.get_ipv6_addr() + + mock_get_iface_addr.return_value = ['2001:db8:1:0:2918:3444:852:5b8a'] + mock_check_out.return_value = IP_OUTPUT_NO_VALID + with nose.tools.assert_raises(Exception): + net_ip.get_ipv6_addr() + + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_addr_w_iface(self, mock_get_iface_addr): + mock_get_iface_addr.return_value = [] + net_ip.get_ipv6_addr(iface='testif', fatal=False) + mock_get_iface_addr.assert_called_once_with(iface='testif', + inet_type='AF_INET6', + inc_aliases=False, + fatal=False, exc_list=None) + + @patch('charmhelpers.contrib.network.ip.unit_get') + @patch('charmhelpers.contrib.network.ip.get_iface_from_addr') + @patch('charmhelpers.contrib.network.ip.get_iface_addr') + def test_get_ipv6_addr_no_iface(self, mock_get_iface_addr, + mock_get_iface_from_addr, mock_unit_get): + mock_unit_get.return_value = '1.2.3.4' + mock_get_iface_addr.return_value = [] + mock_get_iface_from_addr.return_value = "testif" + net_ip.get_ipv6_addr(fatal=False) + mock_get_iface_from_addr.assert_called_once_with('1.2.3.4') + mock_get_iface_addr.assert_called_once_with(iface='testif', + inet_type='AF_INET6', + inc_aliases=False, + fatal=False, exc_list=None) + + @patch('netifaces.interfaces') + @patch('netifaces.ifaddresses') + @patch('charmhelpers.contrib.network.ip.log') + def test_get_iface_from_addr(self, mock_log, mock_ifaddresses, + mock_interfaces): + mock_ifaddresses.side_effect = lambda iface: DUMMY_ADDRESSES[iface] + mock_interfaces.return_value = sorted(DUMMY_ADDRESSES.keys()) + addr = 'fe80::3e97:eff:fe8b:1cf7' + self.assertEqual(net_ip.get_iface_from_addr(addr), 'eth0') + + with nose.tools.assert_raises(Exception): + net_ip.get_iface_from_addr('1.2.3.4') + + def test_is_ip(self): + self.assertTrue(net_ip.is_ip('10.0.0.1')) + self.assertTrue(net_ip.is_ip('2001:db8:1:0:2918:3444:852:5b8a')) + self.assertFalse(net_ip.is_ip('www.ubuntu.com')) + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_host_ip_with_hostname(self, apt_install): + fake_dns = FakeDNS('10.0.0.1') + with patch(builtin_import, side_effect=[fake_dns]): + ip = net_ip.get_host_ip('www.ubuntu.com') + self.assertEquals(ip, '10.0.0.1') + + @patch('charmhelpers.contrib.network.ip.ns_query') + @patch('charmhelpers.contrib.network.ip.socket.gethostbyname') + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_host_ip_with_hostname_no_dns(self, apt_install, socket, + ns_query): + ns_query.return_value = [] + fake_dns = FakeDNS(None) + socket.return_value = '10.0.0.1' + with patch(builtin_import, side_effect=[fake_dns]): + ip = net_ip.get_host_ip('www.ubuntu.com') + self.assertEquals(ip, '10.0.0.1') + + @patch('charmhelpers.contrib.network.ip.log') + @patch('charmhelpers.contrib.network.ip.ns_query') + @patch('charmhelpers.contrib.network.ip.socket.gethostbyname') + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_host_ip_with_hostname_fallback(self, apt_install, socket, + ns_query, *args): + ns_query.return_value = [] + fake_dns = FakeDNS(None) + + def r(): + raise Exception() + + socket.side_effect = r + with patch(builtin_import, side_effect=[fake_dns]): + ip = net_ip.get_host_ip('www.ubuntu.com', fallback='127.0.0.1') + self.assertEquals(ip, '127.0.0.1') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_host_ip_with_ip(self, apt_install): + fake_dns = FakeDNS('5.5.5.5') + with patch(builtin_import, side_effect=[fake_dns]): + ip = net_ip.get_host_ip('4.2.2.1') + self.assertEquals(ip, '4.2.2.1') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_ns_query_trigger_apt_install(self, apt_install): + fake_dns = FakeDNS('5.5.5.5') + with patch(builtin_import, side_effect=[ImportError, fake_dns]): + nsq = net_ip.ns_query('5.5.5.5') + if six.PY2: + apt_install.assert_called_with('python-dnspython', fatal=True) + else: + apt_install.assert_called_with('python3-dnspython', fatal=True) + self.assertEquals(nsq, '5.5.5.5') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_ns_query_ptr_record(self, apt_install): + fake_dns = FakeDNS('127.0.0.1') + with patch(builtin_import, side_effect=[fake_dns]): + nsq = net_ip.ns_query('127.0.0.1') + self.assertEquals(nsq, '127.0.0.1') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_ns_query_a_record(self, apt_install): + fake_dns = FakeDNS('127.0.0.1') + fake_dns_name = FakeDNSName('www.somedomain.tld') + with patch(builtin_import, side_effect=[fake_dns]): + nsq = net_ip.ns_query(fake_dns_name) + self.assertEquals(nsq, '127.0.0.1') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_ns_query_blank_record(self, apt_install): + fake_dns = FakeDNS(None) + with patch(builtin_import, side_effect=[fake_dns, fake_dns]): + nsq = net_ip.ns_query(None) + self.assertEquals(nsq, None) + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_ns_query_lookup_fail(self, apt_install): + fake_dns = FakeDNS('') + with patch(builtin_import, side_effect=[fake_dns, fake_dns]): + nsq = net_ip.ns_query('nonexistant') + self.assertEquals(nsq, None) + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_with_ip(self, apt_install): + fake_dns = FakeDNS('www.ubuntu.com') + with patch(builtin_import, side_effect=[fake_dns, fake_dns]): + hn = net_ip.get_hostname('4.2.2.1') + self.assertEquals(hn, 'www.ubuntu.com') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_with_ip_not_fqdn(self, apt_install): + fake_dns = FakeDNS('packages.ubuntu.com') + with patch(builtin_import, side_effect=[fake_dns, fake_dns]): + hn = net_ip.get_hostname('4.2.2.1', fqdn=False) + self.assertEquals(hn, 'packages') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_with_hostname(self, apt_install): + hn = net_ip.get_hostname('www.ubuntu.com') + self.assertEquals(hn, 'www.ubuntu.com') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_with_hostname_trailingdot(self, apt_install): + hn = net_ip.get_hostname('www.ubuntu.com.') + self.assertEquals(hn, 'www.ubuntu.com') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_with_hostname_not_fqdn(self, apt_install): + hn = net_ip.get_hostname('packages.ubuntu.com', fqdn=False) + self.assertEquals(hn, 'packages') + + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_trigger_apt_install(self, apt_install): + fake_dns = FakeDNS('www.ubuntu.com') + with patch(builtin_import, side_effect=[ImportError, fake_dns, + fake_dns]): + hn = net_ip.get_hostname('4.2.2.1') + if six.PY2: + apt_install.assert_called_with('python-dnspython', fatal=True) + else: + apt_install.assert_called_with('python3-dnspython', fatal=True) + + self.assertEquals(hn, 'www.ubuntu.com') + + @patch('charmhelpers.contrib.network.ip.socket.gethostbyaddr') + @patch('charmhelpers.contrib.network.ip.ns_query') + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_lookup_fail(self, apt_install, ns_query, socket): + fake_dns = FakeDNS('www.ubuntu.com') + ns_query.return_value = [] + socket.return_value = () + with patch(builtin_import, side_effect=[fake_dns, fake_dns]): + hn = net_ip.get_hostname('4.2.2.1') + self.assertEquals(hn, None) + + @patch('charmhelpers.contrib.network.ip.socket.gethostbyaddr') + @patch('charmhelpers.contrib.network.ip.ns_query') + @patch('charmhelpers.contrib.network.ip.apt_install') + def test_get_hostname_lookup_fail_gethostbyaddr_fallback( + self, apt_install, ns_query, socket): + fake_dns = FakeDNS('www.ubuntu.com') + ns_query.return_value = [] + socket.return_value = ("www.ubuntu.com", "", "") + with patch(builtin_import, side_effect=[fake_dns]): + hn = net_ip.get_hostname('4.2.2.1') + self.assertEquals(hn, "www.ubuntu.com") + + @patch('charmhelpers.contrib.network.ip.subprocess.call') + def test_port_has_listener(self, subprocess_call): + subprocess_call.return_value = 1 + self.assertEqual(net_ip.port_has_listener('ip-address', 50), False) + subprocess_call.assert_called_with(['nc', '-z', 'ip-address', '50']) + subprocess_call.return_value = 0 + self.assertEqual(net_ip.port_has_listener('ip-address', 70), True) + subprocess_call.assert_called_with(['nc', '-z', 'ip-address', '70']) + + @patch.object(net_ip, 'log', lambda *args, **kwargs: None) + @patch.object(net_ip, 'config') + @patch.object(net_ip, 'network_get_primary_address') + @patch.object(net_ip, 'get_address_in_network') + @patch.object(net_ip, 'unit_get') + @patch.object(net_ip, 'get_ipv6_addr') + @patch.object(net_ip, 'assert_charm_supports_ipv6') + def test_get_relation_ip(self, assert_charm_supports_ipv6, get_ipv6_addr, + unit_get, get_address_in_network, + network_get_primary_address, config): + ACCESS_IP = '10.50.1.1' + ACCESS_NETWORK = '10.50.1.0/24' + AMQP_IP = '10.200.1.1' + IPV6_IP = '2001:DB8::1' + DEFAULT_IP = '172.16.1.1' + assert_charm_supports_ipv6.return_value = True + get_ipv6_addr.return_value = [IPV6_IP] + unit_get.return_value = DEFAULT_IP + get_address_in_network.return_value = DEFAULT_IP + network_get_primary_address.return_value = AMQP_IP + + # Network-get calls + _config = {'prefer-ipv6': False} + config.side_effect = lambda key: _config.get(key) + + network_get_primary_address.side_effect = NotImplementedError + self.assertEqual(DEFAULT_IP, net_ip.get_relation_ip('amqp')) + + network_get_primary_address.side_effect = net_ip.NoNetworkBinding + self.assertEqual(DEFAULT_IP, net_ip.get_relation_ip('doesnotexist')) + + network_get_primary_address.side_effect = None + self.assertEqual(AMQP_IP, net_ip.get_relation_ip('amqp')) + + self.assertFalse(get_address_in_network.called) + + # Specific CIDR network + get_address_in_network.return_value = ACCESS_IP + network_get_primary_address.return_value = DEFAULT_IP + self.assertEqual( + ACCESS_IP, + net_ip.get_relation_ip('shared-db', + cidr_network=ACCESS_NETWORK)) + get_address_in_network.assert_called_with(ACCESS_NETWORK, DEFAULT_IP) + + self.assertFalse(assert_charm_supports_ipv6.called) + + # IPv6 + _config = {'prefer-ipv6': True} + config.side_effect = lambda key: _config.get(key) + self.assertEqual(IPV6_IP, net_ip.get_relation_ip('amqp')) + assert_charm_supports_ipv6.assert_called_with() diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/test_ovs.py b/nrpe/mod/charmhelpers/tests/contrib/network/test_ovs.py new file mode 100644 index 0000000..221e87a --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/test_ovs.py @@ -0,0 +1,368 @@ +import subprocess +import unittest + +from mock import patch, MagicMock + +import charmhelpers.contrib.network.ovs as ovs + +from tests.helpers import patch_open + + +# NOTE(fnordahl): some functions drectly under the ``contrib.network.ovs`` +# module have their unit tests in the ``test_ovs.py`` module in the +# ``tests.contrib.network.ovs`` package. + + +GOOD_CERT = '''Certificate: + Data: + Version: 1 (0x0) + Serial Number: 13798680962510501282 (0xbf7ec33a136235a2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=CA, L=Palo Alto, O=Open vSwitch, OU=Open vSwitch + Validity + Not Before: Jun 28 17:02:19 2013 GMT + Not After : Jun 28 17:02:19 2019 GMT + Subject: C=US, ST=CA, L=Palo Alto, O=Open vSwitch, OU=Open vSwitch + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e8:a7:db:0a:6d:c0:16:4a:14:96:1d:74:91:15: + 64:3f:ae:2a:54:be:2a:fe:10:14:9a:73:39:d8:58: + 74:7f:ab:d5:f2:39:aa:9a:27:7c:31:82:f8:74:42: + 46:8d:c5:3b:42:55:52:be:75:7f:a5:b1:ec:d5:29: + 9f:62:0e:de:31:27:2b:95:1f:24:0d:ca:8c:48:30: + 96:9f:ba:b7:9d:eb:c1:bd:93:05:e3:d8:ca:66:5a: + e9:cb:a5:7a:3a:8d:27:e2:05:9d:88:fc:a9:ef:af: + 47:4c:66:ce:c6:43:73:1a:85:f4:5f:b9:53:5b:29: + f3:c3:23:1f:0c:20:95:11:50:71:b2:f6:01:23:3f: + 66:0f:5c:43:c2:90:fb:e5:98:73:98:e9:38:bb:1f: + 1b:89:97:1e:dc:d7:98:07:68:32:ec:da:1d:69:0b: + e2:df:40:fb:64:52:e5:e9:40:27:b0:ca:73:21:51: + f6:8f:00:20:c0:2b:1a:d4:01:c2:32:38:9d:d1:8d: + 88:71:46:a9:42:0d:ee:3b:1c:88:db:27:69:49:f9: + 60:34:70:61:3d:60:df:7e:e4:e1:1d:c6:16:89:05: + ba:31:06:eb:88:b5:78:94:5d:8c:9d:88:fe:f2:c2: + 80:a1:04:15:d3:84:85:d3:aa:5a:1d:53:5c:f8:57: + ae:61 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 14:7e:ca:c3:fc:93:60:9f:80:e0:65:2e:ef:41:2d:f9:af:77: + da:6d:e2:e0:11:70:17:fb:e5:67:4c:f0:ad:39:ec:96:ef:fe: + d5:95:94:70:e5:52:31:68:63:8c:ea:b3:a1:8e:02:e2:91:4b: + a8:8c:07:86:fd:80:98:a2:b1:90:2b:9c:2e:ab:f4:73:9d:8f: + fd:31:b9:8f:fe:6c:af:d6:bf:72:44:89:08:93:19:ef:2b:c3: + 7c:ab:ba:bc:57:ca:f1:17:e4:e8:81:40:ca:65:df:84:be:10: + 2c:42:46:af:d2:e0:0d:df:5d:56:53:65:13:e0:20:55:b4:ee: + cd:5e:b5:c4:97:1d:3e:a6:c1:9c:7e:b8:87:ee:64:78:a5:59: + e5:b2:79:47:9a:8e:59:fa:c4:18:ea:27:fd:a2:d5:76:d0:ae: + d9:05:f6:0e:23:ca:7d:66:a1:ba:18:67:f5:6d:bb:51:5a:f5: + 52:e9:17:bb:63:15:24:b4:61:25:9f:d9:9c:89:58:93:9a:c3: + 74:55:72:3e:f9:ff:ef:54:7d:e8:28:78:ba:3c:c7:15:ba:b9: + c6:e3:8c:61:cb:a9:ed:8d:07:16:0d:8d:f6:1c:36:11:69:08: + b8:45:7d:fc:fd:d1:ab:2d:9b:4e:9c:dd:11:78:50:c7:87:9f: + 4a:24:9c:a0 +-----BEGIN CERTIFICATE----- +MIIDwjCCAqoCCQC/fsM6E2I1ojANBgkqhkiG9w0BAQUFADCBojELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAkNBMRIwEAYDVQQHEwlQYWxvIEFsdG8xFTATBgNVBAoTDE9w +ZW4gdlN3aXRjaDEfMB0GA1UECxMWT3BlbiB2U3dpdGNoIGNlcnRpZmllcjE6MDgG +A1UEAxMxb3ZzY2xpZW50IGlkOjU4MTQ5N2E1LWJjMDAtNGVjYy1iNzkwLTU3NTZj +ZWUxNmE0ODAeFw0xMzA2MjgxNzAyMTlaFw0xOTA2MjgxNzAyMTlaMIGiMQswCQYD +VQQGEwJVUzELMAkGA1UECBMCQ0ExEjAQBgNVBAcTCVBhbG8gQWx0bzEVMBMGA1UE +ChMMT3BlbiB2U3dpdGNoMR8wHQYDVQQLExZPcGVuIHZTd2l0Y2ggY2VydGlmaWVy +MTowOAYDVQQDEzFvdnNjbGllbnQgaWQ6NTgxNDk3YTUtYmMwMC00ZWNjLWI3OTAt +NTc1NmNlZTE2YTQ4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6Kfb +Cm3AFkoUlh10kRVkP64qVL4q/hAUmnM52Fh0f6vV8jmqmid8MYL4dEJGjcU7QlVS +vnV/pbHs1SmfYg7eMScrlR8kDcqMSDCWn7q3nevBvZMF49jKZlrpy6V6Oo0n4gWd +iPyp769HTGbOxkNzGoX0X7lTWynzwyMfDCCVEVBxsvYBIz9mD1xDwpD75ZhzmOk4 +ux8biZce3NeYB2gy7NodaQvi30D7ZFLl6UAnsMpzIVH2jwAgwCsa1AHCMjid0Y2I +cUapQg3uOxyI2ydpSflgNHBhPWDffuThHcYWiQW6MQbriLV4lF2MnYj+8sKAoQQV +04SF06paHVNc+FeuYQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAUfsrD/JNgn4Dg +ZS7vQS35r3fabeLgEXAX++VnTPCtOeyW7/7VlZRw5VIxaGOM6rOhjgLikUuojAeG +/YCYorGQK5wuq/RznY/9MbmP/myv1r9yRIkIkxnvK8N8q7q8V8rxF+TogUDKZd+E +vhAsQkav0uAN311WU2UT4CBVtO7NXrXElx0+psGcfriH7mR4pVnlsnlHmo5Z+sQY +6if9otV20K7ZBfYOI8p9ZqG6GGf1bbtRWvVS6Re7YxUktGEln9mciViTmsN0VXI+ ++f/vVH3oKHi6PMcVurnG44xhy6ntjQcWDY32HDYRaQi4RX38/dGrLZtOnN0ReFDH +h59KJJyg +-----END CERTIFICATE----- +''' + +PEM_ENCODED = '''-----BEGIN CERTIFICATE----- +MIIDwjCCAqoCCQC/fsM6E2I1ojANBgkqhkiG9w0BAQUFADCBojELMAkGA1UEBhMC +VVMxCzAJBgNVBAgTAkNBMRIwEAYDVQQHEwlQYWxvIEFsdG8xFTATBgNVBAoTDE9w +ZW4gdlN3aXRjaDEfMB0GA1UECxMWT3BlbiB2U3dpdGNoIGNlcnRpZmllcjE6MDgG +A1UEAxMxb3ZzY2xpZW50IGlkOjU4MTQ5N2E1LWJjMDAtNGVjYy1iNzkwLTU3NTZj +ZWUxNmE0ODAeFw0xMzA2MjgxNzAyMTlaFw0xOTA2MjgxNzAyMTlaMIGiMQswCQYD +VQQGEwJVUzELMAkGA1UECBMCQ0ExEjAQBgNVBAcTCVBhbG8gQWx0bzEVMBMGA1UE +ChMMT3BlbiB2U3dpdGNoMR8wHQYDVQQLExZPcGVuIHZTd2l0Y2ggY2VydGlmaWVy +MTowOAYDVQQDEzFvdnNjbGllbnQgaWQ6NTgxNDk3YTUtYmMwMC00ZWNjLWI3OTAt +NTc1NmNlZTE2YTQ4MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA6Kfb +Cm3AFkoUlh10kRVkP64qVL4q/hAUmnM52Fh0f6vV8jmqmid8MYL4dEJGjcU7QlVS +vnV/pbHs1SmfYg7eMScrlR8kDcqMSDCWn7q3nevBvZMF49jKZlrpy6V6Oo0n4gWd +iPyp769HTGbOxkNzGoX0X7lTWynzwyMfDCCVEVBxsvYBIz9mD1xDwpD75ZhzmOk4 +ux8biZce3NeYB2gy7NodaQvi30D7ZFLl6UAnsMpzIVH2jwAgwCsa1AHCMjid0Y2I +cUapQg3uOxyI2ydpSflgNHBhPWDffuThHcYWiQW6MQbriLV4lF2MnYj+8sKAoQQV +04SF06paHVNc+FeuYQIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQAUfsrD/JNgn4Dg +ZS7vQS35r3fabeLgEXAX++VnTPCtOeyW7/7VlZRw5VIxaGOM6rOhjgLikUuojAeG +/YCYorGQK5wuq/RznY/9MbmP/myv1r9yRIkIkxnvK8N8q7q8V8rxF+TogUDKZd+E +vhAsQkav0uAN311WU2UT4CBVtO7NXrXElx0+psGcfriH7mR4pVnlsnlHmo5Z+sQY +6if9otV20K7ZBfYOI8p9ZqG6GGf1bbtRWvVS6Re7YxUktGEln9mciViTmsN0VXI+ ++f/vVH3oKHi6PMcVurnG44xhy6ntjQcWDY32HDYRaQi4RX38/dGrLZtOnN0ReFDH +h59KJJyg +-----END CERTIFICATE-----''' + +BAD_CERT = ''' NO MARKERS ''' +TO_PATCH = [ + "apt_install", + "log", + "hashlib", +] + + +class OVSHelpersTest(unittest.TestCase): + + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.contrib.network.ovs.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + @patch('subprocess.check_output') + def test_get_bridges(self, check_output): + check_output.return_value = b"br1\n br2 " + self.assertEqual(ovs.get_bridges(), ['br1', 'br2']) + check_output.assert_called_once_with(['ovs-vsctl', 'list-br']) + + @patch('subprocess.check_output') + def test_get_bridge_ports(self, check_output): + check_output.return_value = b"p1\n p2 \np3" + self.assertEqual(ovs.get_bridge_ports('br1'), ['p1', 'p2', 'p3']) + check_output.assert_called_once_with( + ['ovs-vsctl', '--', 'list-ports', 'br1']) + + @patch.object(ovs, 'get_bridges') + @patch.object(ovs, 'get_bridge_ports') + def test_get_bridges_and_ports_map(self, get_bridge_ports, get_bridges): + get_bridges.return_value = ['br1', 'br2'] + get_bridge_ports.side_effect = [ + ['p1', 'p2'], + ['p3']] + self.assertEqual(ovs.get_bridges_and_ports_map(), { + 'br1': ['p1', 'p2'], + 'br2': ['p3'], + }) + + @patch('subprocess.check_call') + def test_del_bridge(self, check_call): + ovs.del_bridge('test') + check_call.assert_called_with(["ovs-vsctl", "--", "--if-exists", + "del-br", 'test']) + self.assertTrue(self.log.call_count == 1) + + @patch.object(ovs, 'port_to_br') + @patch.object(ovs, 'add_bridge_port') + @patch.object(ovs, 'lsb_release') + @patch('os.path.exists') + @patch('subprocess.check_call') + def test_add_ovsbridge_linuxbridge(self, check_call, exists, lsb_release, + add_bridge_port, + port_to_br): + exists.return_value = True + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + port_to_br.return_value = None + if_and_port_data = { + 'external-ids': {'mycharm': 'br-ex'} + } + with patch_open() as (mock_open, mock_file): + ovs.add_ovsbridge_linuxbridge( + 'br-ex', 'br-eno1', ifdata=if_and_port_data, + portdata=if_and_port_data) + + check_call.assert_called_with(['ifup', 'veth-br-eno1']) + add_bridge_port.assert_called_with( + 'br-ex', 'veth-br-eno1', ifdata=if_and_port_data, exclusive=False, + portdata=if_and_port_data) + + @patch.object(ovs, 'port_to_br') + @patch.object(ovs, 'add_bridge_port') + @patch.object(ovs, 'lsb_release') + @patch('os.path.exists') + @patch('subprocess.check_call') + def test_add_ovsbridge_linuxbridge_already_direct_wired( + self, check_call, exists, lsb_release, add_bridge_port, port_to_br): + exists.return_value = True + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + port_to_br.return_value = 'br-ex' + ovs.add_ovsbridge_linuxbridge('br-ex', 'br-eno1') + check_call.assert_not_called() + add_bridge_port.assert_not_called() + + @patch.object(ovs, 'port_to_br') + @patch.object(ovs, 'add_bridge_port') + @patch.object(ovs, 'lsb_release') + @patch('os.path.exists') + @patch('subprocess.check_call') + def test_add_ovsbridge_linuxbridge_longname(self, check_call, exists, + lsb_release, add_bridge_port, + port_to_br): + exists.return_value = True + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + port_to_br.return_value = None + mock_hasher = MagicMock() + mock_hasher.hexdigest.return_value = '12345678901234578910' + self.hashlib.sha256.return_value = mock_hasher + with patch_open() as (mock_open, mock_file): + ovs.add_ovsbridge_linuxbridge('br-ex', 'br-reallylongname') + + check_call.assert_called_with(['ifup', 'cvb12345678-10']) + add_bridge_port.assert_called_with( + 'br-ex', 'cvb12345678-10', ifdata=None, exclusive=False, + portdata=None) + + @patch('os.path.exists') + def test_is_linuxbridge_interface_false(self, exists): + exists.return_value = False + result = ovs.is_linuxbridge_interface('eno1') + self.assertFalse(result) + + @patch('os.path.exists') + def test_is_linuxbridge_interface_true(self, exists): + exists.return_value = True + result = ovs.is_linuxbridge_interface('eno1') + self.assertTrue(result) + + @patch('subprocess.check_call') + def test_set_manager(self, check_call): + ovs.set_manager('manager') + check_call.assert_called_with(['ovs-vsctl', 'set-manager', + 'ssl:manager']) + self.assertTrue(self.log.call_count == 1) + + @patch('subprocess.check_call') + def test_set_Open_vSwitch_column_value(self, check_call): + ovs.set_Open_vSwitch_column_value('other_config:foo=bar') + check_call.assert_called_with(['ovs-vsctl', 'set', + 'Open_vSwitch', '.', 'other_config:foo=bar']) + self.assertTrue(self.log.call_count == 1) + + @patch('os.path.exists') + def test_get_certificate_good_cert(self, exists): + exists.return_value = True + with patch_open() as (mock_open, mock_file): + mock_file.read.return_value = GOOD_CERT + self.assertEqual(ovs.get_certificate(), PEM_ENCODED) + self.assertTrue(self.log.call_count == 1) + + @patch('os.path.exists') + def test_get_certificate_bad_cert(self, exists): + exists.return_value = True + with patch_open() as (mock_open, mock_file): + mock_file.read.return_value = BAD_CERT + self.assertRaises(RuntimeError, ovs.get_certificate) + self.assertTrue(self.log.call_count == 1) + + @patch('os.path.exists') + def test_get_certificate_missing(self, exists): + exists.return_value = False + self.assertIsNone(ovs.get_certificate()) + self.assertTrue(self.log.call_count == 1) + + @patch('os.path.exists') + @patch.object(ovs, 'service') + def test_full_restart(self, service, exists): + exists.return_value = False + ovs.full_restart() + service.assert_called_with('force-reload-kmod', 'openvswitch-switch') + + @patch('os.path.exists') + @patch.object(ovs, 'service') + def test_full_restart_upstart(self, service, exists): + exists.return_value = True + ovs.full_restart() + service.assert_called_with('start', 'openvswitch-force-reload-kmod') + + @patch('subprocess.check_output') + def test_port_to_br(self, check_output): + check_output.return_value = b'br-ex' + self.assertEqual(ovs.port_to_br('br-lb'), + 'br-ex') + + @patch('subprocess.check_output') + def test_port_to_br_not_found(self, check_output): + check_output.side_effect = subprocess.CalledProcessError(1, 'not found') + self.assertEqual(ovs.port_to_br('br-lb'), None) + + @patch('subprocess.check_call') + def test_enable_ipfix_defaults(self, check_call): + ovs.enable_ipfix('br-int', + '10.5.0.10:4739') + check_call.assert_called_once_with([ + 'ovs-vsctl', 'set', 'Bridge', 'br-int', 'ipfix=@i', '--', + '--id=@i', 'create', 'IPFIX', + 'targets="10.5.0.10:4739"', + 'sampling=64', + 'cache_active_timeout=60', + 'cache_max_flows=128', + ]) + + @patch('subprocess.check_call') + def test_enable_ipfix_values(self, check_call): + ovs.enable_ipfix('br-int', + '10.5.0.10:4739', + sampling=120, + cache_max_flows=24, + cache_active_timeout=120) + check_call.assert_called_once_with([ + 'ovs-vsctl', 'set', 'Bridge', 'br-int', 'ipfix=@i', '--', + '--id=@i', 'create', 'IPFIX', + 'targets="10.5.0.10:4739"', + 'sampling=120', + 'cache_active_timeout=120', + 'cache_max_flows=24', + ]) + + @patch('subprocess.check_call') + def test_disable_ipfix(self, check_call): + ovs.disable_ipfix('br-int') + check_call.assert_called_once_with( + ['ovs-vsctl', 'clear', 'Bridge', 'br-int', 'ipfix'] + ) + + @patch.object(ovs, 'lsb_release') + @patch('os.path.exists') + def test_setup_eni_sources_eni_folder(self, exists, lsb_release): + exists.return_value = True + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + with patch_open() as (_, mock_file): + # Mocked initial /etc/network/interfaces file content: + mock_file.__iter__.return_value = [ + 'some line', + 'some other line'] + + ovs.setup_eni() + mock_file.write.assert_called_once_with( + '\nsource /etc/network/interfaces.d/*') + + @patch.object(ovs, 'lsb_release') + @patch('os.path.exists') + def test_setup_eni_wont_source_eni_folder_twice(self, exists, lsb_release): + exists.return_value = True + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + with patch_open() as (_, mock_file): + # Mocked initial /etc/network/interfaces file content: + mock_file.__iter__.return_value = [ + 'some line', + ' source /etc/network/interfaces.d/* ', + 'some other line'] + + ovs.setup_eni() + self.assertFalse(mock_file.write.called) + + @patch.object(ovs, 'lsb_release') + def test_setup_eni_raises_on_focal(self, lsb_release): + lsb_release.return_value = {'DISTRIB_CODENAME': 'focal'} + self.assertRaises(RuntimeError, ovs.setup_eni) diff --git a/nrpe/mod/charmhelpers/tests/contrib/network/test_ufw.py b/nrpe/mod/charmhelpers/tests/contrib/network/test_ufw.py new file mode 100644 index 0000000..6eddc02 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/network/test_ufw.py @@ -0,0 +1,550 @@ +from __future__ import print_function + +import mock +import os +import subprocess +import unittest + +from charmhelpers.contrib.network import ufw + +__author__ = 'Felipe Reyes ' + + +LSMOD_NO_IP6 = """Module Size Used by +raid1 39533 1 +psmouse 106548 0 +raid0 17842 0 +ahci 34062 5 +multipath 13145 0 +r8169 71471 0 +libahci 32424 1 ahci +mii 13934 1 r8169 +linear 12894 0 +""" +LSMOD_IP6 = """Module Size Used by +xt_hl 12521 0 +ip6_tables 27026 0 +ip6t_rt 13537 0 +nf_conntrack_ipv6 18894 0 +nf_defrag_ipv6 34769 1 nf_conntrack_ipv6 +xt_recent 18457 0 +xt_LOG 17702 0 +xt_limit 12711 0 +""" +DEFAULT_POLICY_OUTPUT = """Default incoming policy changed to 'deny' +(be sure to update your rules accordingly) +""" +DEFAULT_POLICY_OUTPUT_OUTGOING = """Default outgoing policy changed to 'allow' +(be sure to update your rules accordingly) +""" + +UFW_STATUS_NUMBERED = """Status: active + + To Action From + -- ------ ---- +[ 1] 6641/tcp ALLOW IN 10.219.3.86 # charm-ovn-central +[12] 6641/tcp REJECT IN Anywhere +[19] 6644/tcp (v6) REJECT IN Anywhere (v6) # charm-ovn-central + +""" + + +class TestUFW(unittest.TestCase): + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_enable_ok(self, modprobe, check_output, log): + msg = 'Firewall is active and enabled on system startup\n' + check_output.return_value = msg + self.assertTrue(ufw.enable()) + + check_output.assert_any_call(['ufw', 'enable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call('ufw enabled', level='INFO') + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_enable_fail(self, modprobe, check_output, log): + msg = 'neneene\n' + check_output.return_value = msg + self.assertFalse(ufw.enable()) + + check_output.assert_any_call(['ufw', 'enable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call("ufw couldn't be enabled", level='WARN') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_disable_ok(self, check_output, log, is_enabled): + is_enabled.return_value = True + msg = 'Firewall stopped and disabled on system startup\n' + check_output.return_value = msg + self.assertTrue(ufw.disable()) + + check_output.assert_any_call(['ufw', 'disable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call('ufw disabled', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_disable_fail(self, check_output, log, is_enabled): + is_enabled.return_value = True + msg = 'neneene\n' + check_output.return_value = msg + self.assertFalse(ufw.disable()) + + check_output.assert_any_call(['ufw', 'disable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call("ufw couldn't be disabled", level='WARN') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_modify_access_ufw_is_disabled(self, check_output, log, + is_enabled): + is_enabled.return_value = False + ufw.modify_access('127.0.0.1') + log.assert_any_call('ufw is disabled, skipping modify_access()', + level='WARN') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_allow(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('127.0.0.1') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', 'to', + 'any'], stdout=subprocess.PIPE) + log.assert_any_call('ufw allow: ufw allow from 127.0.0.1 to any', + level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_allow_set_proto(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('127.0.0.1', proto='udp') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', 'to', + 'any', 'proto', 'udp'], stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw allow from 127.0.0.1 ' + 'to any proto udp'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_allow_set_port(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('127.0.0.1', port='80') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', 'to', + 'any', 'port', '80'], stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw allow from 127.0.0.1 ' + 'to any port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_allow_set_dst(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('127.0.0.1', dst='127.0.0.1', port='80') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', 'to', + '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_allow_ipv6(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('::1', dst='::1', port='80') + popen.assert_any_call(['ufw', 'allow', 'from', '::1', 'to', + '::1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw allow from ::1 ' + 'to ::1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_with_index(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.modify_access('127.0.0.1', dst='127.0.0.1', port='80', index=1) + popen.assert_any_call(['ufw', 'insert', '1', 'allow', 'from', + '127.0.0.1', 'to', '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw insert 1 allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_prepend(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + ufw.modify_access('127.0.0.1', dst='127.0.0.1', port='80', + prepend=True) + popen.assert_any_call(['ufw', 'prepend', 'allow', 'from', '127.0.0.1', + 'to', '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw prepend allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_comment(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + ufw.modify_access('127.0.0.1', dst='127.0.0.1', port='80', + comment='No comment') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', + 'to', '127.0.0.1', 'port', '80', + 'comment', 'No comment'], + stdout=subprocess.PIPE) + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_modify_access_delete_index(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + ufw.modify_access(None, dst=None, action='delete', index=42) + popen.assert_any_call(['ufw', '--force', 'delete', '42'], + stdout=subprocess.PIPE) + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_grant_access(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.grant_access('127.0.0.1', dst='127.0.0.1', port='80') + popen.assert_any_call(['ufw', 'allow', 'from', '127.0.0.1', 'to', + '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_grant_access_with_index(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.grant_access('127.0.0.1', dst='127.0.0.1', port='80', index=1) + popen.assert_any_call(['ufw', 'insert', '1', 'allow', 'from', + '127.0.0.1', 'to', '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw allow: ufw insert 1 allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.Popen') + def test_revoke_access(self, popen, log, is_enabled): + is_enabled.return_value = True + p = mock.Mock() + p.configure_mock(**{'communicate.return_value': ('stdout', 'stderr'), + 'returncode': 0}) + popen.return_value = p + + ufw.revoke_access('127.0.0.1', dst='127.0.0.1', port='80') + popen.assert_any_call(['ufw', 'delete', 'allow', 'from', '127.0.0.1', + 'to', '127.0.0.1', 'port', '80'], + stdout=subprocess.PIPE) + log.assert_any_call(('ufw delete: ufw delete allow from 127.0.0.1 ' + 'to 127.0.0.1 port 80'), level='DEBUG') + log.assert_any_call('stdout', level='INFO') + + @mock.patch('subprocess.check_output') + def test_service_open(self, check_output): + ufw.service('ssh', 'open') + check_output.assert_any_call(['ufw', 'allow', 'ssh'], + universal_newlines=True) + + @mock.patch('subprocess.check_output') + def test_service_close(self, check_output): + ufw.service('ssh', 'close') + check_output.assert_any_call(['ufw', 'delete', 'allow', 'ssh'], + universal_newlines=True) + + @mock.patch('subprocess.check_output') + def test_service_unsupport_action(self, check_output): + self.assertRaises(ufw.UFWError, ufw.service, 'ssh', 'nenene') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('os.path.isdir') + @mock.patch('subprocess.call') + @mock.patch('subprocess.check_output') + def test_no_ipv6(self, check_output, call, isdir, log, is_enabled): + check_output.return_value = ('Firewall is active and enabled ' + 'on system startup\n') + isdir.return_value = False + call.return_value = 0 + is_enabled.return_value = False + ufw.enable() + + call.assert_called_with(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + log.assert_any_call('IPv6 support in ufw disabled', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('os.path.isdir') + @mock.patch('subprocess.call') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_no_ip6_tables(self, modprobe, check_output, call, isdir, log, + is_enabled): + def c(*args, **kwargs): + if args[0] == ['lsmod']: + return LSMOD_NO_IP6 + elif args[0] == ['modprobe', 'ip6_tables']: + return "" + else: + return 'Firewall is active and enabled on system startup\n' + + check_output.side_effect = c + isdir.return_value = True + call.return_value = 0 + + is_enabled.return_value = False + self.assertTrue(ufw.enable()) + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('os.path.isdir') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + @mock.patch('charmhelpers.contrib.network.ufw.is_module_loaded') + def test_no_ip6_tables_fail_to_load(self, is_module_loaded, + modprobe, isdir, log, is_enabled): + is_module_loaded.return_value = False + + def c(m): + raise subprocess.CalledProcessError(1, ['modprobe', + 'ip6_tables'], + "fail to load ip6_tables") + + modprobe.side_effect = c + isdir.return_value = True + is_enabled.return_value = False + + self.assertRaises(ufw.UFWIPv6Error, ufw.enable) + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('os.path.isdir') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + @mock.patch('charmhelpers.contrib.network.ufw.is_module_loaded') + @mock.patch('subprocess.call') + @mock.patch('subprocess.check_output') + def test_no_ip6_tables_fail_to_load_soft_fail(self, check_output, + call, is_module_loaded, + modprobe, + isdir, log, is_enabled): + is_module_loaded.return_value = False + + def c(m): + raise subprocess.CalledProcessError(1, ['modprobe', + 'ip6_tables'], + "fail to load ip6_tables") + + modprobe.side_effect = c + isdir.return_value = True + call.return_value = 0 + check_output.return_value = ("Firewall is active and enabled on " + "system startup\n") + is_enabled.return_value = False + self.assertTrue(ufw.enable(soft_fail=True)) + call.assert_called_with(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + log.assert_any_call('IPv6 support in ufw disabled', level='INFO') + + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('os.path.isdir') + @mock.patch('subprocess.call') + @mock.patch('subprocess.check_output') + def test_no_ipv6_failed_disabling_ufw(self, check_output, call, isdir, + log, is_enabled): + check_output.return_value = ('Firewall is active and enabled ' + 'on system startup\n') + isdir.return_value = False + call.return_value = 1 + is_enabled.return_value = False + self.assertRaises(ufw.UFWError, ufw.enable) + + call.assert_called_with(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + log.assert_any_call("Couldn't disable IPv6 support in ufw", + level="ERROR") + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('charmhelpers.contrib.network.ufw.is_enabled') + @mock.patch('os.path.isdir') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_with_ipv6(self, modprobe, check_output, isdir, is_enabled, log): + def c(*args, **kwargs): + if args[0] == ['lsmod']: + return LSMOD_IP6 + else: + return 'Firewall is active and enabled on system startup\n' + + check_output.side_effect = c + is_enabled.return_value = False + isdir.return_value = True + ufw.enable() + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_change_default_policy(self, check_output, log): + check_output.return_value = DEFAULT_POLICY_OUTPUT + self.assertTrue(ufw.default_policy()) + check_output.asser_any_call(['ufw', 'default', 'deny', 'incoming']) + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_change_default_policy_allow_outgoing(self, check_output, log): + check_output.return_value = DEFAULT_POLICY_OUTPUT_OUTGOING + self.assertTrue(ufw.default_policy('allow', 'outgoing')) + check_output.asser_any_call(['ufw', 'default', 'allow', 'outgoing']) + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_change_default_policy_unexpected_output(self, check_output, log): + check_output.return_value = "asdf" + self.assertFalse(ufw.default_policy()) + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_change_default_policy_wrong_policy(self, check_output, log): + self.assertRaises(ufw.UFWError, ufw.default_policy, 'asdf') + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + def test_change_default_policy_wrong_direction(self, check_output, log): + self.assertRaises(ufw.UFWError, ufw.default_policy, 'allow', 'asdf') + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_reload_ok(self, modprobe, check_output, log): + msg = 'Firewall reloaded\n' + check_output.return_value = msg + self.assertTrue(ufw.reload()) + + check_output.assert_any_call(['ufw', 'reload'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call('ufw reloaded', level='INFO') + + @mock.patch('charmhelpers.core.hookenv.log') + @mock.patch('subprocess.check_output') + @mock.patch('charmhelpers.contrib.network.ufw.modprobe') + def test_reload_fail(self, modprobe, check_output, log): + msg = 'This did not work\n' + check_output.return_value = msg + self.assertFalse(ufw.reload()) + + check_output.assert_any_call(['ufw', 'reload'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + log.assert_any_call(msg, level='DEBUG') + log.assert_any_call("ufw couldn't be reloaded", level='WARN') + + def test_status(self): + with mock.patch('subprocess.check_output') as check_output: + check_output.return_value = UFW_STATUS_NUMBERED + expect = { + 1: {'to': '6641/tcp', 'action': 'allow in', + 'from': '10.219.3.86', 'ipv6': False, + 'comment': 'charm-ovn-central'}, + 12: {'to': '6641/tcp', 'action': 'reject in', + 'from': 'any', 'ipv6': False, + 'comment': ''}, + 19: {'to': '6644/tcp', 'action': 'reject in', + 'from': 'any', 'ipv6': True, + 'comment': 'charm-ovn-central'}, + } + n_rules = 0 + for n, r in ufw.status(): + self.assertDictEqual(r, expect[n]) + n_rules += 1 + self.assertEquals(n_rules, 3) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/ha/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/ha/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/ha/test_ha_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/ha/test_ha_utils.py new file mode 100644 index 0000000..a839632 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/ha/test_ha_utils.py @@ -0,0 +1,445 @@ +from mock import patch +import unittest +import json + +from charmhelpers.contrib.openstack.ha import utils as ha + +IFACE_LOOKUPS = { + '10.5.100.1': 'eth1', + 'ffff::1': 'eth1', + 'ffaa::1': 'eth2', +} + +NETMASK_LOOKUPS = { + '10.5.100.1': '255.255.255.0', + 'ffff::1': '64', + 'ffaa::1': '32', +} + + +class HATests(unittest.TestCase): + def setUp(self): + super(HATests, self).setUp() + [self._patch(m) for m in [ + 'charm_name', + 'config', + 'relation_set', + 'resolve_address', + 'status_set', + 'get_hacluster_config', + 'get_iface_for_address', + 'get_netmask_for_address', + ]] + self.resources = {'res_test_haproxy': 'lsb:haproxy'} + self.resource_params = {'res_test_haproxy': 'op monitor interval="5s"'} + self.conf = {} + self.config.side_effect = lambda key: self.conf.get(key) + self.maxDiff = None + self.get_iface_for_address.side_effect = \ + lambda x: IFACE_LOOKUPS.get(x) + self.get_netmask_for_address.side_effect = \ + lambda x: NETMASK_LOOKUPS.get(x) + + def _patch(self, method): + _m = patch.object(ha, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + @patch.object(ha, 'log', lambda *args, **kwargs: None) + @patch.object(ha, 'assert_charm_supports_dns_ha') + def test_update_dns_ha_resource_params_none(self, + assert_charm_supports_dns_ha): + self.conf = { + 'os-admin-hostname': None, + 'os-internal-hostname': None, + 'os-public-hostname': None, + } + + with self.assertRaises(ha.DNSHAException): + ha.update_dns_ha_resource_params( + relation_id='ha:1', + resources=self.resources, + resource_params=self.resource_params) + + @patch.object(ha, 'log', lambda *args, **kwargs: None) + @patch.object(ha, 'assert_charm_supports_dns_ha') + def test_update_dns_ha_resource_params_one(self, + assert_charm_supports_dns_ha): + EXPECTED_RESOURCES = {'res_test_public_hostname': 'ocf:maas:dns', + 'res_test_haproxy': 'lsb:haproxy'} + EXPECTED_RESOURCE_PARAMS = { + 'res_test_public_hostname': ('params fqdn="test.maas" ' + 'ip_address="10.0.0.1"'), + 'res_test_haproxy': 'op monitor interval="5s"'} + + self.conf = { + 'os-admin-hostname': None, + 'os-internal-hostname': None, + 'os-public-hostname': 'test.maas', + } + + self.charm_name.return_value = 'test' + self.resolve_address.return_value = '10.0.0.1' + ha.update_dns_ha_resource_params(relation_id='ha:1', + resources=self.resources, + resource_params=self.resource_params) + self.assertEqual(self.resources, EXPECTED_RESOURCES) + self.assertEqual(self.resource_params, EXPECTED_RESOURCE_PARAMS) + self.relation_set.assert_called_with( + groups={'grp_test_hostnames': 'res_test_public_hostname'}, + relation_id='ha:1') + + @patch.object(ha, 'log', lambda *args, **kwargs: None) + @patch.object(ha, 'assert_charm_supports_dns_ha') + def test_update_dns_ha_resource_params_all(self, + assert_charm_supports_dns_ha): + EXPECTED_RESOURCES = {'res_test_admin_hostname': 'ocf:maas:dns', + 'res_test_int_hostname': 'ocf:maas:dns', + 'res_test_public_hostname': 'ocf:maas:dns', + 'res_test_haproxy': 'lsb:haproxy'} + EXPECTED_RESOURCE_PARAMS = { + 'res_test_admin_hostname': ('params fqdn="test.admin.maas" ' + 'ip_address="10.0.0.1"'), + 'res_test_int_hostname': ('params fqdn="test.internal.maas" ' + 'ip_address="10.0.0.1"'), + 'res_test_public_hostname': ('params fqdn="test.public.maas" ' + 'ip_address="10.0.0.1"'), + 'res_test_haproxy': 'op monitor interval="5s"'} + + self.conf = { + 'os-admin-hostname': 'test.admin.maas', + 'os-internal-hostname': 'test.internal.maas', + 'os-public-hostname': 'test.public.maas', + } + + self.charm_name.return_value = 'test' + self.resolve_address.return_value = '10.0.0.1' + ha.update_dns_ha_resource_params(relation_id='ha:1', + resources=self.resources, + resource_params=self.resource_params) + self.assertEqual(self.resources, EXPECTED_RESOURCES) + self.assertEqual(self.resource_params, EXPECTED_RESOURCE_PARAMS) + self.relation_set.assert_called_with( + groups={'grp_test_hostnames': + ('res_test_admin_hostname ' + 'res_test_int_hostname ' + 'res_test_public_hostname')}, + relation_id='ha:1') + + @patch.object(ha, 'lsb_release') + def test_assert_charm_supports_dns_ha(self, lsb_release): + lsb_release.return_value = {'DISTRIB_RELEASE': '16.04'} + self.assertTrue(ha.assert_charm_supports_dns_ha()) + + @patch.object(ha, 'lsb_release') + def test_assert_charm_supports_dns_ha_exception(self, lsb_release): + lsb_release.return_value = {'DISTRIB_RELEASE': '12.04'} + self.assertRaises(ha.DNSHAException, + lambda: ha.assert_charm_supports_dns_ha()) + + @patch.object(ha, 'expected_related_units') + def tests_expect_ha(self, expected_related_units): + expected_related_units.return_value = (x for x in []) + self.conf = {'vip': None, + 'dns-ha': None} + self.assertFalse(ha.expect_ha()) + + expected_related_units.return_value = (x for x in ['hacluster-unit/0', + 'hacluster-unit/1', + 'hacluster-unit/2']) + self.conf = {'vip': None, + 'dns-ha': None} + self.assertTrue(ha.expect_ha()) + + expected_related_units.side_effect = NotImplementedError + self.conf = {'vip': '10.0.0.1', + 'dns-ha': None} + self.assertTrue(ha.expect_ha()) + + self.conf = {'vip': None, + 'dns-ha': True} + self.assertTrue(ha.expect_ha()) + + def test_get_vip_settings(self): + self.assertEqual( + ha.get_vip_settings('10.5.100.1'), + ('eth1', '255.255.255.0', False)) + + def test_get_vip_settings_fallback(self): + self.conf = {'vip_iface': 'eth3', + 'vip_cidr': '255.255.0.0'} + self.assertEqual( + ha.get_vip_settings('192.168.100.1'), + ('eth3', '255.255.0.0', True)) + + def test_update_hacluster_vip_single_vip(self): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1' + } + test_data = {'resources': {}, 'resource_params': {}} + expected = { + 'delete_resources': ['res_testservice_eth1_vip'], + 'groups': { + 'grp_testservice_vips': 'res_testservice_242d562_vip' + }, + 'resource_params': { + 'res_testservice_242d562_vip': + ('params ip="10.5.100.1" op monitor ' + 'timeout="20s" interval="10s" depth="0"') + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2' + } + } + ha.update_hacluster_vip('testservice', test_data) + self.assertEqual(test_data, expected) + + def test_update_hacluster_vip_single_vip_fallback(self): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1' + } + test_data = {'resources': {}, 'resource_params': {}} + expected = { + 'delete_resources': ['res_testservice_eth1_vip'], + 'groups': { + 'grp_testservice_vips': 'res_testservice_242d562_vip' + }, + 'resource_params': { + 'res_testservice_242d562_vip': + ('params ip="10.5.100.1" op monitor ' + 'timeout="20s" interval="10s" depth="0"') + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2' + } + } + ha.update_hacluster_vip('testservice', test_data) + self.assertEqual(test_data, expected) + + def test_update_hacluster_config_vip(self): + self.get_iface_for_address.side_effect = lambda x: None + self.get_netmask_for_address.side_effect = lambda x: None + self.conf = {'vip_iface': 'eth1', + 'vip_cidr': '255.255.255.0'} + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1' + } + test_data = {'resources': {}, 'resource_params': {}} + expected = { + 'delete_resources': ['res_testservice_eth1_vip'], + 'groups': { + 'grp_testservice_vips': 'res_testservice_242d562_vip' + }, + 'resource_params': { + 'res_testservice_242d562_vip': ( + 'params ip="10.5.100.1" cidr_netmask="255.255.255.0" ' + 'nic="eth1" op monitor timeout="20s" ' + 'interval="10s" depth="0"') + + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2' + } + } + ha.update_hacluster_vip('testservice', test_data) + self.assertEqual(test_data, expected) + + def test_update_hacluster_vip_multiple_vip(self): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1 ffff::1 ffaa::1' + } + test_data = {'resources': {}, 'resource_params': {}} + expected = { + 'groups': { + 'grp_testservice_vips': ('res_testservice_242d562_vip ' + 'res_testservice_856d56f_vip ' + 'res_testservice_f563c5d_vip') + }, + 'delete_resources': ['res_testservice_eth1_vip', + 'res_testservice_eth1_vip_ipv6addr', + 'res_testservice_eth2_vip'], + 'resource_params': { + 'res_testservice_242d562_vip': + ('params ip="10.5.100.1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_856d56f_vip': + ('params ipv6addr="ffff::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_f563c5d_vip': + ('params ipv6addr="ffaa::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2', + 'res_testservice_856d56f_vip': 'ocf:heartbeat:IPv6addr', + 'res_testservice_f563c5d_vip': 'ocf:heartbeat:IPv6addr', + } + } + ha.update_hacluster_vip('testservice', test_data) + self.assertEqual(test_data, expected) + + def test_generate_ha_relation_data_haproxy_disabled(self): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1 ffff::1 ffaa::1' + } + extra_settings = { + 'colocations': {'vip_cauth': 'inf: res_nova_cauth grp_nova_vips'}, + 'init_services': {'res_nova_cauth': 'nova-cauth'}, + 'delete_resources': ['res_ceilometer_polling'], + 'groups': {'grp_testservice_wombles': 'res_testservice_orinoco'}, + } + expected = { + 'colocations': {'vip_cauth': 'inf: res_nova_cauth grp_nova_vips'}, + 'groups': { + 'grp_testservice_vips': ('res_testservice_242d562_vip ' + 'res_testservice_856d56f_vip ' + 'res_testservice_f563c5d_vip'), + 'grp_testservice_wombles': 'res_testservice_orinoco' + }, + 'resource_params': { + 'res_testservice_242d562_vip': + ('params ip="10.5.100.1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_856d56f_vip': + ('params ipv6addr="ffff::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_f563c5d_vip': + ('params ipv6addr="ffaa::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2', + 'res_testservice_856d56f_vip': 'ocf:heartbeat:IPv6addr', + 'res_testservice_f563c5d_vip': 'ocf:heartbeat:IPv6addr', + }, + 'clones': {}, + 'init_services': { + 'res_nova_cauth': 'nova-cauth' + }, + 'delete_resources': ["res_ceilometer_polling", + "res_testservice_eth1_vip", + "res_testservice_eth1_vip_ipv6addr", + "res_testservice_eth2_vip"], + } + expected = { + 'json_{}'.format(k): json.dumps(v, **ha.JSON_ENCODE_OPTIONS) + for k, v in expected.items() if v + } + self.assertEqual( + ha.generate_ha_relation_data('testservice', + haproxy_enabled=False, + extra_settings=extra_settings), + expected) + + def test_generate_ha_relation_data(self): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1 ffff::1 ffaa::1' + } + extra_settings = { + 'colocations': {'vip_cauth': 'inf: res_nova_cauth grp_nova_vips'}, + 'init_services': {'res_nova_cauth': 'nova-cauth'}, + 'delete_resources': ['res_ceilometer_polling'], + 'groups': {'grp_testservice_wombles': 'res_testservice_orinoco'}, + } + expected = { + 'colocations': {'vip_cauth': 'inf: res_nova_cauth grp_nova_vips'}, + 'groups': { + 'grp_testservice_vips': ('res_testservice_242d562_vip ' + 'res_testservice_856d56f_vip ' + 'res_testservice_f563c5d_vip'), + 'grp_testservice_wombles': 'res_testservice_orinoco' + }, + 'resource_params': { + 'res_testservice_242d562_vip': + ('params ip="10.5.100.1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_856d56f_vip': + ('params ipv6addr="ffff::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_f563c5d_vip': + ('params ipv6addr="ffaa::1" op monitor ' + 'timeout="20s" interval="10s" depth="0"'), + 'res_testservice_haproxy': + ('meta migration-threshold="INFINITY" failure-timeout="5s" ' + 'op monitor interval="5s"'), + }, + 'resources': { + 'res_testservice_242d562_vip': 'ocf:heartbeat:IPaddr2', + 'res_testservice_856d56f_vip': 'ocf:heartbeat:IPv6addr', + 'res_testservice_f563c5d_vip': 'ocf:heartbeat:IPv6addr', + 'res_testservice_haproxy': 'lsb:haproxy', + }, + 'clones': { + 'cl_testservice_haproxy': 'res_testservice_haproxy', + }, + 'init_services': { + 'res_testservice_haproxy': 'haproxy', + 'res_nova_cauth': 'nova-cauth' + }, + 'delete_resources': ["res_ceilometer_polling", + "res_testservice_eth1_vip", + "res_testservice_eth1_vip_ipv6addr", + "res_testservice_eth2_vip"], + } + expected = { + 'json_{}'.format(k): json.dumps(v, **ha.JSON_ENCODE_OPTIONS) + for k, v in expected.items() if v + } + self.assertEqual( + ha.generate_ha_relation_data('testservice', + extra_settings=extra_settings), + expected) + + @patch.object(ha, 'log') + @patch.object(ha, 'assert_charm_supports_dns_ha') + def test_generate_ha_relation_data_dns_ha(self, + assert_charm_supports_dns_ha, + log): + self.get_hacluster_config.return_value = { + 'vip': '10.5.100.1 ffff::1 ffaa::1' + } + self.conf = { + 'os-admin-hostname': 'test.admin.maas', + 'os-internal-hostname': 'test.internal.maas', + 'os-public-hostname': 'test.public.maas', + 'dns-ha': True, + } + self.resolve_address.return_value = '10.0.0.1' + assert_charm_supports_dns_ha.return_value = True + expected = { + 'groups': { + 'grp_testservice_hostnames': ('res_testservice_admin_hostname' + ' res_testservice_int_hostname' + ' res_testservice_public_hostname') + }, + 'resource_params': { + 'res_testservice_admin_hostname': + 'params fqdn="test.admin.maas" ip_address="10.0.0.1"', + 'res_testservice_int_hostname': + 'params fqdn="test.internal.maas" ip_address="10.0.0.1"', + 'res_testservice_public_hostname': + 'params fqdn="test.public.maas" ip_address="10.0.0.1"', + 'res_testservice_haproxy': + ('meta migration-threshold="INFINITY" failure-timeout="5s" ' + 'op monitor interval="5s"'), + }, + 'resources': { + 'res_testservice_admin_hostname': 'ocf:maas:dns', + 'res_testservice_int_hostname': 'ocf:maas:dns', + 'res_testservice_public_hostname': 'ocf:maas:dns', + 'res_testservice_haproxy': 'lsb:haproxy', + }, + 'clones': { + 'cl_testservice_haproxy': 'res_testservice_haproxy', + }, + 'init_services': { + 'res_testservice_haproxy': 'haproxy' + }, + } + expected = { + 'json_{}'.format(k): json.dumps(v, **ha.JSON_ENCODE_OPTIONS) + for k, v in expected.items() if v + } + self.assertEqual(ha.generate_ha_relation_data('testservice'), + expected) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_alternatives.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_alternatives.py new file mode 100644 index 0000000..402f0d4 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_alternatives.py @@ -0,0 +1,76 @@ +from testtools import TestCase +from mock import patch + +import charmhelpers.contrib.openstack.alternatives as alternatives + + +NAME = 'test' +SOURCE = '/var/lib/charm/test/test.conf' +TARGET = '/etc/test/test,conf' + + +class AlternativesTestCase(TestCase): + + @patch('subprocess.os.path') + @patch('subprocess.check_call') + def test_new_alternative(self, _check, _path): + _path.exists.return_value = False + alternatives.install_alternative(NAME, + TARGET, + SOURCE) + _check.assert_called_with( + ['update-alternatives', '--force', '--install', + TARGET, NAME, SOURCE, '50'] + ) + + @patch('subprocess.os.path') + @patch('subprocess.check_call') + def test_priority(self, _check, _path): + _path.exists.return_value = False + alternatives.install_alternative(NAME, + TARGET, + SOURCE, 100) + _check.assert_called_with( + ['update-alternatives', '--force', '--install', + TARGET, NAME, SOURCE, '100'] + ) + + @patch('shutil.move') + @patch('subprocess.os.path') + @patch('subprocess.check_call') + def test_new_alternative_existing_file(self, _check, + _path, _move): + _path.exists.return_value = True + _path.islink.return_value = False + alternatives.install_alternative(NAME, + TARGET, + SOURCE) + _check.assert_called_with( + ['update-alternatives', '--force', '--install', + TARGET, NAME, SOURCE, '50'] + ) + _move.assert_called_with(TARGET, '{}.bak'.format(TARGET)) + + @patch('shutil.move') + @patch('subprocess.os.path') + @patch('subprocess.check_call') + def test_new_alternative_existing_link(self, _check, + _path, _move): + _path.exists.return_value = True + _path.islink.return_value = True + alternatives.install_alternative(NAME, + TARGET, + SOURCE) + _check.assert_called_with( + ['update-alternatives', '--force', '--install', + TARGET, NAME, SOURCE, '50'] + ) + _move.assert_not_called() + + @patch('subprocess.check_call') + def test_remove_alternative(self, _check): + alternatives.remove_alternative(NAME, SOURCE) + _check.assert_called_with( + ['update-alternatives', '--remove', + NAME, SOURCE] + ) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_audits.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_audits.py new file mode 100644 index 0000000..930292d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_audits.py @@ -0,0 +1,329 @@ +from testtools import TestCase, skipIf +from mock import patch, MagicMock, call +import six + +import charmhelpers.contrib.openstack.audits as audits +import charmhelpers.contrib.openstack.audits.openstack_security_guide as guide + + +@skipIf(six.PY2, 'Audits only support Python3') +class AuditTestCase(TestCase): + + @patch('charmhelpers.contrib.openstack.audits._audits', {}) + def test_wrapper(self): + variables = { + 'guard_called': False, + 'test_run': False, + } + + def should_run(audit_options): + variables['guard_called'] = True + return True + + @audits.audit(should_run) + def test(options): + variables['test_run'] = True + + audits.run({}) + self.assertTrue(variables['guard_called']) + self.assertTrue(variables['test_run']) + self.assertEqual(audits._audits['test'], audits.Audit(test, (should_run,))) + + @patch('charmhelpers.contrib.openstack.audits._audits', {}) + def test_wrapper_not_run(self): + variables = { + 'guard_called': False, + 'test_run': False, + } + + def should_run(audit_options): + variables['guard_called'] = True + return False + + @audits.audit(should_run) + def test(options): + variables['test_run'] = True + + audits.run({}) + self.assertTrue(variables['guard_called']) + self.assertFalse(variables['test_run']) + self.assertEqual(audits._audits['test'], audits.Audit(test, (should_run,))) + + @patch('charmhelpers.contrib.openstack.audits._audits', {}) + def test_duplicate_audit(self): + def should_run(audit_options): + return True + + @audits.audit(should_run) + def test(options): + pass + + try: + # Again! + # + # Both of the following '#noqa's are to prevent flake8 from + # noticing the duplicate function `test` The intent in this test + # is for the audits.audit to pick up on the duplicate function. + @audits.audit(should_run) # noqa + def test(options): # noqa + pass + except RuntimeError as e: + self.assertEqual("Test name 'test' used more than once", e.args[0]) + return + self.assertTrue(False, "Duplicate audit should raise an exception") + + @patch('charmhelpers.contrib.openstack.audits._audits', {}) + def test_non_callable_filter(self): + try: + # Again! + @audits.audit(3) + def test(options): + pass + except RuntimeError as e: + self.assertEqual("Configuration includes non-callable filters: [3]", e.args[0]) + return + self.assertTrue(False, "Duplicate audit should raise an exception") + + @patch('charmhelpers.contrib.openstack.audits._audits', {}) + def test_exclude_config(self): + variables = { + 'test_run': False, + } + + @audits.audit() + def test(options): + variables['test_run'] = True + + audits.run({'excludes': ['test']}) + self.assertFalse(variables['test_run']) + + +class AuditsTestCase(TestCase): + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_since_package_less(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = 1 + + verifier = audits.since_package('test', '12.0.0') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_since_package_greater(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = -1 + + verifier = audits.since_package('test', '14.0.0') + self.assertEqual(verifier(), False) + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_since_package_equal(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = 0 + + verifier = audits.since_package('test', '13.0.0') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_since_openstack_less(self, _get_os_codename_package): + _get_os_codename_package.return_value = "icehouse" + + verifier = audits.since_openstack_release('test', 'mitaka') + self.assertEqual(verifier(), False) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_since_openstack_greater(self, _get_os_codename_package): + _get_os_codename_package.return_value = "rocky" + + verifier = audits.since_openstack_release('test', 'queens') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_since_openstack_equal(self, _get_os_codename_package): + _get_os_codename_package.return_value = "mitaka" + + verifier = audits.since_openstack_release('test', 'mitaka') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_before_openstack_less(self, _get_os_codename_package): + _get_os_codename_package.return_value = "icehouse" + + verifier = audits.before_openstack_release('test', 'mitaka') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_before_openstack_greater(self, _get_os_codename_package): + _get_os_codename_package.return_value = "rocky" + + verifier = audits.before_openstack_release('test', 'queens') + self.assertEqual(verifier(), False) + + @patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + def test_before_openstack_equal(self, _get_os_codename_package): + _get_os_codename_package.return_value = "mitaka" + + verifier = audits.before_openstack_release('test', 'mitaka') + self.assertEqual(verifier(), False) + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_before_package_less(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = 1 + + verifier = audits.before_package('test', '12.0.0') + self.assertEqual(verifier(), False) + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_before_package_greater(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = -1 + + verifier = audits.before_package('test', '14.0.0') + self.assertEqual(verifier(), True) + + @patch('charmhelpers.contrib.openstack.audits.cmp_pkgrevno') + def test_before_package_equal(self, _cmp_pkgrevno): + _cmp_pkgrevno.return_value = 0 + + verifier = audits.before_package('test', '13.0.0') + self.assertEqual(verifier(), False) + + def test_is_audit_type_empty(self): + verifier = audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide) + self.assertEqual(verifier({}), False) + + def test_is_audit_type(self): + verifier = audits.is_audit_type(audits.AuditType.OpenStackSecurityGuide) + self.assertEqual(verifier({'audit_type': audits.AuditType.OpenStackSecurityGuide}), True) + + +@skipIf(six.PY2, 'Audits only support Python3') +class OpenstackSecurityGuideTestCase(TestCase): + + @patch('configparser.ConfigParser') + def test_internal_config_parser_is_not_strict(self, _config_parser): + parser = MagicMock() + _config_parser.return_value = parser + guide._config_ini('test') + _config_parser.assert_called_with(strict=False) + parser.read.assert_called_with('test') + + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._stat') + def test_internal_validate_file_ownership(self, _stat): + _stat.return_value = guide.Ownership('test_user', 'test_group', '600') + guide._validate_file_ownership('test_user', 'test_group', 'test-file-name') + _stat.assert_called_with('test-file-name') + pass + + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._stat') + def test_internal_validate_file_mode(self, _stat): + _stat.return_value = guide.Ownership('test_user', 'test_group', '600') + guide._validate_file_mode('600', 'test-file-name') + _stat.assert_called_with('test-file-name') + pass + + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_mode') + def test_validate_file_permissions_defaults(self, _validate_mode, _is_file): + _is_file.return_value = True + config = { + 'files': { + 'test': {} + } + } + guide.validate_file_permissions(config) + _validate_mode.assert_called_once_with('600', 'test', False) + + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_mode') + def test_validate_file_permissions(self, _validate_mode, _is_file): + _is_file.return_value = True + config = { + 'files': { + 'test': { + 'mode': '777' + } + } + } + guide.validate_file_permissions(config) + _validate_mode.assert_called_once_with('777', 'test', False) + + @patch('glob.glob') + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_mode') + def test_validate_file_permissions_glob(self, _validate_mode, _is_file, _glob): + _glob.return_value = ['test'] + _is_file.return_value = True + config = { + 'files': { + '*': { + 'mode': '777' + } + } + } + guide.validate_file_permissions(config) + _validate_mode.assert_called_once_with('777', 'test', False) + + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_ownership') + def test_validate_file_ownership_defaults(self, _validate_owner, _is_file): + _is_file.return_value = True + config = { + 'files': { + 'test': {} + } + } + guide.validate_file_ownership(config) + _validate_owner.assert_called_once_with('root', 'root', 'test', False) + + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_ownership') + def test_validate_file_ownership(self, _validate_owner, _is_file): + _is_file.return_value = True + config = { + 'files': { + 'test': { + 'owner': 'test-user', + 'group': 'test-group', + } + } + } + guide.validate_file_ownership(config) + _validate_owner.assert_called_once_with('test-user', 'test-group', 'test', False) + + @patch('glob.glob') + @patch('os.path.isfile') + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._validate_file_ownership') + def test_validate_file_ownership_glob(self, _validate_owner, _is_file, _glob): + _glob.return_value = ['test'] + _is_file.return_value = True + config = { + 'files': { + '*': { + 'owner': 'test-user', + 'group': 'test-group', + } + } + } + guide.validate_file_ownership(config) + _validate_owner.assert_called_once_with('test-user', 'test-group', 'test', False) + + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._config_section') + def test_validate_uses_keystone(self, _config_section): + _config_section.side_effect = [None, { + 'auth_strategy': 'keystone', + }] + guide.validate_uses_keystone({}) + _config_section.assert_has_calls([call({}, 'api'), call({}, 'DEFAULT')]) + + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._config_section') + def test_validate_uses_tls_for_keystone(self, _config_section): + _config_section.return_value = { + 'auth_uri': 'https://10.10.10.10', + } + guide.validate_uses_tls_for_keystone({}) + _config_section.assert_called_with({}, 'keystone_authtoken') + + @patch('charmhelpers.contrib.openstack.audits.openstack_security_guide._config_section') + def test_validate_uses_tls_for_glance(self, _config_section): + _config_section.return_value = { + 'api_servers': 'https://10.10.10.10', + } + guide.validate_uses_tls_for_glance({}) + _config_section.assert_called_with({}, 'glance') diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_cert_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_cert_utils.py new file mode 100644 index 0000000..7ed43c8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_cert_utils.py @@ -0,0 +1,724 @@ +import json +import mock +import unittest + +import charmhelpers.contrib.openstack.cert_utils as cert_utils + + +class CertUtilsTests(unittest.TestCase): + + def test_CertRequest(self): + cr = cert_utils.CertRequest() + self.assertEqual(cr.entries, []) + self.assertIsNone(cr.hostname_entry) + + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + def test_CertRequest_add_entry(self, local_unit): + cr = cert_utils.CertRequest() + cr.add_entry('admin', 'admin.openstack.local', ['10.10.10.10']) + self.assertEqual( + cr.get_request(), + {'cert_requests': + '{"admin.openstack.local": {"sans": ["10.10.10.10"]}}', + 'unit_name': 'unit_2'}) + + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_CertRequest_add_hostname_cn(self, local_address, get_hostname, + get_vip_in_network, + resolve_network_cidr, local_unit): + resolve_network_cidr.side_effect = lambda x: x + get_vip_in_network.return_value = '10.1.2.100' + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + cr = cert_utils.CertRequest() + cr.add_hostname_cn() + self.assertEqual( + cr.get_request(), + {'cert_requests': + '{"juju-unit-2": {"sans": ["10.1.2.100", "10.1.2.3"]}}', + 'unit_name': 'unit_2'}) + + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_CertRequest_add_hostname_cn_ip(self, local_address, get_hostname, + get_vip_in_network, + resolve_network_cidr, local_unit): + resolve_network_cidr.side_effect = lambda x: x + get_vip_in_network.return_value = '10.1.2.100' + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + cr = cert_utils.CertRequest() + cr.add_hostname_cn() + cr.add_hostname_cn_ip(['10.1.2.4']) + self.assertEqual( + cr.get_request(), + {'cert_requests': + ('{"juju-unit-2": {"sans": ["10.1.2.100", "10.1.2.3", ' + '"10.1.2.4"]}}'), + 'unit_name': 'unit_2'}) + + @mock.patch.object(cert_utils, 'get_certificate_sans') + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'network_get_primary_address') + @mock.patch.object(cert_utils, 'resolve_address') + @mock.patch.object(cert_utils, 'config') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_get_certificate_request(self, local_address, get_hostname, + config, resolve_address, + network_get_primary_address, + get_vip_in_network, resolve_network_cidr, + local_unit, get_certificate_sans): + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + _config = { + 'os-internal-hostname': 'internal.openstack.local', + 'os-admin-hostname': 'admin.openstack.local', + 'os-public-hostname': 'public.openstack.local', + } + _resolve_address = { + 'int': '10.0.0.2', + 'internal': '10.0.0.2', + 'admin': '10.10.0.2', + 'public': '10.20.0.2', + } + _npa = { + 'internal': '10.0.0.3', + 'admin': '10.10.0.3', + 'public': '10.20.0.3', + } + _vips = { + '10.0.0.0/16': '10.0.0.100', + '10.10.0.0/16': '10.10.0.100', + '10.20.0.0/16': '10.20.0.100', + } + _resolve_nets = { + '10.0.0.3': '10.0.0.0/16', + '10.10.0.3': '10.10.0.0/16', + '10.20.0.3': '10.20.0.0/16', + } + get_certificate_sans.return_value = list(set( + list(_resolve_address.values()) + + list(_npa.values()) + + list(_vips.values()))) + expect = { + 'admin.openstack.local': { + 'sans': ['10.10.0.100', '10.10.0.2', '10.10.0.3']}, + 'internal.openstack.local': { + 'sans': ['10.0.0.100', '10.0.0.2', '10.0.0.3']}, + 'juju-unit-2': {'sans': ['10.1.2.3']}, + 'public.openstack.local': { + 'sans': ['10.20.0.100', '10.20.0.2', '10.20.0.3']}} + self.maxDiff = None + config.side_effect = lambda x: _config.get(x) + get_vip_in_network.side_effect = lambda x: _vips.get(x) + resolve_network_cidr.side_effect = lambda x: _resolve_nets.get(x) + network_get_primary_address.side_effect = lambda x: _npa.get(x) + resolve_address.side_effect = \ + lambda endpoint_type: _resolve_address[endpoint_type] + output = json.loads( + cert_utils.get_certificate_request()['cert_requests']) + self.assertEqual( + output, + expect) + get_certificate_sans.assert_called_once_with( + bindings=['internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, 'get_certificate_sans') + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'network_get_primary_address') + @mock.patch.object(cert_utils, 'resolve_address') + @mock.patch.object(cert_utils, 'config') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_get_certificate_request_no_hostnames( + self, local_address, get_hostname, config, resolve_address, + network_get_primary_address, get_vip_in_network, + resolve_network_cidr, local_unit, get_certificate_sans): + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + _config = { + 'os-admin-hostname': 'admin.openstack.local', + 'os-public-hostname': 'public.openstack.local', + } + _resolve_address = { + 'int': '10.0.0.2', + 'internal': '10.0.0.2', + 'admin': '10.10.0.2', + 'public': '10.20.0.2', + } + _npa = { + 'internal': '10.0.0.3', + 'admin': '10.10.0.3', + 'public': '10.20.0.3', + 'mybinding': '10.30.0.3', + } + _vips = { + '10.0.0.0/16': '10.0.0.100', + '10.10.0.0/16': '10.10.0.100', + '10.20.0.0/16': '10.20.0.100', + } + _resolve_nets = { + '10.0.0.3': '10.0.0.0/16', + '10.10.0.3': '10.10.0.0/16', + '10.20.0.3': '10.20.0.0/16', + } + get_certificate_sans.return_value = list(set( + list(_resolve_address.values()) + + list(_npa.values()) + + list(_vips.values()))) + expect = { + 'admin.openstack.local': { + 'sans': ['10.10.0.100', '10.10.0.2', '10.10.0.3']}, + 'juju-unit-2': {'sans': [ + '10.0.0.100', '10.0.0.2', '10.0.0.3', '10.1.2.3', '10.30.0.3']}, + 'public.openstack.local': { + 'sans': ['10.20.0.100', '10.20.0.2', '10.20.0.3']}} + self.maxDiff = None + config.side_effect = lambda x: _config.get(x) + get_vip_in_network.side_effect = lambda x: _vips.get(x) + resolve_network_cidr.side_effect = lambda x: _resolve_nets.get(x) + network_get_primary_address.side_effect = lambda x: _npa.get(x) + resolve_address.side_effect = \ + lambda endpoint_type: _resolve_address[endpoint_type] + output = json.loads( + cert_utils.get_certificate_request( + bindings=['mybinding'])['cert_requests']) + self.assertEqual( + output, + expect) + get_certificate_sans.assert_called_once_with( + bindings=['mybinding', 'internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, 'get_certificate_request') + @mock.patch.object(cert_utils, 'local_address') + @mock.patch.object(cert_utils.os, 'symlink') + @mock.patch.object(cert_utils.os.path, 'isfile') + @mock.patch.object(cert_utils, 'get_hostname') + def test_create_ip_cert_links(self, get_hostname, isfile, + symlink, local_address, get_cert_request): + cert_request = {'cert_requests': { + 'admin.openstack.local': { + 'sans': ['10.10.0.100', '10.10.0.2', '10.10.0.3']}, + 'internal.openstack.local': { + 'sans': ['10.0.0.100', '10.0.0.2', '10.0.0.3']}, + 'juju-unit-2': {'sans': ['10.1.2.3']}, + 'public.openstack.local': { + 'sans': ['10.20.0.100', '10.20.0.2', '10.20.0.3']}}} + get_cert_request.return_value = cert_request + _files = { + '/etc/ssl/cert_juju-unit-2': True, + '/etc/ssl/cert_10.1.2.3': False, + '/etc/ssl/cert_admin.openstack.local': True, + '/etc/ssl/cert_10.10.0.100': False, + '/etc/ssl/cert_10.10.0.2': False, + '/etc/ssl/cert_10.10.0.3': False, + '/etc/ssl/cert_internal.openstack.local': True, + '/etc/ssl/cert_10.0.0.100': False, + '/etc/ssl/cert_10.0.0.2': False, + '/etc/ssl/cert_10.0.0.3': False, + '/etc/ssl/cert_public.openstack.local': True, + '/etc/ssl/cert_10.20.0.100': False, + '/etc/ssl/cert_10.20.0.2': False, + '/etc/ssl/cert_10.20.0.3': False, + '/etc/ssl/cert_funky-name': False, + } + isfile.side_effect = lambda x: _files[x] + expected = [ + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.100'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.100'), + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.2'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.2'), + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.3'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.3'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.100'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.100'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.2'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.2'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.3'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.3'), + mock.call('/etc/ssl/cert_juju-unit-2', '/etc/ssl/cert_10.1.2.3'), + mock.call('/etc/ssl/key_juju-unit-2', '/etc/ssl/key_10.1.2.3'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.100'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.100'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.2'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.2'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.3'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.3')] + cert_utils.create_ip_cert_links('/etc/ssl') + symlink.assert_has_calls(expected, any_order=True) + # Customer hostname + symlink.reset_mock() + get_hostname.return_value = 'juju-unit-2' + cert_utils.create_ip_cert_links( + '/etc/ssl', + custom_hostname_link='funky-name') + expected.extend([ + mock.call('/etc/ssl/cert_juju-unit-2', '/etc/ssl/cert_funky-name'), + mock.call('/etc/ssl/key_juju-unit-2', '/etc/ssl/key_funky-name'), + ]) + symlink.assert_has_calls(expected, any_order=True) + get_cert_request.assert_called_with( + json_encode=False, bindings=['internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, 'get_certificate_request') + @mock.patch.object(cert_utils, 'local_address') + @mock.patch.object(cert_utils.os, 'symlink') + @mock.patch.object(cert_utils.os.path, 'isfile') + @mock.patch.object(cert_utils, 'get_hostname') + def test_create_ip_cert_links_bindings( + self, get_hostname, isfile, symlink, local_address, get_cert_request): + cert_request = {'cert_requests': { + 'admin.openstack.local': { + 'sans': ['10.10.0.100', '10.10.0.2', '10.10.0.3']}, + 'internal.openstack.local': { + 'sans': ['10.0.0.100', '10.0.0.2', '10.0.0.3']}, + 'juju-unit-2': {'sans': ['10.1.2.3']}, + 'public.openstack.local': { + 'sans': ['10.20.0.100', '10.20.0.2', '10.20.0.3']}}} + get_cert_request.return_value = cert_request + _files = { + '/etc/ssl/cert_juju-unit-2': True, + '/etc/ssl/cert_10.1.2.3': False, + '/etc/ssl/cert_admin.openstack.local': True, + '/etc/ssl/cert_10.10.0.100': False, + '/etc/ssl/cert_10.10.0.2': False, + '/etc/ssl/cert_10.10.0.3': False, + '/etc/ssl/cert_internal.openstack.local': True, + '/etc/ssl/cert_10.0.0.100': False, + '/etc/ssl/cert_10.0.0.2': False, + '/etc/ssl/cert_10.0.0.3': False, + '/etc/ssl/cert_public.openstack.local': True, + '/etc/ssl/cert_10.20.0.100': False, + '/etc/ssl/cert_10.20.0.2': False, + '/etc/ssl/cert_10.20.0.3': False, + '/etc/ssl/cert_funky-name': False, + } + isfile.side_effect = lambda x: _files[x] + expected = [ + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.100'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.100'), + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.2'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.2'), + mock.call('/etc/ssl/cert_admin.openstack.local', '/etc/ssl/cert_10.10.0.3'), + mock.call('/etc/ssl/key_admin.openstack.local', '/etc/ssl/key_10.10.0.3'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.100'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.100'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.2'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.2'), + mock.call('/etc/ssl/cert_internal.openstack.local', '/etc/ssl/cert_10.0.0.3'), + mock.call('/etc/ssl/key_internal.openstack.local', '/etc/ssl/key_10.0.0.3'), + mock.call('/etc/ssl/cert_juju-unit-2', '/etc/ssl/cert_10.1.2.3'), + mock.call('/etc/ssl/key_juju-unit-2', '/etc/ssl/key_10.1.2.3'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.100'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.100'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.2'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.2'), + mock.call('/etc/ssl/cert_public.openstack.local', '/etc/ssl/cert_10.20.0.3'), + mock.call('/etc/ssl/key_public.openstack.local', '/etc/ssl/key_10.20.0.3')] + cert_utils.create_ip_cert_links('/etc/ssl', bindings=['mybindings']) + symlink.assert_has_calls(expected, any_order=True) + # Customer hostname + symlink.reset_mock() + get_hostname.return_value = 'juju-unit-2' + cert_utils.create_ip_cert_links( + '/etc/ssl', + custom_hostname_link='funky-name', bindings=['mybinding']) + expected.extend([ + mock.call('/etc/ssl/cert_juju-unit-2', '/etc/ssl/cert_funky-name'), + mock.call('/etc/ssl/key_juju-unit-2', '/etc/ssl/key_funky-name'), + ]) + symlink.assert_has_calls(expected, any_order=True) + get_cert_request.assert_called_with( + json_encode=False, bindings=['mybinding', 'internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, 'write_file') + def test_install_certs(self, write_file): + certs = { + 'admin.openstack.local': { + 'cert': 'ADMINCERT', + 'key': 'ADMINKEY'}} + cert_utils.install_certs('/etc/ssl', certs, chain='CHAIN') + expected = [ + mock.call( + path='/etc/ssl/cert_admin.openstack.local', + content='ADMINCERT\nCHAIN', + owner='root', group='root', + perms=0o640), + mock.call( + path='/etc/ssl/key_admin.openstack.local', + content='ADMINKEY', + owner='root', group='root', + perms=0o640), + ] + write_file.assert_has_calls(expected) + + @mock.patch.object(cert_utils, 'write_file') + def test_install_certs_ca(self, write_file): + certs = { + 'admin.openstack.local': { + 'cert': 'ADMINCERT', + 'key': 'ADMINKEY'}} + ca = 'MYCA' + cert_utils.install_certs('/etc/ssl', certs, ca) + expected = [ + mock.call( + path='/etc/ssl/cert_admin.openstack.local', + content='ADMINCERT\nMYCA', + owner='root', group='root', + perms=0o640), + mock.call( + path='/etc/ssl/key_admin.openstack.local', + content='ADMINKEY', + owner='root', group='root', + perms=0o640), + ] + write_file.assert_has_calls(expected) + + @mock.patch.object(cert_utils, '_manage_ca_certs') + @mock.patch.object(cert_utils, 'remote_service_name') + @mock.patch.object(cert_utils, 'local_unit') + @mock.patch.object(cert_utils, 'create_ip_cert_links') + @mock.patch.object(cert_utils, 'install_certs') + @mock.patch.object(cert_utils, 'install_ca_cert') + @mock.patch.object(cert_utils, 'mkdir') + @mock.patch.object(cert_utils, 'relation_get') + def test_process_certificates(self, relation_get, mkdir, install_ca_cert, + install_certs, create_ip_cert_links, + local_unit, remote_service_name, + _manage_ca_certs): + remote_service_name.return_value = 'vault' + local_unit.return_value = 'devnull/2' + certs = { + 'admin.openstack.local': { + 'cert': 'ADMINCERT', + 'key': 'ADMINKEY'}} + _relation_info = { + 'keystone_2.processed_requests': json.dumps(certs), + 'chain': 'MYCHAIN', + 'ca': 'ROOTCA', + } + relation_get.return_value = _relation_info + self.assertFalse(cert_utils.process_certificates( + 'myservice', + 'certificates:2', + 'vault/0', + custom_hostname_link='funky-name')) + local_unit.return_value = 'keystone/2' + self.assertTrue(cert_utils.process_certificates( + 'myservice', + 'certificates:2', + 'vault/0', + custom_hostname_link='funky-name')) + _manage_ca_certs.assert_called_once_with( + 'ROOTCA', 'certificates:2') + install_certs.assert_called_once_with( + '/etc/apache2/ssl/myservice', + {'admin.openstack.local': { + 'key': 'ADMINKEY', 'cert': 'ADMINCERT'}}, + 'MYCHAIN', user='root', group='root') + create_ip_cert_links.assert_called_once_with( + '/etc/apache2/ssl/myservice', + custom_hostname_link='funky-name', + bindings=['internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, '_manage_ca_certs') + @mock.patch.object(cert_utils, 'remote_service_name') + @mock.patch.object(cert_utils, 'local_unit') + @mock.patch.object(cert_utils, 'create_ip_cert_links') + @mock.patch.object(cert_utils, 'install_certs') + @mock.patch.object(cert_utils, 'install_ca_cert') + @mock.patch.object(cert_utils, 'mkdir') + @mock.patch.object(cert_utils, 'relation_get') + def test_process_certificates_bindings( + self, relation_get, mkdir, install_ca_cert, + install_certs, create_ip_cert_links, + local_unit, remote_service_name, _manage_ca_certs): + remote_service_name.return_value = 'vault' + local_unit.return_value = 'devnull/2' + certs = { + 'admin.openstack.local': { + 'cert': 'ADMINCERT', + 'key': 'ADMINKEY'}} + _relation_info = { + 'keystone_2.processed_requests': json.dumps(certs), + 'chain': 'MYCHAIN', + 'ca': 'ROOTCA', + } + relation_get.return_value = _relation_info + self.assertFalse(cert_utils.process_certificates( + 'myservice', + 'certificates:2', + 'vault/0', + custom_hostname_link='funky-name')) + local_unit.return_value = 'keystone/2' + self.assertTrue(cert_utils.process_certificates( + 'myservice', + 'certificates:2', + 'vault/0', + custom_hostname_link='funky-name', + bindings=['mybinding'])) + _manage_ca_certs.assert_called_once_with( + 'ROOTCA', 'certificates:2') + install_certs.assert_called_once_with( + '/etc/apache2/ssl/myservice', + {'admin.openstack.local': { + 'key': 'ADMINKEY', 'cert': 'ADMINCERT'}}, + 'MYCHAIN', user='root', group='root') + create_ip_cert_links.assert_called_once_with( + '/etc/apache2/ssl/myservice', + custom_hostname_link='funky-name', + bindings=['mybinding', 'internal', 'admin', 'public']) + + @mock.patch.object(cert_utils, 'remote_service_name') + @mock.patch.object(cert_utils, 'relation_ids') + def test_get_cert_relation_ca_name(self, relation_ids, remote_service_name): + remote_service_name.return_value = 'vault' + + # Test with argument: + self.assertEqual(cert_utils.get_cert_relation_ca_name('certificates:1'), + 'vault_juju_ca_cert') + remote_service_name.assert_called_once_with(relid='certificates:1') + remote_service_name.reset_mock() + + # Test without argument: + relation_ids.return_value = ['certificates:2'] + self.assertEqual(cert_utils.get_cert_relation_ca_name(), + 'vault_juju_ca_cert') + remote_service_name.assert_called_once_with(relid='certificates:2') + remote_service_name.reset_mock() + + # Test without argument nor 'certificates' relation: + relation_ids.return_value = [] + self.assertEqual(cert_utils.get_cert_relation_ca_name(), '') + remote_service_name.assert_not_called() + + @mock.patch.object(cert_utils, 'remote_service_name') + @mock.patch.object(cert_utils.os, 'remove') + @mock.patch.object(cert_utils.os.path, 'exists') + @mock.patch.object(cert_utils, 'config') + @mock.patch.object(cert_utils, 'install_ca_cert') + def test__manage_ca_certs(self, install_ca_cert, config, os_exists, + os_remove, remote_service_name): + remote_service_name.return_value = 'vault' + _config = {} + config.side_effect = lambda x: _config.get(x) + os_exists.return_value = False + cert_utils._manage_ca_certs('CA', 'certificates:2') + install_ca_cert.assert_called_once_with( + b'CA', + name='vault_juju_ca_cert') + self.assertFalse(os_remove.called) + # Test old cert removed. + install_ca_cert.reset_mock() + os_exists.reset_mock() + os_exists.return_value = True + cert_utils._manage_ca_certs('CA', 'certificates:2') + install_ca_cert.assert_called_once_with( + b'CA', + name='vault_juju_ca_cert') + os_remove.assert_called_once_with( + '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt') + # Test cert is installed from config + _config['ssl_ca'] = 'Q0FGUk9NQ09ORklHCg==' + install_ca_cert.reset_mock() + os_remove.reset_mock() + os_exists.reset_mock() + os_exists.return_value = True + cert_utils._manage_ca_certs('CA', 'certificates:2') + expected = [ + mock.call(b'CAFROMCONFIG', name='keystone_juju_ca_cert'), + mock.call(b'CA', name='vault_juju_ca_cert')] + install_ca_cert.assert_has_calls(expected) + self.assertFalse(os_remove.called) + + @mock.patch.object(cert_utils, 'local_unit') + @mock.patch.object(cert_utils, 'related_units') + @mock.patch.object(cert_utils, 'relation_ids') + @mock.patch.object(cert_utils, 'relation_get') + def test_get_requests_for_local_unit(self, relation_get, relation_ids, + related_units, local_unit): + local_unit.return_value = 'rabbitmq-server/2' + relation_ids.return_value = ['certificates:12'] + related_units.return_value = ['vault/0'] + certs = { + 'juju-cd4bb3-5.lxd': { + 'cert': 'BASECERT', + 'key': 'BASEKEY'}, + 'juju-cd4bb3-5.internal': { + 'cert': 'INTERNALCERT', + 'key': 'INTERNALKEY'}} + _relation_info = { + 'rabbitmq-server_2.processed_requests': json.dumps(certs), + 'chain': 'MYCHAIN', + 'ca': 'ROOTCA', + } + relation_get.return_value = _relation_info + self.assertEqual( + cert_utils.get_requests_for_local_unit(), + [{ + 'ca': 'ROOTCA', + 'certs': { + 'juju-cd4bb3-5.lxd': { + 'cert': 'BASECERT', + 'key': 'BASEKEY'}, + 'juju-cd4bb3-5.internal': { + 'cert': 'INTERNALCERT', + 'key': 'INTERNALKEY'}}, + 'chain': 'MYCHAIN'}] + ) + + @mock.patch.object(cert_utils, 'get_requests_for_local_unit') + def test_get_bundle_for_cn(self, get_requests_for_local_unit): + get_requests_for_local_unit.return_value = [{ + 'ca': 'ROOTCA', + 'certs': { + 'juju-cd4bb3-5.lxd': { + 'cert': 'BASECERT', + 'key': 'BASEKEY'}, + 'juju-cd4bb3-5.internal': { + 'cert': 'INTERNALCERT', + 'key': 'INTERNALKEY'}}, + 'chain': 'MYCHAIN'}] + self.assertEqual( + cert_utils.get_bundle_for_cn('juju-cd4bb3-5.internal'), + { + 'ca': 'ROOTCA', + 'cert': 'INTERNALCERT', + 'chain': 'MYCHAIN', + 'key': 'INTERNALKEY'}) + + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'get_relation_ip') + @mock.patch.object(cert_utils, 'resolve_address') + @mock.patch.object(cert_utils, 'config') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_get_certificate_sans(self, local_address, get_hostname, + config, resolve_address, + get_relation_ip, + get_vip_in_network, resolve_network_cidr, + local_unit): + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + _config = { + 'os-internal-hostname': 'internal.openstack.local', + 'os-admin-hostname': 'admin.openstack.local', + 'os-public-hostname': 'public.openstack.local', + } + _resolve_address = { + 'int': '10.0.0.2', + 'internal': '10.0.0.2', + 'admin': '10.10.0.2', + 'public': '10.20.0.2', + } + _npa = { + 'internal': '10.0.0.3', + 'admin': '10.10.0.3', + 'public': '10.20.0.3', + } + _vips = { + '10.0.0.0/16': '10.0.0.100', + '10.10.0.0/16': '10.10.0.100', + '10.20.0.0/16': '10.20.0.100', + } + _resolve_nets = { + '10.0.0.3': '10.0.0.0/16', + '10.10.0.3': '10.10.0.0/16', + '10.20.0.3': '10.20.0.0/16', + } + expect = list(set([ + '10.10.0.100', '10.10.0.2', '10.10.0.3', + '10.0.0.100', '10.0.0.2', '10.0.0.3', + '10.1.2.3', + '10.20.0.100', '10.20.0.2', '10.20.0.3'])) + self.maxDiff = None + config.side_effect = lambda x: _config.get(x) + get_vip_in_network.side_effect = lambda x: _vips.get(x) + resolve_network_cidr.side_effect = lambda x: _resolve_nets.get(x) + get_relation_ip.side_effect = lambda x, cidr_network: _npa.get(x) + resolve_address.side_effect = \ + lambda endpoint_type: _resolve_address[endpoint_type] + expected_get_relation_ip_calls = [ + mock.call('internal', cidr_network=None), + mock.call('admin', cidr_network=None), + mock.call('public', cidr_network=None)] + self.assertEqual(cert_utils.get_certificate_sans().sort(), + expect.sort()) + get_relation_ip.assert_has_calls( + expected_get_relation_ip_calls, any_order=True) + + @mock.patch.object(cert_utils, 'local_unit', return_value='unit/2') + @mock.patch.object(cert_utils, 'resolve_network_cidr') + @mock.patch.object(cert_utils, 'get_vip_in_network') + @mock.patch.object(cert_utils, 'get_relation_ip') + @mock.patch.object(cert_utils, 'resolve_address') + @mock.patch.object(cert_utils, 'config') + @mock.patch.object(cert_utils, 'get_hostname') + @mock.patch.object(cert_utils, 'local_address') + def test_get_certificate_sans_bindings( + self, local_address, get_hostname, config, resolve_address, + get_relation_ip, get_vip_in_network, resolve_network_cidr, local_unit): + local_address.return_value = '10.1.2.3' + get_hostname.return_value = 'juju-unit-2' + _config = { + 'os-internal-hostname': 'internal.openstack.local', + 'os-admin-hostname': 'admin.openstack.local', + 'os-public-hostname': 'public.openstack.local', + } + _resolve_address = { + 'int': '10.0.0.2', + 'internal': '10.0.0.2', + 'admin': '10.10.0.2', + 'public': '10.20.0.2', + } + _npa = { + 'internal': '10.0.0.3', + 'admin': '10.10.0.3', + 'public': '10.20.0.3', + } + _vips = { + '10.0.0.0/16': '10.0.0.100', + '10.10.0.0/16': '10.10.0.100', + '10.20.0.0/16': '10.20.0.100', + } + _resolve_nets = { + '10.0.0.3': '10.0.0.0/16', + '10.10.0.3': '10.10.0.0/16', + '10.20.0.3': '10.20.0.0/16', + } + expect = list(set([ + '10.10.0.100', '10.10.0.2', '10.10.0.3', + '10.0.0.100', '10.0.0.2', '10.0.0.3', + '10.1.2.3', + '10.20.0.100', '10.20.0.2', '10.20.0.3'])) + self.maxDiff = None + config.side_effect = lambda x: _config.get(x) + get_vip_in_network.side_effect = lambda x: _vips.get(x) + resolve_network_cidr.side_effect = lambda x: _resolve_nets.get(x) + get_relation_ip.side_effect = lambda x, cidr_network: _npa.get(x) + resolve_address.side_effect = \ + lambda endpoint_type: _resolve_address[endpoint_type] + expected_get_relation_ip_calls = [ + mock.call('internal', cidr_network=None), + mock.call('admin', cidr_network=None), + mock.call('public', cidr_network=None), + mock.call('mybinding', cidr_network=None)] + self.assertEqual( + cert_utils.get_certificate_sans(bindings=['mybinding']).sort(), + expect.sort()) + get_relation_ip.assert_has_calls( + expected_get_relation_ip_calls, any_order=True) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_deferred_events.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_deferred_events.py new file mode 100644 index 0000000..96a2421 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_deferred_events.py @@ -0,0 +1,366 @@ +import contextlib +import copy +import datetime +import json +import tempfile +import shutil +import yaml + +from mock import patch, call + +import charmhelpers.contrib.openstack.deferred_events as deferred_events +import tests.utils + + +class TestDB(object): + '''Test KV store for unitdata testing''' + def __init__(self): + self.data = {} + self.flushed = False + + def get(self, key, default=None): + result = self.data.get(key, default) + if not result: + return default + return json.loads(result) + + def set(self, key, value): + self.data[key] = json.dumps(value) + return value + + def flush(self): + self.flushed = True + + +class TestHookData(object): + + def __init__(self, kv): + self.kv = kv + + @contextlib.contextmanager + def __call__(self): + yield self.kv, True, True + + +class DeferredCharmServiceEventsTestCase(tests.utils.BaseTestCase): + + def setUp(self): + super(DeferredCharmServiceEventsTestCase, self).setUp() + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.tmp_dir)) + self.patch_object(deferred_events.hookenv, 'service_name') + self.service_name.return_value = 'myapp' + self.patch_object(deferred_events.unitdata, 'HookData') + self.db = TestDB() + self.HookData.return_value = TestHookData(self.db) + self.exp_event_a = deferred_events.ServiceEvent( + timestamp=123, + service='svcA', + reason='ReasonA', + action='restart', + policy_requestor_name='myapp', + policy_requestor_type='charm') + self.exp_event_b = deferred_events.ServiceEvent( + timestamp=223, + service='svcB', + reason='ReasonB', + action='restart') + self.exp_event_c = deferred_events.ServiceEvent( + timestamp=323, + service='svcB', + reason='ReasonB', + action='restart') + self.base_expect_events = [ + self.exp_event_a, + self.exp_event_b, + self.exp_event_c] + self.event_file_pair = [] + for index, event in enumerate(self.base_expect_events): + event_file = '{}/{}.deferred'.format('/tmpdir', str(index)) + self.event_file_pair.append(( + event_file, + event)) + + def test_matching_request_event(self): + self.assertTrue( + self.exp_event_b.matching_request( + self.exp_event_c)) + self.assertFalse( + self.exp_event_a.matching_request( + self.exp_event_b)) + + @patch.object(deferred_events.glob, "glob") + def test_deferred_events_files(self, glob): + defer_files = [ + '/var/lib/policy-rc.d/charm-myapp/1612346300.deferred', + '/var/lib/policy-rc.d/charm-myapp/1612346322.deferred', + '/var/lib/policy-rc.d/charm-myapp/1612346360.deferred'] + + glob.return_value = defer_files + self.assertEqual( + deferred_events.deferred_events_files(), + defer_files) + + def test_read_event_file(self): + with tempfile.NamedTemporaryFile('w') as ftmp: + yaml.dump(vars(self.exp_event_a), ftmp) + ftmp.flush() + self.assertEqual( + deferred_events.read_event_file(ftmp.name), + self.exp_event_a) + + @patch.object(deferred_events, "deferred_events_files") + def test_deferred_events(self, deferred_events_files): + event_files = [] + expect = [] + for index, event in enumerate(self.base_expect_events): + event_file = '{}/{}.deferred'.format(self.tmp_dir, str(index)) + with open(event_file, 'w') as f: + yaml.dump(vars(event), f) + event_files.append(event_file) + expect.append(( + event_file, + event)) + deferred_events_files.return_value = event_files + self.assertEqual( + deferred_events.deferred_events(), + expect) + + @patch.object(deferred_events, "deferred_events") + def test_duplicate_event_files(self, _deferred_events): + _deferred_events.return_value = self.event_file_pair + self.assertEqual( + deferred_events.duplicate_event_files(self.exp_event_b), + ['/tmpdir/1.deferred', '/tmpdir/2.deferred']) + self.assertEqual( + deferred_events.duplicate_event_files(deferred_events.ServiceEvent( + timestamp=223, + service='svcX', + reason='ReasonX', + action='restart')), + []) + + @patch.object(deferred_events.uuid, "uuid1") + def test_get_event_record_file(self, uuid1): + uuid1.return_value = '89eb8258' + self.assertEqual( + deferred_events.get_event_record_file( + 'charm', + 'neutron-ovs'), + '/var/lib/policy-rc.d/charm-neutron-ovs-89eb8258.deferred') + + @patch.object(deferred_events, "get_event_record_file") + @patch.object(deferred_events, "duplicate_event_files") + @patch.object(deferred_events, "init_policy_log_dir") + def test_save_event(self, init_policy_log_dir, duplicate_event_files, get_event_record_file): + duplicate_event_files.return_value = [] + test_file = '{}/test_save_event.yaml'.format(self.tmp_dir) + get_event_record_file.return_value = test_file + deferred_events.save_event(self.exp_event_a) + with open(test_file, 'r') as f: + contents = yaml.load(f) + self.assertEqual(contents, vars(self.exp_event_a)) + + @patch.object(deferred_events.os, "remove") + @patch.object(deferred_events, "read_event_file") + @patch.object(deferred_events, "deferred_events_files") + def test_clear_deferred_events(self, deferred_events_files, read_event_file, + remove): + deferred_events_files.return_value = ['/tmp/file1'] + read_event_file.return_value = self.exp_event_a + deferred_events.clear_deferred_events('svcB', 'restart') + self.assertFalse(remove.called) + deferred_events.clear_deferred_events('svcA', 'restart') + remove.assert_called_once_with('/tmp/file1') + + @patch.object(deferred_events.os, "mkdir") + @patch.object(deferred_events.os.path, "exists") + def test_init_policy_log_dir(self, exists, mkdir): + exists.return_value = True + deferred_events.init_policy_log_dir() + self.assertFalse(mkdir.called) + exists.return_value = False + deferred_events.init_policy_log_dir() + mkdir.assert_called_once_with('/var/lib/policy-rc.d') + + @patch.object(deferred_events, "deferred_events") + def test_get_deferred_events(self, _deferred_events): + _deferred_events.return_value = self.event_file_pair + self.assertEqual( + deferred_events.get_deferred_events(), + self.base_expect_events) + + @patch.object(deferred_events, "get_deferred_events") + def test_get_deferred_restarts(self, get_deferred_events): + test_events = copy.deepcopy(self.base_expect_events) + test_events.append( + deferred_events.ServiceEvent( + timestamp=523, + service='svcD', + reason='StopReasonD', + action='stop')) + get_deferred_events.return_value = test_events + self.assertEqual( + deferred_events.get_deferred_restarts(), + self.base_expect_events) + + @patch.object(deferred_events, 'clear_deferred_events') + def test_clear_deferred_restarts(self, clear_deferred_events): + deferred_events.clear_deferred_restarts(['svcA', 'svcB']) + clear_deferred_events.assert_Called_once_with( + ['svcA', 'svcB'], + 'restart') + + @patch.object(deferred_events, 'clear_deferred_restarts') + def test_process_svc_restart(self, clear_deferred_restarts): + deferred_events.process_svc_restart('svcA') + clear_deferred_restarts.assert_called_once_with( + ['svcA']) + + @patch.object(deferred_events.hookenv, 'config') + def test_is_restart_permitted(self, config): + config.return_value = None + self.assertTrue(deferred_events.is_restart_permitted()) + config.return_value = True + self.assertTrue(deferred_events.is_restart_permitted()) + config.return_value = False + self.assertFalse(deferred_events.is_restart_permitted()) + + @patch.object(deferred_events.time, 'time') + @patch.object(deferred_events, 'save_event') + @patch.object(deferred_events, 'is_restart_permitted') + def test_check_and_record_restart_request(self, is_restart_permitted, + save_event, time): + time.return_value = 123 + is_restart_permitted.return_value = False + deferred_events.check_and_record_restart_request( + 'svcA', + ['/tmp/test1.conf', '/tmp/test2.conf']) + save_event.assert_called_once_with(deferred_events.ServiceEvent( + timestamp=123, + service='svcA', + reason='File(s) changed: /tmp/test1.conf, /tmp/test2.conf', + action='restart')) + + @patch.object(deferred_events.time, 'time') + @patch.object(deferred_events, 'save_event') + @patch.object(deferred_events.host, 'service_restart') + @patch.object(deferred_events, 'is_restart_permitted') + def test_deferrable_svc_restart(self, is_restart_permitted, + service_restart, save_event, time): + time.return_value = 123 + is_restart_permitted.return_value = True + deferred_events.deferrable_svc_restart('svcA', reason='ReasonA') + service_restart.assert_called_once_with('svcA') + service_restart.reset_mock() + is_restart_permitted.return_value = False + deferred_events.deferrable_svc_restart('svcA', reason='ReasonA') + self.assertFalse(service_restart.called) + save_event.assert_called_once_with(deferred_events.ServiceEvent( + timestamp=123, + service='svcA', + reason='ReasonA', + action='restart')) + + @patch.object(deferred_events.policy_rcd, 'add_policy_block') + @patch.object(deferred_events.policy_rcd, 'remove_policy_file') + @patch.object(deferred_events, 'is_restart_permitted') + @patch.object(deferred_events.policy_rcd, 'install_policy_rcd') + def test_configure_deferred_restarts(self, install_policy_rcd, + is_restart_permitted, + remove_policy_file, add_policy_block): + is_restart_permitted.return_value = True + deferred_events.configure_deferred_restarts(['svcA', 'svcB']) + remove_policy_file.assert_called_once_with() + install_policy_rcd.assert_called_once_with() + + remove_policy_file.reset_mock() + install_policy_rcd.reset_mock() + is_restart_permitted.return_value = False + deferred_events.configure_deferred_restarts(['svcA', 'svcB']) + self.assertFalse(remove_policy_file.called) + install_policy_rcd.assert_called_once_with() + add_policy_block.assert_has_calls([ + call('svcA', ['stop', 'restart', 'try-restart']), + call('svcB', ['stop', 'restart', 'try-restart'])]) + + @patch.object(deferred_events.subprocess, 'check_output') + def test_get_service_start_time(self, check_output): + check_output.return_value = ( + b'ActiveEnterTimestamp=Tue 2021-02-02 13:19:55 UTC') + expect = datetime.datetime.strptime( + 'Tue 2021-02-02 13:19:55 UTC', + '%a %Y-%m-%d %H:%M:%S %Z') + self.assertEqual( + deferred_events.get_service_start_time('svcA'), + expect) + check_output.assert_called_once_with( + ['systemctl', 'show', 'svcA', '--property=ActiveEnterTimestamp']) + + @patch.object(deferred_events, 'get_deferred_restarts') + @patch.object(deferred_events, 'clear_deferred_restarts') + @patch.object(deferred_events.hookenv, 'log') + @patch.object(deferred_events, 'get_service_start_time') + def test_check_restart_timestamps(self, get_service_start_time, log, + clear_deferred_restarts, + get_deferred_restarts): + deferred_restarts = [ + # 'Tue 2021-02-02 10:19:55 UTC' + deferred_events.ServiceEvent( + timestamp=1612261195.0, + service='svcA', + reason='ReasonA', + action='restart')] + get_deferred_restarts.return_value = deferred_restarts + get_service_start_time.return_value = datetime.datetime.strptime( + 'Tue 2021-02-02 13:19:55 UTC', + '%a %Y-%m-%d %H:%M:%S %Z') + deferred_events.check_restart_timestamps() + clear_deferred_restarts.assert_called_once_with(['svcA']) + + clear_deferred_restarts.reset_mock() + get_service_start_time.return_value = datetime.datetime.strptime( + 'Tue 2021-02-02 10:10:55 UTC', + '%a %Y-%m-%d %H:%M:%S %Z') + deferred_events.check_restart_timestamps() + self.assertFalse(clear_deferred_restarts.called) + log.assert_called_once_with( + ('Restart still required, svcA was started at 2021-02-02 10:10:55,' + ' restart was requested after that at 2021-02-02 10:19:55'), + level='DEBUG') + + def test_set_deferred_hook(self): + deferred_events.set_deferred_hook('config-changed') + self.assertEqual(self.db.get('deferred-hooks'), ['config-changed']) + deferred_events.set_deferred_hook('leader-settings-changed') + self.assertEqual( + self.db.get('deferred-hooks'), + ['config-changed', 'leader-settings-changed']) + + def test_get_deferred_hook(self): + deferred_events.set_deferred_hook('config-changed') + self.assertEqual( + deferred_events.get_deferred_hooks(), + ['config-changed']) + + def test_clear_deferred_hooks(self): + deferred_events.set_deferred_hook('config-changed') + deferred_events.set_deferred_hook('leader-settings-changed') + self.assertEqual( + deferred_events.get_deferred_hooks(), + ['config-changed', 'leader-settings-changed']) + deferred_events.clear_deferred_hooks() + self.assertEqual( + deferred_events.get_deferred_hooks(), + []) + + def test_clear_deferred_hook(self): + deferred_events.set_deferred_hook('config-changed') + deferred_events.set_deferred_hook('leader-settings-changed') + self.assertEqual( + deferred_events.get_deferred_hooks(), + ['config-changed', 'leader-settings-changed']) + deferred_events.clear_deferred_hook('leader-settings-changed') + self.assertEqual( + deferred_events.get_deferred_hooks(), + ['config-changed']) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ip.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ip.py new file mode 100644 index 0000000..a259195 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ip.py @@ -0,0 +1,176 @@ +from testtools import TestCase +from mock import patch, call, MagicMock + +import charmhelpers.core as ch_core +import charmhelpers.contrib.openstack.ip as ip + +TO_PATCH = [ + 'config', + 'unit_get', + 'get_address_in_network', + 'is_clustered', + 'service_name', + 'network_get_primary_address', + 'resolve_network_cidr', +] + + +class TestConfig(): + + def __init__(self): + self.config = {} + + def set(self, key, value): + self.config[key] = value + + def get(self, key): + return self.config.get(key) + + +class IPTestCase(TestCase): + + def setUp(self): + super(IPTestCase, self).setUp() + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + self.test_config = TestConfig() + self.config.side_effect = self.test_config.get + self.network_get_primary_address.side_effect = [ + NotImplementedError, + ch_core.hookenv.NoNetworkBinding, + ] + + def _patch(self, method): + _m = patch('charmhelpers.contrib.openstack.ip.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_resolve_address_default(self): + self.is_clustered.return_value = False + self.unit_get.return_value = 'unit1' + self.get_address_in_network.return_value = 'unit1' + self.assertEquals(ip.resolve_address(), 'unit1') + self.unit_get.assert_called_with('public-address') + calls = [call('os-public-network'), + call('prefer-ipv6')] + self.config.assert_has_calls(calls) + + def test_resolve_address_default_internal(self): + self.is_clustered.return_value = False + self.unit_get.return_value = 'unit1' + self.get_address_in_network.return_value = 'unit1' + self.assertEquals(ip.resolve_address(ip.INTERNAL), 'unit1') + self.unit_get.assert_called_with('private-address') + calls = [call('os-internal-network'), + call('prefer-ipv6')] + self.config.assert_has_calls(calls) + + def test_resolve_address_public_not_clustered(self): + self.is_clustered.return_value = False + self.test_config.set('os-public-network', '192.168.20.0/24') + self.unit_get.return_value = 'unit1' + self.get_address_in_network.return_value = '192.168.20.1' + self.assertEquals(ip.resolve_address(), '192.168.20.1') + self.unit_get.assert_called_with('public-address') + calls = [call('os-public-network'), + call('prefer-ipv6')] + self.config.assert_has_calls(calls) + self.get_address_in_network.assert_called_with( + '192.168.20.0/24', + 'unit1') + + def test_resolve_address_public_clustered(self): + self.is_clustered.return_value = True + self.test_config.set('os-public-network', '192.168.20.0/24') + self.test_config.set('vip', '192.168.20.100 10.5.3.1') + self.assertEquals(ip.resolve_address(), '192.168.20.100') + + def test_resolve_address_default_clustered(self): + self.is_clustered.return_value = True + self.test_config.set('vip', '10.5.3.1') + self.assertEquals(ip.resolve_address(), '10.5.3.1') + self.config.assert_has_calls( + [call('vip'), + call('os-public-network')]) + + def test_resolve_address_public_clustered_inresolvable(self): + self.is_clustered.return_value = True + self.test_config.set('os-public-network', '192.168.20.0/24') + self.test_config.set('vip', '10.5.3.1') + self.assertRaises(ValueError, ip.resolve_address) + + def test_resolve_address_override(self): + self.test_config.set('os-public-hostname', 'public.example.com') + addr = ip.resolve_address() + self.assertEqual('public.example.com', addr) + + @patch.object(ip, '_get_address_override') + def test_resolve_address_no_override(self, _get_address_override): + self.test_config.set('os-public-hostname', 'public.example.com') + self.unit_get.return_value = '10.0.0.1' + addr = ip.resolve_address(override=False) + self.assertFalse(_get_address_override.called) + self.assertEqual('10.0.0.1', addr) + + def test_resolve_address_override_template(self): + self.test_config.set('os-public-hostname', + '{service_name}.example.com') + self.service_name.return_value = 'foo' + addr = ip.resolve_address() + self.assertEqual('foo.example.com', addr) + + @patch.object(ip, 'get_ipv6_addr', lambda *args, **kwargs: ['::1']) + def test_resolve_address_ipv6_fallback(self): + self.test_config.set('prefer-ipv6', True) + self.is_clustered.return_value = False + self.assertEqual(ip.resolve_address(), '::1') + + @patch.object(ip, 'resolve_address') + def test_canonical_url_http(self, resolve_address): + resolve_address.return_value = 'unit1' + configs = MagicMock() + configs.complete_contexts.return_value = [] + self.assertTrue(ip.canonical_url(configs), + 'http://unit1') + + @patch.object(ip, 'resolve_address') + def test_canonical_url_https(self, resolve_address): + resolve_address.return_value = 'unit1' + configs = MagicMock() + configs.complete_contexts.return_value = ['https'] + self.assertTrue(ip.canonical_url(configs), + 'https://unit1') + + @patch.object(ip, 'is_ipv6', lambda *args: True) + @patch.object(ip, 'resolve_address') + def test_canonical_url_ipv6(self, resolve_address): + resolve_address.return_value = 'unit1' + self.assertTrue(ip.canonical_url(None), 'http://[unit1]') + + @patch.object(ip, 'local_address') + def test_resolve_address_network_get(self, local_address): + self.is_clustered.return_value = False + self.unit_get.return_value = 'unit1' + self.network_get_primary_address.side_effect = None + self.network_get_primary_address.return_value = '10.5.60.1' + self.assertEqual(ip.resolve_address(), '10.5.60.1') + local_address.assert_called_once_with( + unit_get_fallback='public-address') + calls = [call('os-public-network'), + call('prefer-ipv6')] + self.config.assert_has_calls(calls) + self.network_get_primary_address.assert_called_with('public') + + def test_resolve_address_network_get_clustered(self): + self.is_clustered.return_value = True + self.test_config.set('vip', '10.5.60.20 192.168.1.20') + self.network_get_primary_address.side_effect = None + self.network_get_primary_address.return_value = '10.5.60.1' + self.resolve_network_cidr.return_value = '10.5.60.1/24' + self.assertEqual(ip.resolve_address(), '10.5.60.20') + calls = [call('os-public-hostname'), + call('vip'), + call('os-public-network')] + self.config.assert_has_calls(calls) + self.network_get_primary_address.assert_called_with('public') diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_keystone_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_keystone_utils.py new file mode 100644 index 0000000..c7bba25 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_keystone_utils.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python + +import unittest + +from mock import patch, PropertyMock + +import charmhelpers.contrib.openstack.keystone as keystone + +TO_PATCH = [ + 'apt_install', + "log", + "ERROR", + "IdentityServiceContext", +] + + +class KeystoneTests(unittest.TestCase): + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.contrib.openstack.keystone.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_get_keystone_manager(self): + manager = keystone.get_keystone_manager( + 'test-endpoint', 2, token="12345" + ) + self.assertTrue(isinstance(manager, keystone.KeystoneManager2)) + + manager = keystone.get_keystone_manager( + 'test-endpoint', 3, token="12345") + + self.assertTrue(isinstance(manager, keystone.KeystoneManager3)) + self.assertRaises(ValueError, keystone.get_keystone_manager, + 'test-endpoint', 4, token="12345") + + def test_resolve_sevice_id_v2(self): + class ServiceList(list): + def __iter__(self): + class Service(object): + _info = { + 'type': 'metering', + 'name': "ceilometer", + 'id': "uuid-uuid", + } + yield Service() + + manager = keystone.get_keystone_manager('test-endpoint', 2, + token="1234") + manager.api.services.list = PropertyMock(return_value=ServiceList()) + self.assertTrue(manager.service_exists(service_name="ceilometer", + service_type="metering")) + self.assertFalse(manager.service_exists(service_name="barbican")) + self.assertFalse(manager.service_exists(service_name="barbican", + service_type="openstack")) + + def test_resolve_sevice_id_v3(self): + class ServiceList(list): + def __iter__(self): + class Service(object): + _info = { + 'type': 'metering', + 'name': "ceilometer", + 'id': "uuid-uuid", + } + yield Service() + + manager = keystone.get_keystone_manager('test-endpoint', 3, + token="12345") + manager.api.services.list = PropertyMock(return_value=ServiceList()) + self.assertTrue(manager.service_exists(service_name="ceilometer", + service_type="metering")) + self.assertFalse(manager.service_exists(service_name="barbican")) + self.assertFalse(manager.service_exists(service_name="barbican", + service_type="openstack")) + + def test_get_api_suffix(self): + self.assertEquals(keystone.get_api_suffix(2), "v2.0") + self.assertEquals(keystone.get_api_suffix(3), "v3") + + def test_format_endpoint(self): + self.assertEquals(keystone.format_endpoint( + "http", "10.0.0.5", "5000", 2), "http://10.0.0.5:5000/v2.0/") + + def test_get_keystone_manager_from_identity_service_context(self): + class FakeIdentityServiceV2(object): + def __call__(self, *args, **kwargs): + return { + "service_protocol": "https", + "service_host": "10.5.0.5", + "service_port": "5000", + "api_version": "2.0", + "admin_user": "amdin", + "admin_password": "admin", + "admin_tenant_name": "admin_tenant" + } + + self.IdentityServiceContext.return_value = FakeIdentityServiceV2() + + manager = keystone.get_keystone_manager_from_identity_service_context() + self.assertIsInstance(manager, keystone.KeystoneManager2) + + class FakeIdentityServiceV3(object): + def __call__(self, *args, **kwargs): + return { + "service_protocol": "https", + "service_host": "10.5.0.5", + "service_port": "5000", + "api_version": "3", + "admin_user": "amdin", + "admin_password": "admin", + "admin_tenant_name": "admin_tenant" + } + + self.IdentityServiceContext.return_value = FakeIdentityServiceV3() + + manager = keystone.get_keystone_manager_from_identity_service_context() + self.assertIsInstance(manager, keystone.KeystoneManager3) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_neutron_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_neutron_utils.py new file mode 100644 index 0000000..a571f17 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_neutron_utils.py @@ -0,0 +1,239 @@ +import unittest +from mock import patch +from nose.tools import raises +import charmhelpers.contrib.openstack.neutron as neutron + +TO_PATCH = [ + 'log', + 'config', + 'os_release', + 'check_output', +] + + +class NeutronTests(unittest.TestCase): + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.contrib.openstack.neutron.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_headers_package(self): + self.check_output.return_value = b'3.13.0-19-generic' + kname = neutron.headers_package() + self.assertEquals(kname, 'linux-headers-3.13.0-19-generic') + + def test_kernel_version(self): + self.check_output.return_value = b'3.13.0-19-generic' + kver_maj, kver_min = neutron.kernel_version() + self.assertEquals((kver_maj, kver_min), (3, 13)) + + @patch.object(neutron, 'kernel_version') + def test_determine_dkms_package_old_kernel(self, _kernel_version): + self.check_output.return_value = b'3.4.0-19-generic' + _kernel_version.return_value = (3, 10) + dkms_package = neutron.determine_dkms_package() + self.assertEquals(dkms_package, ['linux-headers-3.4.0-19-generic', + 'openvswitch-datapath-dkms']) + + @patch.object(neutron, 'kernel_version') + def test_determine_dkms_package_new_kernel(self, _kernel_version): + _kernel_version.return_value = (3, 13) + dkms_package = neutron.determine_dkms_package() + self.assertEquals(dkms_package, []) + + def test_quantum_plugins(self): + self.config.return_value = 'foo' + plugins = neutron.quantum_plugins() + self.assertEquals(plugins['ovs']['services'], + ['quantum-plugin-openvswitch-agent']) + self.assertEquals(plugins['nvp']['services'], []) + + def test_neutron_plugins_preicehouse(self): + self.config.return_value = 'foo' + self.os_release.return_value = 'havana' + plugins = neutron.neutron_plugins() + self.assertEquals(plugins['ovs']['config'], + '/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini') + self.assertEquals(plugins['nvp']['services'], []) + + def test_neutron_plugins(self): + self.config.return_value = 'foo' + self.os_release.return_value = 'icehouse' + plugins = neutron.neutron_plugins() + self.assertEquals(plugins['ovs']['config'], + '/etc/neutron/plugins/ml2/ml2_conf.ini') + self.assertEquals(plugins['nvp']['config'], + '/etc/neutron/plugins/vmware/nsx.ini') + self.assertTrue('neutron-plugin-vmware' in + plugins['nvp']['server_packages']) + self.assertEquals(plugins['n1kv']['config'], + '/etc/neutron/plugins/cisco/cisco_plugins.ini') + self.assertEquals(plugins['Calico']['config'], + '/etc/neutron/plugins/ml2/ml2_conf.ini') + self.assertEquals(plugins['plumgrid']['config'], + '/etc/neutron/plugins/plumgrid/plumgrid.ini') + self.assertEquals(plugins['midonet']['config'], + '/etc/neutron/plugins/midonet/midonet.ini') + + self.assertEquals(plugins['nvp']['services'], []) + self.assertEquals(plugins['nsx'], plugins['nvp']) + + self.os_release.return_value = 'kilo' + plugins = neutron.neutron_plugins() + self.assertEquals(plugins['midonet']['driver'], + 'neutron.plugins.midonet.plugin.MidonetPluginV2') + self.assertEquals(plugins['nsx']['config'], + '/etc/neutron/plugins/vmware/nsx.ini') + + self.os_release.return_value = 'liberty' + self.config.return_value = 'mem-1.9' + plugins = neutron.neutron_plugins() + self.assertEquals(plugins['midonet']['driver'], + 'midonet.neutron.plugin_v1.MidonetPluginV2') + self.assertTrue('python-networking-midonet' in + plugins['midonet']['server_packages']) + + self.os_release.return_value = 'mitaka' + self.config.return_value = 'mem-1.9' + plugins = neutron.neutron_plugins() + self.assertEquals(plugins['nsx']['config'], + '/etc/neutron/nsx.ini') + self.assertTrue('python-vmware-nsx' in + plugins['nsx']['server_packages']) + + @patch.object(neutron, 'network_manager') + def test_neutron_plugin_attribute_quantum(self, _network_manager): + self.config.return_value = 'foo' + _network_manager.return_value = 'quantum' + plugins = neutron.neutron_plugin_attribute('ovs', 'services') + self.assertEquals(plugins, ['quantum-plugin-openvswitch-agent']) + + @patch.object(neutron, 'network_manager') + def test_neutron_plugin_attribute_neutron(self, _network_manager): + self.config.return_value = 'foo' + self.os_release.return_value = 'icehouse' + _network_manager.return_value = 'neutron' + plugins = neutron.neutron_plugin_attribute('ovs', 'services') + self.assertEquals(plugins, ['neutron-plugin-openvswitch-agent']) + + @raises(Exception) + @patch.object(neutron, 'network_manager') + def test_neutron_plugin_attribute_foo(self, _network_manager): + _network_manager.return_value = 'foo' + self.assertRaises(Exception, neutron.neutron_plugin_attribute('ovs', 'services')) + + @raises(Exception) + @patch.object(neutron, 'network_manager') + def test_neutron_plugin_attribute_plugin_keyerror(self, _network_manager): + self.config.return_value = 'foo' + _network_manager.return_value = 'quantum' + self.assertRaises(Exception, neutron.neutron_plugin_attribute('foo', 'foo')) + + @patch.object(neutron, 'network_manager') + def test_neutron_plugin_attribute_attr_keyerror(self, _network_manager): + self.config.return_value = 'foo' + _network_manager.return_value = 'quantum' + plugins = neutron.neutron_plugin_attribute('ovs', 'foo') + self.assertEquals(plugins, None) + + @raises(Exception) + def test_network_manager_essex(self): + essex_cases = { + 'quantum': 'quantum', + 'neutron': 'quantum', + 'newhotness': 'newhotness', + } + self.os_release.return_value = 'essex' + for nwmanager in essex_cases: + self.config.return_value = nwmanager + self.assertRaises(Exception, neutron.network_manager()) + + def test_network_manager_folsom(self): + folsom_cases = { + 'quantum': 'quantum', + 'neutron': 'quantum', + 'newhotness': 'newhotness', + } + self.os_release.return_value = 'folsom' + for nwmanager in folsom_cases: + self.config.return_value = nwmanager + renamed_manager = neutron.network_manager() + self.assertEquals(renamed_manager, folsom_cases[nwmanager]) + + def test_network_manager_grizzly(self): + grizzly_cases = { + 'quantum': 'quantum', + 'neutron': 'quantum', + 'newhotness': 'newhotness', + } + self.os_release.return_value = 'grizzly' + for nwmanager in grizzly_cases: + self.config.return_value = nwmanager + renamed_manager = neutron.network_manager() + self.assertEquals(renamed_manager, grizzly_cases[nwmanager]) + + def test_network_manager_havana(self): + havana_cases = { + 'quantum': 'neutron', + 'neutron': 'neutron', + 'newhotness': 'newhotness', + } + self.os_release.return_value = 'havana' + for nwmanager in havana_cases: + self.config.return_value = nwmanager + renamed_manager = neutron.network_manager() + self.assertEquals(renamed_manager, havana_cases[nwmanager]) + + def test_network_manager_icehouse(self): + icehouse_cases = { + 'quantum': 'neutron', + 'neutron': 'neutron', + 'newhotness': 'newhotness', + } + self.os_release.return_value = 'icehouse' + for nwmanager in icehouse_cases: + self.config.return_value = nwmanager + renamed_manager = neutron.network_manager() + self.assertEquals(renamed_manager, icehouse_cases[nwmanager]) + + def test_parse_bridge_mappings(self): + ret = neutron.parse_bridge_mappings(None) + self.assertEqual(ret, {}) + ret = neutron.parse_bridge_mappings("physnet1:br0") + self.assertEqual(ret, {'physnet1': 'br0'}) + ret = neutron.parse_bridge_mappings("physnet1:br0 physnet2:br1") + self.assertEqual(ret, {'physnet1': 'br0', 'physnet2': 'br1'}) + + def test_parse_data_port_mappings(self): + ret = neutron.parse_data_port_mappings(None) + self.assertEqual(ret, {}) + ret = neutron.parse_data_port_mappings('br0:eth0') + self.assertEqual(ret, {'eth0': 'br0'}) + # Back-compat test + ret = neutron.parse_data_port_mappings('eth0', default_bridge='br0') + self.assertEqual(ret, {'eth0': 'br0'}) + # Multiple mappings + ret = neutron.parse_data_port_mappings('br0:eth0 br1:eth1') + self.assertEqual(ret, {'eth0': 'br0', 'eth1': 'br1'}) + # MultMAC mappings + ret = neutron.parse_data_port_mappings('br0:cb:23:ae:72:f2:33 ' + 'br0:fa:16:3e:12:97:8e') + self.assertEqual(ret, {'cb:23:ae:72:f2:33': 'br0', + 'fa:16:3e:12:97:8e': 'br0'}) + + def test_parse_vlan_range_mappings(self): + ret = neutron.parse_vlan_range_mappings(None) + self.assertEqual(ret, {}) + ret = neutron.parse_vlan_range_mappings('physnet1:1001:2000') + self.assertEqual(ret, {'physnet1': ('1001', '2000')}) + ret = neutron.parse_vlan_range_mappings('physnet1:1001:2000 physnet2:2001:3000') + self.assertEqual(ret, {'physnet1': ('1001', '2000'), + 'physnet2': ('2001', '3000')}) + ret = neutron.parse_vlan_range_mappings('physnet1 physnet2:2001:3000') + self.assertEqual(ret, {'physnet1': ('',), + 'physnet2': ('2001', '3000')}) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_openstack_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_openstack_utils.py new file mode 100644 index 0000000..74d4a73 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_openstack_utils.py @@ -0,0 +1,2526 @@ +import io +import os +import unittest +from copy import copy +from tests.helpers import patch_open, FakeRelation + +from testtools import TestCase +from mock import MagicMock, patch, call + +from charmhelpers.fetch import ubuntu as fetch +from charmhelpers.core.hookenv import WORKLOAD_STATES, flush + +import charmhelpers.contrib.openstack.utils as openstack +import charmhelpers.contrib.openstack.deferred_events as deferred_events +from charmhelpers.contrib.openstack.exceptions import ServiceActionError +import contextlib + +import six + +if not six.PY3: + builtin_open = '__builtin__.open' + builtin_import = '__builtin__.__import__' +else: + builtin_open = 'builtins.open' + builtin_import = 'builtins.__import__' + +FAKE_CODENAME = 'precise' +# mocked return of openstack.lsb_release() +FAKE_RELEASE = { + 'DISTRIB_CODENAME': 'precise', + 'DISTRIB_RELEASE': '12.04', + 'DISTRIB_ID': 'Ubuntu', + 'DISTRIB_DESCRIPTION': '"Ubuntu 12.04"' +} + +FAKE_REPO = { + # liberty patch release + 'neutron-common': { + 'pkg_vers': '2:7.0.1-0ubuntu1', + 'os_release': 'liberty', + 'os_version': '2015.2' + }, + # liberty release version + 'nova-common': { + 'pkg_vers': '2:12.0.0~b1-0ubuntu1', + 'os_release': 'liberty', + 'os_version': '2015.2' + }, + 'nova': { + 'pkg_vers': '2012.2.3-0ubuntu2.1', + 'os_release': 'folsom', + 'os_version': '2012.2' + }, + 'glance-common': { + 'pkg_vers': '2012.1.3+stable-20130423-74b067df-0ubuntu1', + 'os_release': 'essex', + 'os_version': '2012.1' + }, + 'keystone-common': { + 'pkg_vers': '1:2013.1-0ubuntu1.1~cloud0', + 'os_release': 'grizzly', + 'os_version': '2013.1' + }, + # Exercise swift version detection + 'swift-storage': { + 'pkg_vers': '1.8.0-0ubuntu1', + 'os_release': 'grizzly', + 'os_version': '1.8.0' + }, + 'swift-proxy': { + 'pkg_vers': '1.13.1-0ubuntu1', + 'os_release': 'icehouse', + 'os_version': '1.13.1' + }, + 'swift-common': { + 'pkg_vers': '1.10.0~rc1-0ubuntu1', + 'os_release': 'havana', + 'os_version': '1.10.0' + }, + 'swift-mitaka-dev': { + 'pkg_vers': '2.7.1.dev8.201605111703.trusty-0ubuntu1', + 'os_release': 'mitaka', + 'os_version': '2.7.0' + }, + # a package that's available in the cache but is not installed + 'cinder-common': { + 'os_release': 'havana', + 'os_version': '2013.2' + }, + # poorly formed openstack version + 'bad-version': { + 'pkg_vers': '1:2200.1-0ubuntu1.1~cloud0', + 'os_release': None, + 'os_version': None + } +} + +MOUNTS = [ + ['/mnt', '/dev/vdb'] +] + +url = 'deb ' + openstack.CLOUD_ARCHIVE_URL +UCA_SOURCES = [ + ('cloud:precise-folsom/proposed', url + ' precise-proposed/folsom main'), + ('cloud:precise-folsom', url + ' precise-updates/folsom main'), + ('cloud:precise-folsom/updates', url + ' precise-updates/folsom main'), + ('cloud:precise-grizzly/proposed', url + ' precise-proposed/grizzly main'), + ('cloud:precise-grizzly', url + ' precise-updates/grizzly main'), + ('cloud:precise-grizzly/updates', url + ' precise-updates/grizzly main'), + ('cloud:precise-havana/proposed', url + ' precise-proposed/havana main'), + ('cloud:precise-havana', url + ' precise-updates/havana main'), + ('cloud:precise-havana/updates', url + ' precise-updates/havana main'), + ('cloud:precise-icehouse/proposed', + url + ' precise-proposed/icehouse main'), + ('cloud:precise-icehouse', url + ' precise-updates/icehouse main'), + ('cloud:precise-icehouse/updates', url + ' precise-updates/icehouse main'), +] + +# Mock python-dnspython resolver used by get_host_ip() + + +class FakeAnswer(object): + + def __init__(self, ip): + self.ip = ip + + def __str__(self): + return self.ip + + +class FakeResolver(object): + + def __init__(self, ip): + self.ip = ip + + def query(self, hostname, query_type): + if self.ip == '': + return [] + else: + return [FakeAnswer(self.ip)] + + +class FakeReverse(object): + + def from_address(self, address): + return '156.94.189.91.in-addr.arpa' + + +class FakeDNSName(object): + + def __init__(self, dnsname): + pass + + +class FakeDNS(object): + + def __init__(self, ip): + self.resolver = FakeResolver(ip) + self.reversename = FakeReverse() + self.name = MagicMock() + self.name.Name = FakeDNSName + + +class OpenStackHelpersTestCase(TestCase): + + def setUp(self): + super(OpenStackHelpersTestCase, self).setUp() + self.patch(fetch, 'get_apt_dpkg_env', lambda: {}) + # Make sleep() and log() into noops for testing + for funcname in ('charmhelpers.core.decorators.log', 'charmhelpers.core.decorators.time.sleep'): + patcher = patch(funcname, return_value=None) + patcher.start() + self.addCleanup(patcher.stop) + + def _apt_cache(self): + # mocks out the apt cache + def cache_get(package): + pkg = MagicMock() + if package in FAKE_REPO and 'pkg_vers' in FAKE_REPO[package]: + pkg.name = package + pkg.current_ver.ver_str = FAKE_REPO[package]['pkg_vers'] + elif (package in FAKE_REPO and + 'pkg_vers' not in FAKE_REPO[package]): + pkg.name = package + pkg.current_ver = None + else: + raise KeyError + return pkg + cache = MagicMock() + cache.__getitem__.side_effect = cache_get + return cache + + @patch.object(openstack, 'filter_missing_packages') + def test_get_installed_semantic_versioned_packages(self, mock_filter): + def _filter_missing_packages(pkgs): + return [x for x in pkgs if x in ['cinder-common']] + mock_filter.side_effect = _filter_missing_packages + self.assertEquals( + openstack.get_installed_semantic_versioned_packages(), + ['cinder-common']) + + @patch('charmhelpers.contrib.openstack.utils.lsb_release') + def test_os_codename_from_install_source(self, mocked_lsb): + """Test mapping install source to OpenStack release name""" + mocked_lsb.return_value = FAKE_RELEASE + + # the openstack release shipped with respective ubuntu/lsb release. + self.assertEquals(openstack.get_os_codename_install_source('distro'), + 'essex') + # proposed pocket + self.assertEquals(openstack.get_os_codename_install_source( + 'distro-proposed'), + 'essex') + self.assertEquals(openstack.get_os_codename_install_source( + 'proposed'), + 'essex') + + # various cloud archive pockets + src = 'cloud:precise-grizzly' + self.assertEquals(openstack.get_os_codename_install_source(src), + 'grizzly') + src = 'cloud:precise-grizzly/proposed' + self.assertEquals(openstack.get_os_codename_install_source(src), + 'grizzly') + + # ppas and full repo urls. + src = 'ppa:openstack-ubuntu-testing/havana-trunk-testing' + self.assertEquals(openstack.get_os_codename_install_source(src), + 'havana') + src = ('deb http://ubuntu-cloud.archive.canonical.com/ubuntu ' + 'precise-havana main') + self.assertEquals(openstack.get_os_codename_install_source(src), + 'havana') + self.assertEquals(openstack.get_os_codename_install_source(None), + '') + + @patch.object(openstack, 'get_os_version_codename') + @patch.object(openstack, 'get_os_codename_install_source') + def test_os_version_from_install_source(self, codename, version): + codename.return_value = 'grizzly' + openstack.get_os_version_install_source('cloud:precise-grizzly') + version.assert_called_with('grizzly') + + @patch('charmhelpers.contrib.openstack.utils.lsb_release') + def test_os_codename_from_bad_install_source(self, mocked_lsb): + """Test mapping install source to OpenStack release name""" + _fake_release = copy(FAKE_RELEASE) + _fake_release['DISTRIB_CODENAME'] = 'natty' + + mocked_lsb.return_value = _fake_release + _e = 'charmhelpers.contrib.openstack.utils.error_out' + with patch(_e) as mocked_err: + openstack.get_os_codename_install_source('distro') + _er = ('Could not derive openstack release for this Ubuntu ' + 'release: natty') + mocked_err.assert_called_with(_er) + + def test_os_codename_from_version(self): + """Test mapping OpenStack numerical versions to code name""" + self.assertEquals(openstack.get_os_codename_version('2013.1'), + 'grizzly') + + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_codename_from_bad_version(self, mocked_error): + """Test mapping a bad OpenStack numerical versions to code name""" + openstack.get_os_codename_version('2014.5.5') + expected_err = ('Could not determine OpenStack codename for ' + 'version 2014.5.5') + mocked_error.assert_called_with(expected_err) + + def test_os_version_from_codename(self): + """Test mapping a OpenStack codename to numerical version""" + self.assertEquals(openstack.get_os_version_codename('folsom'), + '2012.2') + + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_version_from_bad_codename(self, mocked_error): + """Test mapping a bad OpenStack codename to numerical version""" + openstack.get_os_version_codename('foo') + expected_err = 'Could not derive OpenStack version for codename: foo' + mocked_error.assert_called_with(expected_err) + + def test_os_version_swift_from_codename(self): + """Test mapping a swift codename to numerical version""" + self.assertEquals(openstack.get_os_version_codename_swift('liberty'), + '2.5.0') + + def test_get_swift_codename_single_version_kilo(self): + self.assertEquals(openstack.get_swift_codename('2.2.2'), 'kilo') + + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_version_swift_from_bad_codename(self, mocked_error): + """Test mapping a bad swift codename to numerical version""" + openstack.get_os_version_codename_swift('foo') + expected_err = 'Could not derive swift version for codename: foo' + mocked_error.assert_called_with(expected_err) + + def test_get_swift_codename_multiple_versions_liberty(self): + with patch('subprocess.check_output') as _subp: + _subp.return_value = b"... trusty-updates/liberty/main ..." + self.assertEquals(openstack.get_swift_codename('2.5.0'), 'liberty') + + def test_get_swift_codename_multiple_versions_mitaka(self): + with patch('subprocess.check_output') as _subp: + _subp.return_value = b"... trusty-updates/mitaka/main ..." + self.assertEquals(openstack.get_swift_codename('2.5.0'), 'mitaka') + + def test_get_swift_codename_none(self): + self.assertEquals(openstack.get_swift_codename('1.2.3'), None) + + @patch("charmhelpers.core.hookenv.cache", new={}) + @patch.object(openstack, 'openstack_release') + @patch.object(openstack, 'filter_installed_packages') + @patch.object(openstack, 'apt_install') + def test_get_installed_os_version_no_package(self, mock_apt_install, + mock_filter_installed_packages, + mock_openstack_release): + mock_openstack_release.return_value = {} + self.assertEquals( + openstack.get_installed_os_version(), None) + + @patch("charmhelpers.core.hookenv.cache", new={}) + @patch.object(openstack, 'openstack_release') + @patch.object(openstack, 'filter_installed_packages') + @patch.object(openstack, 'apt_install') + def test_get_installed_os_version_with_package(self, mock_apt_install, + mock_filter_installed_packages, + mock_openstack_release): + mock_openstack_release.return_value = {'OPENSTACK_CODENAME': 'wallaby'} + self.assertEquals( + openstack.get_installed_os_version(), 'wallaby') + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + def test_os_codename_from_package(self, mock_snap_install_requested, + mock_get_installed_os_version): + """Test deriving OpenStack codename from an installed package""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + for pkg, vers in six.iteritems(FAKE_REPO): + # test fake repo for all "installed" packages + if pkg.startswith('bad-'): + continue + if 'pkg_vers' not in vers: + continue + self.assertEquals(openstack.get_os_codename_package(pkg), + vers['os_release']) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_codename_from_bad_package_version(self, mocked_error, + mock_snap_install_requested, + mock_get_installed_os_version): + """Test deriving OpenStack codename for a poorly versioned package""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + openstack.get_os_codename_package('bad-version') + _e = ('Could not determine OpenStack codename for version 2200.1') + mocked_error.assert_called_with(_e) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_codename_from_bad_package(self, mocked_error, + mock_snap_install_requested, + mock_get_installed_os_version): + """Test deriving OpenStack codename from an unavailable package""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + try: + openstack.get_os_codename_package('foo') + except Exception: + # ignore exceptions that raise when error_out is mocked + # and doesn't sys.exit(1) + pass + e = 'Could not determine version of package with no installation '\ + 'candidate: foo' + mocked_error.assert_called_with(e) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + def test_os_codename_from_bad_package_nonfatal( + self, mock_snap_install_requested, + mock_get_installed_os_version): + """Test OpenStack codename from an unavailable package is non-fatal""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + self.assertEquals( + None, + openstack.get_os_codename_package('foo', fatal=False) + ) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_codename_from_uninstalled_package(self, mock_error, + mock_snap_install_requested, + mock_get_installed_os_version): + """Test OpenStack codename from an available but uninstalled pkg""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + try: + openstack.get_os_codename_package('cinder-common', fatal=True) + except Exception: + pass + e = ('Could not determine version of uninstalled package: ' + 'cinder-common') + mock_error.assert_called_with(e) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + def test_os_codename_from_uninstalled_package_nonfatal( + self, mock_snap_install_requested, + mock_get_installed_os_version): + """Test OpenStack codename from avail uninstalled pkg is non fatal""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + self.assertEquals( + None, + openstack.get_os_codename_package('cinder-common', fatal=False) + ) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_version_from_package(self, mocked_error, + mock_snap_install_requested, + mock_get_installed_os_version): + """Test deriving OpenStack version from an installed package""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + for pkg, vers in six.iteritems(FAKE_REPO): + if pkg.startswith('bad-'): + continue + if 'pkg_vers' not in vers: + continue + self.assertEquals(openstack.get_os_version_package(pkg), + vers['os_version']) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_os_version_from_bad_package(self, mocked_error, + mock_snap_install_requested, + mock_get_installed_os_version): + """Test deriving OpenStack version from an uninstalled package""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + try: + openstack.get_os_version_package('foo') + except Exception: + # ignore exceptions that raise when error_out is mocked + # and doesn't sys.exit(1) + pass + e = 'Could not determine version of package with no installation '\ + 'candidate: foo' + mocked_error.assert_called_with(e) + + @patch.object(openstack, 'get_installed_os_version') + @patch.object(openstack, 'snap_install_requested') + def test_os_version_from_bad_package_nonfatal( + self, mock_snap_install_requested, + mock_get_installed_os_version): + """Test OpenStack version from an uninstalled package is non-fatal""" + mock_snap_install_requested.return_value = False + mock_get_installed_os_version.return_value = None + with patch.object(openstack, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + self.assertEquals( + None, + openstack.get_os_version_package('foo', fatal=False) + ) + + @patch.object(openstack, 'lsb_release') + @patch.object(openstack, 'get_os_codename_package') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_os_release_uncached(self, config, get_cn, mock_lsb_release): + openstack._os_rel = None + get_cn.return_value = 'folsom' + mock_lsb_release.return_value = { + 'DISTRIB_CODENAME': 'bionic', + } + self.assertEquals('folsom', openstack.os_release('nova-common')) + + @patch.object(openstack, 'lsb_release') + def test_os_release_cached(self, mock_lsb_release): + openstack._os_rel = 'foo' + mock_lsb_release.return_value = { + 'DISTRIB_CODENAME': 'bionic', + } + self.assertEquals('foo', openstack.os_release('nova-common')) + + @patch.object(openstack, 'juju_log') + @patch('sys.exit') + def test_error_out(self, mocked_exit, juju_log): + """Test erroring out""" + openstack.error_out('Everything broke.') + _log = 'FATAL ERROR: Everything broke.' + juju_log.assert_called_with(_log, level='ERROR') + mocked_exit.assert_called_with(1) + + def test_get_source_and_pgp_key(self): + tests = { + "source|key": ('source', 'key'), + "source|": ('source', None), + "|key": ('', 'key'), + "source": ('source', None), + } + for k, v in six.iteritems(tests): + self.assertEqual(openstack.get_source_and_pgp_key(k), v) + + # These should still work, even though the bulk of the functionality has + # moved to charmhelpers.fetch.add_source() + def test_configure_install_source_distro(self): + """Test configuring installation from distro""" + self.assertIsNone(openstack.configure_installation_source('distro')) + + def test_configure_install_source_ppa(self): + """Test configuring installation source from PPA""" + with patch('subprocess.check_call') as mock: + src = 'ppa:gandelman-a/openstack' + openstack.configure_installation_source(src) + ex_cmd = [ + 'add-apt-repository', '--yes', 'ppa:gandelman-a/openstack'] + mock.assert_called_with(ex_cmd, env={}) + + @patch('subprocess.check_call') + @patch.object(fetch, 'import_key') + def test_configure_install_source_deb_url(self, _import, _spcc): + """Test configuring installation source from deb repo url""" + src = ('deb http://ubuntu-cloud.archive.canonical.com/ubuntu ' + 'precise-havana main|KEYID') + openstack.configure_installation_source(src) + _import.assert_called_with('KEYID') + _spcc.assert_called_once_with( + ['add-apt-repository', '--yes', + 'deb http://ubuntu-cloud.archive.canonical.com/ubuntu ' + 'precise-havana main'], env={}) + + @patch.object(fetch, 'get_distrib_codename') + @patch(builtin_open) + @patch('subprocess.check_call') + def test_configure_install_source_distro_proposed( + self, _spcc, _open, _lsb): + """Test configuring installation source from deb repo url""" + _lsb.return_value = FAKE_CODENAME + _file = MagicMock(spec=io.FileIO) + _open.return_value = _file + openstack.configure_installation_source('distro-proposed') + _file.__enter__().write.assert_called_once_with( + '# Proposed\ndeb http://archive.ubuntu.com/ubuntu ' + 'precise-proposed main universe multiverse restricted\n') + src = ('deb http://archive.ubuntu.com/ubuntu/ precise-proposed ' + 'restricted main multiverse universe') + openstack.configure_installation_source(src) + _spcc.assert_called_once_with( + ['add-apt-repository', '--yes', + 'deb http://archive.ubuntu.com/ubuntu/ precise-proposed ' + 'restricted main multiverse universe'], env={}) + + @patch('charmhelpers.fetch.filter_installed_packages') + @patch('charmhelpers.fetch.apt_install') + @patch.object(openstack, 'error_out') + @patch.object(openstack, 'juju_log') + def test_add_source_cloud_invalid_pocket(self, _log, _out, + apt_install, filter_pkg): + openstack.configure_installation_source("cloud:havana-updates") + _e = ('Invalid Cloud Archive release specified: ' + 'havana-updates on this Ubuntuversion') + _s = _out.call_args[0][0] + self.assertTrue(_s.startswith(_e)) + + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + @patch.object(fetch, 'get_distrib_codename') + def test_add_source_cloud_pocket_style(self, get_distrib_codename, + apt_install, filter_pkg): + source = "cloud:precise-updates/havana" + get_distrib_codename.return_value = 'precise' + result = ( + "# Ubuntu Cloud Archive\n" + "deb http://ubuntu-cloud.archive.canonical.com/ubuntu " + "precise-updates/havana main\n") + with patch_open() as (mock_open, mock_file): + openstack.configure_installation_source(source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + @patch.object(fetch, 'get_distrib_codename') + def test_add_source_cloud_os_style(self, get_distrib_codename, + apt_install, filter_pkg): + source = "cloud:precise-havana" + get_distrib_codename.return_value = 'precise' + result = ( + "# Ubuntu Cloud Archive\n" + "deb http://ubuntu-cloud.archive.canonical.com/ubuntu " + "precise-updates/havana main\n") + with patch_open() as (mock_open, mock_file): + openstack.configure_installation_source(source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + def test_add_source_cloud_distroless_style(self, apt_install, filter_pkg): + source = "cloud:havana" + result = ( + "# Ubuntu Cloud Archive\n" + "deb http://ubuntu-cloud.archive.canonical.com/ubuntu " + "precise-updates/havana main\n") + with patch_open() as (mock_open, mock_file): + openstack.configure_installation_source(source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch('charmhelpers.fetch.ubuntu.log', lambda *args, **kwargs: None) + @patch('charmhelpers.contrib.openstack.utils.juju_log', + lambda *args, **kwargs: None) + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_configure_bad_install_source(self, _error): + openstack.configure_installation_source('foo') + _error.assert_called_with("Unknown source: 'foo'") + + @patch.object(fetch, 'get_distrib_codename') + def test_configure_install_source_uca_staging(self, _lsb): + """Test configuring installation source from UCA staging sources""" + _lsb.return_value = FAKE_CODENAME + # staging pockets are configured as PPAs + with patch('subprocess.check_call') as _subp: + src = 'cloud:precise-folsom/staging' + openstack.configure_installation_source(src) + cmd = ['add-apt-repository', '-y', + 'ppa:ubuntu-cloud-archive/folsom-staging'] + _subp.assert_called_with(cmd, env={}) + + @patch(builtin_open) + @patch.object(fetch, 'apt_install') + @patch.object(fetch, 'get_distrib_codename') + @patch.object(fetch, 'filter_installed_packages') + def test_configure_install_source_uca_repos( + self, _fip, _lsb, _install, _open): + """Test configuring installation source from UCA sources""" + _lsb.return_value = FAKE_CODENAME + _file = MagicMock(spec=io.FileIO) + _open.return_value = _file + _fip.side_effect = lambda x: x + for src, url in UCA_SOURCES: + actual_url = "# Ubuntu Cloud Archive\n{}\n".format(url) + openstack.configure_installation_source(src) + _install.assert_called_with(['ubuntu-cloud-keyring'], + fatal=True) + _open.assert_called_with( + '/etc/apt/sources.list.d/cloud-archive.list', + 'w' + ) + _file.__enter__().write.assert_called_with(actual_url) + + @patch('charmhelpers.contrib.openstack.utils.error_out') + def test_configure_install_source_bad_uca(self, mocked_error): + """Test configuring installation source from bad UCA source""" + try: + openstack.configure_installation_source('cloud:foo-bar') + except Exception: + # ignore exceptions that raise when error_out is mocked + # and doesn't sys.exit(1) + pass + _e = ('Invalid Cloud Archive release specified: foo-bar' + ' on this Ubuntuversion') + _s = mocked_error.call_args[0][0] + self.assertTrue(_s.startswith(_e)) + + @patch.object(openstack, 'fetch_import_key') + def test_import_key_calls_fetch_import_key(self, fetch_import_key): + openstack.import_key('random-string') + fetch_import_key.assert_called_once_with('random-string') + + @patch.object(openstack, 'juju_log', lambda *args, **kwargs: None) + @patch.object(openstack, 'fetch_import_key') + @patch.object(openstack, 'sys') + def test_import_key_calls_sys_exit_on_error(self, mock_sys, + fetch_import_key): + + def raiser(_): + raise openstack.GPGKeyError("an error occurred") + fetch_import_key.side_effect = raiser + openstack.import_key('random failure') + mock_sys.exit.assert_called_once_with(1) + + @patch('os.mkdir') + @patch('os.path.exists') + @patch('charmhelpers.contrib.openstack.utils.charm_dir') + @patch(builtin_open) + def test_save_scriptrc(self, _open, _charm_dir, _exists, _mkdir): + """Test generation of scriptrc from environment""" + scriptrc = ['#!/bin/bash\n', + 'export setting1=foo\n', + 'export setting2=bar\n'] + _file = MagicMock(spec=io.FileIO) + _open.return_value = _file + _charm_dir.return_value = '/var/lib/juju/units/testing-foo-0/charm' + _exists.return_value = False + os.environ['JUJU_UNIT_NAME'] = 'testing-foo/0' + openstack.save_script_rc(setting1='foo', setting2='bar') + rcdir = '/var/lib/juju/units/testing-foo-0/charm/scripts' + _mkdir.assert_called_with(rcdir) + expected_f = '/var/lib/juju/units/testing-foo-0/charm/scripts/scriptrc' + _open.assert_called_with(expected_f, 'wt') + _mkdir.assert_called_with(os.path.dirname(expected_f)) + _file.__enter__().write.assert_has_calls( + list(call(line) for line in scriptrc), any_order=True) + + @patch.object(openstack, 'lsb_release') + @patch.object(openstack, 'get_os_version_package') + @patch.object(openstack, 'get_os_version_codename_swift') + @patch.object(openstack, 'config') + def test_openstack_upgrade_detection_true(self, config, vers_swift, + vers_pkg, lsb): + """Test it detects when an openstack package has available upgrade""" + lsb.return_value = FAKE_RELEASE + config.return_value = 'cloud:precise-havana' + vers_pkg.return_value = '2013.1.1' + self.assertTrue(openstack.openstack_upgrade_available('nova-common')) + # milestone to major release detection + vers_pkg.return_value = '2013.2~b1' + self.assertTrue(openstack.openstack_upgrade_available('nova-common')) + vers_pkg.return_value = '1.9.0' + vers_swift.return_value = '2.5.0' + self.assertTrue(openstack.openstack_upgrade_available('swift-proxy')) + vers_pkg.return_value = '2.5.0' + vers_swift.return_value = '2.10.0' + self.assertTrue(openstack.openstack_upgrade_available('swift-proxy')) + + @patch.object(openstack, 'lsb_release') + @patch.object(openstack, 'get_os_version_package') + @patch.object(openstack, 'config') + def test_openstack_upgrade_detection_false(self, config, vers_pkg, lsb): + """Test it detects when an openstack upgrade is not necessary""" + lsb.return_value = FAKE_RELEASE + config.return_value = 'cloud:precise-folsom' + vers_pkg.return_value = '2013.1.1' + self.assertFalse(openstack.openstack_upgrade_available('nova-common')) + # milestone to majro release detection + vers_pkg.return_value = '2013.1~b1' + self.assertFalse(openstack.openstack_upgrade_available('nova-common')) + # ugly duckling testing + config.return_value = 'cloud:precise-havana' + vers_pkg.return_value = '1.10.0' + self.assertFalse(openstack.openstack_upgrade_available('swift-proxy')) + + @patch.object(openstack, 'is_block_device') + @patch.object(openstack, 'error_out') + def test_ensure_block_device_bad_config(self, err, is_bd): + """Test it doesn't prepare storage with bad config""" + openstack.ensure_block_device(block_device='none') + self.assertTrue(err.called) + + @patch.object(openstack, 'is_block_device') + @patch.object(openstack, 'ensure_loopback_device') + def test_ensure_block_device_loopback(self, ensure_loopback, is_bd): + """Test it ensures loopback device when checking block device""" + defsize = openstack.DEFAULT_LOOPBACK_SIZE + is_bd.return_value = True + + ensure_loopback.return_value = '/tmp/cinder.img' + result = openstack.ensure_block_device('/tmp/cinder.img') + ensure_loopback.assert_called_with('/tmp/cinder.img', defsize) + self.assertEquals(result, '/tmp/cinder.img') + + ensure_loopback.return_value = '/tmp/cinder-2.img' + result = openstack.ensure_block_device('/tmp/cinder-2.img|15G') + ensure_loopback.assert_called_with('/tmp/cinder-2.img', '15G') + self.assertEquals(result, '/tmp/cinder-2.img') + + @patch.object(openstack, 'is_block_device') + def test_ensure_standard_block_device(self, is_bd): + """Test it looks for storage at both relative and full device path""" + for dev in ['vdb', '/dev/vdb']: + openstack.ensure_block_device(dev) + is_bd.assert_called_with('/dev/vdb') + + @patch.object(openstack, 'is_block_device') + @patch.object(openstack, 'error_out') + def test_ensure_nonexistent_block_device(self, error_out, is_bd): + """Test it will not ensure a non-existent block device""" + is_bd.return_value = False + openstack.ensure_block_device(block_device='foo') + self.assertTrue(error_out.called) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'umount') + @patch.object(openstack, 'mounts') + @patch.object(openstack, 'zap_disk') + @patch.object(openstack, 'is_lvm_physical_volume') + def test_clean_storage_unmount(self, is_pv, zap_disk, mounts, umount, log): + """Test it unmounts block device when cleaning storage""" + is_pv.return_value = False + zap_disk.return_value = True + mounts.return_value = MOUNTS + openstack.clean_storage('/dev/vdb') + umount.called_with('/dev/vdb', True) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'remove_lvm_physical_volume') + @patch.object(openstack, 'deactivate_lvm_volume_group') + @patch.object(openstack, 'mounts') + @patch.object(openstack, 'is_lvm_physical_volume') + def test_clean_storage_lvm_wipe(self, is_pv, mounts, rm_lv, rm_vg, log): + """Test it removes traces of LVM when cleaning storage""" + mounts.return_value = [] + is_pv.return_value = True + openstack.clean_storage('/dev/vdb') + rm_lv.assert_called_with('/dev/vdb') + rm_vg .assert_called_with('/dev/vdb') + + @patch.object(openstack, 'zap_disk') + @patch.object(openstack, 'is_lvm_physical_volume') + @patch.object(openstack, 'mounts') + def test_clean_storage_zap_disk(self, mounts, is_pv, zap_disk): + """It removes traces of LVM when cleaning storage""" + mounts.return_value = [] + is_pv.return_value = False + openstack.clean_storage('/dev/vdb') + zap_disk.assert_called_with('/dev/vdb') + + @patch('os.path.isfile') + @patch(builtin_open) + def test_get_matchmaker_map(self, _open, _isfile): + _isfile.return_value = True + mm_data = """ + { + "cinder-scheduler": [ + "juju-t-machine-4" + ] + } + """ + fh = _open.return_value.__enter__.return_value + fh.read.return_value = mm_data + self.assertEqual( + openstack.get_matchmaker_map(), + {'cinder-scheduler': ['juju-t-machine-4']} + ) + + @patch('os.path.isfile') + @patch(builtin_open) + def test_get_matchmaker_map_nofile(self, _open, _isfile): + _isfile.return_value = False + self.assertEqual( + openstack.get_matchmaker_map(), + {} + ) + + def test_incomplete_relation_data(self): + configs = MagicMock() + configs.complete_contexts.return_value = ['pgsql-db', 'amqp'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + expected_result = 'identity' + + result = openstack.incomplete_relation_data( + configs, required_interfaces) + self.assertTrue(expected_result in result.keys()) + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete( + self, is_unit_paused_set, status_set, log): + configs = MagicMock() + configs.complete_contexts.return_value = ['shared-db', + 'amqp', + 'identity-service'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + + openstack.set_os_workload_status(configs, required_interfaces) + status_set.assert_called_with('active', 'Unit is ready') + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.incomplete_relation_data', + return_value={'identity': {'identity-service': {'related': True}}}) + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_related_incomplete( + self, is_unit_paused_set, status_set, + incomplete_relation_data, log): + configs = MagicMock() + configs.complete_contexts.return_value = ['shared-db', 'amqp'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + + openstack.set_os_workload_status(configs, required_interfaces) + status_set.assert_called_with('waiting', + "Incomplete relations: identity") + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.incomplete_relation_data', + return_value={'identity': {'identity-service': {'related': False}}}) + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_absent( + self, is_unit_paused_set, status_set, + incomplete_relation_data, log): + configs = MagicMock() + configs.complete_contexts.return_value = ['shared-db', 'amqp'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + + openstack.set_os_workload_status(configs, required_interfaces) + status_set.assert_called_with('blocked', + 'Missing relations: identity') + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.hook_name', + return_value='identity-service-relation-broken') + @patch('charmhelpers.contrib.openstack.utils.incomplete_relation_data', + return_value={'identity': {'identity-service': {'related': True}}}) + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_related_broken( + self, is_unit_paused_set, status_set, + incomplete_relation_data, hook_name, log): + configs = MagicMock() + configs.complete_contexts.return_value = ['shared-db', 'amqp'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + + openstack.set_os_workload_status(configs, required_interfaces) + status_set.assert_called_with('blocked', + "Missing relations: identity") + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.incomplete_relation_data', + return_value={'identity': + {'identity-service': {'related': True}}, + + 'message': + {'amqp': {'missing_data': ['rabbitmq-password'], + 'related': True}}, + + 'database': + {'shared-db': {'related': False}} + }) + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_mixed( + self, is_unit_paused_set, status_set, + incomplete_relation_data, log): + configs = MagicMock() + configs.complete_contexts.return_value = ['shared-db', 'amqp'] + required_interfaces = { + 'database': ['shared-db', 'pgsql-db'], + 'message': ['amqp', 'zeromq-configuration'], + 'identity': ['identity-service']} + + openstack.set_os_workload_status(configs, required_interfaces) + + args = status_set.call_args + actual_parm1 = args[0][0] + actual_parm2 = args[0][1] + expected1 = ("Missing relations: database; incomplete relations: " + "identity, message") + expected2 = ("Missing relations: database; incomplete relations: " + "message, identity") + self.assertTrue(actual_parm1 == 'blocked') + self.assertTrue(actual_parm2 == expected1 or actual_parm2 == expected2) + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_with_services_list( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = ['database', 'identity'] + # Assume that the service and ports are open. + port_has_listener.return_value = True + service_running.return_value = True + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with('active', 'Unit is ready') + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_services_list_not_running( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = ['database', 'identity'] + port_has_listener.return_value = True + # Fail the identity service + service_running.side_effect = [True, False] + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'blocked', + 'Services not running that should be: identity') + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_with_services( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + # Assume that the service and ports are open. + port_has_listener.return_value = True + service_running.return_value = True + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with('active', 'Unit is ready') + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_service_not_running( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + port_has_listener.return_value = True + # Fail the identity service + service_running.side_effect = [True, False] + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'blocked', + 'Services not running that should be: identity') + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_port_not_open( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + port_has_listener.side_effect = [True, False, True] + # Fail the identity service + service_running.return_value = True + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'blocked', + 'Services with ports not open that should be:' + ' database: [20]') + + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=False) + def test_set_os_workload_status_complete_ports_not_open( + self, is_unit_paused_set, status_set, log, port_has_listener): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + ports = [50, 60, 70] + port_has_listener.side_effect = [True, False, True] + + openstack.set_os_workload_status( + configs, required_interfaces, ports=ports) + status_set.assert_called_with( + 'blocked', + 'Ports which should be open, but are not: 60') + + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_simple( + self, is_unit_paused_set, status_set, log): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + openstack.set_os_workload_status(configs, required_interfaces) + status_set.assert_called_with( + 'maintenance', + "Paused. Use 'resume' action to resume normal service.") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_services_check( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + port_has_listener.return_value = False + service_running.side_effect = [False, False] + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'maintenance', + "Paused. Use 'resume' action to resume normal service.") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_services_fail( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + port_has_listener.return_value = False + # Fail the identity service + service_running.side_effect = [False, True] + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'blocked', + "Services should be paused but these services running: identity") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_services_ports_fail( + self, is_unit_paused_set, status_set, log, + port_has_listener, service_running): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + # make the service 20 port be still listening. + port_has_listener.side_effect = [False, True, False] + service_running.return_value = False + + openstack.set_os_workload_status( + configs, required_interfaces, services=services) + status_set.assert_called_with( + 'blocked', + "Services should be paused but these service:ports are open:" + " database: [20]") + + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_ports_check( + self, is_unit_paused_set, status_set, log, + port_has_listener): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + ports = [50, 60, 70] + port_has_listener.side_effect = [False, False, False] + + openstack.set_os_workload_status( + configs, required_interfaces, ports=ports) + status_set.assert_called_with( + 'maintenance', + "Paused. Use 'resume' action to resume normal service.") + + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + @patch.object(openstack, 'juju_log') + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.is_unit_paused_set', + return_value=True) + def test_set_os_workload_status_paused_ports_fail( + self, is_unit_paused_set, status_set, log, + port_has_listener): + configs = MagicMock() + configs.complete_contexts.return_value = [] + required_interfaces = {} + + # fail port 70 to make it seem to be running + ports = [50, 60, 70] + port_has_listener.side_effect = [False, False, True] + + openstack.set_os_workload_status( + configs, required_interfaces, ports=ports) + status_set.assert_called_with( + 'blocked', + "Services should be paused but " + "these ports which should be closed, but are open: 70") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_simple_services( + self, port_has_listener, service_running): + services = ['database', 'identity'] + port_has_listener.return_value = False + service_running.return_value = False + + state, message = openstack.check_actually_paused( + services) + self.assertEquals(state, None) + self.assertEquals(message, None) + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_simple_services_fail( + self, port_has_listener, service_running): + services = ['database', 'identity'] + port_has_listener.return_value = False + service_running.side_effect = [False, True] + + state, message = openstack.check_actually_paused( + services) + self.assertEquals(state, 'blocked') + self.assertEquals( + message, + "Services should be paused but these services running: identity") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_services_dict( + self, port_has_listener, service_running): + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + # Assume that the service and ports are open. + port_has_listener.return_value = False + service_running.return_value = False + + state, message = openstack.check_actually_paused( + services) + self.assertEquals(state, None) + self.assertEquals(message, None) + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_services_dict_fail( + self, port_has_listener, service_running): + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + # Assume that the service and ports are open. + port_has_listener.return_value = False + service_running.side_effect = [False, True] + + state, message = openstack.check_actually_paused( + services) + self.assertEquals(state, 'blocked') + self.assertEquals( + message, + "Services should be paused but these services running: identity") + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_services_dict_ports_fail( + self, port_has_listener, service_running): + services = [ + {'service': 'database', 'ports': [10, 20]}, + {'service': 'identity', 'ports': [30]}, + ] + # Assume that the service and ports are open. + port_has_listener.side_effect = [False, True, False] + service_running.return_value = False + + state, message = openstack.check_actually_paused( + services) + self.assertEquals(state, 'blocked') + self.assertEquals(message, + 'Services should be paused but these service:ports' + ' are open: database: [20]') + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_ports_okay( + self, port_has_listener, service_running): + port_has_listener.side_effect = [False, False, False] + service_running.return_value = False + ports = [50, 60, 70] + + state, message = openstack.check_actually_paused( + ports=ports) + self.assertEquals(state, None) + self.assertEquals(state, None) + + @patch('charmhelpers.contrib.openstack.utils.service_running') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_check_actually_paused_ports_fail( + self, port_has_listener, service_running): + port_has_listener.side_effect = [False, True, False] + service_running.return_value = False + ports = [50, 60, 70] + + state, message = openstack.check_actually_paused( + ports=ports) + self.assertEquals(state, 'blocked') + self.assertEquals(message, + 'Services should be paused but these ports ' + 'which should be closed, but are open: 60') + + @staticmethod + def _unit_paused_helper(hook_data_mock): + # HookData()() returns a tuple (kv, delta_config, delta_relation) + # but we only want kv in the test. + kv = MagicMock() + + @contextlib.contextmanager + def hook_data__call__(): + yield (kv, True, False) + + hook_data__call__.return_value = (kv, True, False) + hook_data_mock.return_value = hook_data__call__ + return kv + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_set_unit_paused(self, hook_data): + kv = self._unit_paused_helper(hook_data) + openstack.set_unit_paused() + kv.set.assert_called_once_with('unit-paused', True) + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_set_unit_upgrading(self, hook_data): + kv = self._unit_paused_helper(hook_data) + openstack.set_unit_upgrading() + kv.set.assert_called_once_with('unit-upgrading', True) + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_clear_unit_paused(self, hook_data): + kv = self._unit_paused_helper(hook_data) + openstack.clear_unit_paused() + kv.set.assert_called_once_with('unit-paused', False) + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_clear_unit_upgrading(self, hook_data): + kv = self._unit_paused_helper(hook_data) + openstack.clear_unit_upgrading() + kv.set.assert_called_once_with('unit-upgrading', False) + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_is_unit_paused_set(self, hook_data): + kv = self._unit_paused_helper(hook_data) + kv.get.return_value = True + r = openstack.is_unit_paused_set() + kv.get.assert_called_once_with('unit-paused') + self.assertEquals(r, True) + kv.get.return_value = False + r = openstack.is_unit_paused_set() + self.assertEquals(r, False) + + @patch('charmhelpers.contrib.openstack.utils.unitdata.HookData') + def test_is_unit_upgrading_set(self, hook_data): + kv = self._unit_paused_helper(hook_data) + kv.get.return_value = True + r = openstack.is_unit_upgrading_set() + kv.get.assert_called_once_with('unit-upgrading') + self.assertEquals(r, True) + kv.get.return_value = False + r = openstack.is_unit_upgrading_set() + self.assertEquals(r, False) + + @patch.object(openstack, 'config') + @patch.object(openstack, 'is_unit_paused_set') + @patch.object(openstack.deferred_events, 'is_restart_permitted') + @patch.object(openstack.deferred_events, 'set_deferred_hook') + @patch.object(openstack.deferred_events, 'clear_deferred_hook') + def test_is_hook_allowed(self, clear_deferred_hook, set_deferred_hook, + is_restart_permitted, is_unit_paused_set, config): + # Test unit not paused and not checking whether restarts are allowed + is_unit_paused_set.return_value = False + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=False), + (True, '')) + self.assertFalse(clear_deferred_hook.called) + + # Test unit paused and not checking whether restarts are allowed + is_unit_paused_set.return_value = True + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=False), + (False, 'Unit is pause or upgrading. Skipping config-changed')) + + # Test unit not paused and restarts allowed + clear_deferred_hook.reset_mock() + is_unit_paused_set.return_value = False + is_restart_permitted.return_value = True + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=True), + (True, '')) + clear_deferred_hook.assert_called_once_with('config-changed') + + # Test unit not paused and restarts not allowed + # enable-auto-restarts not enabled as part of this hook + clear_deferred_hook.reset_mock() + set_deferred_hook.reset_mock() + is_unit_paused_set.return_value = False + is_restart_permitted.return_value = False + config().changed.return_value = False + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=True), + (False, 'auto restarts are disabled')) + self.assertFalse(clear_deferred_hook.called) + set_deferred_hook.assert_called_once_with('config-changed') + + # Test unit not paused and restarts not allowed. + # enable-auto-restarts enabled as part of this hook + clear_deferred_hook.reset_mock() + set_deferred_hook.reset_mock() + is_unit_paused_set.return_value = False + is_restart_permitted.return_value = False + config().changed.return_value = True + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=True), + (False, 'auto restarts are disabled')) + self.assertFalse(clear_deferred_hook.called) + + # Test unit paused and restarts not allowed + # enable-auto-restarts not enabled as part of this hook + clear_deferred_hook.reset_mock() + set_deferred_hook.reset_mock() + is_unit_paused_set.return_value = True + is_restart_permitted.return_value = False + config().changed.return_value = False + self.assertEqual( + openstack.is_hook_allowed( + 'config-changed', + check_deferred_restarts=True), + (False, 'Unit is pause or upgrading. Skipping config-changed and ' + 'auto restarts are disabled')) + self.assertFalse(clear_deferred_hook.called) + set_deferred_hook.assert_called_once_with('config-changed') + + @patch('charmhelpers.contrib.openstack.utils.service_stop') + def test_manage_payload_services_ok(self, service_stop): + services = ['service1', 'service2'] + service_stop.side_effect = [True, True] + self.assertEqual( + openstack.manage_payload_services('stop', services=services), + (True, [])) + + @patch('charmhelpers.contrib.openstack.utils.service_stop') + def test_manage_payload_services_fails(self, service_stop): + services = ['service1', 'service2'] + service_stop.side_effect = [True, False] + self.assertEqual( + openstack.manage_payload_services('stop', services=services), + (False, ["service2 didn't stop cleanly."])) + + @patch('charmhelpers.contrib.openstack.utils.service_stop') + def test_manage_payload_services_charm_func(self, service_stop): + bespoke_func = MagicMock() + bespoke_func.return_value = None + services = ['service1', 'service2'] + service_stop.side_effect = [True, True] + self.assertEqual( + openstack.manage_payload_services('stop', services=services, + charm_func=bespoke_func), + (True, [])) + bespoke_func.assert_called_once_with() + + @patch('charmhelpers.contrib.openstack.utils.service_stop') + def test_manage_payload_services_charm_func_msg(self, service_stop): + bespoke_func = MagicMock() + bespoke_func.return_value = 'it worked' + services = ['service1', 'service2'] + service_stop.side_effect = [True, True] + self.assertEqual( + openstack.manage_payload_services('stop', services=services, + charm_func=bespoke_func), + (True, ['it worked'])) + bespoke_func.assert_called_once_with() + + @patch('charmhelpers.contrib.openstack.utils.service_stop') + def test_manage_payload_services_charm_func_fails(self, service_stop): + bespoke_func = MagicMock() + bespoke_func.side_effect = Exception('it failed') + services = ['service1', 'service2'] + service_stop.side_effect = [True, True] + self.assertEqual( + openstack.manage_payload_services('stop', services=services, + charm_func=bespoke_func), + (False, ['it failed'])) + bespoke_func.assert_called_once_with() + + def test_manage_payload_services_wrong_action(self): + self.assertRaises( + RuntimeError, + openstack.manage_payload_services, + 'mangle') + + @patch('charmhelpers.contrib.openstack.utils.service_pause') + @patch('charmhelpers.contrib.openstack.utils.set_unit_paused') + def test_pause_unit_okay(self, set_unit_paused, service_pause): + services = ['service1', 'service2'] + service_pause.side_effect = [True, True] + openstack.pause_unit(None, services=services) + set_unit_paused.assert_called_once_with() + self.assertEquals(service_pause.call_count, 2) + + @patch('charmhelpers.contrib.openstack.utils.service_pause') + @patch('charmhelpers.contrib.openstack.utils.set_unit_paused') + def test_pause_unit_service_fails(self, set_unit_paused, service_pause): + services = ['service1', 'service2'] + service_pause.side_effect = [True, True] + openstack.pause_unit(None, services=services) + set_unit_paused.assert_called_once_with() + self.assertEquals(service_pause.call_count, 2) + # Fail the 2nd service + service_pause.side_effect = [True, False] + try: + openstack.pause_unit(None, services=services) + raise Exception("pause_unit should have raised Exception") + except Exception as e: + self.assertEquals(e.args[0], + "Couldn't pause: service2 didn't pause cleanly.") + + @patch('charmhelpers.contrib.openstack.utils.service_pause') + @patch('charmhelpers.contrib.openstack.utils.set_unit_paused') + def test_pause_unit_service_charm_func( + self, set_unit_paused, service_pause): + services = ['service1', 'service2'] + service_pause.return_value = True + charm_func = MagicMock() + charm_func.return_value = None + openstack.pause_unit(None, services=services, charm_func=charm_func) + charm_func.assert_called_once_with() + # fail the charm_func + charm_func.return_value = "Custom charm failed" + try: + openstack.pause_unit( + None, services=services, charm_func=charm_func) + raise Exception("pause_unit should have raised Exception") + except Exception as e: + self.assertEquals(e.args[0], + "Couldn't pause: Custom charm failed") + + @patch('charmhelpers.contrib.openstack.utils.service_pause') + @patch('charmhelpers.contrib.openstack.utils.set_unit_paused') + def test_pause_unit_assess_status_func( + self, set_unit_paused, service_pause): + services = ['service1', 'service2'] + service_pause.return_value = True + assess_status_func = MagicMock() + assess_status_func.return_value = None + openstack.pause_unit(assess_status_func, services=services) + assess_status_func.assert_called_once_with() + # fail the assess_status_func + assess_status_func.return_value = "assess_status_func failed" + try: + openstack.pause_unit(assess_status_func, services=services) + raise Exception("pause_unit should have raised Exception") + except Exception as e: + self.assertEquals(e.args[0], + "Couldn't pause: assess_status_func failed") + + @patch('charmhelpers.contrib.openstack.utils.service_pause') + @patch('charmhelpers.contrib.openstack.utils.set_unit_paused') + @patch('charmhelpers.contrib.openstack.utils.port_has_listener') + def test_pause_unit_retry_port_check_retries( + self, port_has_listener, set_unit_paused, service_pause): + service_pause.return_value = True + port_has_listener.side_effect = [True, False] + wait_for_ports_func = openstack.make_wait_for_ports_barrier([77]) + openstack.pause_unit(None, services=['service1'], ports=[77], charm_func=wait_for_ports_func) + port_has_listener.assert_has_calls([call('0.0.0.0', 77), call('0.0.0.0', 77)]) + + @patch('charmhelpers.contrib.openstack.utils.service_resume') + @patch('charmhelpers.contrib.openstack.utils.clear_unit_paused') + def test_resume_unit_okay(self, clear_unit_paused, service_resume): + services = ['service1', 'service2'] + service_resume.side_effect = [True, True] + openstack.resume_unit(None, services=services) + clear_unit_paused.assert_called_once_with() + self.assertEquals(service_resume.call_count, 2) + + @patch('charmhelpers.contrib.openstack.utils.service_resume') + @patch('charmhelpers.contrib.openstack.utils.clear_unit_paused') + def test_resume_unit_service_fails( + self, clear_unit_paused, service_resume): + services = ['service1', 'service2'] + service_resume.side_effect = [True, True] + openstack.resume_unit(None, services=services) + clear_unit_paused.assert_called_once_with() + self.assertEquals(service_resume.call_count, 2) + # Fail the 2nd service + service_resume.side_effect = [True, False] + try: + openstack.resume_unit(None, services=services) + raise Exception("resume_unit should have raised Exception") + except Exception as e: + self.assertEquals( + e.args[0], "Couldn't resume: service2 didn't resume cleanly.") + + @patch('charmhelpers.contrib.openstack.utils.service_resume') + @patch('charmhelpers.contrib.openstack.utils.clear_unit_paused') + def test_resume_unit_service_charm_func( + self, clear_unit_paused, service_resume): + services = ['service1', 'service2'] + service_resume.return_value = True + charm_func = MagicMock() + charm_func.return_value = None + openstack.resume_unit(None, services=services, charm_func=charm_func) + charm_func.assert_called_once_with() + # fail the charm_func + charm_func.return_value = "Custom charm failed" + try: + openstack.resume_unit( + None, services=services, charm_func=charm_func) + raise Exception("resume_unit should have raised Exception") + except Exception as e: + self.assertEquals(e.args[0], + "Couldn't resume: Custom charm failed") + + @patch('charmhelpers.contrib.openstack.utils.service_resume') + @patch('charmhelpers.contrib.openstack.utils.clear_unit_paused') + def test_resume_unit_assess_status_func( + self, clear_unit_paused, service_resume): + services = ['service1', 'service2'] + service_resume.return_value = True + assess_status_func = MagicMock() + assess_status_func.return_value = None + openstack.resume_unit(assess_status_func, services=services) + assess_status_func.assert_called_once_with() + # fail the assess_status_func + assess_status_func.return_value = "assess_status_func failed" + try: + openstack.resume_unit(assess_status_func, services=services) + raise Exception("resume_unit should have raised Exception") + except Exception as e: + self.assertEquals(e.args[0], + "Couldn't resume: assess_status_func failed") + + @patch('charmhelpers.contrib.openstack.utils.status_set') + @patch('charmhelpers.contrib.openstack.utils.' + '_determine_os_workload_status') + def test_make_assess_status_func(self, _determine_os_workload_status, + status_set): + _determine_os_workload_status.return_value = ('active', 'fine') + f = openstack.make_assess_status_func('one', 'two', three='three') + r = f() + self.assertEquals(r, None) + _determine_os_workload_status.assert_called_once_with( + 'one', 'two', three='three') + status_set.assert_called_once_with('active', 'fine') + # return something other than 'active' or 'maintenance' + _determine_os_workload_status.return_value = ('broken', 'damaged') + r = f() + self.assertEquals(r, 'damaged') + + # TODO(ajkavanagh) -- there should be a test for + # _determine_os_workload_status() as the policyd override code has changed + # it, but there wasn't a test previously. + + @patch.object(openstack, 'restart_on_change_helper') + @patch.object(openstack, 'is_unit_paused_set') + def test_pausable_restart_on_change( + self, is_unit_paused_set, restart_on_change_helper): + @openstack.pausable_restart_on_change({}) + def test_func(): + pass + + # test with pause: restart_on_change_helper should not be called. + is_unit_paused_set.return_value = True + test_func() + self.assertEquals(restart_on_change_helper.call_count, 0) + + # test without pause: restart_on_change_helper should be called. + is_unit_paused_set.return_value = False + test_func() + self.assertEquals(restart_on_change_helper.call_count, 1) + + @patch.object(openstack, 'restart_on_change_helper') + @patch.object(openstack, 'is_unit_paused_set') + def test_pausable_restart_on_change_with_callable( + self, is_unit_paused_set, restart_on_change_helper): + mock_test = MagicMock() + mock_test.called_set = False + + def _restart_map(): + mock_test.called_set = True + return {"a": "b"} + + @openstack.pausable_restart_on_change(_restart_map) + def test_func(): + pass + + self.assertFalse(mock_test.called_set) + is_unit_paused_set.return_value = False + test_func() + self.assertEquals(restart_on_change_helper.call_count, 1) + self.assertTrue(mock_test.called_set) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'action_set') + @patch.object(openstack, 'action_fail') + @patch.object(openstack, 'openstack_upgrade_available') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_openstack_upgrade(self, config, openstack_upgrade_available, + action_fail, action_set, log): + def do_openstack_upgrade(configs): + pass + + openstack_upgrade_available.return_value = True + + # action-managed-upgrade=True + config.side_effect = [True] + + openstack.do_action_openstack_upgrade('package-xyz', + do_openstack_upgrade, + None) + + self.assertTrue(openstack_upgrade_available.called) + msg = ('success, upgrade completed.') + action_set.assert_called_with({'outcome': msg}) + self.assertFalse(action_fail.called) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'action_set') + @patch.object(openstack, 'action_fail') + @patch.object(openstack, 'openstack_upgrade_available') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_openstack_upgrade_not_avail(self, config, + openstack_upgrade_available, + action_fail, action_set, log): + def do_openstack_upgrade(configs): + pass + + openstack_upgrade_available.return_value = False + + openstack.do_action_openstack_upgrade('package-xyz', + do_openstack_upgrade, + None) + + self.assertTrue(openstack_upgrade_available.called) + msg = ('no upgrade available.') + action_set.assert_called_with({'outcome': msg}) + self.assertFalse(action_fail.called) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'action_set') + @patch.object(openstack, 'action_fail') + @patch.object(openstack, 'openstack_upgrade_available') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_openstack_upgrade_config_false(self, config, + openstack_upgrade_available, + action_fail, action_set, log): + def do_openstack_upgrade(configs): + pass + + openstack_upgrade_available.return_value = True + + # action-managed-upgrade=False + config.side_effect = [False] + + openstack.do_action_openstack_upgrade('package-xyz', + do_openstack_upgrade, + None) + + self.assertTrue(openstack_upgrade_available.called) + msg = ('action-managed-upgrade config is False, skipped upgrade.') + action_set.assert_called_with({'outcome': msg}) + self.assertFalse(action_fail.called) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'action_set') + @patch.object(openstack, 'action_fail') + @patch.object(openstack, 'openstack_upgrade_available') + @patch('traceback.format_exc') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_openstack_upgrade_traceback(self, config, traceback, + openstack_upgrade_available, + action_fail, action_set, log): + def do_openstack_upgrade(configs): + oops() # noqa + + openstack_upgrade_available.return_value = True + + # action-managed-upgrade=False + config.side_effect = [True] + + openstack.do_action_openstack_upgrade('package-xyz', + do_openstack_upgrade, + None) + + self.assertTrue(openstack_upgrade_available.called) + msg = 'do_openstack_upgrade resulted in an unexpected error' + action_fail.assert_called_with(msg) + self.assertTrue(action_set.called) + self.assertTrue(traceback.called) + + @patch.object(openstack, 'os_release') + @patch.object(openstack, 'application_version_set') + def test_os_application_version_set(self, + mock_application_version_set, + mock_os_release): + with patch.object(fetch, 'apt_cache') as cache: + cache.return_value = self._apt_cache() + mock_os_release.return_value = 'mitaka' + openstack.os_application_version_set('neutron-common') + mock_application_version_set.assert_called_with('7.0.1') + openstack.os_application_version_set('cinder-common') + mock_application_version_set.assert_called_with('mitaka') + + @patch.object(openstack, 'valid_snap_channel') + @patch('charmhelpers.contrib.openstack.utils.config') + def test_snap_install_requested(self, config, valid_snap_channel): + valid_snap_channel.return_value = True + # Expect True + flush('snap_install_requested') + config.return_value = 'snap:ocata/edge' + self.assertTrue(openstack.snap_install_requested()) + valid_snap_channel.assert_called_with('edge') + flush('snap_install_requested') + config.return_value = 'snap:pike' + self.assertTrue(openstack.snap_install_requested()) + valid_snap_channel.assert_called_with('stable') + flush('snap_install_requested') + config.return_value = 'snap:pike/stable/jamespage' + self.assertTrue(openstack.snap_install_requested()) + valid_snap_channel.assert_called_with('stable') + # Expect False + flush('snap_install_requested') + config.return_value = 'cloud:xenial-ocata' + self.assertFalse(openstack.snap_install_requested()) + + def test_get_snaps_install_info_from_origin(self): + snaps = ['os_project'] + mode = 'jailmode' + + # snap:track/channel + src = 'snap:ocata/beta' + expected = {snaps[0]: {'mode': mode, + 'channel': '--channel=ocata/beta'}} + self.assertEqual( + expected, + openstack.get_snaps_install_info_from_origin(snaps, src, + mode=mode)) + + # snap:track/channel/branch + src = 'snap:ocata/beta/jamespage' + expected = {snaps[0]: {'mode': mode, + 'channel': '--channel=ocata/beta/jamespage'}} + self.assertEqual( + expected, + openstack.get_snaps_install_info_from_origin(snaps, src, + mode=mode)) + # snap:track + src = 'snap:pike' + expected = {snaps[0]: {'mode': mode, + 'channel': '--channel=pike'}} + self.assertEqual( + expected, + openstack.get_snaps_install_info_from_origin(snaps, src, + mode=mode)) + + @patch.object(openstack, 'snap_install') + def test_install_os_snaps(self, mock_snap_install): + snaps = ['os_project'] + mode = 'jailmode' + + # snap:track/channel + src = 'snap:ocata/beta' + openstack.install_os_snaps( + openstack.get_snaps_install_info_from_origin( + snaps, src, mode=mode)) + mock_snap_install.assert_called_with( + 'os_project', '--channel=ocata/beta', '--jailmode') + + # snap:track + src = 'snap:pike' + openstack.install_os_snaps( + openstack.get_snaps_install_info_from_origin( + snaps, src, mode=mode)) + mock_snap_install.assert_called_with( + 'os_project', '--channel=pike', '--jailmode') + + @patch.object(openstack, 'set_unit_upgrading') + @patch.object(openstack, 'is_unit_paused_set') + def test_series_upgrade_prepare( + self, is_unit_paused_set, set_unit_upgrading): + is_unit_paused_set.return_value = False + fake_pause_helper = MagicMock() + fake_configs = MagicMock() + openstack.series_upgrade_prepare(fake_pause_helper, fake_configs) + set_unit_upgrading.assert_called_once() + fake_pause_helper.assert_called_once_with(fake_configs) + + @patch.object(openstack, 'set_unit_upgrading') + @patch.object(openstack, 'is_unit_paused_set') + def test_series_upgrade_prepare_no_pause( + self, is_unit_paused_set, set_unit_upgrading): + is_unit_paused_set.return_value = True + fake_pause_helper = MagicMock() + fake_configs = MagicMock() + openstack.series_upgrade_prepare(fake_pause_helper, fake_configs) + set_unit_upgrading.assert_called_once() + fake_pause_helper.assert_not_called() + + @patch.object(openstack, 'clear_unit_upgrading') + @patch.object(openstack, 'clear_unit_paused') + def test_series_upgrade_complete( + self, clear_unit_paused, clear_unit_upgrading): + fake_resume_helper = MagicMock() + fake_configs = MagicMock() + openstack.series_upgrade_complete(fake_resume_helper, fake_configs) + clear_unit_upgrading.assert_called_once() + clear_unit_paused.assert_called_once() + fake_configs.write_all.assert_called_once() + fake_resume_helper.assert_called_once_with(fake_configs) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'leader_get') + def test_is_db_initialised(self, leader_get, juju_log): + leader_get.return_value = 'True' + self.assertTrue(openstack.is_db_initialised()) + leader_get.return_value = 'False' + self.assertFalse(openstack.is_db_initialised()) + leader_get.return_value = None + self.assertFalse(openstack.is_db_initialised()) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'leader_set') + def test_set_db_initialised(self, leader_set, juju_log): + openstack.set_db_initialised() + leader_set.assert_called_once_with({'db-initialised': True}) + + @patch.object(openstack, 'juju_log') + @patch.object(openstack, 'relation_ids') + @patch.object(openstack, 'related_units') + @patch.object(openstack, 'relation_get') + def test_is_db_maintenance_mode(self, relation_get, related_units, + relation_ids, juju_log): + relation_ids.return_value = ['rid:1'] + related_units.return_value = ['unit/0', 'unit/2'] + rsettings = { + 'rid:1': { + 'unit/0': { + 'private-ip': '1.2.3.4', + 'cluster-series-upgrading': 'True'}, + 'unit/2': { + 'private-ip': '1.2.3.5'}}} + relation_get.side_effect = lambda unit, rid: rsettings[rid][unit] + self.assertTrue(openstack.is_db_maintenance_mode()) + rsettings = { + 'rid:1': { + 'unit/0': { + 'private-ip': '1.2.3.4'}, + 'unit/2': { + 'private-ip': '1.2.3.5'}}} + self.assertFalse(openstack.is_db_maintenance_mode()) + rsettings = { + 'rid:1': { + 'unit/0': { + 'private-ip': '1.2.3.4', + 'cluster-series-upgrading': 'False'}, + 'unit/2': { + 'private-ip': '1.2.3.5'}}} + self.assertFalse(openstack.is_db_maintenance_mode()) + rsettings = { + 'rid:1': { + 'unit/0': { + 'private-ip': '1.2.3.4', + 'cluster-series-upgrading': 'lskjfsd'}, + 'unit/2': { + 'private-ip': '1.2.3.5'}}} + self.assertFalse(openstack.is_db_maintenance_mode()) + + def test_get_endpoint_key(self): + self.assertEqual( + openstack.get_endpoint_key('placement', 'is:2', 'keystone/0'), + 'placement-is_2-keystone_0') + + @patch.object(openstack, 'relation_get') + @patch.object(openstack, 'related_units') + @patch.object(openstack, 'relation_ids') + def test_get_endpoint_notifications(self, relation_ids, related_units, + relation_get): + id_svc_rel_units = { + 'identity-service:3': ['keystone/0', 'keystone/1', 'keystone/2'] + } + + def _related_units(relid): + return id_svc_rel_units[relid] + + id_svc_rel_data = { + 'keystone/0': { + 'ep_changed': '{"placement": "d5c3"}'}, + 'keystone/1': { + 'ep_changed': '{"nova": "4d06", "neutron": "2aa6"}'}, + 'keystone/2': {}} + + def _relation_get(unit, rid, attribute): + return id_svc_rel_data[unit].get(attribute) + + relation_ids.return_value = id_svc_rel_units.keys() + related_units.side_effect = _related_units + relation_get.side_effect = _relation_get + self.assertEqual( + openstack.get_endpoint_notifications(['neutron']), + { + 'neutron-identity-service_3-keystone_1': '2aa6'}) + self.assertEqual( + openstack.get_endpoint_notifications(['placement', 'neutron']), + { + 'neutron-identity-service_3-keystone_1': '2aa6', + 'placement-identity-service_3-keystone_0': 'd5c3'}) + + @patch.object(openstack, 'get_endpoint_notifications') + @patch.object(openstack.unitdata, 'HookData') + def test_endpoint_changed(self, HookData, get_endpoint_notifications): + self.kv_data = {} + + def _kv_get(key): + return self.kv_data.get(key) + kv = self._unit_paused_helper(HookData) + kv.get.side_effect = _kv_get + # Check endpoint_changed returns True when there are new notifications. + get_endpoint_notifications.return_value = { + 'neutron-identity-service_3-keystone_1': '2aa6', + 'placement-identity-service_3-keystone_0': 'd5c3'} + self.assertTrue(openstack.endpoint_changed('placement')) + # Check endpoint_changed returns False when there are new + # notifications but they are not the ones being looked for. + self.assertTrue(openstack.endpoint_changed('nova')) + # Check endpoint_changed returns False if the notification + # has already been seen + get_endpoint_notifications.return_value = { + 'placement-identity-service_3-keystone_0': 'd5c3'} + self.kv_data = { + 'placement-identity-service_3-keystone_0': 'd5c3'} + self.assertFalse(openstack.endpoint_changed('placement')) + + @patch.object(openstack, 'get_endpoint_notifications') + @patch.object(openstack.unitdata, 'HookData') + def test_save_endpoint_changed_triggers(self, HookData, + get_endpoint_notifications): + kv = self._unit_paused_helper(HookData) + get_endpoint_notifications.return_value = { + 'neutron-identity-service_3-keystone_1': '2aa6', + 'placement-identity-service_3-keystone_0': 'd5c3'} + openstack.save_endpoint_changed_triggers(['neutron', 'placement']) + kv_set_calls = [ + call('neutron-identity-service_3-keystone_1', '2aa6'), + call('placement-identity-service_3-keystone_0', 'd5c3')] + kv.set.assert_has_calls(kv_set_calls, any_order=True) + + +class OpenStackUtilsAdditionalTests(TestCase): + SHARED_DB_RELATIONS = { + 'shared-db:8': { + 'mysql-svc1/0': { + 'allowed_units': 'client/0', + }, + 'mysql-svc1/1': {}, + 'mysql-svc1/2': { + 'allowed_units': 'client/0 client/1', + }, + }, + 'shared-db:12': { + 'mysql-svc2/0': { + 'allowed_units': 'client/1', + }, + 'mysql-svc2/1': { + 'allowed_units': 'client/3', + }, + 'mysql-svc2/2': { + 'allowed_units': {}, + }, + } + } + SCALE_RELATIONS = { + 'cluster:2': { + 'keystone/1': {}, + 'keystone/2': {}}, + 'shared-db:12': { + 'mysql-svc2/0': { + 'allowed_units': 'client/1', + }, + 'mysql-svc2/1': { + 'allowed_units': 'client/3', + }, + 'mysql-svc2/2': { + 'allowed_units': {}, + }}, + } + SCALE_RELATIONS_HA = { + 'cluster:2': { + 'keystone/1': {'unit-state-keystone-1': 'READY'}, + 'keystone/2': {}}, + 'shared-db:12': { + 'mysql-svc2/0': { + 'allowed_units': 'client/1', + }, + 'mysql-svc2/1': { + 'allowed_units': 'client/3', + }, + 'mysql-svc2/2': { + 'allowed_units': {}, + }}, + 'ha:32': { + 'hacluster-keystone/1': {}} + } + All_PEERS_READY = { + 'cluster:2': { + 'keystone/1': {'unit-state-keystone-1': 'READY'}, + 'keystone/2': {'unit-state-keystone-2': 'READY'}}} + PEERS_NOT_READY = { + 'cluster:2': { + 'keystone/1': {'unit-state-keystone-1': 'READY'}, + 'keystone/2': {}}} + + def setUp(self): + super(OpenStackUtilsAdditionalTests, self).setUp() + [self._patch(m) for m in [ + 'expect_ha', + 'expected_peer_units', + 'expected_related_units', + 'juju_log', + 'metadata', + 'related_units', + 'relation_get', + 'relation_id', + 'relation_ids', + 'relation_set', + 'local_unit', + ]] + + def _patch(self, method): + _m = patch.object(openstack, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def setup_relation(self, relation_map): + relation = FakeRelation(relation_map) + self.relation_id.side_effect = relation.relation_id + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.related_units + return relation + + def test_is_db_ready(self): + relation = self.setup_relation(self.SHARED_DB_RELATIONS) + + # Check unit allowed in 1st relation + self.local_unit.return_value = 'client/0' + self.assertTrue(openstack.is_db_ready()) + + # Check unit allowed in 2nd relation + self.local_unit.return_value = 'client/3' + self.assertTrue(openstack.is_db_ready()) + + # Check unit not allowed in any list + self.local_unit.return_value = 'client/5' + self.assertFalse(openstack.is_db_ready()) + + # Check call with an invalid relation + self.local_unit.return_value = 'client/3' + # None returned if not in a relation context (eg update-status) + relation.clear_relation_context() + self.assertRaises( + Exception, + openstack.is_db_ready, + use_current_context=True) + + # Check unit allowed using current relation context + relation.set_relation_context('mysql-svc2/0', 'shared-db:12') + self.local_unit.return_value = 'client/1' + self.assertTrue(openstack.is_db_ready(use_current_context=True)) + + # Check unit not allowed using current relation context + relation.set_relation_context('mysql-svc2/0', 'shared-db:12') + self.local_unit.return_value = 'client/0' + self.assertFalse(openstack.is_db_ready(use_current_context=True)) + + @patch.object(openstack, 'container_scoped_relations') + def test_is_expected_scale_noha(self, container_scoped_relations): + self.setup_relation(self.SCALE_RELATIONS) + self.expect_ha.return_value = False + eru = { + 'shared-db': ['mysql/0', 'mysql/1', 'mysql/2']} + + def _expected_related_units(reltype): + return eru[reltype] + self.expected_related_units.side_effect = _expected_related_units + container_scoped_relations.return_value = ['ha', 'domain-backend'] + + # All peer and db units are present + self.expected_peer_units.return_value = ['keystone/0', 'keystone/2'] + self.assertTrue(openstack.is_expected_scale()) + + # db units are present but a peer is missing + self.expected_peer_units.return_value = ['keystone/0', 'keystone/2', 'keystone/3'] + self.assertFalse(openstack.is_expected_scale()) + + # peer units are present but a db unit is missing + eru['shared-db'].append('mysql/3') + self.expected_peer_units.return_value = ['keystone/0', 'keystone/2'] + self.assertFalse(openstack.is_expected_scale()) + eru['shared-db'].remove('mysql/3') + + # Expect ha but ha unit is missing + self.expect_ha.return_value = True + self.expected_peer_units.return_value = ['keystone/0', 'keystone/2'] + self.assertFalse(openstack.is_expected_scale()) + + @patch.object(openstack, 'container_scoped_relations') + def test_is_expected_scale_ha(self, container_scoped_relations): + self.setup_relation(self.SCALE_RELATIONS_HA) + eru = { + 'shared-db': ['mysql/0', 'mysql/1', 'mysql/2']} + + def _expected_related_units(reltype): + return eru[reltype] + self.expected_related_units.side_effect = _expected_related_units + container_scoped_relations.return_value = ['ha', 'domain-backend'] + self.expect_ha.return_value = True + self.expected_peer_units.return_value = ['keystone/0', 'keystone/2'] + self.assertTrue(openstack.is_expected_scale()) + + def test_container_scoped_relations(self): + _metadata = { + 'provides': { + 'amqp': {'interface': 'rabbitmq'}, + 'identity-service': {'interface': 'keystone'}, + 'ha': { + 'interface': 'hacluster', + 'scope': 'container'}}, + 'peers': { + 'cluster': {'interface': 'openstack-ha'}}} + self.metadata.return_value = _metadata + self.assertEqual(openstack.container_scoped_relations(), ['ha']) + + def test_get_peer_key(self): + self.assertEqual( + openstack.get_peer_key('cinder/0'), + 'unit-state-cinder-0') + + def test_inform_peers_unit_state(self): + self.local_unit.return_value = 'client/0' + self.setup_relation(self.All_PEERS_READY) + openstack.inform_peers_unit_state('READY') + self.relation_set.assert_called_once_with( + relation_id='cluster:2', + relation_settings={'unit-state-client-0': 'READY'}) + + def test_get_peers_unit_state(self): + self.setup_relation(self.All_PEERS_READY) + self.assertEqual( + openstack.get_peers_unit_state(), + {'keystone/1': 'READY', 'keystone/2': 'READY'}) + self.setup_relation(self.PEERS_NOT_READY) + self.assertEqual( + openstack.get_peers_unit_state(), + {'keystone/1': 'READY', 'keystone/2': 'UNKNOWN'}) + + def test_are_peers_ready(self): + self.setup_relation(self.All_PEERS_READY) + self.assertTrue(openstack.are_peers_ready()) + self.setup_relation(self.PEERS_NOT_READY) + self.assertFalse(openstack.are_peers_ready()) + + @patch.object(openstack, 'inform_peers_unit_state') + def test_inform_peers_if_ready(self, inform_peers_unit_state): + self.setup_relation(self.All_PEERS_READY) + + def _not_ready(): + return False, "Its all gone wrong" + + def _ready(): + return True, "Hurray!" + openstack.inform_peers_if_ready(_not_ready) + inform_peers_unit_state.assert_called_once_with('NOTREADY', 'cluster') + inform_peers_unit_state.reset_mock() + openstack.inform_peers_if_ready(_ready) + inform_peers_unit_state.assert_called_once_with('READY', 'cluster') + + @patch.object(openstack, 'is_expected_scale') + @patch.object(openstack, 'is_db_initialised') + @patch.object(openstack, 'is_db_ready') + @patch.object(openstack, 'is_unit_paused_set') + @patch.object(openstack, 'is_db_maintenance_mode') + def test_check_api_unit_ready(self, is_db_maintenance_mode, + is_unit_paused_set, is_db_ready, + is_db_initialised, is_expected_scale): + is_db_maintenance_mode.return_value = True + self.assertFalse(openstack.check_api_unit_ready()[0]) + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = True + self.assertFalse(openstack.check_api_unit_ready()[0]) + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = False + self.assertFalse(openstack.check_api_unit_ready()[0]) + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = False + self.assertFalse(openstack.check_api_unit_ready()[0]) + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = True + is_expected_scale.return_value = False + self.assertFalse(openstack.check_api_unit_ready()[0]) + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = True + is_expected_scale.return_value = True + self.assertTrue(openstack.check_api_unit_ready()[0]) + + @patch.object(openstack, 'is_expected_scale') + @patch.object(openstack, 'is_db_initialised') + @patch.object(openstack, 'is_db_ready') + @patch.object(openstack, 'is_unit_paused_set') + @patch.object(openstack, 'is_db_maintenance_mode') + def test_get_api_unit_status(self, is_db_maintenance_mode, + is_unit_paused_set, is_db_ready, + is_db_initialised, is_expected_scale): + is_db_maintenance_mode.return_value = True + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'maintenance') + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = True + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'blocked') + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = False + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'waiting') + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = False + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'waiting') + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = True + is_expected_scale.return_value = False + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'waiting') + + is_db_maintenance_mode.return_value = False + is_unit_paused_set.return_value = False + is_db_ready.return_value = True + is_db_initialised.return_value = True + is_expected_scale.return_value = True + self.assertEqual( + openstack.get_api_unit_status()[0].value, + 'active') + + @patch.object(openstack, 'get_api_unit_status') + def test_check_api_application_ready(self, get_api_unit_status): + get_api_unit_status.return_value = (WORKLOAD_STATES.ACTIVE, 'Hurray') + self.assertTrue(openstack.check_api_application_ready()[0]) + get_api_unit_status.return_value = (WORKLOAD_STATES.BLOCKED, ':-(') + self.assertFalse(openstack.check_api_application_ready()[0]) + + @patch.object(openstack, 'get_api_unit_status') + def test_get_api_application_status(self, get_api_unit_status): + get_api_unit_status.return_value = (WORKLOAD_STATES.ACTIVE, 'Hurray') + self.assertEqual( + openstack.get_api_application_status()[0].value, + 'active') + get_api_unit_status.return_value = (WORKLOAD_STATES.BLOCKED, ':-(') + self.assertEqual( + openstack.get_api_application_status()[0].value, + 'blocked') + + @patch.object(openstack.deferred_events, 'clear_deferred_restarts') + @patch.object(openstack, 'manage_payload_services') + @patch.object(openstack.deferred_events, 'get_deferred_restarts') + def test_restart_services_action(self, get_deferred_restarts, + manage_payload_services, + clear_deferred_restarts): + deferred_restarts = [ + deferred_events.ServiceEvent( + timestamp=123, + service='svcA', + reason='ReasonA', + action='restart')] + get_deferred_restarts.return_value = deferred_restarts + manage_payload_services.return_value = (None, None) + openstack.restart_services_action(deferred_only=True) + manage_payload_services.assert_has_calls([ + call('stop', services=['svcA'], charm_func=None), + call('start', services=['svcA'])]) + clear_deferred_restarts.assert_called_once_with(['svcA']) + self.assertRaises( + ValueError, + openstack.restart_services_action, + ['svcA'], + deferred_only=True) + + manage_payload_services.return_value = (None, 'something went wrong') + self.assertRaises( + ServiceActionError, + openstack.restart_services_action, + ['svcA']) + + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_contexts.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_contexts.py new file mode 100644 index 0000000..2d19ff1 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_contexts.py @@ -0,0 +1,5142 @@ +import collections +import copy +import json +import mock +import six +import unittest +import yaml + +from mock import ( + patch, + Mock, + MagicMock, + call +) + +from tests.helpers import patch_open + +import tests.utils + +import charmhelpers.contrib.openstack.context as context + + +if not six.PY3: + open_builtin = '__builtin__.open' +else: + open_builtin = 'builtins.open' + + +class FakeRelation(object): + + ''' + A fake relation class. Lets tests specify simple relation data + for a default relation + unit (foo:0, foo/0, set in setUp()), eg: + + rel = { + 'private-address': 'foo', + 'password': 'passwd', + } + relation = FakeRelation(rel) + self.relation_get.side_effect = relation.get + passwd = self.relation_get('password') + + or more complex relations meant to be addressed by explicit relation id + + unit id combos: + + rel = { + 'mysql:0': { + 'mysql/0': { + 'private-address': 'foo', + 'password': 'passwd', + } + } + } + relation = FakeRelation(rel) + self.relation_get.side_affect = relation.get + passwd = self.relation_get('password', rid='mysql:0', unit='mysql/0') + ''' + + def __init__(self, relation_data): + self.relation_data = relation_data + + def get(self, attribute=None, unit=None, rid=None): + if not rid or rid == 'foo:0': + if attribute is None: + return self.relation_data + elif attribute in self.relation_data: + return self.relation_data[attribute] + return None + else: + if rid not in self.relation_data: + return None + try: + relation = self.relation_data[rid][unit] + except KeyError: + return None + if attribute is None: + return relation + if attribute in relation: + return relation[attribute] + return None + + def relation_ids(self, relation): + rids = [] + for rid in sorted(self.relation_data.keys()): + if relation + ':' in rid: + rids.append(rid) + return rids + + def relation_units(self, relation_id): + if relation_id not in self.relation_data: + return None + return sorted(self.relation_data[relation_id].keys()) + + +SHARED_DB_RELATION = { + 'db_host': 'dbserver.local', + 'password': 'foo' +} + +SHARED_DB_RELATION_W_PORT = { + 'db_host': 'dbserver.local', + 'password': 'foo', + 'db_port': 3306, +} + +SHARED_DB_RELATION_ALT_RID = { + 'mysql-alt:0': { + 'mysql-alt/0': { + 'db_host': 'dbserver-alt.local', + 'password': 'flump'}}} + +SHARED_DB_RELATION_SSL = { + 'db_host': 'dbserver.local', + 'password': 'foo', + 'ssl_ca': 'Zm9vCg==', + 'ssl_cert': 'YmFyCg==', + 'ssl_key': 'Zm9vYmFyCg==', +} + +SHARED_DB_CONFIG = { + 'database-user': 'adam', + 'database': 'foodb', +} + +SHARED_DB_RELATION_NAMESPACED = { + 'db_host': 'bar', + 'quantum_password': 'bar2' +} + +SHARED_DB_RELATION_ACCESS_NETWORK = { + 'db_host': 'dbserver.local', + 'password': 'foo', + 'access-network': '10.5.5.0/24', + 'hostname': 'bar', +} + + +IDENTITY_SERVICE_RELATION_HTTP = { + 'service_port': '5000', + 'service_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'service_domain': 'admin_domain', + 'service_tenant': 'admin', + 'service_tenant_id': '123456', + 'service_password': 'foo', + 'service_username': 'adam', + 'service_protocol': 'http', + 'auth_protocol': 'http', + 'internal_protocol': 'http', +} + +IDENTITY_SERVICE_RELATION_UNSET = { + 'service_port': '5000', + 'service_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'service_domain': 'admin_domain', + 'service_tenant': 'admin', + 'service_password': 'foo', + 'service_username': 'adam', +} + +IDENTITY_CREDENTIALS_RELATION_UNSET = { + 'credentials_port': '5000', + 'credentials_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'domain': 'admin_domain', + 'credentials_project': 'admin', + 'credentials_project_id': '123456', + 'credentials_password': 'foo', + 'credentials_username': 'adam', + 'credentials_protocol': 'https', +} + + +APIIDENTITY_SERVICE_RELATION_UNSET = { + 'neutron-plugin-api:0': { + 'neutron-api/0': { + 'service_port': '5000', + 'service_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'internal_port': '5000', + 'internal_host': 'keystone-internal.local', + 'service_domain': 'admin_domain', + 'service_tenant': 'admin', + 'service_password': 'foo', + 'service_username': 'adam', + } + } +} + +IDENTITY_SERVICE_RELATION_HTTPS = { + 'service_port': '5000', + 'service_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'service_domain': 'admin_domain', + 'service_tenant': 'admin', + 'service_password': 'foo', + 'service_username': 'adam', + 'service_protocol': 'https', + 'auth_protocol': 'https', + 'internal_protocol': 'https', +} + +IDENTITY_SERVICE_RELATION_VERSIONED = { + 'api_version': '3', + 'service_tenant_id': 'svc-proj-id', + 'service_domain_id': 'svc-dom-id', +} +IDENTITY_SERVICE_RELATION_VERSIONED.update(IDENTITY_SERVICE_RELATION_HTTPS) + +IDENTITY_CREDENTIALS_RELATION_VERSIONED = { + 'api_version': '3', + 'service_tenant_id': 'svc-proj-id', + 'service_domain_id': 'svc-dom-id', +} +IDENTITY_CREDENTIALS_RELATION_VERSIONED.update(IDENTITY_CREDENTIALS_RELATION_UNSET) + +POSTGRESQL_DB_RELATION = { + 'host': 'dbserver.local', + 'user': 'adam', + 'password': 'foo', +} + +POSTGRESQL_DB_CONFIG = { + 'database': 'foodb', +} + +IDENTITY_SERVICE_RELATION = { + 'service_port': '5000', + 'service_host': 'keystonehost.local', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'service_domain': 'admin_domain', + 'service_tenant': 'admin', + 'service_password': 'foo', + 'service_username': 'adam', +} + +AMQP_RELATION = { + 'private-address': 'rabbithost', + 'password': 'foobar', + 'vip': '10.0.0.1', +} + +AMQP_RELATION_ALT_RID = { + 'amqp-alt:0': { + 'rabbitmq-alt/0': { + 'private-address': 'rabbitalthost1', + 'password': 'flump', + }, + } +} + +AMQP_RELATION_WITH_SSL = { + 'private-address': 'rabbithost', + 'password': 'foobar', + 'vip': '10.0.0.1', + 'ssl_port': 5671, + 'ssl_ca': 'cert', + 'ha_queues': 'queues', +} + +AMQP_AA_RELATION = { + 'amqp:0': { + 'rabbitmq/0': { + 'private-address': 'rabbithost1', + 'password': 'foobar', + }, + 'rabbitmq/1': { + 'private-address': 'rabbithost2', + 'password': 'foobar', + }, + 'rabbitmq/2': { # Should be ignored because password is missing. + 'private-address': 'rabbithost3', + } + } +} + +AMQP_CONFIG = { + 'rabbit-user': 'adam', + 'rabbit-vhost': 'foo', +} + +AMQP_OSLO_CONFIG = { + 'oslo-messaging-flags': ("rabbit_max_retries=1" + ",rabbit_retry_backoff=1" + ",rabbit_retry_interval=1"), + 'oslo-messaging-driver': 'log' +} + +AMQP_NOTIFICATION_FORMAT = { + 'notification-format': 'both' +} + +AMQP_NOTIFICATION_TOPICS = { + 'notification-topics': 'foo,bar' +} + +AMQP_NOTIFICATIONS_LOGS = { + 'send-notifications-to-logs': True +} + +AMQP_NOVA_CONFIG = { + 'nova-rabbit-user': 'adam', + 'nova-rabbit-vhost': 'foo', +} + +HAPROXY_CONFIG = { + 'haproxy-server-timeout': 50000, + 'haproxy-client-timeout': 50000, +} + +CEPH_RELATION = { + 'ceph:0': { + 'ceph/0': { + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true' + }, + 'ceph/1': { + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'false' + }, + } +} + +CEPH_RELATION_WITH_PUBLIC_ADDR = { + 'ceph:0': { + 'ceph/0': { + 'ceph-public-address': '192.168.1.10', + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar' + }, + 'ceph/1': { + 'ceph-public-address': '192.168.1.11', + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar' + }, + } +} + +CEPH_REL_WITH_PUBLIC_ADDR_PORT = { + 'ceph:0': { + 'ceph/0': { + 'ceph-public-address': '192.168.1.10:1234', + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar' + }, + 'ceph/1': { + 'ceph-public-address': '192.168.1.11:4321', + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar' + }, + } +} + +CEPH_REL_WITH_PUBLIC_IPv6_ADDR = { + 'ceph:0': { + 'ceph/0': { + 'ceph-public-address': '2001:5c0:9168::1', + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar' + }, + 'ceph/1': { + 'ceph-public-address': '2001:5c0:9168::2', + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar' + }, + } +} + +CEPH_REL_WITH_PUBLIC_IPv6_ADDR_PORT = { + 'ceph:0': { + 'ceph/0': { + 'ceph-public-address': '[2001:5c0:9168::1]:1234', + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar' + }, + 'ceph/1': { + 'ceph-public-address': '[2001:5c0:9168::2]:4321', + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar' + }, + } +} + +CEPH_REL_WITH_MULTI_PUBLIC_ADDR = { + 'ceph:0': { + 'ceph/0': { + 'ceph-public-address': '192.168.1.10 192.168.1.20', + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar' + }, + 'ceph/1': { + 'ceph-public-address': '192.168.1.11 192.168.1.21', + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar' + }, + } +} + +CEPH_REL_WITH_DEFAULT_FEATURES = { + 'ceph:0': { + 'ceph/0': { + 'private-address': 'ceph_node1', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + 'rbd-features': '1' + }, + 'ceph/1': { + 'private-address': 'ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'false', + 'rbd-features': '1' + }, + } +} + + +IDENTITY_RELATION_NO_CERT = { + 'identity-service:0': { + 'keystone/0': { + 'private-address': 'keystone1', + }, + } +} + +IDENTITY_RELATION_SINGLE_CERT = { + 'identity-service:0': { + 'keystone/0': { + 'private-address': 'keystone1', + 'ssl_cert_cinderhost1': 'certa', + 'ssl_key_cinderhost1': 'keya', + }, + } +} + +IDENTITY_RELATION_MULTIPLE_CERT = { + 'identity-service:0': { + 'keystone/0': { + 'private-address': 'keystone1', + 'ssl_cert_cinderhost1-int-network': 'certa', + 'ssl_key_cinderhost1-int-network': 'keya', + 'ssl_cert_cinderhost1-pub-network': 'certa', + 'ssl_key_cinderhost1-pub-network': 'keya', + 'ssl_cert_cinderhost1-adm-network': 'certa', + 'ssl_key_cinderhost1-adm-network': 'keya', + }, + } +} + +QUANTUM_NETWORK_SERVICE_RELATION = { + 'quantum-network-service:0': { + 'unit/0': { + 'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion' + }, + } +} + +QUANTUM_NETWORK_SERVICE_RELATION_VERSIONED = { + 'quantum-network-service:0': { + 'unit/0': { + 'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion', + 'api_version': '3', + }, + } +} + +SUB_CONFIG = """ +nova: + /etc/nova/nova.conf: + sections: + DEFAULT: + - [nova-key1, value1] + - [nova-key2, value2] +glance: + /etc/glance/glance.conf: + sections: + DEFAULT: + - [glance-key1, value1] + - [glance-key2, value2] +""" + +NOVA_SUB_CONFIG1 = """ +nova: + /etc/nova/nova.conf: + sections: + DEFAULT: + - [nova-key1, value1] + - [nova-key2, value2] +""" + + +NOVA_SUB_CONFIG2 = """ +nova-compute: + /etc/nova/nova.conf: + sections: + DEFAULT: + - [nova-key3, value3] + - [nova-key4, value4] +""" + +NOVA_SUB_CONFIG3 = """ +nova-compute: + /etc/nova/nova.conf: + sections: + DEFAULT: + - [nova-key5, value5] + - [nova-key6, value6] +""" + +CINDER_SUB_CONFIG1 = """ +cinder: + /etc/cinder/cinder.conf: + sections: + cinder-1-section: + - [key1, value1] +""" + +CINDER_SUB_CONFIG2 = """ +cinder: + /etc/cinder/cinder.conf: + sections: + cinder-2-section: + - [key2, value2] + not-a-section: + 1234 +""" + +SUB_CONFIG_RELATION = { + 'nova-subordinate:0': { + 'nova-subordinate/0': { + 'private-address': 'nova_node1', + 'subordinate_configuration': json.dumps(yaml.safe_load(SUB_CONFIG)), + }, + }, + 'glance-subordinate:0': { + 'glance-subordinate/0': { + 'private-address': 'glance_node1', + 'subordinate_configuration': json.dumps(yaml.safe_load(SUB_CONFIG)), + }, + }, + 'foo-subordinate:0': { + 'foo-subordinate/0': { + 'private-address': 'foo_node1', + 'subordinate_configuration': 'ea8e09324jkadsfh', + }, + }, + 'cinder-subordinate:0': { + 'cinder-subordinate/0': { + 'private-address': 'cinder_node1', + 'subordinate_configuration': json.dumps( + yaml.safe_load(CINDER_SUB_CONFIG1)), + }, + }, + 'cinder-subordinate:1': { + 'cinder-subordinate/1': { + 'private-address': 'cinder_node1', + 'subordinate_configuration': json.dumps( + yaml.safe_load(CINDER_SUB_CONFIG2)), + }, + }, + 'empty:0': {}, +} + +SUB_CONFIG_RELATION2 = { + 'nova-ceilometer:6': { + 'ceilometer-agent/0': { + 'private-address': 'nova_node1', + 'subordinate_configuration': json.dumps( + yaml.safe_load(NOVA_SUB_CONFIG1)), + }, + }, + 'neutron-plugin:3': { + 'neutron-ovs-plugin/0': { + 'private-address': 'nova_node1', + 'subordinate_configuration': json.dumps( + yaml.safe_load(NOVA_SUB_CONFIG2)), + }, + }, + 'neutron-plugin:4': { + 'neutron-other-plugin/0': { + 'private-address': 'nova_node1', + 'subordinate_configuration': json.dumps( + yaml.safe_load(NOVA_SUB_CONFIG3)), + }, + } +} + +NONET_CONFIG = { + 'vip': 'cinderhost1vip', + 'os-internal-network': None, + 'os-admin-network': None, + 'os-public-network': None +} + +FULLNET_CONFIG = { + 'vip': '10.5.1.1 10.5.2.1 10.5.3.1', + 'os-internal-network': "10.5.1.0/24", + 'os-admin-network': "10.5.2.0/24", + 'os-public-network': "10.5.3.0/24" +} + +MACHINE_MACS = { + 'eth0': 'fe:c5:ce:8e:2b:00', + 'eth1': 'fe:c5:ce:8e:2b:01', + 'eth2': 'fe:c5:ce:8e:2b:02', + 'eth3': 'fe:c5:ce:8e:2b:03', +} + +MACHINE_NICS = { + 'eth0': ['192.168.0.1'], + 'eth1': ['192.168.0.2'], + 'eth2': [], + 'eth3': [], +} + +ABSENT_MACS = "aa:a5:ae:ae:ab:a4 " + +# Imported in contexts.py and needs patching in setUp() +TO_PATCH = [ + 'b64decode', + 'check_call', + 'get_cert', + 'get_ca_cert', + 'install_ca_cert', + 'log', + 'config', + 'relation_get', + 'relation_ids', + 'related_units', + 'is_relation_made', + 'relation_set', + 'local_address', + 'https', + 'determine_api_port', + 'determine_apache_port', + 'is_clustered', + 'time', + 'https', + 'get_address_in_network', + 'get_netmask_for_address', + 'local_unit', + 'get_ipv6_addr', + 'mkdir', + 'write_file', + 'get_relation_ip', + 'charm_name', + 'sysctl_create', + 'kv', + 'pwgen', + 'lsb_release', + 'network_get_primary_address', + 'resolve_address', + 'is_ipv6_disabled', +] + + +class fake_config(object): + + def __init__(self, data): + self.data = data + + def __call__(self, attr): + if attr in self.data: + return self.data[attr] + return None + + +class fake_is_relation_made(): + def __init__(self, relations): + self.relations = relations + + def rel_made(self, relation): + return self.relations[relation] + + +class TestDB(object): + '''Test KV store for unitdata testing''' + def __init__(self): + self.data = {} + self.flushed = False + + def get(self, key, default=None): + return self.data.get(key, default) + + def set(self, key, value): + self.data[key] = value + return value + + def flush(self): + self.flushed = True + + +class ContextTests(unittest.TestCase): + + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + # mock at least a single relation + unit + self.relation_ids.return_value = ['foo:0'] + self.related_units.return_value = ['foo/0'] + self.local_unit.return_value = 'localunit' + self.kv.side_effect = TestDB + self.pwgen.return_value = 'testpassword' + self.lsb_release.return_value = {'DISTRIB_RELEASE': '16.04'} + self.network_get_primary_address.side_effect = NotImplementedError() + self.resolve_address.return_value = '10.5.1.50' + self.maxDiff = None + + def _patch(self, method): + _m = patch('charmhelpers.contrib.openstack.context.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_base_class_not_implemented(self): + base = context.OSContextGenerator() + self.assertRaises(NotImplementedError, base) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_data(self, os_codename): + '''Test shared-db context with all required data''' + os_codename.return_value = 'queens' + relation = FakeRelation(relation_data=SHARED_DB_RELATION) + self.relation_get.side_effect = relation.get + self.get_address_in_network.return_value = '' + self.config.side_effect = fake_config(SHARED_DB_CONFIG) + shared_db = context.SharedDBContext() + result = shared_db() + expected = { + 'database_host': 'dbserver.local', + 'database': 'foodb', + 'database_user': 'adam', + 'database_password': 'foo', + 'database_type': 'mysql+pymysql', + } + self.assertEquals(result, expected) + + def test_shared_db_context_with_data_and_access_net_mismatch(self): + """Mismatch between hostname and hostname for access net - defers + execution""" + relation = FakeRelation( + relation_data=SHARED_DB_RELATION_ACCESS_NETWORK) + self.relation_get.side_effect = relation.get + self.get_address_in_network.return_value = '10.5.5.1' + self.config.side_effect = fake_config(SHARED_DB_CONFIG) + shared_db = context.SharedDBContext() + result = shared_db() + self.assertEquals(result, None) + self.relation_set.assert_called_with( + relation_settings={ + 'hostname': '10.5.5.1'}) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_data_and_access_net_match(self, + os_codename): + """Correctly set hostname for access net returns complete context""" + os_codename.return_value = 'queens' + relation = FakeRelation( + relation_data=SHARED_DB_RELATION_ACCESS_NETWORK) + self.relation_get.side_effect = relation.get + self.get_address_in_network.return_value = 'bar' + self.config.side_effect = fake_config(SHARED_DB_CONFIG) + shared_db = context.SharedDBContext() + result = shared_db() + expected = { + 'database_host': 'dbserver.local', + 'database': 'foodb', + 'database_user': 'adam', + 'database_password': 'foo', + 'database_type': 'mysql+pymysql', + } + self.assertEquals(result, expected) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_explicit_relation_id(self, os_codename): + '''Test shared-db context setting the relation_id''' + os_codename.return_value = 'queens' + relation = FakeRelation(relation_data=SHARED_DB_RELATION_ALT_RID) + self.related_units.return_value = ['mysql-alt/0'] + self.relation_get.side_effect = relation.get + self.get_address_in_network.return_value = '' + self.config.side_effect = fake_config(SHARED_DB_CONFIG) + shared_db = context.SharedDBContext(relation_id='mysql-alt:0') + result = shared_db() + expected = { + 'database_host': 'dbserver-alt.local', + 'database': 'foodb', + 'database_user': 'adam', + 'database_password': 'flump', + 'database_type': 'mysql+pymysql', + } + self.assertEquals(result, expected) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_port(self, os_codename): + '''Test shared-db context with all required data''' + os_codename.return_value = 'queens' + relation = FakeRelation(relation_data=SHARED_DB_RELATION_W_PORT) + self.relation_get.side_effect = relation.get + self.get_address_in_network.return_value = '' + self.config.side_effect = fake_config(SHARED_DB_CONFIG) + shared_db = context.SharedDBContext() + result = shared_db() + expected = { + 'database_host': 'dbserver.local', + 'database': 'foodb', + 'database_user': 'adam', + 'database_password': 'foo', + 'database_type': 'mysql+pymysql', + 'database_port': 3306, + } + self.assertEquals(result, expected) + + @patch('os.path.exists') + @patch(open_builtin) + def test_db_ssl(self, _open, osexists): + osexists.return_value = False + ssl_dir = '/etc/dbssl' + db_ssl_ctxt = context.db_ssl(SHARED_DB_RELATION_SSL, {}, ssl_dir) + expected = { + 'database_ssl_ca': ssl_dir + '/db-client.ca', + 'database_ssl_cert': ssl_dir + '/db-client.cert', + 'database_ssl_key': ssl_dir + '/db-client.key', + } + files = [ + call(expected['database_ssl_ca'], 'wb'), + call(expected['database_ssl_cert'], 'wb'), + call(expected['database_ssl_key'], 'wb') + ] + for f in files: + self.assertIn(f, _open.call_args_list) + self.assertEquals(db_ssl_ctxt, expected) + decode = [ + call(SHARED_DB_RELATION_SSL['ssl_ca']), + call(SHARED_DB_RELATION_SSL['ssl_cert']), + call(SHARED_DB_RELATION_SSL['ssl_key']) + ] + self.assertEquals(decode, self.b64decode.call_args_list) + + def test_db_ssl_nossldir(self): + db_ssl_ctxt = context.db_ssl(SHARED_DB_RELATION_SSL, {}, None) + self.assertEquals(db_ssl_ctxt, {}) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_missing_relation(self, os_codename): + '''Test shared-db context missing relation data''' + os_codename.return_value = 'stein' + incomplete_relation = copy.copy(SHARED_DB_RELATION) + incomplete_relation['password'] = None + relation = FakeRelation(relation_data=incomplete_relation) + self.relation_get.side_effect = relation.get + self.config.return_value = SHARED_DB_CONFIG + shared_db = context.SharedDBContext() + result = shared_db() + self.assertEquals(result, {}) + + def test_shared_db_context_with_missing_config(self): + '''Test shared-db context missing relation data''' + incomplete_config = copy.copy(SHARED_DB_CONFIG) + del incomplete_config['database-user'] + self.config.side_effect = fake_config(incomplete_config) + relation = FakeRelation(relation_data=SHARED_DB_RELATION) + self.relation_get.side_effect = relation.get + self.config.return_value = incomplete_config + shared_db = context.SharedDBContext() + self.assertRaises(context.OSContextError, shared_db) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_params(self, os_codename): + '''Test shared-db context with object parameters''' + os_codename.return_value = 'stein' + shared_db = context.SharedDBContext( + database='quantum', user='quantum', relation_prefix='quantum') + relation = FakeRelation(relation_data=SHARED_DB_RELATION_NAMESPACED) + self.relation_get.side_effect = relation.get + result = shared_db() + self.assertIn( + call(rid='foo:0', unit='foo/0'), + self.relation_get.call_args_list) + self.assertEquals( + result, {'database': 'quantum', + 'database_user': 'quantum', + 'database_password': 'bar2', + 'database_host': 'bar', + 'database_type': 'mysql+pymysql'}) + + @patch.object(context, 'get_os_codename_install_source') + def test_shared_db_context_with_params_pike(self, os_codename): + '''Test shared-db context with object parameters''' + os_codename.return_value = 'pike' + shared_db = context.SharedDBContext( + database='quantum', user='quantum', relation_prefix='quantum') + relation = FakeRelation(relation_data=SHARED_DB_RELATION_NAMESPACED) + self.relation_get.side_effect = relation.get + result = shared_db() + self.assertIn( + call(rid='foo:0', unit='foo/0'), + self.relation_get.call_args_list) + self.assertEquals( + result, {'database': 'quantum', + 'database_user': 'quantum', + 'database_password': 'bar2', + 'database_host': 'bar', + 'database_type': 'mysql'}) + + @patch.object(context, 'get_os_codename_install_source') + @patch('charmhelpers.contrib.openstack.context.format_ipv6_addr') + def test_shared_db_context_with_ipv6(self, format_ipv6_addr, os_codename): + '''Test shared-db context with ipv6''' + shared_db = context.SharedDBContext( + database='quantum', user='quantum', relation_prefix='quantum') + os_codename.return_value = 'stein' + relation = FakeRelation(relation_data=SHARED_DB_RELATION_NAMESPACED) + self.relation_get.side_effect = relation.get + format_ipv6_addr.return_value = '[2001:db8:1::1]' + result = shared_db() + self.assertIn( + call(rid='foo:0', unit='foo/0'), + self.relation_get.call_args_list) + self.assertEquals( + result, {'database': 'quantum', + 'database_user': 'quantum', + 'database_password': 'bar2', + 'database_host': '[2001:db8:1::1]', + 'database_type': 'mysql+pymysql'}) + + def test_postgresql_db_context_with_data(self): + '''Test postgresql-db context with all required data''' + relation = FakeRelation(relation_data=POSTGRESQL_DB_RELATION) + self.relation_get.side_effect = relation.get + self.config.side_effect = fake_config(POSTGRESQL_DB_CONFIG) + postgresql_db = context.PostgresqlDBContext() + result = postgresql_db() + expected = { + 'database_host': 'dbserver.local', + 'database': 'foodb', + 'database_user': 'adam', + 'database_password': 'foo', + 'database_type': 'postgresql', + } + self.assertEquals(result, expected) + + def test_postgresql_db_context_with_missing_relation(self): + '''Test postgresql-db context missing relation data''' + incomplete_relation = copy.copy(POSTGRESQL_DB_RELATION) + incomplete_relation['password'] = None + relation = FakeRelation(relation_data=incomplete_relation) + self.relation_get.side_effect = relation.get + self.config.return_value = POSTGRESQL_DB_CONFIG + postgresql_db = context.PostgresqlDBContext() + result = postgresql_db() + self.assertEquals(result, {}) + + def test_postgresql_db_context_with_missing_config(self): + '''Test postgresql-db context missing relation data''' + incomplete_config = copy.copy(POSTGRESQL_DB_CONFIG) + del incomplete_config['database'] + self.config.side_effect = fake_config(incomplete_config) + relation = FakeRelation(relation_data=POSTGRESQL_DB_RELATION) + self.relation_get.side_effect = relation.get + self.config.return_value = incomplete_config + postgresql_db = context.PostgresqlDBContext() + self.assertRaises(context.OSContextError, postgresql_db) + + def test_postgresql_db_context_with_params(self): + '''Test postgresql-db context with object parameters''' + postgresql_db = context.PostgresqlDBContext(database='quantum') + result = postgresql_db() + self.assertEquals(result['database'], 'quantum') + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_data(self, *args): + '''Test shared-db context with all required data''' + relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION_UNSET) + self.relation_get.side_effect = relation.get + identity_service = context.IdentityServiceContext() + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': None, + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'http', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'http', + 'api_version': '2.0', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + def test_identity_credentials_context_with_data(self): + '''Test identity-credentials context with all required data''' + relation = FakeRelation(relation_data=IDENTITY_CREDENTIALS_RELATION_UNSET) + self.relation_get.side_effect = relation.get + identity_credentials = context.IdentityCredentialsContext() + result = identity_credentials() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': '123456', + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'https', + 'api_version': '2.0', + } + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_altname(self, *args): + '''Test identity context when using an explicit relation name''' + relation = FakeRelation( + relation_data=APIIDENTITY_SERVICE_RELATION_UNSET + ) + self.relation_get.side_effect = relation.get + self.relation_ids.return_value = ['neutron-plugin-api:0'] + self.related_units.return_value = ['neutron-api/0'] + identity_service = context.IdentityServiceContext( + rel_name='neutron-plugin-api' + ) + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': None, + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'http', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'http', + 'api_version': '2.0', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_cache(self, *args): + '''Test shared-db context with signing cache info''' + relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION_UNSET) + self.relation_get.side_effect = relation.get + svc = 'cinder' + identity_service = context.IdentityServiceContext(service=svc, + service_user=svc) + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': None, + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'http', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'http', + 'signing_dir': '/var/cache/cinder', + 'api_version': '2.0', + } + self.assertTrue(self.mkdir.called) + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_data_http(self, *args): + '''Test shared-db context with all required data''' + relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION_HTTP) + self.relation_get.side_effect = relation.get + identity_service = context.IdentityServiceContext() + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': '123456', + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'http', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'http', + 'api_version': '2.0', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_data_https(self, *args): + '''Test shared-db context with all required data''' + relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION_HTTPS) + self.relation_get.side_effect = relation.get + identity_service = context.IdentityServiceContext() + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': None, + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'https', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'https', + 'api_version': '2.0', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_data_versioned(self, *args): + '''Test shared-db context with api version supplied from keystone''' + relation = FakeRelation( + relation_data=IDENTITY_SERVICE_RELATION_VERSIONED) + self.relation_get.side_effect = relation.get + identity_service = context.IdentityServiceContext() + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_domain_name': 'admin_domain', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': 'svc-proj-id', + 'admin_domain_id': 'svc-dom-id', + 'service_project_id': 'svc-proj-id', + 'service_domain_id': 'svc-dom-id', + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'https', + 'internal_host': 'keystone-internal.local', + 'internal_port': '5000', + 'internal_protocol': 'https', + 'api_version': '3', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + def test_identity_credentials_context_with_data_versioned(self): + '''Test identity-credentials context with api version supplied from keystone''' + relation = FakeRelation( + relation_data=IDENTITY_CREDENTIALS_RELATION_VERSIONED) + self.relation_get.side_effect = relation.get + identity_credentials = context.IdentityCredentialsContext() + result = identity_credentials() + expected = { + 'admin_password': 'foo', + 'admin_domain_name': 'admin_domain', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': '123456', + 'admin_user': 'adam', + 'auth_host': 'keystone-host.local', + 'auth_port': '35357', + 'auth_protocol': 'https', + 'service_host': 'keystonehost.local', + 'service_port': '5000', + 'service_protocol': 'https', + 'api_version': '3', + } + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + @patch('charmhelpers.contrib.openstack.context.format_ipv6_addr') + def test_identity_service_context_with_ipv6(self, format_ipv6_addr, *args): + '''Test identity-service context with ipv6''' + relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION_HTTP) + self.relation_get.side_effect = relation.get + format_ipv6_addr.return_value = '[2001:db8:1::1]' + identity_service = context.IdentityServiceContext() + result = identity_service() + expected = { + 'admin_password': 'foo', + 'admin_tenant_name': 'admin', + 'admin_tenant_id': '123456', + 'admin_domain_id': None, + 'admin_user': 'adam', + 'auth_host': '[2001:db8:1::1]', + 'auth_port': '35357', + 'auth_protocol': 'http', + 'service_host': '[2001:db8:1::1]', + 'service_port': '5000', + 'service_protocol': 'http', + 'internal_host': '[2001:db8:1::1]', + 'internal_port': '5000', + 'internal_protocol': 'http', + 'api_version': '2.0', + } + result.pop('keystone_authtoken') + self.assertEquals(result, expected) + + @patch.object(context, 'filter_installed_packages', return_value=[]) + @patch.object(context, 'os_release', return_value='rocky') + def test_identity_service_context_with_missing_relation(self, *args): + '''Test shared-db context missing relation data''' + incomplete_relation = copy.copy(IDENTITY_SERVICE_RELATION_UNSET) + incomplete_relation['service_password'] = None + relation = FakeRelation(relation_data=incomplete_relation) + self.relation_get.side_effect = relation.get + identity_service = context.IdentityServiceContext() + result = identity_service() + self.assertEquals(result, {}) + + @patch.object(context, 'filter_installed_packages') + @patch.object(context, 'os_release') + def test_keystone_authtoken_www_authenticate_uri_stein_apiv3(self, mock_os_release, mock_filter_installed_packages): + relation_data = copy.deepcopy(IDENTITY_SERVICE_RELATION_VERSIONED) + relation = FakeRelation(relation_data=relation_data) + self.relation_get.side_effect = relation.get + + mock_filter_installed_packages.return_value = [] + mock_os_release.return_value = 'stein' + + identity_service = context.IdentityServiceContext() + + cfg_ctx = identity_service() + + keystone_authtoken = cfg_ctx.get('keystone_authtoken', {}) + + expected = collections.OrderedDict(( + ('auth_type', 'password'), + ('www_authenticate_uri', 'https://keystonehost.local:5000/v3'), + ('auth_url', 'https://keystone-host.local:35357/v3'), + ('project_domain_name', 'admin_domain'), + ('user_domain_name', 'admin_domain'), + ('project_name', 'admin'), + ('username', 'adam'), + ('password', 'foo'), + ('signing_dir', ''), + )) + + self.assertEquals(keystone_authtoken, expected) + + def test_amqp_context_with_data(self): + '''Test amqp context with all required data''' + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo' + } + self.assertEquals(result, expected) + + def test_amqp_context_explicit_relation_id(self): + '''Test amqp context setting the relation_id''' + relation = FakeRelation(relation_data=AMQP_RELATION_ALT_RID) + self.relation_get.side_effect = relation.get + self.related_units.return_value = ['rabbitmq-alt/0'] + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext(relation_id='amqp-alt:0') + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbitalthost1', + 'rabbitmq_password': 'flump', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'transport_url': 'rabbit://adam:flump@rabbitalthost1:5672/foo' + } + self.assertEquals(result, expected) + + def test_amqp_context_with_data_altname(self): + '''Test amqp context with alternative relation name''' + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_NOVA_CONFIG + amqp = context.AMQPContext( + rel_name='amqp-nova', + relation_prefix='nova') + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo' + } + self.assertEquals(result, expected) + + @patch(open_builtin) + def test_amqp_context_with_data_ssl(self, _open): + '''Test amqp context with all required data and ssl''' + relation = FakeRelation(relation_data=AMQP_RELATION_WITH_SSL) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_CONFIG + ssl_dir = '/etc/sslamqp' + amqp = context.AMQPContext(ssl_dir=ssl_dir) + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbit_ssl_port': 5671, + 'rabbitmq_virtual_host': 'foo', + 'rabbit_ssl_ca': ssl_dir + '/rabbit-client-ca.pem', + 'rabbitmq_ha_queues': True, + 'transport_url': 'rabbit://adam:foobar@rabbithost:5671/foo' + } + _open.assert_called_once_with(ssl_dir + '/rabbit-client-ca.pem', 'wb') + self.assertEquals(result, expected) + self.assertEquals([call(AMQP_RELATION_WITH_SSL['ssl_ca'])], + self.b64decode.call_args_list) + + def test_amqp_context_with_data_ssl_noca(self): + '''Test amqp context with all required data with ssl but missing ca''' + relation = FakeRelation(relation_data=AMQP_RELATION_WITH_SSL) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbit_ssl_port': 5671, + 'rabbitmq_virtual_host': 'foo', + 'rabbit_ssl_ca': 'cert', + 'rabbitmq_ha_queues': True, + 'transport_url': 'rabbit://adam:foobar@rabbithost:5671/foo' + } + self.assertEquals(result, expected) + + def test_amqp_context_with_data_clustered(self): + '''Test amqp context with all required data with clustered rabbit''' + relation_data = copy.copy(AMQP_RELATION) + relation_data['clustered'] = 'yes' + relation = FakeRelation(relation_data=relation_data) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'clustered': True, + 'rabbitmq_host': relation_data['vip'], + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'transport_url': 'rabbit://adam:foobar@10.0.0.1:5672/foo' + } + self.assertEquals(result, expected) + + def test_amqp_context_with_data_active_active(self): + '''Test amqp context with required data with active/active rabbit''' + relation_data = copy.copy(AMQP_AA_RELATION) + relation = FakeRelation(relation_data=relation_data) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost1', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'rabbitmq_hosts': 'rabbithost1,rabbithost2', + 'transport_url': ('rabbit://adam:foobar@rabbithost1:5672' + ',adam:foobar@rabbithost2:5672/foo') + } + self.assertEquals(result, expected) + + def test_amqp_context_with_missing_relation(self): + '''Test amqp context missing relation data''' + incomplete_relation = copy.copy(AMQP_RELATION) + incomplete_relation['password'] = '' + relation = FakeRelation(relation_data=incomplete_relation) + self.relation_get.side_effect = relation.get + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + self.assertEquals({}, result) + + def test_amqp_context_with_missing_config(self): + '''Test amqp context missing relation data''' + incomplete_config = copy.copy(AMQP_CONFIG) + del incomplete_config['rabbit-user'] + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + self.config.return_value = incomplete_config + amqp = context.AMQPContext() + self.assertRaises(context.OSContextError, amqp) + + @patch('charmhelpers.contrib.openstack.context.format_ipv6_addr') + def test_amqp_context_with_ipv6(self, format_ipv6_addr): + '''Test amqp context with ipv6''' + relation_data = copy.copy(AMQP_AA_RELATION) + relation = FakeRelation(relation_data=relation_data) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + format_ipv6_addr.return_value = '[2001:db8:1::1]' + self.config.return_value = AMQP_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': '[2001:db8:1::1]', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'rabbitmq_hosts': '[2001:db8:1::1],[2001:db8:1::1]', + 'transport_url': ('rabbit://adam:foobar@[2001:db8:1::1]:5672' + ',adam:foobar@[2001:db8:1::1]:5672/foo') + } + self.assertEquals(result, expected) + + def test_amqp_context_with_oslo_messaging(self): + """Test amqp context with oslo-messaging-flags option""" + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + AMQP_OSLO_CONFIG.update(AMQP_CONFIG) + self.config.return_value = AMQP_OSLO_CONFIG + amqp = context.AMQPContext() + result = amqp() + expected = { + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'oslo_messaging_flags': { + 'rabbit_max_retries': '1', + 'rabbit_retry_backoff': '1', + 'rabbit_retry_interval': '1' + }, + 'oslo_messaging_driver': 'log', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo' + } + + self.assertEquals(result, expected) + + def test_amqp_context_with_notification_format(self): + """Test amqp context with notification_format option""" + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + AMQP_NOTIFICATION_FORMAT.update(AMQP_CONFIG) + self.config.return_value = AMQP_NOTIFICATION_FORMAT + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'notification_format': 'both', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo' + } + + self.assertEquals(result, expected) + + def test_amqp_context_with_notification_topics(self): + """Test amqp context with notification_topics option""" + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + AMQP_NOTIFICATION_TOPICS.update(AMQP_CONFIG) + self.config.return_value = AMQP_NOTIFICATION_TOPICS + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'notification_topics': 'foo,bar', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo' + } + + self.assertEquals(result, expected) + + def test_amqp_context_with_notifications_to_logs(self): + """Test amqp context with send_notifications_to_logs""" + relation = FakeRelation(relation_data=AMQP_RELATION) + self.relation_get.side_effect = relation.get + AMQP_NOTIFICATIONS_LOGS.update(AMQP_CONFIG) + self.config.return_value = AMQP_NOTIFICATIONS_LOGS + amqp = context.AMQPContext() + result = amqp() + expected = { + 'oslo_messaging_driver': 'messagingv2', + 'rabbitmq_host': 'rabbithost', + 'rabbitmq_password': 'foobar', + 'rabbitmq_user': 'adam', + 'rabbitmq_virtual_host': 'foo', + 'transport_url': 'rabbit://adam:foobar@rabbithost:5672/foo', + 'send_notifications_to_logs': True, + } + + self.assertEquals(result, expected) + + def test_libvirt_config_flags(self): + self.config.side_effect = fake_config({ + 'libvirt-flags': 'iscsi_use_multipath=True,chap_auth=False', + }) + + results = context.LibvirtConfigFlagsContext()() + self.assertEquals(results, { + 'libvirt_flags': { + 'chap_auth': 'False', + 'iscsi_use_multipath': 'True' + } + }) + + def test_ceph_no_relids(self): + '''Test empty ceph realtion''' + relation = FakeRelation(relation_data={}) + self.relation_ids.side_effect = relation.get + ceph = context.CephContext() + result = ceph() + self.assertEquals(result, {}) + + def test_ceph_rel_with_no_units(self): + '''Test ceph context with missing related units''' + relation = FakeRelation(relation_data={}) + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = [] + ceph = context.CephContext() + result = ceph() + self.assertEquals(result, {}) + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_data(self, ensure_packages, mkdir, isdir, + mock_config): + '''Test ceph context with all relation data''' + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + isdir.return_value = False + relation = FakeRelation(relation_data=CEPH_RELATION) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_missing_data(self, ensure_packages, mkdir): + '''Test ceph context with missing relation data''' + relation = copy.deepcopy(CEPH_RELATION) + for k, v in six.iteritems(relation): + for u in six.iterkeys(v): + del relation[k][u]['auth'] + relation = FakeRelation(relation_data=relation) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + self.assertEquals(result, {}) + self.assertFalse(ensure_packages.called) + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_partial_missing_data(self, ensure_packages, mkdir, + isdir, config): + '''Test ceph context last unit missing data + + Tests a fix to a previously bug which meant only the config from + last unit was returned so if a valid value was supplied from an + earlier unit it would be ignored''' + config.side_effect = fake_config({'use-syslog': 'True'}) + relation = copy.deepcopy(CEPH_RELATION) + for k, v in six.iteritems(relation): + last_unit = sorted(six.iterkeys(v))[-1] + unit_data = relation[k][last_unit] + del unit_data['auth'] + relation[k][last_unit] = unit_data + relation = FakeRelation(relation_data=relation) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_public_addr( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_RELATION_WITH_PUBLIC_ADDR) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': '192.168.1.10 192.168.1.11', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_public_addr_and_port( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_PUBLIC_ADDR_PORT) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': '192.168.1.10:1234 192.168.1.11:4321', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_public_ipv6_addr(self, ensure_packages, mkdir, + isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_PUBLIC_IPv6_ADDR) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': '[2001:5c0:9168::1] [2001:5c0:9168::2]', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_public_ipv6_addr_port( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation( + relation_data=CEPH_REL_WITH_PUBLIC_IPv6_ADDR_PORT) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': '[2001:5c0:9168::1]:1234 [2001:5c0:9168::2]:4321', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_multi_public_addr( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_MULTI_PUBLIC_ADDR) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': '192.168.1.10 192.168.1.11 192.168.1.20 192.168.1.21', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_default_features( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with all + relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_DEFAULT_FEATURES) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + 'rbd_features': '1', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_ec_pool_no_rbd_pool( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context with erasure coded pools''' + isdir.return_value = False + config_dict = { + 'use-syslog': True, + 'pool-type': 'erasure-coded' + } + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_DEFAULT_FEATURES) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + 'rbd_features': '1', + 'rbd_default_data_pool': 'testing-foo', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_ec_pool_rbd_pool( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context with erasure coded pools''' + isdir.return_value = False + config_dict = { + 'use-syslog': True, + 'pool-type': 'erasure-coded', + 'rbd-pool': 'glance' + } + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_DEFAULT_FEATURES) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + 'rbd_features': '1', + 'rbd_default_data_pool': 'glance', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_ec_pool_rbd_pool_name( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context with erasure coded pools''' + isdir.return_value = False + config_dict = { + 'use-syslog': True, + 'pool-type': 'erasure-coded', + 'rbd-pool-name': 'nova' + } + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_REL_WITH_DEFAULT_FEATURES) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + result = ceph() + expected = { + 'mon_hosts': 'ceph_node1 ceph_node2', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + 'rbd_features': '1', + 'rbd_default_data_pool': 'nova', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_with_rbd_cache(self, ensure_packages, mkdir, isdir, + mock_config): + isdir.return_value = False + config_dict = {'rbd-client-cache': 'enabled', + 'use-syslog': False} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = FakeRelation(relation_data=CEPH_RELATION_WITH_PUBLIC_ADDR) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + + class CephContextWithRBDCache(context.CephContext): + def __call__(self): + ctxt = super(CephContextWithRBDCache, self).__call__() + + rbd_cache = fake_config('rbd-client-cache') or "" + if rbd_cache.lower() == "enabled": + ctxt['rbd_client_cache_settings'] = \ + {'rbd cache': 'true', + 'rbd cache writethrough until flush': 'true'} + elif rbd_cache.lower() == "disabled": + ctxt['rbd_client_cache_settings'] = \ + {'rbd cache': 'false'} + + return ctxt + + ceph = CephContextWithRBDCache() + result = ceph() + expected = { + 'mon_hosts': '192.168.1.10 192.168.1.11', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'false', + } + expected['rbd_client_cache_settings'] = \ + {'rbd cache': 'true', + 'rbd cache writethrough until flush': 'true'} + + self.assertDictEqual(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch.object(context, 'config') + def test_sysctl_context_with_config(self, config): + self.charm_name.return_value = 'test-charm' + config.return_value = '{ kernel.max_pid: "1337"}' + self.sysctl_create.return_value = True + ctxt = context.SysctlContext() + result = ctxt() + self.sysctl_create.assert_called_with( + config.return_value, + "/etc/sysctl.d/50-test-charm.conf") + + self.assertTrue(result, {'sysctl': config.return_value}) + + @patch.object(context, 'config') + def test_sysctl_context_without_config(self, config): + self.charm_name.return_value = 'test-charm' + config.return_value = None + self.sysctl_create.return_value = True + ctxt = context.SysctlContext() + result = ctxt() + self.assertTrue(self.sysctl_create.called == 0) + self.assertTrue(result, {'sysctl': config.return_value}) + + @patch.object(context, 'config') + @patch('os.path.isdir') + @patch('os.mkdir') + @patch.object(context, 'ensure_packages') + def test_ceph_context_missing_public_addr( + self, ensure_packages, mkdir, isdir, mock_config): + '''Test ceph context in host with multiple networks with no + ceph-public-addr in relation data''' + isdir.return_value = False + config_dict = {'use-syslog': True} + + def fake_config(key): + return config_dict.get(key) + + mock_config.side_effect = fake_config + relation = copy.deepcopy(CEPH_RELATION_WITH_PUBLIC_ADDR) + del relation['ceph:0']['ceph/0']['ceph-public-address'] + relation = FakeRelation(relation_data=relation) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + ceph = context.CephContext() + + result = ceph() + expected = { + 'mon_hosts': '192.168.1.11 ceph_node1', + 'auth': 'foo', + 'key': 'bar', + 'use_syslog': 'true', + } + self.assertEquals(result, expected) + ensure_packages.assert_called_with(['ceph-common']) + mkdir.assert_called_with('/etc/ceph') + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_data(self, local_unit, local_address): + '''Test haproxy context with all relation data''' + cluster_relation = { + 'cluster:0': { + 'peer/1': { + 'private-address': 'cluster-peer1.localnet', + }, + 'peer/2': { + 'private-address': 'cluster-peer2.localnet', + }, + }, + } + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, + 'cluster-peer0.localnet'] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.get_netmask_for_address.return_value = '255.255.0.0' + self.config.return_value = False + self.maxDiff = None + self.is_ipv6_disabled.return_value = True + haproxy = context.HAProxyContext() + with patch_open() as (_open, _file): + result = haproxy() + ex = { + 'frontends': { + 'cluster-peer0.localnet': { + 'network': 'cluster-peer0.localnet/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.localnet'), + ('peer-1', 'cluster-peer1.localnet'), + ('peer-2', 'cluster-peer2.localnet'), + ]), + }, + }, + 'default_backend': 'cluster-peer0.localnet', + 'local_host': '127.0.0.1', + 'haproxy_host': '0.0.0.0', + 'ipv6_enabled': False, + 'stat_password': 'testpassword', + 'stat_port': '8888', + } + # the context gets generated. + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('admin', False), + call('internal', False), + call('public', False), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_data_timeout(self, local_unit, local_address): + '''Test haproxy context with all relation data and timeout''' + cluster_relation = { + 'cluster:0': { + 'peer/1': { + 'private-address': 'cluster-peer1.localnet', + }, + 'peer/2': { + 'private-address': 'cluster-peer2.localnet', + }, + }, + } + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, + 'cluster-peer0.localnet'] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.get_netmask_for_address.return_value = '255.255.0.0' + self.config.return_value = False + self.maxDiff = None + c = fake_config(HAPROXY_CONFIG) + c.data['prefer-ipv6'] = False + self.config.side_effect = c + self.is_ipv6_disabled.return_value = True + haproxy = context.HAProxyContext() + with patch_open() as (_open, _file): + result = haproxy() + ex = { + 'frontends': { + 'cluster-peer0.localnet': { + 'network': 'cluster-peer0.localnet/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.localnet'), + ('peer-1', 'cluster-peer1.localnet'), + ('peer-2', 'cluster-peer2.localnet'), + ]), + } + }, + 'default_backend': 'cluster-peer0.localnet', + 'local_host': '127.0.0.1', + 'haproxy_host': '0.0.0.0', + 'ipv6_enabled': False, + 'stat_password': 'testpassword', + 'stat_port': '8888', + 'haproxy_client_timeout': 50000, + 'haproxy_server_timeout': 50000, + } + # the context gets generated. + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('admin', None), + call('internal', None), + call('public', None), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_data_multinet(self, local_unit, local_address): + '''Test haproxy context with all relation data for network splits''' + cluster_relation = { + 'cluster:0': { + 'peer/1': { + 'private-address': 'cluster-peer1.localnet', + 'admin-address': 'cluster-peer1.admin', + 'internal-address': 'cluster-peer1.internal', + 'public-address': 'cluster-peer1.public', + }, + 'peer/2': { + 'private-address': 'cluster-peer2.localnet', + 'admin-address': 'cluster-peer2.admin', + 'internal-address': 'cluster-peer2.internal', + 'public-address': 'cluster-peer2.public', + }, + }, + } + + local_unit.return_value = 'peer/0' + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = ['cluster-peer0.admin', + 'cluster-peer0.internal', + 'cluster-peer0.public', + 'cluster-peer0.localnet'] + self.get_netmask_for_address.return_value = '255.255.0.0' + self.config.return_value = False + self.maxDiff = None + self.is_ipv6_disabled.return_value = True + haproxy = context.HAProxyContext() + with patch_open() as (_open, _file): + result = haproxy() + ex = { + 'frontends': { + 'cluster-peer0.admin': { + 'network': 'cluster-peer0.admin/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.admin'), + ('peer-1', 'cluster-peer1.admin'), + ('peer-2', 'cluster-peer2.admin'), + ]), + }, + 'cluster-peer0.internal': { + 'network': 'cluster-peer0.internal/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.internal'), + ('peer-1', 'cluster-peer1.internal'), + ('peer-2', 'cluster-peer2.internal'), + ]), + }, + 'cluster-peer0.public': { + 'network': 'cluster-peer0.public/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.public'), + ('peer-1', 'cluster-peer1.public'), + ('peer-2', 'cluster-peer2.public'), + ]), + }, + 'cluster-peer0.localnet': { + 'network': 'cluster-peer0.localnet/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.localnet'), + ('peer-1', 'cluster-peer1.localnet'), + ('peer-2', 'cluster-peer2.localnet'), + ]), + } + }, + 'default_backend': 'cluster-peer0.localnet', + 'local_host': '127.0.0.1', + 'haproxy_host': '0.0.0.0', + 'ipv6_enabled': False, + 'stat_password': 'testpassword', + 'stat_port': '8888', + } + # the context gets generated. + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('admin', False), + call('internal', False), + call('public', False), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_data_public_only(self, local_unit, local_address): + '''Test haproxy context with with openstack-dashboard public only binding''' + cluster_relation = { + 'cluster:0': { + 'peer/1': { + 'private-address': 'cluster-peer1.localnet', + 'public-address': 'cluster-peer1.public', + }, + 'peer/2': { + 'private-address': 'cluster-peer2.localnet', + 'public-address': 'cluster-peer2.public', + }, + }, + } + + local_unit.return_value = 'peer/0' + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + _network_get_map = { + 'public': 'cluster-peer0.public', + 'cluster': 'cluster-peer0.localnet', + } + self.get_relation_ip.side_effect = ( + lambda binding, config_opt=None: + _network_get_map[binding] + ) + self.get_netmask_for_address.return_value = '255.255.0.0' + self.config.return_value = None + self.maxDiff = None + self.is_ipv6_disabled.return_value = True + haproxy = context.HAProxyContext(address_types=['public']) + with patch_open() as (_open, _file): + result = haproxy() + ex = { + 'frontends': { + 'cluster-peer0.public': { + 'network': 'cluster-peer0.public/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.public'), + ('peer-1', 'cluster-peer1.public'), + ('peer-2', 'cluster-peer2.public'), + ]), + }, + 'cluster-peer0.localnet': { + 'network': 'cluster-peer0.localnet/255.255.0.0', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.localnet'), + ('peer-1', 'cluster-peer1.localnet'), + ('peer-2', 'cluster-peer2.localnet'), + ]), + } + }, + 'default_backend': 'cluster-peer0.localnet', + 'local_host': '127.0.0.1', + 'haproxy_host': '0.0.0.0', + 'ipv6_enabled': False, + 'stat_password': 'testpassword', + 'stat_port': '8888', + } + # the context gets generated. + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('public', None), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_data_ipv6(self, local_unit, local_address): + '''Test haproxy context with all relation data ipv6''' + cluster_relation = { + 'cluster:0': { + 'peer/1': { + 'private-address': 'cluster-peer1.localnet', + }, + 'peer/2': { + 'private-address': 'cluster-peer2.localnet', + }, + }, + } + + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, + 'cluster-peer0.localnet'] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.get_address_in_network.return_value = None + self.get_netmask_for_address.return_value = \ + 'FFFF:FFFF:FFFF:FFFF:0000:0000:0000:0000' + self.get_ipv6_addr.return_value = ['cluster-peer0.localnet'] + c = fake_config(HAPROXY_CONFIG) + c.data['prefer-ipv6'] = True + self.config.side_effect = c + self.maxDiff = None + self.is_ipv6_disabled.return_value = False + haproxy = context.HAProxyContext() + with patch_open() as (_open, _file): + result = haproxy() + ex = { + 'frontends': { + 'cluster-peer0.localnet': { + 'network': 'cluster-peer0.localnet/' + 'FFFF:FFFF:FFFF:FFFF:0000:0000:0000:0000', + 'backends': collections.OrderedDict([ + ('peer-0', 'cluster-peer0.localnet'), + ('peer-1', 'cluster-peer1.localnet'), + ('peer-2', 'cluster-peer2.localnet'), + ]), + } + }, + 'default_backend': 'cluster-peer0.localnet', + 'local_host': 'ip6-localhost', + 'haproxy_server_timeout': 50000, + 'haproxy_client_timeout': 50000, + 'haproxy_host': '::', + 'ipv6_enabled': True, + 'stat_password': 'testpassword', + 'stat_port': '8888', + } + # the context gets generated. + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('admin', None), + call('internal', None), + call('public', None), + call('cluster')]) + + def test_haproxy_context_with_missing_data(self): + '''Test haproxy context with missing relation data''' + self.relation_ids.return_value = [] + haproxy = context.HAProxyContext() + self.assertEquals({}, haproxy()) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_no_peers(self, local_unit, local_address): + '''Test haproxy context with single unit''' + # peer relations always show at least one peer relation, even + # if unit is alone. should be an incomplete context. + cluster_relation = { + 'cluster:0': { + 'peer/0': { + 'private-address': 'lonely.clusterpeer.howsad', + }, + }, + } + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, None] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.config.return_value = False + haproxy = context.HAProxyContext() + self.assertEquals({}, haproxy()) + self.get_relation_ip.assert_has_calls([call('admin', False), + call('internal', False), + call('public', False), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_net_override(self, local_unit, local_address): + '''Test haproxy context with single unit''' + # peer relations always show at least one peer relation, even + # if unit is alone. should be an incomplete context. + cluster_relation = { + 'cluster:0': { + 'peer/0': { + 'private-address': 'lonely.clusterpeer.howsad', + }, + }, + } + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, None] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.config.return_value = False + c = fake_config(HAPROXY_CONFIG) + c.data['os-admin-network'] = '192.168.10.0/24' + c.data['os-internal-network'] = '192.168.20.0/24' + c.data['os-public-network'] = '192.168.30.0/24' + self.config.side_effect = c + haproxy = context.HAProxyContext() + self.assertEquals({}, haproxy()) + self.get_relation_ip.assert_has_calls([call('admin', '192.168.10.0/24'), + call('internal', '192.168.20.0/24'), + call('public', '192.168.30.0/24'), + call('cluster')]) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch('charmhelpers.contrib.openstack.context.local_unit') + def test_haproxy_context_with_no_peers_singlemode(self, local_unit, local_address): + '''Test haproxy context with single unit''' + # peer relations always show at least one peer relation, even + # if unit is alone. should be an incomplete context. + cluster_relation = { + 'cluster:0': { + 'peer/0': { + 'private-address': 'lonely.clusterpeer.howsad', + }, + }, + } + local_unit.return_value = 'peer/0' + # We are only using get_relation_ip. + # Setup the values it returns on each subsequent call. + self.get_relation_ip.side_effect = [None, None, None, + 'lonely.clusterpeer.howsad'] + relation = FakeRelation(cluster_relation) + self.relation_ids.side_effect = relation.relation_ids + self.relation_get.side_effect = relation.get + self.related_units.side_effect = relation.relation_units + self.config.return_value = False + self.get_address_in_network.return_value = None + self.get_netmask_for_address.return_value = '255.255.0.0' + self.is_ipv6_disabled.return_value = True + with patch_open() as (_open, _file): + result = context.HAProxyContext(singlenode_mode=True)() + ex = { + 'frontends': { + 'lonely.clusterpeer.howsad': { + 'backends': collections.OrderedDict([ + ('peer-0', 'lonely.clusterpeer.howsad')]), + 'network': 'lonely.clusterpeer.howsad/255.255.0.0' + }, + }, + 'default_backend': 'lonely.clusterpeer.howsad', + 'haproxy_host': '0.0.0.0', + 'local_host': '127.0.0.1', + 'ipv6_enabled': False, + 'stat_port': '8888', + 'stat_password': 'testpassword', + } + self.assertEquals(ex, result) + # and /etc/default/haproxy is updated. + self.assertEquals(_file.write.call_args_list, + [call('ENABLED=1\n')]) + self.get_relation_ip.assert_has_calls([call('admin', False), + call('internal', False), + call('public', False), + call('cluster')]) + + def test_https_context_with_no_https(self): + '''Test apache2 https when no https data available''' + apache = context.ApacheSSLContext() + self.https.return_value = False + self.assertEquals({}, apache()) + + def _https_context_setup(self): + ''' + Helper for test_https_context* tests. + + ''' + self.https.return_value = True + self.determine_api_port.return_value = 8756 + self.determine_apache_port.return_value = 8766 + + apache = context.ApacheSSLContext() + apache.configure_cert = MagicMock() + apache.enable_modules = MagicMock() + apache.configure_ca = MagicMock() + apache.canonical_names = MagicMock() + apache.canonical_names.return_value = [ + '10.5.1.1', + '10.5.2.1', + '10.5.3.1', + ] + apache.get_network_addresses = MagicMock() + apache.get_network_addresses.return_value = [ + ('10.5.1.100', '10.5.1.1'), + ('10.5.2.100', '10.5.2.1'), + ('10.5.3.100', '10.5.3.1'), + ] + apache.external_ports = '8776' + apache.service_namespace = 'cinder' + + ex = { + 'namespace': 'cinder', + 'endpoints': [('10.5.1.100', '10.5.1.1', 8766, 8756), + ('10.5.2.100', '10.5.2.1', 8766, 8756), + ('10.5.3.100', '10.5.3.1', 8766, 8756)], + 'ext_ports': [8766] + } + + return apache, ex + + def test_https_context(self): + self.relation_ids.return_value = [] + + apache, ex = self._https_context_setup() + + self.assertEquals(ex, apache()) + + apache.configure_cert.assert_has_calls([ + call('10.5.1.1'), + call('10.5.2.1'), + call('10.5.3.1') + ]) + + self.assertTrue(apache.configure_ca.called) + self.assertTrue(apache.enable_modules.called) + self.assertTrue(apache.configure_cert.called) + + def test_https_context_vault_relation(self): + self.relation_ids.return_value = ['certificates:2'] + self.related_units.return_value = 'vault/0' + + apache, ex = self._https_context_setup() + + self.assertEquals(ex, apache()) + + self.assertFalse(apache.configure_cert.called) + self.assertFalse(apache.configure_ca.called) + + def test_https_context_no_canonical_names(self): + self.relation_ids.return_value = [] + + apache, ex = self._https_context_setup() + apache.canonical_names.return_value = [] + + self.resolve_address.side_effect = ( + '10.5.1.4', '10.5.2.5', '10.5.3.6') + + self.assertEquals(ex, apache()) + + apache.configure_cert.assert_has_calls([ + call('10.5.1.4'), + call('10.5.2.5'), + call('10.5.3.6') + ]) + + self.resolve_address.assert_has_calls([ + call(endpoint_type=context.INTERNAL), + call(endpoint_type=context.ADMIN), + call(endpoint_type=context.PUBLIC), + ]) + + self.assertTrue(apache.configure_ca.called) + self.assertTrue(apache.enable_modules.called) + self.assertTrue(apache.configure_cert.called) + + def test_https_context_loads_correct_apache_mods(self): + # Test apache2 context also loads required apache modules + apache = context.ApacheSSLContext() + apache.enable_modules() + ex_cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers'] + self.check_call.assert_called_with(ex_cmd) + + def test_https_configure_cert(self): + # Test apache2 properly installs certs and keys to disk + self.get_cert.return_value = ('SSL_CERT', 'SSL_KEY') + self.b64decode.side_effect = [b'SSL_CERT', b'SSL_KEY'] + apache = context.ApacheSSLContext() + apache.service_namespace = 'cinder' + apache.configure_cert('test-cn') + # appropriate directories are created. + self.mkdir.assert_called_with(path='/etc/apache2/ssl/cinder') + # appropriate files are written. + files = [call(path='/etc/apache2/ssl/cinder/cert_test-cn', + content=b'SSL_CERT', owner='root', group='root', + perms=0o640), + call(path='/etc/apache2/ssl/cinder/key_test-cn', + content=b'SSL_KEY', owner='root', group='root', + perms=0o640)] + self.write_file.assert_has_calls(files) + # appropriate bits are b64decoded. + decode = [call('SSL_CERT'), call('SSL_KEY')] + self.assertEquals(decode, self.b64decode.call_args_list) + + def test_https_configure_cert_deprecated(self): + # Test apache2 properly installs certs and keys to disk + self.get_cert.return_value = ('SSL_CERT', 'SSL_KEY') + self.b64decode.side_effect = ['SSL_CERT', 'SSL_KEY'] + apache = context.ApacheSSLContext() + apache.service_namespace = 'cinder' + apache.configure_cert() + # appropriate directories are created. + self.mkdir.assert_called_with(path='/etc/apache2/ssl/cinder') + # appropriate files are written. + files = [call(path='/etc/apache2/ssl/cinder/cert', + content='SSL_CERT', owner='root', group='root', + perms=0o640), + call(path='/etc/apache2/ssl/cinder/key', + content='SSL_KEY', owner='root', group='root', + perms=0o640)] + self.write_file.assert_has_calls(files) + # appropriate bits are b64decoded. + decode = [call('SSL_CERT'), call('SSL_KEY')] + self.assertEquals(decode, self.b64decode.call_args_list) + + def test_https_canonical_names(self): + rel = FakeRelation(IDENTITY_RELATION_SINGLE_CERT) + self.relation_ids.side_effect = rel.relation_ids + self.related_units.side_effect = rel.relation_units + self.relation_get.side_effect = rel.get + apache = context.ApacheSSLContext() + self.assertEquals(apache.canonical_names(), ['cinderhost1']) + rel.relation_data = IDENTITY_RELATION_MULTIPLE_CERT + self.assertEquals(apache.canonical_names(), + sorted(['cinderhost1-adm-network', + 'cinderhost1-int-network', + 'cinderhost1-pub-network'])) + rel.relation_data = IDENTITY_RELATION_NO_CERT + self.assertEquals(apache.canonical_names(), []) + + def test_image_service_context_missing_data(self): + '''Test image-service with missing relation and missing data''' + image_service = context.ImageServiceContext() + self.relation_ids.return_value = [] + self.assertEquals({}, image_service()) + self.relation_ids.return_value = ['image-service:0'] + self.related_units.return_value = ['glance/0'] + self.relation_get.return_value = None + self.assertEquals({}, image_service()) + + def test_image_service_context_with_data(self): + '''Test image-service with required data''' + image_service = context.ImageServiceContext() + self.relation_ids.return_value = ['image-service:0'] + self.related_units.return_value = ['glance/0'] + self.relation_get.return_value = 'http://glancehost:9292' + self.assertEquals({'glance_api_servers': 'http://glancehost:9292'}, + image_service()) + + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_context_base_properties(self, attr): + '''Test neutron context base properties''' + neutron = context.NeutronContext() + attr.return_value = 'quantum-plugin-package' + self.assertEquals(None, neutron.plugin) + self.assertEquals(None, neutron.network_manager) + self.assertEquals(None, neutron.neutron_security_groups) + self.assertEquals('quantum-plugin-package', neutron.packages) + + @patch.object(context, 'neutron_plugin_attribute') + @patch.object(context, 'apt_install') + @patch.object(context, 'filter_installed_packages') + def test_neutron_ensure_package(self, _filter, _install, _packages): + '''Test neutron context installed required packages''' + _filter.return_value = ['quantum-plugin-package'] + _packages.return_value = [['quantum-plugin-package']] + neutron = context.NeutronContext() + neutron._ensure_packages() + _install.assert_called_with(['quantum-plugin-package'], fatal=True) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_ovs_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'ovs', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.ovs_ctxt()) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_nvp_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'nvp', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.nvp_ctxt()) + + @patch.object(context, 'config') + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_n1kv_plugin_context(self, attr, ip, sec_groups, config): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + config.return_value = 'n1kv' + neutron = context.NeutronContext() + self.assertEquals({ + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'n1kv', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1', + 'config': 'some.quantum.driver.class', + 'vsm_ip': 'n1kv', + 'vsm_username': 'n1kv', + 'vsm_password': 'n1kv', + 'user_config_flags': {}, + 'restrict_policy_profiles': 'n1kv', + }, neutron.n1kv_ctxt()) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_calico_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'Calico', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.calico_ctxt()) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_plumgrid_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'plumgrid', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.pg_ctxt()) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_nuage_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'vsp', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.nuage_ctxt()) + + @patch.object(context.NeutronContext, 'neutron_security_groups') + @patch.object(context, 'unit_private_ip') + @patch.object(context, 'neutron_plugin_attribute') + def test_neutron_midonet_plugin_context(self, attr, ip, sec_groups): + ip.return_value = '10.0.0.1' + sec_groups.__get__ = MagicMock(return_value=True) + attr.return_value = 'some.quantum.driver.class' + neutron = context.NeutronContext() + self.assertEquals({ + 'config': 'some.quantum.driver.class', + 'core_plugin': 'some.quantum.driver.class', + 'neutron_plugin': 'midonet', + 'neutron_security_groups': True, + 'local_ip': '10.0.0.1'}, neutron.midonet_ctxt()) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch.object(context.NeutronContext, 'network_manager') + def test_neutron_neutron_ctxt(self, mock_network_manager, + mock_local_address): + vip = '88.11.22.33' + priv_addr = '10.0.0.1' + mock_local_address.return_value = priv_addr + neutron = context.NeutronContext() + + config = {'vip': vip} + self.config.side_effect = lambda key: config[key] + mock_network_manager.__get__ = Mock(return_value='neutron') + + self.is_clustered.return_value = False + self.assertEquals( + {'network_manager': 'neutron', + 'neutron_url': 'https://%s:9696' % (priv_addr)}, + neutron.neutron_ctxt() + ) + + self.is_clustered.return_value = True + self.assertEquals( + {'network_manager': 'neutron', + 'neutron_url': 'https://%s:9696' % (vip)}, + neutron.neutron_ctxt() + ) + + @patch('charmhelpers.contrib.openstack.context.local_address') + @patch.object(context.NeutronContext, 'network_manager') + def test_neutron_neutron_ctxt_http(self, mock_network_manager, + mock_local_address): + vip = '88.11.22.33' + priv_addr = '10.0.0.1' + mock_local_address.return_value = priv_addr + neutron = context.NeutronContext() + + config = {'vip': vip} + self.config.side_effect = lambda key: config[key] + self.https.return_value = False + mock_network_manager.__get__ = Mock(return_value='neutron') + + self.is_clustered.return_value = False + self.assertEquals( + {'network_manager': 'neutron', + 'neutron_url': 'http://%s:9696' % (priv_addr)}, + neutron.neutron_ctxt() + ) + + self.is_clustered.return_value = True + self.assertEquals( + {'network_manager': 'neutron', + 'neutron_url': 'http://%s:9696' % (vip)}, + neutron.neutron_ctxt() + ) + + @patch.object(context.NeutronContext, 'neutron_ctxt') + @patch.object(context.NeutronContext, 'ovs_ctxt') + @patch.object(context.NeutronContext, 'plugin') + @patch.object(context.NeutronContext, '_ensure_packages') + @patch.object(context.NeutronContext, 'network_manager') + def test_neutron_main_context_generation(self, mock_network_manager, + mock_ensure_packages, + mock_plugin, mock_ovs_ctxt, + mock_neutron_ctxt): + + mock_neutron_ctxt.return_value = {'network_manager': 'neutron', + 'neutron_url': 'https://foo:9696'} + config = {'neutron-alchemy-flags': None} + self.config.side_effect = lambda key: config[key] + neutron = context.NeutronContext() + + mock_network_manager.__get__ = Mock(return_value='flatdhcpmanager') + mock_plugin.__get__ = Mock() + + self.assertEquals({}, neutron()) + self.assertTrue(mock_network_manager.__get__.called) + self.assertFalse(mock_plugin.__get__.called) + + mock_network_manager.__get__.return_value = 'neutron' + mock_plugin.__get__ = Mock(return_value=None) + self.assertEquals({}, neutron()) + self.assertTrue(mock_plugin.__get__.called) + + mock_ovs_ctxt.return_value = {'ovs': 'ovs_context'} + mock_plugin.__get__.return_value = 'ovs' + self.assertEquals( + {'network_manager': 'neutron', + 'ovs': 'ovs_context', + 'neutron_url': 'https://foo:9696'}, + neutron() + ) + + @patch.object(context.NeutronContext, 'neutron_ctxt') + @patch.object(context.NeutronContext, 'nvp_ctxt') + @patch.object(context.NeutronContext, 'plugin') + @patch.object(context.NeutronContext, '_ensure_packages') + @patch.object(context.NeutronContext, 'network_manager') + def test_neutron_main_context_gen_nvp_and_alchemy(self, + mock_network_manager, + mock_ensure_packages, + mock_plugin, + mock_nvp_ctxt, + mock_neutron_ctxt): + + mock_neutron_ctxt.return_value = {'network_manager': 'neutron', + 'neutron_url': 'https://foo:9696'} + config = {'neutron-alchemy-flags': 'pool_size=20'} + self.config.side_effect = lambda key: config[key] + neutron = context.NeutronContext() + + mock_network_manager.__get__ = Mock(return_value='flatdhcpmanager') + mock_plugin.__get__ = Mock() + + self.assertEquals({}, neutron()) + self.assertTrue(mock_network_manager.__get__.called) + self.assertFalse(mock_plugin.__get__.called) + + mock_network_manager.__get__.return_value = 'neutron' + mock_plugin.__get__ = Mock(return_value=None) + self.assertEquals({}, neutron()) + self.assertTrue(mock_plugin.__get__.called) + + mock_nvp_ctxt.return_value = {'nvp': 'nvp_context'} + mock_plugin.__get__.return_value = 'nvp' + self.assertEquals( + {'network_manager': 'neutron', + 'nvp': 'nvp_context', + 'neutron_alchemy_flags': {'pool_size': '20'}, + 'neutron_url': 'https://foo:9696'}, + neutron() + ) + + @patch.object(context.NeutronContext, 'neutron_ctxt') + @patch.object(context.NeutronContext, 'calico_ctxt') + @patch.object(context.NeutronContext, 'plugin') + @patch.object(context.NeutronContext, '_ensure_packages') + @patch.object(context.NeutronContext, 'network_manager') + def test_neutron_main_context_gen_calico(self, mock_network_manager, + mock_ensure_packages, + mock_plugin, mock_ovs_ctxt, + mock_neutron_ctxt): + + mock_neutron_ctxt.return_value = {'network_manager': 'neutron', + 'neutron_url': 'https://foo:9696'} + config = {'neutron-alchemy-flags': None} + self.config.side_effect = lambda key: config[key] + neutron = context.NeutronContext() + + mock_network_manager.__get__ = Mock(return_value='flatdhcpmanager') + mock_plugin.__get__ = Mock() + + self.assertEquals({}, neutron()) + self.assertTrue(mock_network_manager.__get__.called) + self.assertFalse(mock_plugin.__get__.called) + + mock_network_manager.__get__.return_value = 'neutron' + mock_plugin.__get__ = Mock(return_value=None) + self.assertEquals({}, neutron()) + self.assertTrue(mock_plugin.__get__.called) + + mock_ovs_ctxt.return_value = {'Calico': 'calico_context'} + mock_plugin.__get__.return_value = 'Calico' + self.assertEquals( + {'network_manager': 'neutron', + 'Calico': 'calico_context', + 'neutron_url': 'https://foo:9696'}, + neutron() + ) + + @patch('charmhelpers.contrib.openstack.utils.juju_log', + lambda *args, **kwargs: None) + @patch.object(context, 'config') + def test_os_configflag_context(self, config): + flags = context.OSConfigFlagContext() + + # single + config.return_value = 'deadbeef=True' + self.assertEquals({ + 'user_config_flags': { + 'deadbeef': 'True', + } + }, flags()) + + # multi + config.return_value = 'floating_ip=True,use_virtio=False,max=5' + self.assertEquals({ + 'user_config_flags': { + 'floating_ip': 'True', + 'use_virtio': 'False', + 'max': '5', + } + }, flags()) + + for empty in [None, '']: + config.return_value = empty + self.assertEquals({}, flags()) + + # multi with commas + config.return_value = 'good_flag=woot,badflag,great_flag=w00t' + self.assertEquals({ + 'user_config_flags': { + 'good_flag': 'woot,badflag', + 'great_flag': 'w00t', + } + }, flags()) + + # missing key + config.return_value = 'good_flag=woot=toow' + self.assertRaises(context.OSContextError, flags) + + # bad value + config.return_value = 'good_flag=woot==' + self.assertRaises(context.OSContextError, flags) + + @patch.object(context, 'config') + def test_os_configflag_context_custom(self, config): + flags = context.OSConfigFlagContext( + charm_flag='api-config-flags', + template_flag='api_config_flags') + + # single + config.return_value = 'deadbeef=True' + self.assertEquals({ + 'api_config_flags': { + 'deadbeef': 'True', + } + }, flags()) + + def test_os_subordinate_config_context(self): + relation = FakeRelation(relation_data=SUB_CONFIG_RELATION) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + nova_sub_ctxt = context.SubordinateConfigContext( + service='nova', + config_file='/etc/nova/nova.conf', + interface='nova-subordinate', + ) + glance_sub_ctxt = context.SubordinateConfigContext( + service='glance', + config_file='/etc/glance/glance.conf', + interface='glance-subordinate', + ) + cinder_sub_ctxt = context.SubordinateConfigContext( + service='cinder', + config_file='/etc/cinder/cinder.conf', + interface='cinder-subordinate', + ) + foo_sub_ctxt = context.SubordinateConfigContext( + service='foo', + config_file='/etc/foo/foo.conf', + interface='foo-subordinate', + ) + empty_sub_ctxt = context.SubordinateConfigContext( + service='empty', + config_file='/etc/foo/foo.conf', + interface='empty-subordinate', + ) + self.assertEquals( + nova_sub_ctxt(), + {'sections': { + 'DEFAULT': [ + ['nova-key1', 'value1'], + ['nova-key2', 'value2']] + }} + ) + self.assertEquals( + glance_sub_ctxt(), + {'sections': { + 'DEFAULT': [ + ['glance-key1', 'value1'], + ['glance-key2', 'value2']] + }} + ) + self.assertEquals( + cinder_sub_ctxt(), + {'sections': { + 'cinder-1-section': [ + ['key1', 'value1']], + 'cinder-2-section': [ + ['key2', 'value2']] + + }, 'not-a-section': 1234} + ) + self.assertTrue( + cinder_sub_ctxt.context_complete(cinder_sub_ctxt())) + + # subrodinate supplies nothing for given config + glance_sub_ctxt.config_file = '/etc/glance/glance-api-paste.ini' + self.assertEquals(glance_sub_ctxt(), {}) + + # subordinate supplies bad input + self.assertEquals(foo_sub_ctxt(), {}) + self.assertEquals(empty_sub_ctxt(), {}) + self.assertFalse( + empty_sub_ctxt.context_complete(empty_sub_ctxt())) + + def test_os_subordinate_config_context_multiple(self): + relation = FakeRelation(relation_data=SUB_CONFIG_RELATION2) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.relation_units + nova_sub_ctxt = context.SubordinateConfigContext( + service=['nova', 'nova-compute'], + config_file='/etc/nova/nova.conf', + interface=['nova-ceilometer', 'neutron-plugin'], + ) + self.assertEquals( + nova_sub_ctxt(), + {'sections': { + 'DEFAULT': [ + ['nova-key1', 'value1'], + ['nova-key2', 'value2'], + ['nova-key3', 'value3'], + ['nova-key4', 'value4'], + ['nova-key5', 'value5'], + ['nova-key6', 'value6']] + }} + ) + + def test_syslog_context(self): + self.config.side_effect = fake_config({'use-syslog': 'foo'}) + syslog = context.SyslogContext() + result = syslog() + expected = { + 'use_syslog': 'foo', + } + self.assertEquals(result, expected) + + def test_loglevel_context_set(self): + self.config.side_effect = fake_config({ + 'debug': True, + 'verbose': True, + }) + syslog = context.LogLevelContext() + result = syslog() + expected = { + 'debug': True, + 'verbose': True, + } + self.assertEquals(result, expected) + + def test_loglevel_context_unset(self): + self.config.side_effect = fake_config({ + 'debug': None, + 'verbose': None, + }) + syslog = context.LogLevelContext() + result = syslog() + expected = { + 'debug': False, + 'verbose': False, + } + self.assertEquals(result, expected) + + @patch.object(context, '_calculate_workers') + def test_wsgi_worker_config_context(self, + _calculate_workers): + self.config.return_value = 2 # worker-multiplier=2 + _calculate_workers.return_value = 8 + service_name = 'service-name' + script = '/usr/bin/script' + ctxt = context.WSGIWorkerConfigContext(name=service_name, + script=script) + expect = { + "service_name": service_name, + "user": service_name, + "group": service_name, + "script": script, + "admin_script": None, + "public_script": None, + "processes": 8, + "admin_processes": 2, + "public_processes": 6, + "threads": 1, + } + self.assertEqual(expect, ctxt()) + + @patch.object(context, '_calculate_workers') + def test_wsgi_worker_config_context_user_and_group(self, + _calculate_workers): + self.config.return_value = 1 + _calculate_workers.return_value = 1 + service_name = 'service-name' + script = '/usr/bin/script' + user = 'nova' + group = 'nobody' + ctxt = context.WSGIWorkerConfigContext(name=service_name, + user=user, + group=group, + script=script) + expect = { + "service_name": service_name, + "user": user, + "group": group, + "script": script, + "admin_script": None, + "public_script": None, + "processes": 1, + "admin_processes": 1, + "public_processes": 1, + "threads": 1, + } + self.assertEqual(expect, ctxt()) + + def test_zeromq_context_unrelated(self): + self.is_relation_made.return_value = False + self.assertEquals(context.ZeroMQContext()(), {}) + + def test_zeromq_context_related(self): + self.is_relation_made.return_value = True + self.relation_ids.return_value = ['zeromq-configuration:1'] + self.related_units.return_value = ['openstack-zeromq/0'] + self.relation_get.side_effect = ['nonce-data', 'hostname', 'redis'] + self.assertEquals(context.ZeroMQContext()(), + {'zmq_host': 'hostname', + 'zmq_nonce': 'nonce-data', + 'zmq_redis_address': 'redis'}) + + def test_notificationdriver_context_nomsg(self): + relations = { + 'zeromq-configuration': False, + 'amqp': False, + } + rels = fake_is_relation_made(relations=relations) + self.is_relation_made.side_effect = rels.rel_made + self.assertEquals(context.NotificationDriverContext()(), + {'notifications': 'False'}) + + def test_notificationdriver_context_zmq_nometer(self): + relations = { + 'zeromq-configuration': True, + 'amqp': False, + } + rels = fake_is_relation_made(relations=relations) + self.is_relation_made.side_effect = rels.rel_made + self.assertEquals(context.NotificationDriverContext()(), + {'notifications': 'False'}) + + def test_notificationdriver_context_zmq_meter(self): + relations = { + 'zeromq-configuration': True, + 'amqp': False, + } + rels = fake_is_relation_made(relations=relations) + self.is_relation_made.side_effect = rels.rel_made + self.assertEquals(context.NotificationDriverContext()(), + {'notifications': 'False'}) + + def test_notificationdriver_context_amq(self): + relations = { + 'zeromq-configuration': False, + 'amqp': True, + } + rels = fake_is_relation_made(relations=relations) + self.is_relation_made.side_effect = rels.rel_made + self.assertEquals(context.NotificationDriverContext()(), + {'notifications': 'True'}) + + @patch.object(context, 'psutil') + def test_num_cpus_xenial(self, _psutil): + _psutil.cpu_count.return_value = 4 + self.assertEqual(context._num_cpus(), 4) + + @patch.object(context, 'psutil') + def test_num_cpus_trusty(self, _psutil): + _psutil.cpu_count.side_effect = AttributeError + _psutil.NUM_CPUS = 4 + self.assertEqual(context._num_cpus(), 4) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_float(self, _num_cpus): + self.config.side_effect = fake_config({ + 'worker-multiplier': 0.3 + }) + _num_cpus.return_value = 8 + self.assertEqual(context._calculate_workers(), 2) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_float_negative(self, _num_cpus): + self.config.side_effect = fake_config({ + 'worker-multiplier': -4.0 + }) + _num_cpus.return_value = 8 + self.assertEqual(context._calculate_workers(), 1) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_not_quite_0(self, _num_cpus): + # Make sure that the multiplier evaluating to somewhere between + # 0 and 1 in the floating point range still has at least one + # worker. + self.config.side_effect = fake_config({ + 'worker-multiplier': 0.001 + }) + _num_cpus.return_value = 100 + self.assertEqual(context._calculate_workers(), 1) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_0(self, _num_cpus): + self.config.side_effect = fake_config({ + 'worker-multiplier': 0 + }) + _num_cpus.return_value = 2 + self.assertEqual(context._calculate_workers(), 1) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_noconfig(self, _num_cpus): + self.config.return_value = None + _num_cpus.return_value = 1 + self.assertEqual(context._calculate_workers(), 2) + + @patch.object(context, '_num_cpus') + def test_calculate_workers_noconfig_lotsa_cpus(self, _num_cpus): + self.config.return_value = None + _num_cpus.return_value = 32 + self.assertEqual(context._calculate_workers(), 4) + + @patch.object(context, '_calculate_workers', return_value=256) + def test_worker_context(self, calculate_workers): + self.assertEqual(context.WorkerConfigContext()(), + {'workers': 256}) + + def test_apache_get_addresses_no_network_config(self): + self.config.side_effect = fake_config({ + 'os-internal-network': None, + 'os-admin-network': None, + 'os-public-network': None + }) + self.resolve_address.return_value = '10.5.1.50' + self.local_address.return_value = '10.5.1.50' + + apache = context.ApacheSSLContext() + apache.external_ports = '8776' + + addresses = apache.get_network_addresses() + expected = [('10.5.1.50', '10.5.1.50')] + + self.assertEqual(addresses, expected) + + self.get_address_in_network.assert_not_called() + self.resolve_address.assert_has_calls([ + call(context.INTERNAL), + call(context.ADMIN), + call(context.PUBLIC) + ]) + + def test_apache_get_addresses_with_network_config(self): + self.config.side_effect = fake_config({ + 'os-internal-network': '10.5.1.0/24', + 'os-admin-network': '10.5.2.0/24', + 'os-public-network': '10.5.3.0/24', + }) + _base_addresses = ['10.5.1.100', + '10.5.2.100', + '10.5.3.100'] + self.get_address_in_network.side_effect = _base_addresses + self.resolve_address.side_effect = _base_addresses + self.local_address.return_value = '10.5.1.50' + + apache = context.ApacheSSLContext() + + addresses = apache.get_network_addresses() + expected = [('10.5.1.100', '10.5.1.100'), + ('10.5.2.100', '10.5.2.100'), + ('10.5.3.100', '10.5.3.100')] + self.assertEqual(addresses, expected) + + calls = [call('10.5.1.0/24', '10.5.1.50'), + call('10.5.2.0/24', '10.5.1.50'), + call('10.5.3.0/24', '10.5.1.50')] + self.get_address_in_network.assert_has_calls(calls) + self.resolve_address.assert_has_calls([ + call(context.INTERNAL), + call(context.ADMIN), + call(context.PUBLIC) + ]) + + def test_apache_get_addresses_network_spaces(self): + self.config.side_effect = fake_config({ + 'os-internal-network': None, + 'os-admin-network': None, + 'os-public-network': None + }) + self.network_get_primary_address.side_effect = None + self.network_get_primary_address.return_value = '10.5.2.50' + self.resolve_address.return_value = '10.5.2.100' + self.local_address.return_value = '10.5.1.50' + + apache = context.ApacheSSLContext() + apache.external_ports = '8776' + + addresses = apache.get_network_addresses() + expected = [('10.5.2.50', '10.5.2.100')] + + self.assertEqual(addresses, expected) + + self.get_address_in_network.assert_not_called() + self.resolve_address.assert_has_calls([ + call(context.INTERNAL), + call(context.ADMIN), + call(context.PUBLIC) + ]) + + def test_config_flag_parsing_simple(self): + # Standard key=value checks... + flags = context.config_flags_parser('key1=value1, key2=value2') + self.assertEqual(flags, {'key1': 'value1', 'key2': 'value2'}) + + # Check for multiple values to a single key + flags = context.config_flags_parser('key1=value1, ' + 'key2=value2,value3,value4') + self.assertEqual(flags, {'key1': 'value1', + 'key2': 'value2,value3,value4'}) + + # Check for yaml formatted key value pairings for more complex + # assignment options. + flags = context.config_flags_parser('key1: subkey1=value1,' + 'subkey2=value2') + self.assertEqual(flags, {'key1': 'subkey1=value1,subkey2=value2'}) + + # Check for good measure the ldap formats + test_string = ('user_tree_dn: ou=ABC General,' + 'ou=User Accounts,dc=example,dc=com') + flags = context.config_flags_parser(test_string) + self.assertEqual(flags, {'user_tree_dn': ('ou=ABC General,' + 'ou=User Accounts,' + 'dc=example,dc=com')}) + + def _fake_get_hwaddr(self, arg): + return MACHINE_MACS[arg] + + def _fake_get_ipv4(self, arg, fatal=False): + return MACHINE_NICS[arg] + + @patch('charmhelpers.contrib.openstack.context.config') + def test_no_ext_port(self, mock_config): + self.config.side_effect = config = fake_config({}) + mock_config.side_effect = config + self.assertEquals(context.ExternalPortContext()(), {}) + + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_eth(self, mock_config, mock_list_nics): + config = fake_config({'ext-port': 'eth1010'}) + self.config.side_effect = config + mock_config.side_effect = config + mock_list_nics.return_value = ['eth1010'] + self.assertEquals(context.ExternalPortContext()(), + {'ext_port': 'eth1010'}) + + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_eth_non_existent(self, mock_config, mock_list_nics): + config = fake_config({'ext-port': 'eth1010'}) + self.config.side_effect = config + mock_config.side_effect = config + mock_list_nics.return_value = [] + self.assertEquals(context.ExternalPortContext()(), {}) + + @patch('charmhelpers.contrib.openstack.context.is_phy_iface', + lambda arg: True) + @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') + @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_mac(self, mock_config, mock_get_ipv4_addr, + mock_get_ipv6_addr, mock_list_nics, + mock_get_nic_hwaddr): + config_macs = ABSENT_MACS + " " + MACHINE_MACS['eth2'] + config = fake_config({'ext-port': config_macs}) + self.config.side_effect = config + mock_config.side_effect = config + + mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 + mock_get_ipv6_addr.return_value = [] + mock_list_nics.return_value = MACHINE_MACS.keys() + mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr + + self.assertEquals(context.ExternalPortContext()(), + {'ext_port': 'eth2'}) + + config = fake_config({'ext-port': ABSENT_MACS}) + self.config.side_effect = config + mock_config.side_effect = config + + self.assertEquals(context.ExternalPortContext()(), {}) + + @patch('charmhelpers.contrib.openstack.context.is_phy_iface', + lambda arg: True) + @patch('charmhelpers.contrib.openstack.context.get_nic_hwaddr') + @patch('charmhelpers.contrib.openstack.context.list_nics') + @patch('charmhelpers.contrib.openstack.context.get_ipv6_addr') + @patch('charmhelpers.contrib.openstack.context.get_ipv4_addr') + @patch('charmhelpers.contrib.openstack.context.config') + def test_ext_port_mac_one_used_nic(self, mock_config, + mock_get_ipv4_addr, + mock_get_ipv6_addr, mock_list_nics, + mock_get_nic_hwaddr): + + self.relation_ids.return_value = ['neutron-plugin-api:1'] + self.related_units.return_value = ['neutron-api/0'] + self.relation_get.return_value = {'network-device-mtu': 1234, + 'l2-population': 'False'} + config_macs = "%s %s" % (MACHINE_MACS['eth1'], + MACHINE_MACS['eth2']) + + mock_get_ipv4_addr.side_effect = self._fake_get_ipv4 + mock_get_ipv6_addr.return_value = [] + mock_list_nics.return_value = MACHINE_MACS.keys() + mock_get_nic_hwaddr.side_effect = self._fake_get_hwaddr + + config = fake_config({'ext-port': config_macs}) + self.config.side_effect = config + mock_config.side_effect = config + self.assertEquals(context.ExternalPortContext()(), + {'ext_port': 'eth2', 'ext_port_mtu': 1234}) + + @patch('charmhelpers.contrib.openstack.context.NeutronPortContext.' + 'resolve_ports') + def test_data_port_eth(self, mock_resolve): + self.config.side_effect = fake_config({'data-port': + 'phybr1:eth1010 ' + 'phybr1:eth1011'}) + mock_resolve.side_effect = lambda ports: ['eth1010'] + self.assertEquals(context.DataPortContext()(), + {'eth1010': 'phybr1'}) + + @patch.object(context, 'get_nic_hwaddr') + @patch.object(context.NeutronPortContext, 'resolve_ports') + def test_data_port_mac(self, mock_resolve, mock_get_nic_hwaddr): + extant_mac = 'cb:23:ae:72:f2:33' + non_extant_mac = 'fa:16:3e:12:97:8e' + self.config.side_effect = fake_config({'data-port': + 'phybr1:%s phybr1:%s' % + (non_extant_mac, extant_mac)}) + + def fake_resolve(ports): + resolved = [] + for port in ports: + if port == extant_mac: + resolved.append('eth1010') + + return resolved + + mock_get_nic_hwaddr.side_effect = lambda nic: extant_mac + mock_resolve.side_effect = fake_resolve + + self.assertEquals(context.DataPortContext()(), + {'eth1010': 'phybr1'}) + + @patch.object(context.NeutronAPIContext, '__call__', lambda *args: + {'network_device_mtu': 5000}) + @patch.object(context, 'get_nic_hwaddr', lambda inst, port: port) + @patch.object(context.NeutronPortContext, 'resolve_ports', + lambda inst, ports: ports) + def test_phy_nic_mtu_context(self): + self.config.side_effect = fake_config({'data-port': + 'phybr1:eth0'}) + ctxt = context.PhyNICMTUContext()() + self.assertEqual(ctxt, {'devs': 'eth0', 'mtu': 5000}) + + @patch.object(context.glob, 'glob') + @patch.object(context.NeutronAPIContext, '__call__', lambda *args: + {'network_device_mtu': 5000}) + @patch.object(context, 'get_nic_hwaddr', lambda inst, port: port) + @patch.object(context.NeutronPortContext, 'resolve_ports', + lambda inst, ports: ports) + def test_phy_nic_mtu_context_vlan(self, mock_glob): + self.config.side_effect = fake_config({'data-port': + 'phybr1:eth0.100'}) + mock_glob.return_value = ['/sys/class/net/eth0.100/lower_eth0'] + ctxt = context.PhyNICMTUContext()() + self.assertEqual(ctxt, {'devs': 'eth0\\neth0.100', 'mtu': 5000}) + + @patch.object(context.glob, 'glob') + @patch.object(context.NeutronAPIContext, '__call__', lambda *args: + {'network_device_mtu': 5000}) + @patch.object(context, 'get_nic_hwaddr', lambda inst, port: port) + @patch.object(context.NeutronPortContext, 'resolve_ports', + lambda inst, ports: ports) + def test_phy_nic_mtu_context_vlan_w_duplicate_raw(self, mock_glob): + self.config.side_effect = fake_config({'data-port': + 'phybr1:eth0.100 ' + 'phybr1:eth0.200'}) + + def fake_glob(wcard): + if 'eth0.100' in wcard: + return ['/sys/class/net/eth0.100/lower_eth0'] + elif 'eth0.200' in wcard: + return ['/sys/class/net/eth0.200/lower_eth0'] + + raise Exception("Unexpeced key '%s'" % (wcard)) + + mock_glob.side_effect = fake_glob + ctxt = context.PhyNICMTUContext()() + self.assertEqual(ctxt, {'devs': 'eth0\\neth0.100\\neth0.200', + 'mtu': 5000}) + + def test_neutronapicontext_defaults(self): + self.relation_ids.return_value = [] + expected_keys = [ + 'l2_population', 'enable_dvr', 'enable_l3ha', + 'overlay_network_type', 'network_device_mtu', + 'enable_qos', 'enable_nsg_logging', 'global_physnet_mtu', + 'physical_network_mtus' + ] + api_ctxt = context.NeutronAPIContext()() + for key in expected_keys: + self.assertTrue(key in api_ctxt) + self.assertEquals(api_ctxt['polling_interval'], 2) + self.assertEquals(api_ctxt['rpc_response_timeout'], 60) + self.assertEquals(api_ctxt['report_interval'], 30) + self.assertEquals(api_ctxt['enable_nsg_logging'], False) + self.assertEquals(api_ctxt['global_physnet_mtu'], 1500) + self.assertIsNone(api_ctxt['physical_network_mtus']) + + def setup_neutron_api_context_relation(self, cfg): + self.relation_ids.return_value = ['neutron-plugin-api:1'] + self.related_units.return_value = ['neutron-api/0'] + # The l2-population key is used by the context as a way of checking if + # the api service on the other end is sending data in a recent format. + self.relation_get.return_value = cfg + + def test_neutronapicontext_extension_drivers_qos_on(self): + self.setup_neutron_api_context_relation({ + 'enable-qos': 'True', + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertTrue(api_ctxt['enable_qos']) + self.assertEquals(api_ctxt['extension_drivers'], 'qos') + + def test_neutronapicontext_extension_drivers_qos_off(self): + self.setup_neutron_api_context_relation({ + 'enable-qos': 'False', + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertFalse(api_ctxt['enable_qos']) + self.assertEquals(api_ctxt['extension_drivers'], '') + + def test_neutronapicontext_extension_drivers_qos_absent(self): + self.setup_neutron_api_context_relation({ + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertFalse(api_ctxt['enable_qos']) + self.assertEquals(api_ctxt['extension_drivers'], '') + + def test_neutronapicontext_extension_drivers_log_off(self): + self.setup_neutron_api_context_relation({ + 'enable-nsg-logging': 'False', + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['extension_drivers'], '') + + def test_neutronapicontext_extension_drivers_log_on(self): + self.setup_neutron_api_context_relation({ + 'enable-nsg-logging': 'True', + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['extension_drivers'], 'log') + + def test_neutronapicontext_extension_drivers_log_qos_on(self): + self.setup_neutron_api_context_relation({ + 'enable-qos': 'True', + 'enable-nsg-logging': 'True', + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['extension_drivers'], 'qos,log') + + def test_neutronapicontext_firewall_group_logging_on(self): + self.setup_neutron_api_context_relation({ + 'enable-nfg-logging': 'True', + 'l2-population': 'True' + }) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['enable_nfg_logging'], True) + + def test_neutronapicontext_firewall_group_logging_off(self): + self.setup_neutron_api_context_relation({ + 'enable-nfg-logging': 'False', + 'l2-population': 'True' + }) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['enable_nfg_logging'], False) + + def test_neutronapicontext_port_forwarding_on(self): + self.setup_neutron_api_context_relation({ + 'enable-port-forwarding': 'True', + 'l2-population': 'True' + }) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['enable_port_forwarding'], True) + + def test_neutronapicontext_port_forwarding_off(self): + self.setup_neutron_api_context_relation({ + 'enable-port-forwarding': 'False', + 'l2-population': 'True' + }) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['enable_port_forwarding'], False) + + def test_neutronapicontext_string_converted(self): + self.setup_neutron_api_context_relation({ + 'l2-population': 'True'}) + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['l2_population'], True) + + def test_neutronapicontext_none(self): + self.relation_ids.return_value = ['neutron-plugin-api:1'] + self.related_units.return_value = ['neutron-api/0'] + self.relation_get.return_value = {'l2-population': 'True'} + api_ctxt = context.NeutronAPIContext()() + self.assertEquals(api_ctxt['network_device_mtu'], None) + + def test_network_service_ctxt_no_units(self): + self.relation_ids.return_value = [] + self.relation_ids.return_value = ['foo'] + self.related_units.return_value = [] + self.assertEquals(context.NetworkServiceContext()(), {}) + + @patch.object(context.OSContextGenerator, 'context_complete') + def test_network_service_ctxt_no_data(self, mock_context_complete): + rel = FakeRelation(QUANTUM_NETWORK_SERVICE_RELATION) + self.relation_ids.side_effect = rel.relation_ids + self.related_units.side_effect = rel.relation_units + relation = FakeRelation(relation_data=QUANTUM_NETWORK_SERVICE_RELATION) + self.relation_get.side_effect = relation.get + mock_context_complete.return_value = False + self.assertEquals(context.NetworkServiceContext()(), {}) + + def test_network_service_ctxt_data(self): + data_result = { + 'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion', + 'service_protocol': 'http', + 'auth_protocol': 'http', + 'api_version': '2.0', + } + rel = FakeRelation(QUANTUM_NETWORK_SERVICE_RELATION) + self.relation_ids.side_effect = rel.relation_ids + self.related_units.side_effect = rel.relation_units + relation = FakeRelation(relation_data=QUANTUM_NETWORK_SERVICE_RELATION) + self.relation_get.side_effect = relation.get + self.assertEquals(context.NetworkServiceContext()(), data_result) + + def test_network_service_ctxt_data_api_version(self): + data_result = { + 'keystone_host': '10.5.0.1', + 'service_port': '5000', + 'auth_port': '20000', + 'service_tenant': 'tenant', + 'service_username': 'username', + 'service_password': 'password', + 'quantum_host': '10.5.0.2', + 'quantum_port': '9696', + 'quantum_url': 'http://10.5.0.2:9696/v2', + 'region': 'aregion', + 'service_protocol': 'http', + 'auth_protocol': 'http', + 'api_version': '3', + } + rel = FakeRelation(QUANTUM_NETWORK_SERVICE_RELATION_VERSIONED) + self.relation_ids.side_effect = rel.relation_ids + self.related_units.side_effect = rel.relation_units + relation = FakeRelation( + relation_data=QUANTUM_NETWORK_SERVICE_RELATION_VERSIONED) + self.relation_get.side_effect = relation.get + self.assertEquals(context.NetworkServiceContext()(), data_result) + + def test_internal_endpoint_context(self): + config = {'use-internal-endpoints': False} + self.config.side_effect = fake_config(config) + ctxt = context.InternalEndpointContext() + self.assertFalse(ctxt()['use_internal_endpoints']) + config = {'use-internal-endpoints': True} + self.config.side_effect = fake_config(config) + self.assertTrue(ctxt()['use_internal_endpoints']) + + @patch.object(context, 'os_release') + def test_volume_api_context(self, mock_os_release): + mock_os_release.return_value = 'ocata' + config = {'use-internal-endpoints': False} + self.config.side_effect = fake_config(config) + ctxt = context.VolumeAPIContext('cinder-common') + c = ctxt() + self.assertEqual(c['volume_api_version'], '2') + self.assertEqual(c['volume_catalog_info'], + 'volumev2:cinderv2:publicURL') + + mock_os_release.return_value = 'pike' + config['use-internal-endpoints'] = True + self.config.side_effect = fake_config(config) + ctxt = context.VolumeAPIContext('cinder-common') + c = ctxt() + self.assertEqual(c['volume_api_version'], '3') + self.assertEqual(c['volume_catalog_info'], + 'volumev3:cinderv3:internalURL') + + def test_volume_api_context_no_pkg(self): + self.assertRaises(ValueError, context.VolumeAPIContext, "") + self.assertRaises(ValueError, context.VolumeAPIContext, None) + + def test_apparmor_context_call_not_valid(self): + ''' Tests for the apparmor context''' + mock_aa_object = context.AppArmorContext() + # Test with invalid config + self.config.return_value = 'NOTVALID' + self.assertEquals(mock_aa_object.__call__(), None) + + def test_apparmor_context_call_complain(self): + ''' Tests for the apparmor context''' + mock_aa_object = context.AppArmorContext() + # Test complain mode + self.config.return_value = 'complain' + self.assertEquals(mock_aa_object.__call__(), + {'aa_profile_mode': 'complain', + 'ubuntu_release': '16.04'}) + + def test_apparmor_context_call_enforce(self): + ''' Tests for the apparmor context''' + mock_aa_object = context.AppArmorContext() + # Test enforce mode + self.config.return_value = 'enforce' + self.assertEquals(mock_aa_object.__call__(), + {'aa_profile_mode': 'enforce', + 'ubuntu_release': '16.04'}) + + def test_apparmor_context_call_disable(self): + ''' Tests for the apparmor context''' + mock_aa_object = context.AppArmorContext() + # Test complain mode + self.config.return_value = 'disable' + self.assertEquals(mock_aa_object.__call__(), + {'aa_profile_mode': 'disable', + 'ubuntu_release': '16.04'}) + + def test_apparmor_setup_complain(self): + ''' Tests for the apparmor setup''' + AA = context.AppArmorContext(profile_name='fake-aa-profile') + AA.install_aa_utils = MagicMock() + AA.manually_disable_aa_profile = MagicMock() + # Test complain mode + self.config.return_value = 'complain' + AA.setup_aa_profile() + AA.install_aa_utils.assert_called_with() + self.check_call.assert_called_with(['aa-complain', 'fake-aa-profile']) + self.assertFalse(AA.manually_disable_aa_profile.called) + + def test_apparmor_setup_enforce(self): + ''' Tests for the apparmor setup''' + AA = context.AppArmorContext(profile_name='fake-aa-profile') + AA.install_aa_utils = MagicMock() + AA.manually_disable_aa_profile = MagicMock() + # Test enforce mode + self.config.return_value = 'enforce' + AA.setup_aa_profile() + self.check_call.assert_called_with(['aa-enforce', 'fake-aa-profile']) + self.assertFalse(AA.manually_disable_aa_profile.called) + + def test_apparmor_setup_disable(self): + ''' Tests for the apparmor setup''' + AA = context.AppArmorContext(profile_name='fake-aa-profile') + AA.install_aa_utils = MagicMock() + AA.manually_disable_aa_profile = MagicMock() + # Test disable mode + self.config.return_value = 'disable' + AA.setup_aa_profile() + self.check_call.assert_called_with(['aa-disable', 'fake-aa-profile']) + self.assertFalse(AA.manually_disable_aa_profile.called) + # Test failed to disable + from subprocess import CalledProcessError + self.check_call.side_effect = CalledProcessError(0, 0, 0) + AA.setup_aa_profile() + self.check_call.assert_called_with(['aa-disable', 'fake-aa-profile']) + AA.manually_disable_aa_profile.assert_called_with() + + @patch.object(context, 'enable_memcache') + @patch.object(context, 'is_ipv6_disabled') + def test_memcache_context_ipv6(self, _is_ipv6_disabled, _enable_memcache): + self.lsb_release.return_value = {'DISTRIB_CODENAME': 'xenial'} + _enable_memcache.return_value = True + _is_ipv6_disabled.return_value = False + config = { + 'openstack-origin': 'distro', + } + self.config.side_effect = fake_config(config) + ctxt = context.MemcacheContext() + self.assertTrue(ctxt()['use_memcache']) + expect = { + 'memcache_port': '11211', + 'memcache_server': '::1', + 'memcache_server_formatted': '[::1]', + 'memcache_url': 'inet6:[::1]:11211', + 'use_memcache': True} + self.assertEqual(ctxt(), expect) + self.lsb_release.return_value = {'DISTRIB_CODENAME': 'trusty'} + expect['memcache_server'] = 'ip6-localhost' + ctxt = context.MemcacheContext() + self.assertEqual(ctxt(), expect) + + @patch.object(context, 'enable_memcache') + @patch.object(context, 'is_ipv6_disabled') + def test_memcache_context_ipv4(self, _is_ipv6_disabled, _enable_memcache): + self.lsb_release.return_value = {'DISTRIB_CODENAME': 'xenial'} + _enable_memcache.return_value = True + _is_ipv6_disabled.return_value = True + config = { + 'openstack-origin': 'distro', + } + self.config.side_effect = fake_config(config) + ctxt = context.MemcacheContext() + self.assertTrue(ctxt()['use_memcache']) + expect = { + 'memcache_port': '11211', + 'memcache_server': '127.0.0.1', + 'memcache_server_formatted': '127.0.0.1', + 'memcache_url': '127.0.0.1:11211', + 'use_memcache': True} + self.assertEqual(ctxt(), expect) + self.lsb_release.return_value = {'DISTRIB_CODENAME': 'trusty'} + expect['memcache_server'] = 'localhost' + ctxt = context.MemcacheContext() + self.assertEqual(ctxt(), expect) + + @patch.object(context, 'enable_memcache') + def test_memcache_off_context(self, _enable_memcache): + _enable_memcache.return_value = False + config = {'openstack-origin': 'distro'} + self.config.side_effect = fake_config(config) + ctxt = context.MemcacheContext() + self.assertFalse(ctxt()['use_memcache']) + self.assertEqual(ctxt(), {'use_memcache': False}) + + @patch('charmhelpers.contrib.openstack.context.mkdir') + def test_ensure_dir_ctx(self, mkdir): + dirname = '/etc/keystone/policy.d' + owner = 'someuser' + group = 'somegroup' + perms = 0o555 + force = False + ctxt = context.EnsureDirContext(dirname, owner=owner, + group=group, perms=perms, + force=force) + ctxt() + mkdir.assert_called_with(dirname, owner=owner, group=group, + perms=perms, force=force) + + @patch.object(context, 'os_release') + def test_VersionsContext(self, os_release): + self.lsb_release.return_value = {'DISTRIB_CODENAME': 'xenial'} + os_release.return_value = 'essex' + self.assertEqual( + context.VersionsContext()(), + { + 'openstack_release': 'essex', + 'operating_system_release': 'xenial'}) + os_release.assert_called_once_with('python-keystone') + self.lsb_release.assert_called_once_with() + + def test_logrotate_context_unset(self): + logrotate = context.LogrotateContext(location='nova', + interval='weekly', + count=4) + ctxt = logrotate() + expected_ctxt = { + 'logrotate_logs_location': 'nova', + 'logrotate_interval': 'weekly', + 'logrotate_count': 'rotate 4', + } + self.assertEquals(ctxt, expected_ctxt) + + @patch.object(context, 'os_release') + def test_vendordata_static(self, os_release): + _vdata = '{"good": "json"}' + os_release.return_value = 'rocky' + self.config.side_effect = [_vdata, None] + ctxt = context.NovaVendorMetadataContext('nova-common')() + + self.assertTrue(ctxt['vendor_data']) + self.assertEqual('StaticJSON', ctxt['vendordata_providers']) + self.assertNotIn('vendor_data_url', ctxt) + + @patch.object(context, 'os_release') + def test_vendordata_dynamic(self, os_release): + _vdata_url = 'http://example.org/vdata' + os_release.return_value = 'rocky' + + self.config.side_effect = [None, _vdata_url] + ctxt = context.NovaVendorMetadataContext('nova-common')() + + self.assertEqual(_vdata_url, ctxt['vendor_data_url']) + self.assertEqual('DynamicJSON', ctxt['vendordata_providers']) + self.assertFalse(ctxt['vendor_data']) + + @patch.object(context, 'os_release') + def test_vendordata_static_and_dynamic(self, os_release): + os_release.return_value = 'rocky' + _vdata = '{"good": "json"}' + _vdata_url = 'http://example.org/vdata' + + self.config.side_effect = [_vdata, _vdata_url] + ctxt = context.NovaVendorMetadataContext('nova-common')() + + self.assertTrue(ctxt['vendor_data']) + self.assertEqual(_vdata_url, ctxt['vendor_data_url']) + self.assertEqual('StaticJSON,DynamicJSON', + ctxt['vendordata_providers']) + + @patch.object(context, 'log') + @patch.object(context, 'os_release') + def test_vendordata_static_invalid_and_dynamic(self, os_release, log): + os_release.return_value = 'rocky' + _vdata = '{bad: json}' + _vdata_url = 'http://example.org/vdata' + + self.config.side_effect = [_vdata, _vdata_url] + ctxt = context.NovaVendorMetadataContext('nova-common')() + + self.assertFalse(ctxt['vendor_data']) + self.assertEqual(_vdata_url, ctxt['vendor_data_url']) + self.assertEqual('DynamicJSON', ctxt['vendordata_providers']) + self.assertTrue(log.called) + + @patch('charmhelpers.contrib.openstack.context.log') + @patch.object(context, 'os_release') + def test_vendordata_static_and_dynamic_mitaka(self, os_release, log): + os_release.return_value = 'mitaka' + _vdata = '{"good": "json"}' + _vdata_url = 'http://example.org/vdata' + + self.config.side_effect = [_vdata, _vdata_url] + ctxt = context.NovaVendorMetadataContext('nova-common')() + + self.assertTrue(log.called) + self.assertTrue(ctxt['vendor_data']) + self.assertNotIn('vendor_data_url', ctxt) + self.assertNotIn('vendordata_providers', ctxt) + + @patch.object(context, 'log') + def test_vendordata_json_valid(self, log): + _vdata = '{"good": "json"}' + self.config.side_effect = [_vdata] + + ctxt = context.NovaVendorMetadataJSONContext('nova-common')() + + self.assertEqual({'vendor_data_json': _vdata}, ctxt) + self.assertFalse(log.called) + + @patch.object(context, 'log') + def test_vendordata_json_invalid(self, log): + _vdata = '{bad: json}' + self.config.side_effect = [_vdata] + + ctxt = context.NovaVendorMetadataJSONContext('nova-common')() + + self.assertEqual({'vendor_data_json': '{}'}, ctxt) + self.assertTrue(log.called) + + @patch.object(context, 'log') + def test_vendordata_json_empty(self, log): + self.config.side_effect = [None] + + ctxt = context.NovaVendorMetadataJSONContext('nova-common')() + + self.assertEqual({'vendor_data_json': '{}'}, ctxt) + self.assertFalse(log.called) + + @patch.object(context, 'socket') + def test_host_info_context(self, _socket): + _socket.getaddrinfo.return_value = [(None, None, None, 'myhost.mydomain', None)] + _socket.gethostname.return_value = 'myhost' + ctxt = context.HostInfoContext()() + self.assertEqual({ + 'host_fqdn': 'myhost.mydomain', + 'host': 'myhost', + 'use_fqdn_hint': False}, + ctxt) + ctxt = context.HostInfoContext(use_fqdn_hint_cb=lambda: True)() + self.assertEqual({ + 'host_fqdn': 'myhost.mydomain', + 'host': 'myhost', + 'use_fqdn_hint': True}, + ctxt) + # if getaddrinfo is unable to find the canonical name we should return + # the shortname to match the behaviour of the original implementation. + _socket.getaddrinfo.return_value = [(None, None, None, 'localhost', None)] + ctxt = context.HostInfoContext()() + self.assertEqual({ + 'host_fqdn': 'myhost', + 'host': 'myhost', + 'use_fqdn_hint': False}, + ctxt) + if six.PY2: + _socket.error = Exception + _socket.getaddrinfo.side_effect = Exception + else: + _socket.getaddrinfo.side_effect = OSError + _socket.gethostname.return_value = 'myhost' + ctxt = context.HostInfoContext()() + self.assertEqual({ + 'host_fqdn': 'myhost', + 'host': 'myhost', + 'use_fqdn_hint': False}, + ctxt) + + @patch.object(context, "DHCPAgentContext") + def test_validate_ovs_use_veth(self, _context): + # No existing dhcp_agent.ini and no config + _context.get_existing_ovs_use_veth.return_value = None + _context.parse_ovs_use_veth.return_value = None + self.assertEqual((None, None), context.validate_ovs_use_veth()) + + # No existing dhcp_agent.ini and config set + _context.get_existing_ovs_use_veth.return_value = None + _context.parse_ovs_use_veth.return_value = True + self.assertEqual((None, None), context.validate_ovs_use_veth()) + + # Existing dhcp_agent.ini and no config + _context.get_existing_ovs_use_veth.return_value = True + _context.parse_ovs_use_veth.return_value = None + self.assertEqual((None, None), context.validate_ovs_use_veth()) + + # Check for agreement with existing dhcp_agent.ini + _context.get_existing_ovs_use_veth.return_value = False + _context.parse_ovs_use_veth.return_value = False + self.assertEqual((None, None), context.validate_ovs_use_veth()) + + # Check for disagreement with existing dhcp_agent.ini + _context.get_existing_ovs_use_veth.return_value = True + _context.parse_ovs_use_veth.return_value = False + self.assertEqual( + ("blocked", + "Mismatched existing and configured ovs-use-veth. See log."), + context.validate_ovs_use_veth()) + + def test_dhcp_agent_context(self): + # Defaults + _config = { + "debug": False, + "dns-servers": None, + "enable-isolated-metadata": None, + "enable-metadata-network": None, + "instance-mtu": None, + "ovs-use-veth": None} + _expect = { + "append_ovs_config": False, + "debug": False, + "dns_servers": None, + "enable_isolated_metadata": None, + "enable_metadata_network": None, + "instance_mtu": None, + "ovs_use_veth": False} + self.config.side_effect = fake_config(_config) + _get_ovs_use_veth = MagicMock() + _get_ovs_use_veth.return_value = False + ctx_object = context.DHCPAgentContext() + ctx_object.get_ovs_use_veth = _get_ovs_use_veth + ctxt = ctx_object() + self.assertEqual(_expect, ctxt) + + # Non-defaults + _dns = "10.5.0.2" + _mtu = 8950 + _config = { + "debug": True, + "dns-servers": _dns, + "enable-isolated-metadata": True, + "enable-metadata-network": True, + "instance-mtu": _mtu, + "ovs-use-veth": True} + _expect = { + "append_ovs_config": False, + "debug": True, + "dns_servers": _dns, + "enable_isolated_metadata": True, + "enable_metadata_network": True, + "instance_mtu": _mtu, + "ovs_use_veth": True} + self.config.side_effect = fake_config(_config) + _get_ovs_use_veth.return_value = True + ctxt = ctx_object() + self.assertEqual(_expect, ctxt) + + def test_dhcp_agent_context_no_dns_domain(self): + _config = {"dns-servers": '8.8.8.8'} + self.config.side_effect = fake_config(_config) + self.relation_ids.return_value = ['rid1'] + self.related_units.return_value = ['nova-compute/0'] + self.relation_get.return_value = 'nova' + self.assertEqual( + context.DHCPAgentContext()(), + {'instance_mtu': None, + 'dns_servers': '8.8.8.8', + 'ovs_use_veth': False, + "enable_isolated_metadata": None, + "enable_metadata_network": None, + "debug": None, + "append_ovs_config": False} + ) + + def test_dhcp_agent_context_dnsmasq_flags(self): + _config = {'dnsmasq-flags': 'dhcp-userclass=set:ipxe,iPXE,' + 'dhcp-match=set:ipxe,175,' + 'server=1.2.3.4'} + self.config.side_effect = fake_config(_config) + self.assertEqual( + context.DHCPAgentContext()(), + { + 'dnsmasq_flags': collections.OrderedDict( + [('dhcp-userclass', 'set:ipxe,iPXE'), + ('dhcp-match', 'set:ipxe,175'), + ('server', '1.2.3.4')]), + 'instance_mtu': None, + 'dns_servers': None, + 'ovs_use_veth': False, + "enable_isolated_metadata": None, + "enable_metadata_network": None, + "debug": None, + "append_ovs_config": False, + } + ) + + def test_get_ovs_use_veth(self): + _get_existing_ovs_use_veth = MagicMock() + _parse_ovs_use_veth = MagicMock() + ctx_object = context.DHCPAgentContext() + ctx_object.get_existing_ovs_use_veth = _get_existing_ovs_use_veth + ctx_object.parse_ovs_use_veth = _parse_ovs_use_veth + + # Default + _get_existing_ovs_use_veth.return_value = None + _parse_ovs_use_veth.return_value = None + self.assertEqual(False, ctx_object.get_ovs_use_veth()) + + # Existing dhcp_agent.ini and no config + _get_existing_ovs_use_veth.return_value = True + _parse_ovs_use_veth.return_value = None + self.assertEqual(True, ctx_object.get_ovs_use_veth()) + + # No existing dhcp_agent.ini and config set + _get_existing_ovs_use_veth.return_value = None + _parse_ovs_use_veth.return_value = False + self.assertEqual(False, ctx_object.get_ovs_use_veth()) + + # Both set matching + _get_existing_ovs_use_veth.return_value = True + _parse_ovs_use_veth.return_value = True + self.assertEqual(True, ctx_object.get_ovs_use_veth()) + + # Both set mismatch: existing overrides + _get_existing_ovs_use_veth.return_value = False + _parse_ovs_use_veth.return_value = True + self.assertEqual(False, ctx_object.get_ovs_use_veth()) + + # Both set mismatch: existing overrides + _get_existing_ovs_use_veth.return_value = True + _parse_ovs_use_veth.return_value = False + self.assertEqual(True, ctx_object.get_ovs_use_veth()) + + @patch.object(context, 'config_ini') + @patch.object(context.os.path, 'isfile') + def test_get_existing_ovs_use_veth(self, _is_file, _config_ini): + _config = {"ovs-use-veth": None} + self.config.side_effect = fake_config(_config) + + ctx_object = context.DHCPAgentContext() + + # Default + _is_file.return_value = False + self.assertEqual(None, ctx_object.get_existing_ovs_use_veth()) + + # Existing + _is_file.return_value = True + _config_ini.return_value = {"DEFAULT": {"ovs_use_veth": True}} + self.assertEqual(True, ctx_object.get_existing_ovs_use_veth()) + + # Existing config_ini returns string + _is_file.return_value = True + _config_ini.return_value = {"DEFAULT": {"ovs_use_veth": "False"}} + self.assertEqual(False, ctx_object.get_existing_ovs_use_veth()) + + @patch.object(context, 'bool_from_string') + def test_parse_ovs_use_veth(self, _bool_from_string): + _config = {"ovs-use-veth": None} + self.config.side_effect = fake_config(_config) + + ctx_object = context.DHCPAgentContext() + + # Unset + self.assertEqual(None, ctx_object.parse_ovs_use_veth()) + _bool_from_string.assert_not_called() + + # Consider empty string unset + _config = {"ovs-use-veth": ""} + self.config.side_effect = fake_config(_config) + self.assertEqual(None, ctx_object.parse_ovs_use_veth()) + _bool_from_string.assert_not_called() + + # Lower true + _bool_from_string.return_value = True + _config = {"ovs-use-veth": "true"} + self.config.side_effect = fake_config(_config) + self.assertEqual(True, ctx_object.parse_ovs_use_veth()) + _bool_from_string.assert_called_with("true") + + # Lower false + _bool_from_string.return_value = False + _bool_from_string.reset_mock() + _config = {"ovs-use-veth": "false"} + self.config.side_effect = fake_config(_config) + self.assertEqual(False, ctx_object.parse_ovs_use_veth()) + _bool_from_string.assert_called_with("false") + + # Upper True + _bool_from_string.return_value = True + _bool_from_string.reset_mock() + _config = {"ovs-use-veth": "True"} + self.config.side_effect = fake_config(_config) + self.assertEqual(True, ctx_object.parse_ovs_use_veth()) + _bool_from_string.assert_called_with("True") + + # Invalid + _bool_from_string.reset_mock() + _config = {"ovs-use-veth": "Invalid"} + self.config.side_effect = fake_config(_config) + _bool_from_string.side_effect = ValueError + with self.assertRaises(ValueError): + ctx_object.parse_ovs_use_veth() + _bool_from_string.assert_called_with("Invalid") + + +class MockPCIDevice(object): + """Simple wrapper to mock pci.PCINetDevice class""" + def __init__(self, address): + self.pci_address = address + + +TEST_CPULIST_1 = "0-3" +TEST_CPULIST_2 = "0-7,16-23" +TEST_CPULIST_3 = "0,4,8,12,16,20,24" +DPDK_DATA_PORTS = ( + "br-phynet3:fe:16:41:df:23:fe " + "br-phynet1:fe:16:41:df:23:fd " + "br-phynet2:fe:f2:d0:45:dc:66" +) +BOND_MAPPINGS = ( + "bond0:fe:16:41:df:23:fe " + "bond0:fe:16:41:df:23:fd " + "bond1:fe:f2:d0:45:dc:66" +) +PCI_DEVICE_MAP = { + 'fe:16:41:df:23:fd': MockPCIDevice('0000:00:1c.0'), + 'fe:16:41:df:23:fe': MockPCIDevice('0000:00:1d.0'), +} + + +class TestDPDKUtils(tests.utils.BaseTestCase): + + def test_resolve_pci_from_mapping_config(self): + # FIXME: need to mock out the unit key value store + self.patch_object(context, 'config') + self.config.side_effect = lambda x: { + 'data-port': DPDK_DATA_PORTS, + 'dpdk-bond-mappings': BOND_MAPPINGS, + }.get(x) + _pci_devices = Mock() + _pci_devices.get_device_from_mac.side_effect = PCI_DEVICE_MAP.get + self.patch_object(context, 'pci') + self.pci.PCINetDevices.return_value = _pci_devices + self.assertDictEqual( + context.resolve_pci_from_mapping_config('data-port'), + { + '0000:00:1c.0': context.EntityMac( + 'br-phynet1', 'fe:16:41:df:23:fd'), + '0000:00:1d.0': context.EntityMac( + 'br-phynet3', 'fe:16:41:df:23:fe'), + }) + self.config.assert_called_once_with('data-port') + self.config.reset_mock() + self.assertDictEqual( + context.resolve_pci_from_mapping_config('dpdk-bond-mappings'), + { + '0000:00:1c.0': context.EntityMac( + 'bond0', 'fe:16:41:df:23:fd'), + '0000:00:1d.0': context.EntityMac( + 'bond0', 'fe:16:41:df:23:fe'), + }) + self.config.assert_called_once_with('dpdk-bond-mappings') + + +DPDK_PATCH = [ + 'resolve_pci_from_mapping_config', + 'glob', +] + +NUMA_CORES_SINGLE = { + '0': [0, 1, 2, 3] +} + +NUMA_CORES_MULTI = { + '0': [0, 1, 2, 3], + '1': [4, 5, 6, 7] +} + +LSCPU_ONE_SOCKET = b""" +# The following is the parsable format, which can be fed to other +# programs. Each different item in every column has an unique ID +# starting from zero. +# Socket +0 +0 +0 +0 +""" + +LSCPU_TWO_SOCKET = b""" +# The following is the parsable format, which can be fed to other +# programs. Each different item in every column has an unique ID +# starting from zero. +# Socket +0 +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 +1 +""" + + +class TestOVSDPDKDeviceContext(tests.utils.BaseTestCase): + + def setUp(self): + super(TestOVSDPDKDeviceContext, self).setUp() + self.patch_object(context, 'config') + self.config.side_effect = lambda x: { + 'enable-dpdk': True, + } + self.target = context.OVSDPDKDeviceContext() + + def patch_target(self, attr, return_value=None): + mocked = mock.patch.object(self.target, attr) + self._patches[attr] = mocked + started = mocked.start() + started.return_value = return_value + self._patches_start[attr] = started + setattr(self, attr, started) + + def test__parse_cpu_list(self): + self.assertEqual(self.target._parse_cpu_list(TEST_CPULIST_1), + [0, 1, 2, 3]) + self.assertEqual(self.target._parse_cpu_list(TEST_CPULIST_2), + [0, 1, 2, 3, 4, 5, 6, 7, + 16, 17, 18, 19, 20, 21, 22, 23]) + self.assertEqual(self.target._parse_cpu_list(TEST_CPULIST_3), + [0, 4, 8, 12, 16, 20, 24]) + + def test__numa_node_cores(self): + self.patch_target('_parse_cpu_list') + self._parse_cpu_list.return_value = [0, 1, 2, 3] + self.patch_object(context, 'glob') + self.glob.glob.return_value = [ + '/sys/devices/system/node/node0' + ] + with patch_open() as (_, mock_file): + mock_file.read.return_value = TEST_CPULIST_1 + self.target._numa_node_cores() + self.assertEqual(self.target._numa_node_cores(), + {'0': [0, 1, 2, 3]}) + self.glob.glob.assert_called_with('/sys/devices/system/node/node*') + self._parse_cpu_list.assert_called_with(TEST_CPULIST_1) + + def test_device_whitelist(self): + """Test device whitelist generation""" + self.patch_object( + context, 'resolve_pci_from_mapping_config', + return_value=collections.OrderedDict( + sorted({ + '0000:00:1c.0': 'br-data', + '0000:00:1d.0': 'br-data', + }.items()))) + self.assertEqual(self.target.device_whitelist(), + '-w 0000:00:1c.0 -w 0000:00:1d.0') + self.resolve_pci_from_mapping_config.assert_has_calls([ + call('data-port'), + call('dpdk-bond-mappings'), + ]) + + def test_socket_memory(self): + """Test socket memory configuration""" + self.patch_object(context, 'check_output') + self.patch_object(context, 'config') + self.config.side_effect = lambda x: { + 'dpdk-socket-memory': 1024, + }.get(x) + self.check_output.return_value = LSCPU_ONE_SOCKET + self.assertEqual(self.target.socket_memory(), + '1024') + + self.check_output.return_value = LSCPU_TWO_SOCKET + self.assertEqual(self.target.socket_memory(), + '1024,1024') + + self.config.side_effect = lambda x: { + 'dpdk-socket-memory': 2048, + }.get(x) + self.assertEqual(self.target.socket_memory(), + '2048,2048') + + def test_cpu_mask(self): + """Test generation of hex CPU masks""" + self.patch_target('_numa_node_cores') + self._numa_node_cores.return_value = NUMA_CORES_SINGLE + self.config.side_effect = lambda x: { + 'dpdk-socket-cores': 1, + }.get(x) + self.assertEqual(self.target.cpu_mask(), '0x01') + + self._numa_node_cores.return_value = NUMA_CORES_MULTI + self.assertEqual(self.target.cpu_mask(), '0x11') + + self.config.side_effect = lambda x: { + 'dpdk-socket-cores': 2, + }.get(x) + self.assertEqual(self.target.cpu_mask(), '0x33') + + def test_cpu_masks(self): + self.patch_target('_numa_node_cores') + self._numa_node_cores.return_value = NUMA_CORES_MULTI + self.config.side_effect = lambda x: { + 'dpdk-socket-cores': 1, + 'pmd-socket-cores': 2, + }.get(x) + self.assertEqual( + self.target.cpu_masks(), + {'dpdk_lcore_mask': '0x11', 'pmd_cpu_mask': '0x66'}) + + def test_context_no_devices(self): + """Ensure that DPDK is disable when no devices detected""" + self.patch_object(context, 'resolve_pci_from_mapping_config') + self.resolve_pci_from_mapping_config.return_value = {} + self.assertEqual(self.target(), {}) + self.resolve_pci_from_mapping_config.assert_has_calls([ + call('data-port'), + call('dpdk-bond-mappings'), + ]) + + def test_context_devices(self): + """Ensure DPDK is enabled when devices are detected""" + self.patch_target('_numa_node_cores') + self.patch_target('devices') + self.devices.return_value = collections.OrderedDict(sorted({ + '0000:00:1c.0': 'br-data', + '0000:00:1d.0': 'br-data', + }.items())) + self._numa_node_cores.return_value = NUMA_CORES_SINGLE + self.patch_object(context, 'check_output') + self.check_output.return_value = LSCPU_ONE_SOCKET + self.config.side_effect = lambda x: { + 'dpdk-socket-cores': 1, + 'dpdk-socket-memory': 1024, + 'enable-dpdk': True, + }.get(x) + self.assertEqual(self.target(), { + 'cpu_mask': '0x01', + 'device_whitelist': '-w 0000:00:1c.0 -w 0000:00:1d.0', + 'dpdk_enabled': True, + 'socket_memory': '1024' + }) + + +class TestDPDKDeviceContext(tests.utils.BaseTestCase): + + _dpdk_bridges = { + '0000:00:1c.0': 'br-data', + '0000:00:1d.0': 'br-physnet1', + } + _dpdk_bonds = { + '0000:00:1c.1': 'dpdk-bond0', + '0000:00:1d.1': 'dpdk-bond0', + } + + def setUp(self): + super(TestDPDKDeviceContext, self).setUp() + self.target = context.DPDKDeviceContext() + self.patch_object(context, 'resolve_pci_from_mapping_config') + self.resolve_pci_from_mapping_config.side_effect = [ + self._dpdk_bridges, + self._dpdk_bonds, + ] + + def test_context(self): + self.patch_object(context, 'config') + self.config.side_effect = lambda x: { + 'dpdk-driver': 'uio_pci_generic', + }.get(x) + devices = copy.deepcopy(self._dpdk_bridges) + devices.update(self._dpdk_bonds) + self.assertEqual(self.target(), { + 'devices': devices, + 'driver': 'uio_pci_generic' + }) + self.config.assert_called_with('dpdk-driver') + + def test_context_none_driver(self): + self.patch_object(context, 'config') + self.config.return_value = None + self.assertEqual(self.target(), {}) + self.config.assert_called_with('dpdk-driver') + + +class TestBridgePortInterfaceMap(tests.utils.BaseTestCase): + + def test__init__(self): + self.maxDiff = None + self.patch_object(context, 'config') + # system with three interfaces (eth0, eth1 and eth2) where + # eth0 and eth1 is part of linux bond bond0. + # Bridge mapping br-ex:eth2, br-provider1:bond0 + self.config.side_effect = lambda x: { + 'data-port': ( + 'br-ex:eth2 ' + 'br-provider1:00:00:5e:00:00:41 ' + 'br-provider1:00:00:5e:00:00:40'), + 'dpdk-bond-mappings': '', + }.get(x) + self.patch_object(context, 'resolve_pci_from_mapping_config') + self.resolve_pci_from_mapping_config.side_effect = [ + { + '0000:00:1c.0': context.EntityMac( + 'br-ex', '00:00:5e:00:00:42'), + }, + {}, + ] + self.patch_object(context, 'list_nics') + self.list_nics.return_value = ['bond0', 'eth0', 'eth1', 'eth2'] + self.patch_object(context, 'is_phy_iface') + self.is_phy_iface.side_effect = lambda x: True if not x.startswith( + 'bond') else False + self.patch_object(context, 'get_bond_master') + self.get_bond_master.side_effect = lambda x: 'bond0' if x in ( + 'eth0', 'eth1') else None + self.patch_object(context, 'get_nic_hwaddr') + self.get_nic_hwaddr.side_effect = lambda x: { + 'bond0': '00:00:5e:00:00:24', + 'eth0': '00:00:5e:00:00:40', + 'eth1': '00:00:5e:00:00:41', + 'eth2': '00:00:5e:00:00:42', + }.get(x) + bpi = context.BridgePortInterfaceMap() + self.maxDiff = None + expect = { + 'br-provider1': { + 'bond0': { + 'bond0': { + 'type': 'system', + }, + }, + }, + 'br-ex': { + 'eth2': { + 'eth2': { + 'type': 'system', + }, + }, + }, + } + self.assertDictEqual(bpi._map, expect) + # do it again but this time use the linux bond name instead of mac + # addresses. + self.config.side_effect = lambda x: { + 'data-port': ( + 'br-ex:eth2 ' + 'br-provider1:bond0'), + 'dpdk-bond-mappings': '', + }.get(x) + bpi = context.BridgePortInterfaceMap() + self.assertDictEqual(bpi._map, expect) + # and if a user asks for a purely virtual interface let's not stop them + expect = { + 'br-provider1': { + 'bond0.1234': { + 'bond0.1234': { + 'type': 'system', + }, + }, + }, + 'br-ex': { + 'eth2': { + 'eth2': { + 'type': 'system', + }, + }, + }, + } + self.config.side_effect = lambda x: { + 'data-port': ( + 'br-ex:eth2 ' + 'br-provider1:bond0.1234'), + 'dpdk-bond-mappings': '', + }.get(x) + bpi = context.BridgePortInterfaceMap() + self.assertDictEqual(bpi._map, expect) + # system with three interfaces (eth0, eth1 and eth2) where we should + # enable DPDK and create OVS bond of eth0 and eth1. + # Bridge mapping br-ex:eth2 br-provider1:dpdk-bond0 + self.config.side_effect = lambda x: { + 'enable-dpdk': True, + 'data-port': ( + 'br-ex:00:00:5e:00:00:42 ' + 'br-provider1:dpdk-bond0'), + 'dpdk-bond-mappings': ( + 'dpdk-bond0:00:00:5e:00:00:40 ' + 'dpdk-bond0:00:00:5e:00:00:41'), + }.get(x) + self.resolve_pci_from_mapping_config.side_effect = [ + { + '0000:00:1c.0': context.EntityMac( + 'br-ex', '00:00:5e:00:00:42'), + }, + { + '0000:00:1d.0': context.EntityMac( + 'dpdk-bond0', '00:00:5e:00:00:40'), + '0000:00:1e.0': context.EntityMac( + 'dpdk-bond0', '00:00:5e:00:00:41'), + }, + ] + # once devices are bound to DPDK they disappear from the system list + # of interfaces + self.list_nics.return_value = [] + bpi = context.BridgePortInterfaceMap(global_mtu=1500) + self.assertDictEqual(bpi._map, { + 'br-provider1': { + 'dpdk-bond0': { + 'dpdk-600a59e': { + 'pci-address': '0000:00:1d.0', + 'type': 'dpdk', + 'mtu-request': '1500', + }, + 'dpdk-5fc1d91': { + 'pci-address': '0000:00:1e.0', + 'type': 'dpdk', + 'mtu-request': '1500', + }, + }, + }, + 'br-ex': { + 'dpdk-6204d33': { + 'dpdk-6204d33': { + 'pci-address': '0000:00:1c.0', + 'type': 'dpdk', + 'mtu-request': '1500', + }, + }, + }, + }) + + def test_wrong_bridges_keys_pattern(self): + self.patch_object(context, 'config') + # check "" pattern + self.config.side_effect = lambda x: { + 'data-port': ( + 'incorrect_pattern'), + 'dpdk-bond-mappings': '', + }.get(x) + with self.assertRaises(ValueError): + context.BridgePortInterfaceMap() + + # check ": " pattern + self.config.side_effect = lambda x: { + 'data-port': ( + 'br-ex:eth2 ' + 'br-provider1'), + 'dpdk-bond-mappings': '', + }.get(x) + with self.assertRaises(ValueError): + context.BridgePortInterfaceMap() + + def test_add_interface(self): + self.patch_object(context, 'config') + self.config.return_value = '' + ctx = context.BridgePortInterfaceMap() + ctx.add_interface("br1", "bond1", "port1", ctx.interface_type.dpdk, + "00:00:00:00:00:01", 1500) + ctx.add_interface("br1", "bond1", "port2", ctx.interface_type.dpdk, + "00:00:00:00:00:02", 1500) + ctx.add_interface("br1", "bond2", "port3", ctx.interface_type.dpdk, + "00:00:00:00:00:03", 1500) + ctx.add_interface("br1", "bond2", "port4", ctx.interface_type.dpdk, + "00:00:00:00:00:04", 1500) + + expected = ( + 'br1', { + 'bond1': { + 'port1': { + 'type': 'dpdk', + 'pci-address': '00:00:00:00:00:01', + 'mtu-request': '1500', + }, + 'port2': { + 'type': 'dpdk', + 'pci-address': '00:00:00:00:00:02', + 'mtu-request': '1500', + }, + }, + 'bond2': { + 'port3': { + 'type': 'dpdk', + 'pci-address': '00:00:00:00:00:03', + 'mtu-request': '1500', + }, + 'port4': { + 'type': 'dpdk', + 'pci-address': '00:00:00:00:00:04', + 'mtu-request': '1500', + }, + }, + }, + ) + for br, bonds in ctx.items(): + self.maxDiff = None + self.assertEqual(br, expected[0]) + self.assertDictEqual(bonds, expected[1]) + + +class TestBondConfig(tests.utils.BaseTestCase): + + def test_get_bond_config(self): + self.patch_object(context, 'config') + self.config.side_effect = lambda x: { + 'dpdk-bond-config': ':active-backup bond1:balance-slb:off', + }.get(x) + bonds_config = context.BondConfig() + + self.assertEqual(bonds_config.get_bond_config('bond0'), + {'mode': 'active-backup', + 'lacp': 'active', + 'lacp-time': 'fast' + }) + self.assertEqual(bonds_config.get_bond_config('bond1'), + {'mode': 'balance-slb', + 'lacp': 'off', + 'lacp-time': 'fast' + }) + + +class TestSRIOVContext(tests.utils.BaseTestCase): + + class ObjectView(object): + + def __init__(self, _dict): + self.__dict__ = _dict + + def test___init__(self): + self.patch_object(context.pci, 'PCINetDevices') + pci_devices = self.ObjectView({ + 'pci_devices': [ + self.ObjectView({ + 'pci_address': '0000:81:00.0', + 'sriov': True, + 'interface_name': 'eth0', + 'sriov_totalvfs': 16, + }), + self.ObjectView({ + 'pci_address': '0000:81:00.1', + 'sriov': True, + 'interface_name': 'eth1', + 'sriov_totalvfs': 32, + }), + self.ObjectView({ + 'pci_address': '0000:3:00.0', + 'sriov': False, + 'interface_name': 'eth2', + }), + ] + }) + self.PCINetDevices.return_value = pci_devices + self.patch_object(context, 'config') + # auto sets up numvfs = totalvfs + self.config.return_value = { + 'sriov-numvfs': 'auto', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth0': 16, + 'eth1': 32, + }) + # when sriov-device-mappings is used only listed devices are set up + self.config.return_value = { + 'sriov-numvfs': 'auto', + 'sriov-device-mappings': 'physnet1:eth0', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth0': 16, + }) + self.config.return_value = { + 'sriov-numvfs': 'eth0:8', + 'sriov-device-mappings': 'physnet1:eth0', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth0': 8, + }) + self.config.return_value = { + 'sriov-numvfs': 'eth1:8', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth1': 8, + }) + # setting a numvfs value higher than a nic supports will revert to + # the nics max value + self.config.return_value = { + 'sriov-numvfs': 'eth1:64', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth1': 32, + }) + # devices listed in sriov-numvfs have precedence over + # sriov-device-mappings and the limiter still works when both are used + self.config.return_value = { + 'sriov-numvfs': 'eth1:64', + 'sriov-device-mappings': 'physnet:eth0', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth1': 32, + }) + # alternate config keys have effect + self.config.return_value = { + 'my-own-sriov-numvfs': 'auto', + 'my-own-sriov-device-mappings': 'physnet1:eth0', + } + self.assertDictEqual( + context.SRIOVContext( + numvfs_key='my-own-sriov-numvfs', + device_mappings_key='my-own-sriov-device-mappings')(), + { + 'eth0': 16, + }) + # blanket configuration works and respects limits + self.config.return_value = { + 'sriov-numvfs': '24', + } + self.assertDictEqual(context.SRIOVContext()(), { + 'eth0': 16, + 'eth1': 24, + }) + + def test___call__(self): + self.patch_object(context.pci, 'PCINetDevices') + pci_devices = self.ObjectView({'pci_devices': []}) + self.PCINetDevices.return_value = pci_devices + self.patch_object(context, 'config') + self.config.return_value = {'sriov-numvfs': 'auto'} + ctxt_obj = context.SRIOVContext() + ctxt_obj._map = {} + self.assertDictEqual(ctxt_obj(), {}) + + def test_get_map(self): + self.patch_object(context.pci, 'PCINetDevices') + pci_devices = self.ObjectView({ + 'pci_devices': [ + self.ObjectView({ + 'pci_address': '0000:81:00.0', + 'sriov': True, + 'interface_name': 'eth0', + 'sriov_totalvfs': 16, + }), + self.ObjectView({ + 'pci_address': '0000:81:00.1', + 'sriov': True, + 'interface_name': 'eth1', + 'sriov_totalvfs': 32, + }), + self.ObjectView({ + 'pci_address': '0000:3:00.0', + 'sriov': False, + 'interface_name': 'eth2', + }), + ] + }) + self.PCINetDevices.return_value = pci_devices + self.patch_object(context, 'config') + self.config.return_value = { + 'sriov-numvfs': 'auto', + } + self.assertDictEqual(context.SRIOVContext().get_map, { + '0000:81:00.0': context.SRIOVContext.PCIDeviceNumVFs( + mock.ANY, 16), + '0000:81:00.1': context.SRIOVContext.PCIDeviceNumVFs( + mock.ANY, 32), + }) + + +class TestCephBlueStoreContext(tests.utils.BaseTestCase): + + def setUp(self): + super(TestCephBlueStoreContext, self,).setUp() + self.expected_config_map = { + 'bluestore-compression-algorithm': 'fake-bca', + 'bluestore-compression-mode': 'fake-bcm', + 'bluestore-compression-required-ratio': 'fake-bcrr', + 'bluestore-compression-min-blob-size': 'fake-bcmibs', + 'bluestore-compression-min-blob-size-hdd': 'fake-bcmibsh', + 'bluestore-compression-min-blob-size-ssd': 'fake-bcmibss', + 'bluestore-compression-max-blob-size': 'fake-bcmabs', + 'bluestore-compression-max-blob-size-hdd': 'fake-bcmabsh', + 'bluestore-compression-max-blob-size-ssd': 'fake-bcmabss', + } + self.expected_op = { + key.replace('bluestore-', ''): value + for key, value in self.expected_config_map.items() + } + self.patch_object(context, 'config') + self.config.return_value = self.expected_config_map + + def test___call__(self): + ctxt = context.CephBlueStoreCompressionContext() + self.assertDictEqual(ctxt(), { + key.replace('-', '_'): value + for key, value in self.expected_config_map.items() + }) + + def test_get_op(self): + ctxt = context.CephBlueStoreCompressionContext() + self.assertDictEqual(ctxt.get_op(), self.expected_op) + + def test_get_kwargs(self): + ctxt = context.CephBlueStoreCompressionContext() + for arg in ctxt.get_kwargs().keys(): + self.assertNotIn('-', arg, "get_kwargs() returned '-' in the key") + + def test_validate(self): + self.patch_object(context.ch_ceph, 'BasePool') + pool = MagicMock() + self.BasePool.return_value = pool + ctxt = context.CephBlueStoreCompressionContext() + ctxt.validate() + # the order for the Dict argument is unpredictable, match on ANY and + # do separate check against call_args_list with assertDictEqual. + self.BasePool.assert_called_once_with('dummy-service', op=mock.ANY) + expected_op = self.expected_op.copy() + expected_op.update({'name': 'dummy-name'}) + self.assertDictEqual( + self.BasePool.call_args_list[0][1]['op'], expected_op) + pool.validate.assert_called_once_with() diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_templating.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_templating.py new file mode 100644 index 0000000..b30dc31 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_templating.py @@ -0,0 +1,374 @@ + +import os +import unittest + +from mock import patch, call, MagicMock + +import charmhelpers.contrib.openstack.templating as templating + +from jinja2.exceptions import TemplateNotFound + +import six +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + + +class FakeContextGenerator(object): + interfaces = None + + def set(self, interfaces, context): + self.interfaces = interfaces + self.context = context + + def __call__(self): + return self.context + + +class FakeLoader(object): + def set(self, template): + self.template = template + + def get(self, name): + return self.template + + +class MockFSLoader(object): + def __init__(self, dirs): + self.searchpath = [dirs] + + +class MockChoiceLoader(object): + def __init__(self, loaders): + self.loaders = loaders + + +def MockTemplate(): + templ = MagicMock() + templ.render = MagicMock() + return templ + + +class TemplatingTests(unittest.TestCase): + def setUp(self): + path = os.path.dirname(__file__) + self.loader = FakeLoader() + self.context = FakeContextGenerator() + + self.addCleanup(patch.object(templating, 'apt_install').start().stop()) + self.addCleanup(patch.object(templating, 'log').start().stop()) + + templating.FileSystemLoader = MockFSLoader + templating.ChoiceLoader = MockChoiceLoader + templating.Environment = MagicMock + + self.renderer = templating.OSConfigRenderer(templates_dir=path, + openstack_release='folsom') + + @patch.object(templating, 'apt_install') + def test_initializing_a_render_ensures_jinja2_present(self, apt): + '''Creatinga new renderer object installs jinja2 if needed''' + # temp. undo the patching from setUp + templating.FileSystemLoader = None + templating.ChoiceLoader = None + templating.Environment = None + templating.OSConfigRenderer(templates_dir='/tmp', + openstack_release='foo') + templating.FileSystemLoader = MockFSLoader + templating.ChoiceLoader = MockChoiceLoader + templating.Environment = MagicMock + if six.PY2: + apt.assert_called_with('python-jinja2') + else: + apt.assert_called_with('python3-jinja2') + + def test_create_renderer_invalid_templates_dir(self): + '''Ensure OSConfigRenderer checks templates_dir''' + self.assertRaises(templating.OSConfigException, + templating.OSConfigRenderer, + templates_dir='/tmp/foooo0', + openstack_release='grizzly') + + def test_render_unregistered_config(self): + '''Ensure cannot render an unregistered config file''' + self.assertRaises(templating.OSConfigException, + self.renderer.render, + config_file='/tmp/foo') + + def test_write_unregistered_config(self): + '''Ensure cannot write an unregistered config file''' + self.assertRaises(templating.OSConfigException, + self.renderer.write, + config_file='/tmp/foo') + + def test_render_complete_context(self): + '''It renders a template when provided a complete context''' + self.loader.set('{{ foo }}') + self.context.set(interfaces=['fooservice'], context={'foo': 'bar'}) + self.renderer.register('/tmp/foo', [self.context]) + with patch.object(self.renderer, '_get_template') as _get_t: + fake_tmpl = MockTemplate() + _get_t.return_value = fake_tmpl + self.renderer.render('/tmp/foo') + fake_tmpl.render.assert_called_with(self.context()) + self.assertIn('fooservice', self.renderer.complete_contexts()) + + def test_render_incomplete_context_with_template(self): + '''It renders a template when provided an incomplete context''' + self.context.set(interfaces=['fooservice'], context={}) + self.renderer.register('/tmp/foo', [self.context]) + with patch.object(self.renderer, '_get_template') as _get_t: + fake_tmpl = MockTemplate() + _get_t.return_value = fake_tmpl + self.renderer.render('/tmp/foo') + fake_tmpl.render.assert_called_with({}) + self.assertNotIn('fooservice', self.renderer.complete_contexts()) + + def test_render_template_registered_but_not_found(self): + '''It loads a template by basename of config file first''' + path = os.path.dirname(__file__) + renderer = templating.OSConfigRenderer(templates_dir=path, + openstack_release='folsom') + e = TemplateNotFound('') + renderer._get_template = MagicMock() + renderer._get_template.side_effect = e + renderer.register('/etc/nova/nova.conf', contexts=[]) + self.assertRaises( + TemplateNotFound, renderer.render, '/etc/nova/nova.conf') + + def test_render_template_by_basename_first(self): + '''It loads a template by basename of config file first''' + path = os.path.dirname(__file__) + renderer = templating.OSConfigRenderer(templates_dir=path, + openstack_release='folsom') + renderer._get_template = MagicMock() + renderer.register('/etc/nova/nova.conf', contexts=[]) + renderer.render('/etc/nova/nova.conf') + self.assertEquals(1, len(renderer._get_template.call_args_list)) + self.assertEquals( + [call('nova.conf')], renderer._get_template.call_args_list) + + def test_render_template_by_munged_full_path_last(self): + '''It loads a template by full path of config file second''' + path = os.path.dirname(__file__) + renderer = templating.OSConfigRenderer(templates_dir=path, + openstack_release='folsom') + tmp = MagicMock() + tmp.render = MagicMock() + e = TemplateNotFound('') + renderer._get_template = MagicMock() + renderer._get_template.side_effect = [e, tmp] + renderer.register('/etc/nova/nova.conf', contexts=[]) + renderer.render('/etc/nova/nova.conf') + self.assertEquals(2, len(renderer._get_template.call_args_list)) + self.assertEquals( + [call('nova.conf'), call('etc_nova_nova.conf')], + renderer._get_template.call_args_list) + + def test_render_template_by_basename(self): + '''It renders template if it finds it by config file basename''' + + @patch(builtin_open) + @patch.object(templating, 'get_loader') + def test_write_out_config(self, loader, _open): + '''It writes a templated config when provided a complete context''' + self.context.set(interfaces=['fooservice'], context={'foo': 'bar'}) + self.renderer.register('/tmp/foo', [self.context]) + with patch.object(self.renderer, '_get_template') as _get_t: + fake_tmpl = MockTemplate() + _get_t.return_value = fake_tmpl + self.renderer.write('/tmp/foo') + _open.assert_called_with('/tmp/foo', 'wb') + + def test_write_all(self): + '''It writes out all configuration files at once''' + self.context.set(interfaces=['fooservice'], context={'foo': 'bar'}) + self.renderer.register('/tmp/foo', [self.context]) + self.renderer.register('/tmp/bar', [self.context]) + ex_calls = [ + call('/tmp/bar'), + call('/tmp/foo'), + ] + with patch.object(self.renderer, 'write') as _write: + self.renderer.write_all() + self.assertEquals(sorted(ex_calls), sorted(_write.call_args_list)) + pass + + @patch.object(templating, 'get_loader') + def test_reset_template_loader_for_new_os_release(self, loader): + self.loader.set('') + self.context.set(interfaces=['fooservice'], context={}) + loader.return_value = MockFSLoader('/tmp/foo') + self.renderer.register('/tmp/foo', [self.context]) + self.renderer.render('/tmp/foo') + loader.assert_called_with(os.path.dirname(__file__), 'folsom') + self.renderer.set_release(openstack_release='grizzly') + self.renderer.render('/tmp/foo') + loader.assert_called_with(os.path.dirname(__file__), 'grizzly') + + @patch.object(templating, 'get_loader') + def test_incomplete_context_not_reported_complete(self, loader): + '''It does not recognize an incomplete context as a complete context''' + self.context.set(interfaces=['fooservice'], context={}) + self.renderer.register('/tmp/foo', [self.context]) + self.assertNotIn('fooservice', self.renderer.complete_contexts()) + + @patch.object(templating, 'get_loader') + def test_complete_context_reported_complete(self, loader): + '''It recognizes a complete context as a complete context''' + self.context.set(interfaces=['fooservice'], context={'foo': 'bar'}) + self.renderer.register('/tmp/foo', [self.context]) + self.assertIn('fooservice', self.renderer.complete_contexts()) + + @patch('os.path.isdir') + def test_get_loader_no_templates_dir(self, isdir): + '''Ensure getting loader fails with no template dir''' + isdir.return_value = False + self.assertRaises(templating.OSConfigException, + templating.get_loader, + templates_dir='/tmp/foo', os_release='foo') + + @patch('os.path.isdir') + def test_get_loader_all_search_paths(self, isdir): + '''Ensure loader reverse searches of all release template dirs''' + isdir.return_value = True + choice_loader = templating.get_loader('/tmp/foo', + os_release='icehouse') + dirs = [l.searchpath for l in choice_loader.loaders] + + common_tmplts = os.path.join(os.path.dirname(templating.__file__), + 'templates') + expected = [['/tmp/foo/icehouse'], + ['/tmp/foo/havana'], + ['/tmp/foo/grizzly'], + ['/tmp/foo/folsom'], + ['/tmp/foo/essex'], + ['/tmp/foo/diablo'], + ['/tmp/foo'], + [common_tmplts]] + self.assertEquals(dirs, expected) + + @patch('os.path.isdir') + def test_get_loader_some_search_paths(self, isdir): + '''Ensure loader reverse searches of some release template dirs''' + isdir.return_value = True + choice_loader = templating.get_loader('/tmp/foo', os_release='grizzly') + dirs = [l.searchpath for l in choice_loader.loaders] + + common_tmplts = os.path.join(os.path.dirname(templating.__file__), + 'templates') + + expected = [['/tmp/foo/grizzly'], + ['/tmp/foo/folsom'], + ['/tmp/foo/essex'], + ['/tmp/foo/diablo'], + ['/tmp/foo'], + [common_tmplts]] + self.assertEquals(dirs, expected) + + def test_register_template_with_list_of_contexts(self): + '''Ensure registering a template with a list of context generators''' + def _c1(): + pass + + def _c2(): + pass + tmpl = templating.OSConfigTemplate(config_file='/tmp/foo', + contexts=[_c1, _c2]) + self.assertEquals(tmpl.contexts, [_c1, _c2]) + + def test_register_template_with_single_context(self): + '''Ensure registering a template with a single non-list context''' + def _c1(): + pass + tmpl = templating.OSConfigTemplate(config_file='/tmp/foo', + contexts=_c1) + self.assertEquals(tmpl.contexts, [_c1]) + + +class TemplatingStringTests(unittest.TestCase): + def setUp(self): + path = os.path.dirname(__file__) + self.loader = FakeLoader() + self.context = FakeContextGenerator() + + self.addCleanup(patch.object(templating, + 'apt_install').start().stop()) + self.addCleanup(patch.object(templating, 'log').start().stop()) + + templating.FileSystemLoader = MockFSLoader + templating.ChoiceLoader = MockChoiceLoader + + self.config_file = '/etc/confd/extensible.d/drop-in.conf' + self.config_template = 'use: {{ fake_key }}' + self.renderer = templating.OSConfigRenderer(templates_dir=path, + openstack_release='folsom') + + def test_render_template_from_string_full_context(self): + ''' + Test rendering a specified config file with a string template + and a context. + ''' + + context = {'fake_key': 'fake_val'} + self.context.set( + interfaces=['fooservice'], + context=context + ) + + expected_output = 'use: {}'.format(context['fake_key']) + + self.renderer.register( + config_file=self.config_file, + contexts=[self.context], + config_template=self.config_template + ) + + # should return a string given we render from an in-memory + # template source + output = self.renderer.render(self.config_file) + + self.assertEquals(output, expected_output) + + def test_render_template_from_string_incomplete_context(self): + ''' + Test rendering a specified config file with a string template + and a context. + ''' + + self.context.set( + interfaces=['fooservice'], + context={} + ) + + expected_output = 'use: ' + + self.renderer.register( + config_file=self.config_file, + contexts=[self.context], + config_template=self.config_template + ) + + # should return a string given we render from an in-memory + # template source + output = self.renderer.render(self.config_file) + + self.assertEquals(output, expected_output) + + def test_register_string_template_with_single_context(self): + '''Template rendering from a provided string with a context''' + def _c1(): + pass + + config_file = '/etc/confdir/custom-drop-in.conf' + config_template = 'use: {{ key_available_in_c1 }}' + tmpl = templating.OSConfigTemplate( + config_file=config_file, + contexts=_c1, + config_template=config_template + ) + + self.assertEquals(tmpl.contexts, [_c1]) + self.assertEquals(tmpl.config_file, config_file) + self.assertEquals(tmpl.config_template, config_template) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_utils.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_utils.py new file mode 100644 index 0000000..f84f762 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_os_utils.py @@ -0,0 +1,383 @@ +import collections +import copy +import json +import mock +import six +import unittest + +from charmhelpers.contrib.openstack import utils + +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + + +class UtilsTests(unittest.TestCase): + def setUp(self): + super(UtilsTests, self).setUp() + + def test_compare_openstack_comparator(self): + self.assertTrue(utils.CompareOpenStackReleases('mitaka') < 'newton') + self.assertTrue(utils.CompareOpenStackReleases('pike') > 'essex') + + @mock.patch.object(utils, 'config') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_set') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_ids') + @mock.patch('charmhelpers.contrib.openstack.utils.get_ipv6_addr') + def test_sync_db_with_multi_ipv6_addresses(self, mock_get_ipv6_addr, + mock_relation_ids, + mock_relation_set, + mock_config): + mock_config.return_value = None + addr1 = '2001:db8:1:0:f816:3eff:fe45:7c/64' + addr2 = '2001:db8:1:0:d0cf:528c:23eb:5000/64' + mock_get_ipv6_addr.return_value = [addr1, addr2] + mock_relation_ids.return_value = ['shared-db'] + + utils.sync_db_with_multi_ipv6_addresses('testdb', 'testdbuser') + hosts = json.dumps([addr1, addr2]) + mock_relation_set.assert_called_with(relation_id='shared-db', + database='testdb', + username='testdbuser', + hostname=hosts) + + @mock.patch.object(utils, 'config') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_set') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_ids') + @mock.patch('charmhelpers.contrib.openstack.utils.get_ipv6_addr') + def test_sync_db_with_multi_ipv6_addresses_single(self, mock_get_ipv6_addr, + mock_relation_ids, + mock_relation_set, + mock_config): + mock_config.return_value = None + addr1 = '2001:db8:1:0:f816:3eff:fe45:7c/64' + mock_get_ipv6_addr.return_value = [addr1] + mock_relation_ids.return_value = ['shared-db'] + + utils.sync_db_with_multi_ipv6_addresses('testdb', 'testdbuser') + hosts = json.dumps([addr1]) + mock_relation_set.assert_called_with(relation_id='shared-db', + database='testdb', + username='testdbuser', + hostname=hosts) + + @mock.patch.object(utils, 'config') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_set') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_ids') + @mock.patch('charmhelpers.contrib.openstack.utils.get_ipv6_addr') + def test_sync_db_with_multi_ipv6_addresses_w_prefix(self, + mock_get_ipv6_addr, + mock_relation_ids, + mock_relation_set, + mock_config): + mock_config.return_value = None + addr1 = '2001:db8:1:0:f816:3eff:fe45:7c/64' + mock_get_ipv6_addr.return_value = [addr1] + mock_relation_ids.return_value = ['shared-db'] + + utils.sync_db_with_multi_ipv6_addresses('testdb', 'testdbuser', + relation_prefix='bungabunga') + hosts = json.dumps([addr1]) + mock_relation_set.assert_called_with(relation_id='shared-db', + bungabunga_database='testdb', + bungabunga_username='testdbuser', + bungabunga_hostname=hosts) + + @mock.patch.object(utils, 'config') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_set') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_ids') + @mock.patch('charmhelpers.contrib.openstack.utils.get_ipv6_addr') + def test_sync_db_with_multi_ipv6_addresses_vips(self, mock_get_ipv6_addr, + mock_relation_ids, + mock_relation_set, + mock_config): + addr1 = '2001:db8:1:0:f816:3eff:fe45:7c/64' + addr2 = '2001:db8:1:0:d0cf:528c:23eb:5000/64' + vip1 = '2001:db8:1:0:f816:3eff:32b3:7c' + vip2 = '2001:db8:1:0:f816:3eff:32b3:7d' + mock_config.return_value = '%s 10.0.0.1 %s' % (vip1, vip2) + + mock_get_ipv6_addr.return_value = [addr1, addr2] + mock_relation_ids.return_value = ['shared-db'] + + utils.sync_db_with_multi_ipv6_addresses('testdb', 'testdbuser') + hosts = json.dumps([addr1, addr2, vip1, vip2]) + mock_relation_set.assert_called_with(relation_id='shared-db', + database='testdb', + username='testdbuser', + hostname=hosts) + + @mock.patch('uuid.uuid4') + @mock.patch('charmhelpers.contrib.openstack.utils.related_units') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_set') + @mock.patch('charmhelpers.contrib.openstack.utils.relation_ids') + def test_remote_restart(self, mock_relation_ids, mock_relation_set, + mock_related_units, mock_uuid4): + mock_relation_ids.return_value = ['neutron-plugin-api-subordinate:8'] + mock_related_units.return_value = ['neutron-api/0'] + mock_uuid4.return_value = 'uuid4' + utils.remote_restart('neutron-plugin-api-subordinate') + mock_relation_set.assert_called_with( + relation_id='neutron-plugin-api-subordinate:8', + relation_settings={'restart-trigger': 'uuid4'} + ) + + @mock.patch.object(utils, 'lsb_release') + @mock.patch.object(utils, 'config') + @mock.patch('charmhelpers.contrib.openstack.utils.get_os_codename_package') + @mock.patch('charmhelpers.contrib.openstack.utils.' + 'get_os_codename_install_source') + def test_os_release(self, mock_get_os_codename_install_source, + mock_get_os_codename_package, + mock_config, mock_lsb_release): + # Wipe the modules cached os_rel + utils._os_rel = None + mock_lsb_release.return_value = {"DISTRIB_CODENAME": "trusty"} + mock_get_os_codename_install_source.return_value = None + mock_get_os_codename_package.return_value = None + mock_config.return_value = 'cloud-pocket' + self.assertEqual(utils.os_release('my-pkg'), 'icehouse') + mock_get_os_codename_install_source.assert_called_once_with( + 'cloud-pocket') + mock_get_os_codename_package.assert_called_once_with( + 'my-pkg', fatal=False) + mock_config.assert_called_once_with('openstack-origin') + # Next call to os_release should pickup cached version + mock_get_os_codename_install_source.reset_mock() + mock_get_os_codename_package.reset_mock() + self.assertEqual(utils.os_release('my-pkg'), 'icehouse') + self.assertFalse(mock_get_os_codename_install_source.called) + self.assertFalse(mock_get_os_codename_package.called) + # Call os_release and bypass cache + mock_lsb_release.return_value = {"DISTRIB_CODENAME": "xenial"} + mock_get_os_codename_install_source.reset_mock() + mock_get_os_codename_package.reset_mock() + self.assertEqual(utils.os_release('my-pkg', reset_cache=True), + 'mitaka') + mock_get_os_codename_install_source.assert_called_once_with( + 'cloud-pocket') + mock_get_os_codename_package.assert_called_once_with( + 'my-pkg', fatal=False) + # Override base + mock_lsb_release.return_value = {"DISTRIB_CODENAME": "xenial"} + mock_get_os_codename_install_source.reset_mock() + mock_get_os_codename_package.reset_mock() + self.assertEqual(utils.os_release('my-pkg', reset_cache=True, base="ocata"), + 'ocata') + mock_get_os_codename_install_source.assert_called_once_with( + 'cloud-pocket') + mock_get_os_codename_package.assert_called_once_with( + 'my-pkg', fatal=False) + # Override source key + mock_config.reset_mock() + mock_get_os_codename_install_source.reset_mock() + mock_get_os_codename_package.reset_mock() + mock_get_os_codename_package.return_value = None + utils.os_release('my-pkg', reset_cache=True, source_key='source') + mock_config.assert_called_once_with('source') + mock_get_os_codename_install_source.assert_called_once_with( + 'cloud-pocket') + + @mock.patch.object(utils, 'os_release') + @mock.patch.object(utils, 'get_os_codename_install_source') + def test_enable_memcache(self, _get_os_codename_install_source, + _os_release): + # Check call with 'release' + self.assertFalse(utils.enable_memcache(release='icehouse')) + self.assertTrue(utils.enable_memcache(release='ocata')) + # Check call with 'source' + _os_release.return_value = None + _get_os_codename_install_source.return_value = 'icehouse' + self.assertFalse(utils.enable_memcache(source='distro')) + _os_release.return_value = None + _get_os_codename_install_source.return_value = 'ocata' + self.assertTrue(utils.enable_memcache(source='distro')) + # Check call with 'package' + _os_release.return_value = 'icehouse' + _get_os_codename_install_source.return_value = None + self.assertFalse(utils.enable_memcache(package='pkg1')) + _os_release.return_value = 'ocata' + _get_os_codename_install_source.return_value = None + self.assertTrue(utils.enable_memcache(package='pkg1')) + + @mock.patch.object(utils, 'enable_memcache') + def test_enable_token_cache_pkgs(self, _enable_memcache): + _enable_memcache.return_value = False + self.assertEqual(utils.token_cache_pkgs(source='distro'), []) + _enable_memcache.return_value = True + self.assertEqual(utils.token_cache_pkgs(source='distro'), + ['memcached', 'python-memcache']) + + def test_update_json_file(self): + TEST_POLICY = """{ + "delete_image_location": "", + "get_image_location": "", + "set_image_location": "", + "extra_property": "False" + }""" + + TEST_POLICY_FILE = "/etc/glance/policy.json" + + items_to_update = { + "get_image_location": "role:admin", + "extra_policy": "extra", + } + + mock_open = mock.mock_open(read_data=TEST_POLICY) + with mock.patch(builtin_open, mock_open) as mock_file: + utils.update_json_file(TEST_POLICY_FILE, {}) + self.assertFalse(mock_file.called) + + utils.update_json_file(TEST_POLICY_FILE, items_to_update) + mock_file.assert_has_calls([ + mock.call(TEST_POLICY_FILE), + mock.call(TEST_POLICY_FILE, 'w'), + ], any_order=True) + + modified_policy = json.loads(TEST_POLICY) + modified_policy.update(items_to_update) + mock_open().write.assert_called_with( + json.dumps(modified_policy, indent=4, sort_keys=True)) + + tmp = json.loads(TEST_POLICY) + tmp.update(items_to_update) + TEST_POLICY = json.dumps(tmp) + mock_open = mock.mock_open(read_data=TEST_POLICY) + with mock.patch(builtin_open, mock_open) as mock_file: + utils.update_json_file(TEST_POLICY_FILE, items_to_update) + mock_file.assert_has_calls([ + mock.call(TEST_POLICY_FILE), + ], any_order=True) + + def test_ordered(self): + data = {'one': 1, 'two': 2, 'three': 3} + expected = [('one', 1), ('three', 3), ('two', 2)] + self.assertSequenceEqual(expected, + [x for x in utils.ordered(data).items()]) + + data = { + 'one': 1, + 'two': 2, + 'three': { + 'uno': 1, + 'dos': 2, + 'tres': 3 + } + } + expected = collections.OrderedDict() + expected['one'] = 1 + nested = collections.OrderedDict() + nested['dos'] = 2 + nested['tres'] = 3 + nested['uno'] = 1 + expected['three'] = nested + expected['two'] = 2 + self.assertEqual(expected, utils.ordered(data)) + + self.assertRaises(ValueError, utils.ordered, "foo") + + def test_sequence_status_check_functions(self): + # all messages are reported and the highest priority status "wins" + f1 = mock.Mock(return_value=('blocked', 'status 1')) + f2 = mock.Mock(return_value=('', 'status 2')) + f3 = mock.Mock(return_value=('maintenance', 'status 3')) + f = utils.sequence_status_check_functions(f1, f2, f3) + expected = ('blocked', 'status 1, status 2, status 3') + result = f(mock.Mock()) + self.assertEquals(result, expected) + # empty status must be replaced by "unknown" + f4 = mock.Mock(return_value=('', 'status 4')) + f5 = mock.Mock(return_value=('', 'status 5')) + f = utils.sequence_status_check_functions(f4, f5) + expected = ('unknown', 'status 4, status 5') + result = f(mock.Mock()) + self.assertEquals(result, expected) + # sequencing 0 status checks must return state 'unknown', '' + f = utils.sequence_status_check_functions() + expected = ('unknown', '') + result = f(mock.Mock()) + self.assertEquals(result, expected) + + @mock.patch.object(utils, 'relation_get') + @mock.patch.object(utils, 'related_units') + @mock.patch.object(utils, 'relation_ids') + @mock.patch.object(utils, 'container_scoped_relations') + def test_container_scoped_relation_get( + self, + mock_container_scoped_relations, + mock_relation_ids, + mock_related_units, + mock_relation_get): + mock_container_scoped_relations.return_value = [ + 'relation1', 'relation2'] + mock_relation_ids.return_value = ['rid'] + mock_related_units.return_value = ['unit'] + + for rdata in utils.container_scoped_relation_get(): + pass + mock_relation_ids.assert_has_calls([ + mock.call('relation1'), + mock.call('relation2')]) + mock_relation_get.assert_has_calls([ + mock.call(attribute=None, unit='unit', rid='rid'), + mock.call(attribute=None, unit='unit', rid='rid')]) + + mock_relation_get.reset_mock() + for rdata in utils.container_scoped_relation_get(attribute='attr'): + pass + mock_relation_get.assert_has_calls([ + mock.call(attribute='attr', unit='unit', rid='rid'), + mock.call(attribute='attr', unit='unit', rid='rid')]) + + @mock.patch.object(utils, 'container_scoped_relation_get') + def test_get_subordinate_release_packages( + self, + mock_container_scoped_relation_get): + rdata = { + 'queens': {'snap': {'install': ['q_inst'], 'purge': ['q_purg']}}, + 'stein': {'deb': {'install': ['s_inst'], 'purge': ['s_purg']}}} + mock_container_scoped_relation_get.return_value = [ + json.dumps(rdata), + json.dumps(rdata), + ] + # None of the subordinate relations have information about rocky or + # earlier for deb installations + self.assertEquals( + utils.get_subordinate_release_packages('rocky'), + utils.SubordinatePackages(set(), set())) + # Information on most recent earlier release with matching package + # type will be provided when requesting a release not specifically + # provided by subordinates + self.assertEquals( + utils.get_subordinate_release_packages( + 'rocky', package_type='snap'), + utils.SubordinatePackages( + {'q_inst'}, {'q_purg'})) + self.assertEquals( + utils.get_subordinate_release_packages('train'), + utils.SubordinatePackages( + {'s_inst'}, {'s_purg'})) + # Confirm operation when each subordinate has different release package + # information + rdata2 = copy.deepcopy(rdata) + rdata2.update({ + 'train': {'deb': {'install': ['t_inst'], 'purge': ['t_purg']}}}) + mock_container_scoped_relation_get.return_value = [ + json.dumps(rdata), + json.dumps(rdata2), + ] + self.assertEquals( + utils.get_subordinate_release_packages('train'), + utils.SubordinatePackages( + {'s_inst', 't_inst'}, {'s_purg', 't_purg'})) + # Confirm operation when one of the subordinate relations does not + # implement sharing the package information + mock_container_scoped_relation_get.return_value = [ + json.dumps(rdata), + None, + ] + self.assertEquals( + utils.get_subordinate_release_packages('train'), + utils.SubordinatePackages( + {'s_inst'}, {'s_purg'})) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rc_d_file.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rc_d_file.py new file mode 100644 index 0000000..a768ae4 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rc_d_file.py @@ -0,0 +1,183 @@ +import shutil +import tempfile +import yaml + +import tests.utils + +import charmhelpers.contrib.openstack.files.policy_rc_d_script as policy_rc_d + + +NOVS_POLICY_FILE_CONTENTS = """ +blocked_actions: + neutron-dhcp-agent: [restart, stop, try-restart] + neutron-l3-agent: [restart, stop, try-restart] +policy_requestor_name: neutron-openvswitch +policy_requestor_type: charm +""" + +COMPUTE_POLICY_FILE_CONTENTS = """ +blocked_actions: + libvirt: [restart, stop, try-restart] + nova-compute: [restart, stop, try-restart] +policy_requestor_name: nova-compute +policy_requestor_type: charm +""" + +ANOTHER_POLICY_FILE_CONTENTS = """ +blocked_actions: + libvirt: [restart, stop, try-restart] +policy_requestor_name: mycharm +policy_requestor_type: charm +""" + + +class PolicyRCDScriptTestCase(tests.utils.BaseTestCase): + + def setUp(self): + super(PolicyRCDScriptTestCase, self).setUp() + self.test_dir = tempfile.mkdtemp() + + def tearDown(self): + super(PolicyRCDScriptTestCase, self).tearDown() + shutil.rmtree(self.test_dir) + + def write_policy(self, pfile_name, contents): + test_policy_file = '{}/{}'.format(self.test_dir, pfile_name) + with open(test_policy_file, 'w') as f: + f.write(contents) + return test_policy_file + + def test_read_policy_file(self): + test_policy_file = self.write_policy( + 'novs.policy', + NOVS_POLICY_FILE_CONTENTS) + policy = policy_rc_d.read_policy_file(test_policy_file) + self.assertEqual( + policy, + [ + policy_rc_d.SystemPolicy( + policy_requestor_name='neutron-openvswitch', + policy_requestor_type='charm', + service='neutron-dhcp-agent', + blocked_actions=['restart', 'stop', 'try-restart']), + policy_rc_d.SystemPolicy( + policy_requestor_name='neutron-openvswitch', + policy_requestor_type='charm', + service='neutron-l3-agent', + blocked_actions=['restart', 'stop', 'try-restart'])]) + + def test_get_policies(self): + self.write_policy('novs.policy', NOVS_POLICY_FILE_CONTENTS) + self.write_policy('compute.policy', COMPUTE_POLICY_FILE_CONTENTS) + policies = policy_rc_d.get_policies(self.test_dir) + self.maxDiff = None + self.assertEqual( + sorted(policies), + [ + policy_rc_d.SystemPolicy( + policy_requestor_name='neutron-openvswitch', + policy_requestor_type='charm', + service='neutron-dhcp-agent', + blocked_actions=['restart', 'stop', 'try-restart']), + policy_rc_d.SystemPolicy( + policy_requestor_name='neutron-openvswitch', + policy_requestor_type='charm', + service='neutron-l3-agent', + blocked_actions=['restart', 'stop', 'try-restart']), + policy_rc_d.SystemPolicy( + policy_requestor_name='nova-compute', + policy_requestor_type='charm', + service='libvirt', + blocked_actions=['restart', 'stop', 'try-restart']), + policy_rc_d.SystemPolicy( + policy_requestor_name='nova-compute', + policy_requestor_type='charm', + service='nova-compute', + blocked_actions=['restart', 'stop', 'try-restart'])]) + + def test_record_blocked_action(self): + self.patch_object(policy_rc_d.time, 'time') + self.time.return_value = 456 + self.patch_object(policy_rc_d.uuid, 'uuid1') + uuids = ['uuid1', 'uuid2'] + self.uuid1.side_effect = lambda: uuids.pop() + blocking_policies = [ + policy_rc_d.SystemPolicy( + policy_requestor_name='cinder', + policy_requestor_type='charm', + service='cinder', + blocked_actions=['restart', 'stop', 'try-restart']), + policy_rc_d.SystemPolicy( + policy_requestor_name='cinder-ceph', + policy_requestor_type='charm', + service='cinder', + blocked_actions=['restart', 'stop', 'try-restart'])] + policy_rc_d.record_blocked_action( + 'cinder-api', + 'restart', + blocking_policies, + self.test_dir) + expect = [ + ( + '{}/charm-cinder-uuid2.deferred'.format(self.test_dir), + { + 'action': 'restart', + 'reason': 'Package update', + 'policy_requestor_name': 'cinder', + 'policy_requestor_type': 'charm', + 'service': 'cinder-api', + 'timestamp': 456.0}), + ( + '{}/charm-cinder-ceph-uuid1.deferred'.format(self.test_dir), + { + 'action': 'restart', + 'reason': 'Package update', + 'policy_requestor_name': 'cinder-ceph', + 'policy_requestor_type': 'charm', + 'service': 'cinder-api', + 'timestamp': 456.0})] + for defer_file, contents in expect: + with open(defer_file, 'r') as f: + self.assertEqual( + yaml.safe_load(f), + contents) + + def test_get_blocking_policies(self): + self.write_policy('novs.policy', NOVS_POLICY_FILE_CONTENTS) + self.write_policy('compute.policy', COMPUTE_POLICY_FILE_CONTENTS) + policies = policy_rc_d.get_blocking_policies( + 'libvirt', + 'restart', + self.test_dir) + self.assertEqual( + policies, + [ + policy_rc_d.SystemPolicy( + policy_requestor_name='nova-compute', + policy_requestor_type='charm', + service='libvirt', + blocked_actions=['restart', 'stop', 'try-restart'])]) + + def test_process_action_request(self): + self.write_policy('novs.policy', NOVS_POLICY_FILE_CONTENTS) + self.write_policy('compute.policy', COMPUTE_POLICY_FILE_CONTENTS) + self.write_policy('another.policy', ANOTHER_POLICY_FILE_CONTENTS) + self.assertEqual( + policy_rc_d.process_action_request( + 'libvirt', + 'restart', + self.test_dir, + self.test_dir), + ( + False, + ('restart of libvirt blocked by charm mycharm, ' + 'charm nova-compute'))) + self.assertEqual( + policy_rc_d.process_action_request( + 'glance-api', + 'restart', + self.test_dir, + self.test_dir), + ( + True, + 'Permitting glance-api restart')) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rcd.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rcd.py new file mode 100644 index 0000000..8c90270 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policy_rcd.py @@ -0,0 +1,126 @@ +import tempfile +import copy +import mock +import unittest +import shutil +import yaml + +from charmhelpers.contrib.openstack import policy_rcd + +TEST_POLICY = { + 'blocked_actions': { + 'neutron-dhcp-agent': ['restart', 'stop', 'try-restart'], + 'neutron-l3-agent': ['restart', 'stop', 'try-restart'], + 'neutron-metadata-agent': ['restart', 'stop', 'try-restart'], + 'neutron-openvswitch-agent': ['restart', 'stop', 'try-restart'], + 'openvswitch-switch': ['restart', 'stop', 'try-restart'], + 'ovs-vswitchd': ['restart', 'stop', 'try-restart'], + 'ovs-vswitchd-dpdk': ['restart', 'stop', 'try-restart'], + 'ovsdb-server': ['restart', 'stop', 'try-restart']}, + 'policy_requestor_name': 'neutron-openvswitch', + 'policy_requestor_type': 'charm'} + + +class PolicyRCDTests(unittest.TestCase): + + def setUp(self): + super(PolicyRCDTests, self).setUp() + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.tmp_dir)) + + @mock.patch.object(policy_rcd.hookenv, "service_name") + def test_get_policy_file_name(self, service_name): + service_name.return_value = 'mysvc' + self.assertEqual( + policy_rcd.get_policy_file_name(), + '/etc/policy-rc.d/charm-mysvc.policy') + + @mock.patch.object(policy_rcd, "get_policy_file_name") + def test_read_default_policy_file(self, get_policy_file_name): + with tempfile.NamedTemporaryFile('w') as ftmp: + policy_rcd.write_policy_file(ftmp.name, TEST_POLICY) + get_policy_file_name.return_value = ftmp.name + self.assertEqual( + policy_rcd.read_default_policy_file(), + TEST_POLICY) + + def test_write_policy_file(self): + with tempfile.NamedTemporaryFile('w') as ftmp: + policy_rcd.write_policy_file(ftmp.name, TEST_POLICY) + with open(ftmp.name, 'r') as f: + policy = yaml.load(f) + self.assertEqual(policy, TEST_POLICY) + + @mock.patch.object(policy_rcd.os, "remove") + @mock.patch.object(policy_rcd.hookenv, "service_name") + def test_remove_policy_file(self, service_name, remove): + service_name.return_value = 'mysvc' + policy_rcd.remove_policy_file() + remove.assert_called_once_with('/etc/policy-rc.d/charm-mysvc.policy') + + @mock.patch.object(policy_rcd.os.path, "exists") + @mock.patch.object(policy_rcd.shutil, "copy2") + @mock.patch.object(policy_rcd.host, "mkdir") + @mock.patch.object(policy_rcd.alternatives, "install_alternative") + @mock.patch.object(policy_rcd.hookenv, "service_name") + @mock.patch.object(policy_rcd.os.path, "dirname") + def test_install_policy_rcd(self, dirname, service_name, + install_alternative, mkdir, copy2, exists): + dirs = ['/dir1', '/dir2'] + service_name.return_value = 'mysvc' + dirname.side_effect = lambda x: dirs.pop() + exists.return_value = False + policy_rcd.install_policy_rcd() + install_alternative.assert_called_once_with( + 'policy-rc.d', + '/usr/sbin/policy-rc.d', + '/var/lib/charm/mysvc/policy-rc.d') + mkdir.assert_has_calls([ + mock.call('/dir1'), + mock.call('/etc/policy-rc.d') + ]) + copy2.assert_called_once_with( + '/dir2/policy_rc_d_script.py', + '/var/lib/charm/mysvc/policy-rc.d') + + @mock.patch.object(policy_rcd.hookenv, "service_name") + def test_get_default_policy(self, service_name): + service_name.return_value = 'mysvc' + self.assertEqual( + policy_rcd.get_default_policy(), + { + 'policy_requestor_name': 'mysvc', + 'policy_requestor_type': 'charm', + 'blocked_actions': {}}) + + @mock.patch.object(policy_rcd, "write_policy_file") + @mock.patch.object(policy_rcd.hookenv, "service_name") + @mock.patch.object(policy_rcd, "read_default_policy_file") + def test_add_policy_block(self, read_default_policy_file, service_name, + write_policy_file): + service_name.return_value = 'mysvc' + old_policy = copy.deepcopy(TEST_POLICY) + read_default_policy_file.return_value = old_policy + policy_rcd.add_policy_block('apache2', ['restart']) + expect_policy = copy.deepcopy(TEST_POLICY) + expect_policy['blocked_actions']['apache2'] = ['restart'] + write_policy_file.assert_called_once_with( + '/etc/policy-rc.d/charm-mysvc.policy', + expect_policy) + + @mock.patch.object(policy_rcd, "write_policy_file") + @mock.patch.object(policy_rcd.hookenv, "service_name") + @mock.patch.object(policy_rcd, "read_default_policy_file") + def test_remove_policy_block(self, read_default_policy_file, service_name, + write_policy_file): + service_name.return_value = 'mysvc' + old_policy = copy.deepcopy(TEST_POLICY) + read_default_policy_file.return_value = old_policy + policy_rcd.remove_policy_block( + 'neutron-dhcp-agent', + ['try-restart', 'restart']) + expect_policy = copy.deepcopy(TEST_POLICY) + expect_policy['blocked_actions']['neutron-dhcp-agent'] = ['stop'] + write_policy_file.assert_called_once_with( + '/etc/policy-rc.d/charm-mysvc.policy', + expect_policy) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policyd.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policyd.py new file mode 100644 index 0000000..378e58d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_policyd.py @@ -0,0 +1,467 @@ +import contextlib +import copy +import io +import os +import mock +import six +import unittest + +from charmhelpers.contrib.openstack import policyd + + +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + + +class PolicydTests(unittest.TestCase): + def setUp(self): + super(PolicydTests, self).setUp() + + def test_is_policyd_override_valid_on_this_release(self): + self.assertTrue( + policyd.is_policyd_override_valid_on_this_release("queens")) + self.assertTrue( + policyd.is_policyd_override_valid_on_this_release("rocky")) + self.assertFalse( + policyd.is_policyd_override_valid_on_this_release("pike")) + + @mock.patch.object(policyd, "clean_policyd_dir_for") + @mock.patch.object(policyd, "remove_policy_success_file") + @mock.patch.object(policyd, "process_policy_resource_file") + @mock.patch.object(policyd, "get_policy_resource_filename") + @mock.patch.object(policyd, "is_policyd_override_valid_on_this_release") + @mock.patch.object(policyd, "_policy_success_file") + @mock.patch("os.path.isfile") + @mock.patch.object(policyd.hookenv, "config") + @mock.patch("charmhelpers.core.hookenv.log") + def test_maybe_do_policyd_overrides( + self, + mock_log, + mock_config, + mock_isfile, + mock__policy_success_file, + mock_is_policyd_override_valid_on_this_release, + mock_get_policy_resource_filename, + mock_process_policy_resource_file, + mock_remove_policy_success_file, + mock_clean_policyd_dir_for, + ): + mock_isfile.return_value = False + mock__policy_success_file.return_value = "s-return" + # test success condition + mock_config.return_value = {policyd.POLICYD_CONFIG_NAME: True} + mock_is_policyd_override_valid_on_this_release.return_value = True + mock_get_policy_resource_filename.return_value = "resource.zip" + mock_process_policy_resource_file.return_value = True + mod_fn = mock.Mock() + restart_handler = mock.Mock() + policyd.maybe_do_policyd_overrides( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler) + mock_is_policyd_override_valid_on_this_release.assert_called_once_with( + "arelease") + mock_get_policy_resource_filename.assert_called_once_with() + mock_process_policy_resource_file.assert_called_once_with( + "resource.zip", "aservice", ["a"], ["b"], mod_fn) + restart_handler.assert_called_once_with() + # test process_policy_resource_file is not called if not valid on the + # release. + mock_process_policy_resource_file.reset_mock() + restart_handler.reset_mock() + mock_is_policyd_override_valid_on_this_release.return_value = False + policyd.maybe_do_policyd_overrides( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler) + mock_process_policy_resource_file.assert_not_called() + restart_handler.assert_not_called() + # test restart_handler is not called if not needed. + mock_is_policyd_override_valid_on_this_release.return_value = True + mock_process_policy_resource_file.return_value = False + policyd.maybe_do_policyd_overrides( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler) + mock_process_policy_resource_file.assert_called_once_with( + "resource.zip", "aservice", ["a"], ["b"], mod_fn) + restart_handler.assert_not_called() + # test that directory gets cleaned if the config is not set + mock_config.return_value = {policyd.POLICYD_CONFIG_NAME: False} + mock_process_policy_resource_file.reset_mock() + policyd.maybe_do_policyd_overrides( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler) + mock_process_policy_resource_file.assert_not_called() + mock_remove_policy_success_file.assert_called_once_with() + mock_clean_policyd_dir_for.assert_called_once_with( + "aservice", ["a"], user='aservice', group='aservice') + + @mock.patch.object(policyd, "maybe_do_policyd_overrides") + def test_maybe_do_policyd_overrides_with_config_changed( + self, + mock_maybe_do_policyd_overrides, + ): + mod_fn = mock.Mock() + restart_handler = mock.Mock() + policyd.maybe_do_policyd_overrides_on_config_changed( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler) + mock_maybe_do_policyd_overrides.assert_called_once_with( + "arelease", "aservice", ["a"], ["b"], mod_fn, restart_handler, + config_changed=True) + + @mock.patch("charmhelpers.core.hookenv.resource_get") + def test_get_policy_resource_filename(self, mock_resource_get): + mock_resource_get.return_value = "test-file" + self.assertEqual(policyd.get_policy_resource_filename(), + "test-file") + mock_resource_get.assert_called_once_with( + policyd.POLICYD_RESOURCE_NAME) + + # check that if an error is raised, that None is returned. + def go_bang(*args): + raise Exception("bang") + + mock_resource_get.side_effect = go_bang + self.assertEqual(policyd.get_policy_resource_filename(), None) + + @mock.patch.object(policyd, "_yamlfiles") + @mock.patch.object(policyd.zipfile, "ZipFile") + def test_open_and_filter_yaml_files(self, mock_ZipFile, mock__yamlfiles): + mock__yamlfiles.return_value = [ + ("file1", ".yaml", "file1.yaml", None), + ("file2", ".yml", "file2.YML", None)] + mock_ZipFile.return_value.__enter__.return_value = "zfp" + # test a valid zip file + with policyd.open_and_filter_yaml_files("some-file") as (zfp, files): + self.assertEqual(zfp, "zfp") + mock_ZipFile.assert_called_once_with("some-file", "r") + self.assertEqual(files, [ + ("file1", ".yaml", "file1.yaml", None), + ("file2", ".yml", "file2.YML", None)]) + # ensure that there must be at least one file. + mock__yamlfiles.return_value = [] + with self.assertRaises(policyd.BadPolicyZipFile): + with policyd.open_and_filter_yaml_files("some-file"): + pass + # ensure that it picks up duplicates + mock__yamlfiles.return_value = [ + ("file1", ".yaml", "file1.yaml", None), + ("file2", ".yml", "file2.yml", None), + ("file1", ".yml", "file1.yml", None)] + with self.assertRaises(policyd.BadPolicyZipFile): + with policyd.open_and_filter_yaml_files("some-file"): + pass + + def test__yamlfiles(self): + class MockZipFile(object): + def __init__(self, infolist): + self._infolist = infolist + + def infolist(self): + return self._infolist + + class MockInfoListItem(object): + def __init__(self, is_dir, filename): + self.filename = filename + self._is_dir = is_dir + + def is_dir(self): + return self._is_dir + + def __repr__(self): + return "MockInfoListItem({}, {})".format(self._is_dir, + self.filename) + + zipfile = MockZipFile([ + MockInfoListItem(False, "file1.yaml"), + MockInfoListItem(False, "file2.md"), + MockInfoListItem(False, "file3.YML"), + MockInfoListItem(False, "file4.Yaml"), + MockInfoListItem(True, "file5"), + MockInfoListItem(True, "file6.yaml"), + MockInfoListItem(False, "file7"), + MockInfoListItem(False, "file8.j2")]) + + self.assertEqual(list(policyd._yamlfiles(zipfile)), + [("file1", ".yaml", "file1.yaml", mock.ANY), + ("file3", ".yml", "file3.YML", mock.ANY), + ("file4", ".yaml", "file4.Yaml", mock.ANY), + ("file8", ".j2", "file8.j2", mock.ANY)]) + + @mock.patch.object(policyd.yaml, "safe_load") + def test_read_and_validate_yaml(self, mock_safe_load): + # test a valid document + good_doc = { + "key1": "rule1", + "key2": "rule2", + } + mock_safe_load.return_value = copy.deepcopy(good_doc) + doc = policyd.read_and_validate_yaml("test-stream") + self.assertEqual(doc, good_doc) + mock_safe_load.assert_called_once_with("test-stream") + # test an invalid document - return a string + mock_safe_load.return_value = "wrong" + with self.assertRaises(policyd.BadPolicyYamlFile): + policyd.read_and_validate_yaml("test-stream") + # test for black-listed keys + with self.assertRaises(policyd.BadPolicyYamlFile): + mock_safe_load.return_value = copy.deepcopy(good_doc) + policyd.read_and_validate_yaml("test-stream", ["key1"]) + # test for non string keys + bad_key_doc = { + (1,): "rule1", + "key2": "rule2", + } + with self.assertRaises(policyd.BadPolicyYamlFile): + mock_safe_load.return_value = copy.deepcopy(bad_key_doc) + policyd.read_and_validate_yaml("test-stream", ["key1"]) + # test for non string values (i.e. no nested keys) + bad_key_doc2 = { + "key1": "rule1", + "key2": {"sub_key": "rule2"}, + } + with self.assertRaises(policyd.BadPolicyYamlFile): + mock_safe_load.return_value = copy.deepcopy(bad_key_doc2) + policyd.read_and_validate_yaml("test-stream", ["key1"]) + + def test_policyd_dir_for(self): + self.assertEqual(policyd.policyd_dir_for('thing'), + "/etc/thing/policy.d") + + @mock.patch.object(policyd.hookenv, 'log') + @mock.patch("os.remove") + @mock.patch("shutil.rmtree") + @mock.patch("charmhelpers.core.host.mkdir") + @mock.patch("os.path.exists") + @mock.patch.object(policyd, "policyd_dir_for") + def test_clean_policyd_dir_for(self, + mock_policyd_dir_for, + mock_os_path_exists, + mock_mkdir, + mock_shutil_rmtree, + mock_os_remove, + mock_log): + if hasattr(os, 'scandir'): + mock_scan_dir_parts = (mock.patch, ["os.scandir"]) + else: + mock_scan_dir_parts = (mock.patch.object, + [policyd, "_fallback_scandir"]) + + class MockDirEntry(object): + def __init__(self, path, is_dir): + self.path = path + self._is_dir = is_dir + + def is_dir(self): + return self._is_dir + + # list of scanned objects + directory_contents = [ + MockDirEntry("one", False), + MockDirEntry("two", False), + MockDirEntry("three", True), + MockDirEntry("four", False)] + + mock_policyd_dir_for.return_value = "the-path" + + # Initial conditions + mock_os_path_exists.return_value = False + + # call the function + with mock_scan_dir_parts[0](*mock_scan_dir_parts[1]) as \ + mock_os_scandir: + mock_os_scandir.return_value = directory_contents + policyd.clean_policyd_dir_for("aservice") + + # check it did the right thing + mock_policyd_dir_for.assert_called_once_with("aservice") + mock_os_path_exists.assert_called_once_with("the-path") + mock_mkdir.assert_called_once_with("the-path", + owner="aservice", + group="aservice", + perms=0o775) + mock_shutil_rmtree.assert_called_once_with("three") + mock_os_remove.assert_has_calls([ + mock.call("one"), mock.call("two"), mock.call("four")]) + + # check also that we can omit paths ... reset everything + mock_os_remove.reset_mock() + mock_shutil_rmtree.reset_mock() + mock_os_path_exists.reset_mock() + mock_os_path_exists.return_value = True + mock_mkdir.reset_mock() + + with mock_scan_dir_parts[0](*mock_scan_dir_parts[1]) as \ + mock_os_scandir: + mock_os_scandir.return_value = directory_contents + policyd.clean_policyd_dir_for("aservice", + keep_paths=["one", "three"]) + + # verify all worked as we expected + mock_mkdir.assert_not_called() + mock_shutil_rmtree.assert_not_called() + mock_os_remove.assert_has_calls([mock.call("two"), mock.call("four")]) + + def test_path_for_policy_file(self): + self.assertEqual(policyd.path_for_policy_file('this', 'that'), + "/etc/this/policy.d/that.yaml") + + @mock.patch("charmhelpers.core.hookenv.charm_dir") + def test__policy_success_file(self, mock_charm_dir): + mock_charm_dir.return_value = "/this" + self.assertEqual(policyd._policy_success_file(), + "/this/{}".format(policyd.POLICYD_SUCCESS_FILENAME)) + + @mock.patch("os.remove") + @mock.patch.object(policyd, "_policy_success_file") + def test_remove_policy_success_file(self, mock_file, mock_os_remove): + mock_file.return_value = "the-path" + policyd.remove_policy_success_file() + mock_os_remove.assert_called_once_with("the-path") + + # now test that failure doesn't fail the function + def go_bang(*args): + raise Exception("bang") + + mock_os_remove.side_effect = go_bang + policyd.remove_policy_success_file() + + @mock.patch("os.path.isfile") + @mock.patch.object(policyd, "_policy_success_file") + def test_policyd_status_message_prefix(self, mock_file, mock_is_file): + mock_file.return_value = "the-path" + mock_is_file.return_value = True + self.assertEqual(policyd.policyd_status_message_prefix(), "PO:") + mock_is_file.return_value = False + self.assertEqual( + policyd.policyd_status_message_prefix(), "PO (broken):") + + @mock.patch("yaml.dump") + @mock.patch.object(policyd, "_policy_success_file") + @mock.patch.object(policyd.hookenv, "log") + @mock.patch.object(policyd, "read_and_validate_yaml") + @mock.patch.object(policyd, "path_for_policy_file") + @mock.patch.object(policyd, "clean_policyd_dir_for") + @mock.patch.object(policyd, "remove_policy_success_file") + @mock.patch.object(policyd, "open_and_filter_yaml_files") + @mock.patch.object(policyd.ch_host, 'write_file') + @mock.patch.object(policyd, "maybe_create_directory_for") + def test_process_policy_resource_file( + self, + mock_maybe_create_directory_for, + mock_write_file, + mock_open_and_filter_yaml_files, + mock_remove_policy_success_file, + mock_clean_policyd_dir_for, + mock_path_for_policy_file, + mock_read_and_validate_yaml, + mock_log, + mock__policy_success_file, + mock_yaml_dump, + ): + mock_zfp = mock.MagicMock() + mod_fn = mock.Mock() + mock_path_for_policy_file.side_effect = lambda s, n: s + "/" + n + gen = [ + ("file1", ".yaml", "file1.yaml", "file1-zipinfo"), + ("file2", ".yml", "file2.yml", "file2-zipinfo")] + mock_open_and_filter_yaml_files.return_value.__enter__.return_value = \ + (mock_zfp, gen) + # first verify that we can blacklist a file + res = policyd.process_policy_resource_file( + "resource.zip", "aservice", ["aservice/file1"], [], mod_fn) + self.assertFalse(res) + mock_remove_policy_success_file.assert_called_once_with() + mock_clean_policyd_dir_for.assert_has_calls([ + mock.call("aservice", + ["aservice/file1"], + user='aservice', + group='aservice'), + mock.call("aservice", + ["aservice/file1"], + user='aservice', + group='aservice')]) + mock_zfp.open.assert_not_called() + mod_fn.assert_not_called() + mock_log.assert_any_call("Processing resource.zip failed: policy.d" + " name aservice/file1 is blacklisted", + level=policyd.POLICYD_LOG_LEVEL_DEFAULT) + + # now test for success + @contextlib.contextmanager + def _patch_open(): + '''Patch open() to allow mocking both open() itself and the file that is + yielded. + + Yields the mock for "open" and "file", respectively.''' + mock_open = mock.MagicMock(spec=open) + mock_file = mock.MagicMock(spec=io.FileIO) + + with mock.patch(builtin_open, mock_open): + yield mock_open, mock_file + + mock_clean_policyd_dir_for.reset_mock() + mock_zfp.reset_mock() + mock_fp = mock.MagicMock() + mock_fp.read.return_value = '{"rule1": "value1"}' + mock_zfp.open.return_value.__enter__.return_value = mock_fp + gen = [("file1", ".j2", "file1.j2", "file1-zipinfo")] + mock_open_and_filter_yaml_files.return_value.__enter__.return_value = \ + (mock_zfp, gen) + mock_read_and_validate_yaml.return_value = {"rule1": "modded_value1"} + mod_fn.return_value = '{"rule1": "modded_value1"}' + mock__policy_success_file.return_value = "policy-success-file" + mock_yaml_dump.return_value = "dumped-file" + with _patch_open() as (mock_open, mock_file): + res = policyd.process_policy_resource_file( + "resource.zip", "aservice", [], ["key"], mod_fn) + self.assertTrue(res) + # mock_open.assert_any_call("aservice/file1", "wt") + mock_write_file.assert_called_once_with( + "aservice/file1", + b'dumped-file', + "aservice", + "aservice") + mock_open.assert_any_call("policy-success-file", "w") + mock_yaml_dump.assert_called_once_with({"rule1": "modded_value1"}) + mock_zfp.open.assert_called_once_with("file1-zipinfo") + mock_read_and_validate_yaml.assert_called_once_with( + '{"rule1": "modded_value1"}', ["key"]) + mod_fn.assert_called_once_with('{"rule1": "value1"}') + + # raise a BadPolicyZipFile if we have a template, but there is no + # template function + mock_log.reset_mock() + with _patch_open() as (mock_open, mock_file): + res = policyd.process_policy_resource_file( + "resource.zip", "aservice", [], ["key"], + template_function=None) + self.assertFalse(res) + mock_log.assert_any_call( + "Processing resource.zip failed: Template file1.j2 " + "but no template_function is available", + level=policyd.POLICYD_LOG_LEVEL_DEFAULT) + + # raise the IOError to validate that code path + def raise_ioerror(*args): + raise IOError("bang") + + mock_open_and_filter_yaml_files.side_effect = raise_ioerror + mock_log.reset_mock() + res = policyd.process_policy_resource_file( + "resource.zip", "aservice", [], ["key"], mod_fn) + self.assertFalse(res, False) + mock_log.assert_any_call( + "File resource.zip failed with IOError. " + "This really shouldn't happen -- error: bang", + level=policyd.POLICYD_LOG_LEVEL_DEFAULT) + # raise a general exception, so that is caught and logged too. + + def raise_exception(*args): + raise Exception("bang2") + + mock_open_and_filter_yaml_files.reset_mock() + mock_open_and_filter_yaml_files.side_effect = raise_exception + mock_log.reset_mock() + res = policyd.process_policy_resource_file( + "resource.zip", "aservice", [], ["key"], mod_fn) + self.assertFalse(res, False) + mock_log.assert_any_call( + "General Exception(bang2) during policyd processing", + level=policyd.POLICYD_LOG_LEVEL_DEFAULT) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ssh_migrations.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ssh_migrations.py new file mode 100644 index 0000000..5a32442 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_ssh_migrations.py @@ -0,0 +1,582 @@ +import mock +import six +import subprocess +import unittest + +from tests.helpers import patch_open, mock_open + +import charmhelpers.contrib.openstack.ssh_migrations as ssh_migrations + +if not six.PY3: + builtin_open = '__builtin__.open' + builtin_import = '__builtin__.__import__' +else: + builtin_open = 'builtins.open' + builtin_import = 'builtins.__import__' + + +UNIT1_HOST_KEY_1 = """|1|EaIiWNsBsaSke5T5bdDlaV5xKPU=|WKMu3Va+oNwRjXmPGOZ+mrpWbM8= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZdZdR7I35ymFdspruN1CIez/0m62sJeld2nLuOGaNbdl/rk5bGrWUAZh6c9p9H53FAqGAXBD/1C8dZ5dgIAGdTs7PAZq7owXCpgUPQcGOYVAtBwv8qfnWyI1W+Vpi6vnb2sgYr6XGbB9b84i4vrd98IIpXIleC9qd0VUTSYgd7+NPaFNoK0HZmqcNEf5leaa8sgSf4t5F+BTWEXzU3ql/3isFT8lEpJ9N8wOvNzAoFEQcxqauvOJn72QQ6kUrQT3NdQFUMHquS/s+nBrQNPbUmzqrvSOed75Qk8359zqU1Rce7U39cqc0scYi1ak3oJdojwfLFKJw4TMPn/Pq7JnT""" +UNIT1_HOST_KEY_2 = """|1|mCyYWqJl8loqV6LCY84lu2rpqLA=|51m+M+0ES3jYVzr3Kco3CDg8hEY= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZdZdR7I35ymFdspruN1CIez/0m62sJeld2nLuOGaNbdl/rk5bGrWUAZh6c9p9H53FAqGAXBD/1C8dZ5dgIAGdTs7PAZq7owXCpgUPQcGOYVAtBwv8qfnWyI1W+Vpi6vnb2sgYr6XGbB9b84i4vrd98IIpXIleC9qd0VUTSYgd7+NPaFNoK0HZmqcNEf5leaa8sgSf4t5F+BTWEXzU3ql/3isFT8lEpJ9N8wOvNzAoFEQcxqauvOJn72QQ6kUrQT3NdQFUMHquS/s+nBrQNPbUmzqrvSOed75Qk8359zqU1Rce7U39cqc0scYi1ak3oJdojwfLFKJw4TMPn/Pq7JnT""" +UNIT2_HOST_KEY_1 = """|1|eWagMqrN7XmX7NdVpZbqMZ2cb4Q=|3jgGiFEU9SMhXwdX0w0kkG54CZc= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC27Lv4wtAiPIOTsrUCFOU4qaNsov+LZSVHtxlu0aERuD+oU3ZILXOITXJlDohweXN6YuP4hFg49FF119gmMag+hDiA8/BztQmsplkwHrWEPuWKpReLRNLBU+Nt78jrJkjTK8Egwxbaxu8fAPZkCgGyeLkIH4ghrdWlOaWYXzwuxXkYWSpQOgF6E/T+19JKVKNpt2i6w7q9vVwZEjwVr30ubs1bNdPzE9ylNLQRrGa7c38SKsEos5RtZJjEuZGTC9KI0QdEgwnxxNMlT/CIgwWA1V38vLsosF2pHKxbmtCNvBuPNtrBDgXhukVqyEh825RhTAyQYGshMCbAbxl/M9c3""" +UNIT2_HOST_KEY_2 = """|1|zRH8troNwhVzrkMx86E5Ibevw5s=|gESlgkwUumP8q0A6l+CoRlFRpTw= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC27Lv4wtAiPIOTsrUCFOU4qaNsov+LZSVHtxlu0aERuD+oU3ZILXOITXJlDohweXN6YuP4hFg49FF119gmMag+hDiA8/BztQmsplkwHrWEPuWKpReLRNLBU+Nt78jrJkjTK8Egwxbaxu8fAPZkCgGyeLkIH4ghrdWlOaWYXzwuxXkYWSpQOgF6E/T+19JKVKNpt2i6w7q9vVwZEjwVr30ubs1bNdPzE9ylNLQRrGa7c38SKsEos5RtZJjEuZGTC9KI0QdEgwnxxNMlT/CIgwWA1V38vLsosF2pHKxbmtCNvBuPNtrBDgXhukVqyEh825RhTAyQYGshMCbAbxl/M9c3""" +HOST_KEYS = [UNIT1_HOST_KEY_1, UNIT1_HOST_KEY_2, + UNIT2_HOST_KEY_1, UNIT2_HOST_KEY_2] +UNIT1_PUBKEY_1 = """ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDqtnB3zh3sMufZd1khi44su0hTg/LqLb3ma2iyueTULZikDYa65UidVxzsa6r0Y9jkHwknGlh7fNnGdmc3S8EE+rVUNF4r3JF2Zd/pdfCBia/BmKJcO7+NyRWc8ihlrA3xYUSm+Yg8ZIpqoSb1LKjgAdYISh9HQQaXut2sXtHESdpilNpDf42AZfuQM+B0op0v7bq86ZXOM1rvdJriI6BduHaAOux+d9HDNvV5AxYTICrUkXqIvdHnoRyOFfhTcKun0EtuUxpDiAi0im9C+i+MPwMvA6AmRbot6Tqt2xZPRBYY8+WF7I5cBoovES/dWKP5TZwaGBr+WNv+z2JJhvlN root@juju-4665be-20180716142533-8""" +UNIT2_PUBKEY_1 = """ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCkWfkVrG7wTnfifvL0GkmDj6L33PKrWjzN2hOwZb9EoxwNzFGTMTBIpepTAnO6hdFBwtus1Ej/L12K6L/0YRDZAKjE7yTWOsh1kUxPZ1INRCqLILiefE5A/LPNx8NDb+d/2ryc5QmOQXUALs6mC5VDNchImUp9L7l0RIzPOgPXZCqMC1nZLqqX+eI9EUaf29/+NztYw59rFAa3hWNe8RJCSFeU+iWirWP8rfX9jsLzD9hO3nuZjP23M6tv1jX9LQD+8qkx0WSMa2WrIjkMiclP6tkyCJOZogyoPzZm/+dUhLeY9bIizbZCQKH/b4gOl5m/PkWoqEFshfqGzUIPkAJp root@juju-4665be-20180716142533-9""" +PUB_KEYS = [UNIT1_PUBKEY_1, UNIT2_PUBKEY_1] + + +class SSHMigrationsTests(unittest.TestCase): + + def setUp(self): + self._patches = {} + self._patches_start = {} + + def tearDown(self): + """Run teardown of patches.""" + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def patch_object(self, obj, attr, return_value=None, name=None, new=None, + **kwargs): + """Patch the given object.""" + if name is None: + name = attr + if new is not None: + mocked = mock.patch.object(obj, attr, new=new, **kwargs) + else: + mocked = mock.patch.object(obj, attr, **kwargs) + self._patches[name] = mocked + started = mocked.start() + if new is None: + started.return_value = return_value + self._patches_start[name] = started + setattr(self, name, started) + + def setup_mocks_ssh_directory_for_unit(self, app_name, ssh_dir_exists, + app_dir_exists, auth_keys_exists, + known_hosts_exists, user=None): + def _isdir(x): + return { + ssh_dir + '/': ssh_dir_exists, + app_dir: app_dir_exists}[x] + + def _isfile(x): + return { + '{}/authorized_keys'.format(app_dir): auth_keys_exists, + '{}/known_hosts'.format(app_dir): known_hosts_exists}[x] + + if user: + app_name = "{}_{}".format(app_name, user) + ssh_dir = '/etc/nova/compute_ssh' + app_dir = '{}/{}'.format(ssh_dir, app_name) + + self.patch_object(ssh_migrations.os, 'mkdir') + self.patch_object(ssh_migrations.os.path, 'isdir', side_effect=_isdir) + self.patch_object(ssh_migrations.os.path, 'isfile', + side_effect=_isfile) + + def test_ssh_directory_for_unit(self): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=True, + app_dir_exists=True, + auth_keys_exists=True, + known_hosts_exists=True) + self.assertEqual( + ssh_migrations.ssh_directory_for_unit('nova-compute-lxd'), + '/etc/nova/compute_ssh/nova-compute-lxd') + self.assertFalse(self.mkdir.called) + + def test_ssh_directory_for_unit_user(self): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=True, + app_dir_exists=True, + auth_keys_exists=True, + known_hosts_exists=True, + user='nova') + self.assertEqual( + ssh_migrations.ssh_directory_for_unit( + 'nova-compute-lxd', + user='nova'), + '/etc/nova/compute_ssh/nova-compute-lxd_nova') + self.assertFalse(self.mkdir.called) + + def test_ssh_directory_missing_dir(self): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=False, + app_dir_exists=True, + auth_keys_exists=True, + known_hosts_exists=True) + self.assertEqual( + ssh_migrations.ssh_directory_for_unit('nova-compute-lxd'), + '/etc/nova/compute_ssh/nova-compute-lxd') + self.mkdir.assert_called_once_with('/etc/nova/compute_ssh/') + + def test_ssh_directory_missing_dirs(self): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=False, + app_dir_exists=False, + auth_keys_exists=True, + known_hosts_exists=True) + self.assertEqual( + ssh_migrations.ssh_directory_for_unit('nova-compute-lxd'), + '/etc/nova/compute_ssh/nova-compute-lxd') + mkdir_calls = [ + mock.call('/etc/nova/compute_ssh/'), + mock.call('/etc/nova/compute_ssh/nova-compute-lxd')] + self.mkdir.assert_has_calls(mkdir_calls) + + @mock.patch(builtin_open) + def test_ssh_directory_missing_file(self, _open): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=True, + app_dir_exists=True, + auth_keys_exists=False, + known_hosts_exists=True) + self.assertEqual( + ssh_migrations.ssh_directory_for_unit('nova-compute-lxd'), + '/etc/nova/compute_ssh/nova-compute-lxd') + _open.assert_called_once_with( + '/etc/nova/compute_ssh/nova-compute-lxd/authorized_keys', + 'w') + self.assertFalse(self.mkdir.called) + + @mock.patch(builtin_open) + def test_ssh_directory_missing_files(self, _open): + self.setup_mocks_ssh_directory_for_unit( + 'nova-compute-lxd', + ssh_dir_exists=True, + app_dir_exists=True, + auth_keys_exists=False, + known_hosts_exists=False) + self.assertEqual( + ssh_migrations.ssh_directory_for_unit('nova-compute-lxd'), + '/etc/nova/compute_ssh/nova-compute-lxd') + open_calls = [ + mock.call( + '/etc/nova/compute_ssh/nova-compute-lxd/authorized_keys', + 'w'), + mock.call().close(), + mock.call( + '/etc/nova/compute_ssh/nova-compute-lxd/known_hosts', + 'w'), + mock.call().close()] + _open.assert_has_calls(open_calls) + self.assertFalse(self.mkdir.called) + + def setup_ssh_directory_for_unit_mocks(self): + self.patch_object( + ssh_migrations, + 'ssh_directory_for_unit', + return_value='/somedir') + + def test_known_hosts(self): + self.setup_ssh_directory_for_unit_mocks() + self.assertEqual( + ssh_migrations.known_hosts('nova-compute-lxd'), + '/somedir/known_hosts') + + def test_authorized_keys(self): + self.setup_ssh_directory_for_unit_mocks() + self.assertEqual( + ssh_migrations.authorized_keys('nova-compute-lxd'), + '/somedir/authorized_keys') + + @mock.patch('subprocess.check_output') + def test_ssh_known_host_key(self, _check_output): + self.setup_ssh_directory_for_unit_mocks() + _check_output.return_value = UNIT1_HOST_KEY_1 + self.assertEqual( + ssh_migrations.ssh_known_host_key( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd'), + UNIT1_HOST_KEY_1) + + @mock.patch('subprocess.check_output') + def test_ssh_known_host_key_multi_match(self, _check_output): + self.setup_ssh_directory_for_unit_mocks() + _check_output.return_value = '{}\n{}\n'.format(UNIT1_HOST_KEY_1, + UNIT1_HOST_KEY_2) + self.assertEqual( + ssh_migrations.ssh_known_host_key( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd'), + UNIT1_HOST_KEY_1) + + @mock.patch('subprocess.check_output') + def test_ssh_known_host_key_rc1(self, _check_output): + self.setup_ssh_directory_for_unit_mocks() + _check_output.side_effect = subprocess.CalledProcessError( + cmd=['anything'], + returncode=1, + output=UNIT1_HOST_KEY_1) + self.assertEqual( + ssh_migrations.ssh_known_host_key( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd'), + UNIT1_HOST_KEY_1) + + @mock.patch('subprocess.check_output') + def test_ssh_known_host_key_rc2(self, _check_output): + self.setup_ssh_directory_for_unit_mocks() + _check_output.side_effect = subprocess.CalledProcessError( + cmd=['anything'], + returncode=2, + output='') + with self.assertRaises(subprocess.CalledProcessError): + ssh_migrations.ssh_known_host_key( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + + @mock.patch('subprocess.check_output') + def test_ssh_known_host_key_no_match(self, _check_output): + self.setup_ssh_directory_for_unit_mocks() + _check_output.return_value = '' + self.assertIsNone( + ssh_migrations.ssh_known_host_key( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd')) + + @mock.patch('subprocess.check_call') + def test_remove_known_host(self, _check_call): + self.patch_object(ssh_migrations, 'log') + self.setup_ssh_directory_for_unit_mocks() + ssh_migrations.remove_known_host( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + _check_call.assert_called_once_with([ + 'ssh-keygen', + '-f', + '/somedir/known_hosts', + '-R', + 'juju-4665be-20180716142533-8']) + + def test_is_same_key(self): + self.assertTrue( + ssh_migrations.is_same_key(UNIT1_HOST_KEY_1, UNIT1_HOST_KEY_2)) + + def test_is_same_key_false(self): + self.assertFalse( + ssh_migrations.is_same_key(UNIT1_HOST_KEY_1, UNIT2_HOST_KEY_1)) + + def setup_mocks_add_known_host(self): + self.setup_ssh_directory_for_unit_mocks() + self.patch_object(ssh_migrations.subprocess, 'check_output') + self.patch_object(ssh_migrations, 'log') + self.patch_object(ssh_migrations, 'ssh_known_host_key') + self.patch_object(ssh_migrations, 'remove_known_host') + + def test_add_known_host(self): + self.setup_mocks_add_known_host() + self.check_output.return_value = UNIT1_HOST_KEY_1 + self.ssh_known_host_key.return_value = '' + with patch_open() as (mock_open, mock_file): + ssh_migrations.add_known_host( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + mock_file.write.assert_called_with(UNIT1_HOST_KEY_1 + '\n') + mock_open.assert_called_with('/somedir/known_hosts', 'a') + self.assertFalse(self.remove_known_host.called) + + def test_add_known_host_existing_invalid_key(self): + self.setup_mocks_add_known_host() + self.check_output.return_value = UNIT1_HOST_KEY_1 + self.ssh_known_host_key.return_value = UNIT2_HOST_KEY_1 + with patch_open() as (mock_open, mock_file): + ssh_migrations.add_known_host( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + mock_file.write.assert_called_with(UNIT1_HOST_KEY_1 + '\n') + mock_open.assert_called_with('/somedir/known_hosts', 'a') + self.remove_known_host.assert_called_once_wth( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + + def test_add_known_host_existing_valid_key(self): + self.setup_mocks_add_known_host() + self.check_output.return_value = UNIT2_HOST_KEY_1 + self.ssh_known_host_key.return_value = UNIT2_HOST_KEY_1 + with patch_open() as (mock_open, mock_file): + ssh_migrations.add_known_host( + 'juju-4665be-20180716142533-8', + 'nova-compute-lxd') + self.assertFalse(mock_open.called) + self.assertFalse(self.remove_known_host.called) + + def test_ssh_authorized_key_exists(self): + self.setup_mocks_add_known_host() + contents = '{}\n{}\n'.format(UNIT1_PUBKEY_1, UNIT2_PUBKEY_1) + with mock_open('/somedir/authorized_keys', contents=contents): + self.assertTrue( + ssh_migrations.ssh_authorized_key_exists( + UNIT1_PUBKEY_1, + 'nova-compute-lxd')) + + def test_ssh_authorized_key_exists_false(self): + self.setup_mocks_add_known_host() + contents = '{}\n'.format(UNIT1_PUBKEY_1) + with mock_open('/somedir/authorized_keys', contents=contents): + self.assertFalse( + ssh_migrations.ssh_authorized_key_exists( + UNIT2_PUBKEY_1, + 'nova-compute-lxd')) + + def test_add_authorized_key(self): + self.setup_mocks_add_known_host() + with patch_open() as (mock_open, mock_file): + ssh_migrations.add_authorized_key( + UNIT1_PUBKEY_1, + 'nova-compute-lxd') + mock_file.write.assert_called_with(UNIT1_PUBKEY_1 + '\n') + mock_open.assert_called_with('/somedir/authorized_keys', 'a') + + def setup_mocks_ssh_compute_add_host_and_key(self): + self.setup_ssh_directory_for_unit_mocks() + self.patch_object(ssh_migrations, 'log') + self.patch_object(ssh_migrations, 'get_hostname') + self.patch_object(ssh_migrations, 'get_host_ip') + self.patch_object(ssh_migrations, 'ns_query') + self.patch_object(ssh_migrations, 'add_known_host') + self.patch_object(ssh_migrations, 'ssh_authorized_key_exists') + self.patch_object(ssh_migrations, 'add_authorized_key') + + def test_ssh_compute_add_host_and_key(self): + self.setup_mocks_ssh_compute_add_host_and_key() + self.get_hostname.return_value = 'alt-hostname.project.serverstack' + self.ns_query.return_value = '10.6.0.17' + ssh_migrations.ssh_compute_add_host_and_key( + UNIT1_PUBKEY_1, + 'juju-4665be-20180716142533-8.project.serverstack', + '10.5.0.17', + 'nova-compute-lxd') + expect_hosts = [ + 'juju-4665be-20180716142533-8.project.serverstack', + 'alt-hostname.project.serverstack', + 'alt-hostname'] + add_known_host_calls = [] + for host in expect_hosts: + add_known_host_calls.append( + mock.call(host, 'nova-compute-lxd', None)) + self.add_known_host.assert_has_calls( + add_known_host_calls, + any_order=True) + self.add_authorized_key.assert_called_once_with( + UNIT1_PUBKEY_1, + 'nova-compute-lxd', + None) + + def test_ssh_compute_add_host_and_key_priv_addr_not_ip(self): + self.setup_mocks_ssh_compute_add_host_and_key() + self.get_hostname.return_value = 'alt-hostname.project.serverstack' + self.ns_query.return_value = '10.6.0.17' + self.get_host_ip.return_value = '10.6.0.17' + ssh_migrations.ssh_compute_add_host_and_key( + UNIT1_PUBKEY_1, + 'juju-4665be-20180716142533-8.project.serverstack', + 'bob.maas', + 'nova-compute-lxd') + expect_hosts = [ + 'bob.maas', + 'juju-4665be-20180716142533-8.project.serverstack', + '10.6.0.17', + 'bob'] + add_known_host_calls = [] + for host in expect_hosts: + add_known_host_calls.append( + mock.call(host, 'nova-compute-lxd', None)) + self.add_known_host.assert_has_calls( + add_known_host_calls, + any_order=True) + self.add_authorized_key.assert_called_once_with( + UNIT1_PUBKEY_1, + 'nova-compute-lxd', + None) + + def test_ssh_compute_add_host_and_key_ipv6(self): + self.setup_mocks_ssh_compute_add_host_and_key() + ssh_migrations.ssh_compute_add_host_and_key( + UNIT1_PUBKEY_1, + 'juju-4665be-20180716142533-8.project.serverstack', + 'fe80::8842:a9ff:fe53:72e4', + 'nova-compute-lxd') + self.add_known_host.assert_called_once_with( + 'fe80::8842:a9ff:fe53:72e4', + 'nova-compute-lxd', + None) + self.add_authorized_key.assert_called_once_with( + UNIT1_PUBKEY_1, + 'nova-compute-lxd', + None) + + @mock.patch.object(ssh_migrations, 'ssh_compute_add_host_and_key') + @mock.patch.object(ssh_migrations, 'relation_get') + def test_ssh_compute_add(self, _relation_get, + _ssh_compute_add_host_and_key): + _relation_get.return_value = { + 'hostname': 'juju-4665be-20180716142533-8.project.serverstack', + 'private-address': '10.5.0.17', + } + ssh_migrations.ssh_compute_add( + UNIT1_PUBKEY_1, + 'nova-compute-lxd', + rid='cloud-compute:23', + unit='nova-compute-lxd/2') + _ssh_compute_add_host_and_key.assert_called_once_with( + UNIT1_PUBKEY_1, + 'juju-4665be-20180716142533-8.project.serverstack', + '10.5.0.17', + 'nova-compute-lxd', + user=None) + + @mock.patch.object(ssh_migrations, 'known_hosts') + def test_ssh_known_hosts_lines(self, _known_hosts): + _known_hosts.return_value = '/somedir/known_hosts' + contents = '\n'.join(HOST_KEYS) + with mock_open('/somedir/known_hosts', contents=contents): + self.assertEqual( + ssh_migrations.ssh_known_hosts_lines('nova-compute-lxd'), + HOST_KEYS) + + @mock.patch.object(ssh_migrations, 'authorized_keys') + def test_ssh_authorized_keys_lines(self, _authorized_keys): + _authorized_keys.return_value = '/somedir/authorized_keys' + contents = '\n'.join(PUB_KEYS) + with mock_open('/somedir/authorized_keys', contents=contents): + self.assertEqual( + ssh_migrations.ssh_authorized_keys_lines('nova-compute-lxd'), + PUB_KEYS) + + def setup_mocks_ssh_compute_remove(self, isfile, authorized_keys_lines): + self.patch_object( + ssh_migrations, + 'ssh_authorized_keys_lines', + return_value=authorized_keys_lines) + self.patch_object(ssh_migrations, 'known_hosts') + self.patch_object( + ssh_migrations, + 'authorized_keys', + return_value='/somedir/authorized_keys') + self.patch_object( + ssh_migrations.os.path, + 'isfile', + return_value=isfile) + + def test_ssh_compute_remove(self): + self.setup_mocks_ssh_compute_remove( + isfile=True, + authorized_keys_lines=PUB_KEYS) + with patch_open() as (mock_open, mock_file): + ssh_migrations.ssh_compute_remove( + UNIT1_PUBKEY_1, + 'nova-compute-lxd') + mock_file.write.assert_called_with(UNIT2_PUBKEY_1 + '\n') + mock_open.assert_called_with('/somedir/authorized_keys', 'w') + + def test_ssh_compute_remove_missing_file(self): + self.setup_mocks_ssh_compute_remove( + isfile=False, + authorized_keys_lines=PUB_KEYS) + with patch_open() as (mock_open, mock_file): + ssh_migrations.ssh_compute_remove( + UNIT1_PUBKEY_1, + 'nova-compute-lxd') + self.assertFalse(mock_file.write.called) + + def test_ssh_compute_remove_missing_key(self): + self.setup_mocks_ssh_compute_remove( + isfile=False, + authorized_keys_lines=[UNIT2_PUBKEY_1]) + with patch_open() as (mock_open, mock_file): + ssh_migrations.ssh_compute_remove( + UNIT1_PUBKEY_1, + 'nova-compute-lxd') + self.assertFalse(mock_file.write.called) + + @mock.patch.object(ssh_migrations, 'ssh_known_hosts_lines') + @mock.patch.object(ssh_migrations, 'ssh_authorized_keys_lines') + def test_get_ssh_settings(self, _ssh_authorized_keys_lines, + _ssh_known_hosts_lines): + _ssh_authorized_keys_lines.return_value = PUB_KEYS + _ssh_known_hosts_lines.return_value = HOST_KEYS + expect = { + 'known_hosts_0': UNIT1_HOST_KEY_1, + 'known_hosts_1': UNIT1_HOST_KEY_2, + 'known_hosts_2': UNIT2_HOST_KEY_1, + 'known_hosts_3': UNIT2_HOST_KEY_2, + 'known_hosts_max_index': 4, + 'authorized_keys_0': UNIT1_PUBKEY_1, + 'authorized_keys_1': UNIT2_PUBKEY_1, + 'authorized_keys_max_index': 2, + } + self.assertEqual( + ssh_migrations.get_ssh_settings('nova-compute-lxd'), + expect) + + @mock.patch.object(ssh_migrations, 'ssh_known_hosts_lines') + @mock.patch.object(ssh_migrations, 'ssh_authorized_keys_lines') + def test_get_ssh_settings_user(self, _ssh_authorized_keys_lines, + _ssh_known_hosts_lines): + _ssh_authorized_keys_lines.return_value = PUB_KEYS + _ssh_known_hosts_lines.return_value = HOST_KEYS + expect = { + 'nova_known_hosts_0': UNIT1_HOST_KEY_1, + 'nova_known_hosts_1': UNIT1_HOST_KEY_2, + 'nova_known_hosts_2': UNIT2_HOST_KEY_1, + 'nova_known_hosts_3': UNIT2_HOST_KEY_2, + 'nova_known_hosts_max_index': 4, + 'nova_authorized_keys_0': UNIT1_PUBKEY_1, + 'nova_authorized_keys_1': UNIT2_PUBKEY_1, + 'nova_authorized_keys_max_index': 2, + } + self.assertEqual( + ssh_migrations.get_ssh_settings('nova-compute-lxd', user='nova'), + expect) + + @mock.patch.object(ssh_migrations, 'ssh_known_hosts_lines') + @mock.patch.object(ssh_migrations, 'ssh_authorized_keys_lines') + def test_get_ssh_settings_empty(self, _ssh_authorized_keys_lines, + _ssh_known_hosts_lines): + _ssh_authorized_keys_lines.return_value = [] + _ssh_known_hosts_lines.return_value = [] + self.assertEqual( + ssh_migrations.get_ssh_settings('nova-compute-lxd'), + {}) + + @mock.patch.object(ssh_migrations, 'get_ssh_settings') + def test_get_all_user_ssh_settings(self, _get_ssh_settings): + def ssh_setiings(application_name, user=None): + base_settings = { + 'known_hosts_0': UNIT1_HOST_KEY_1, + 'known_hosts_max_index': 1, + 'authorized_keys_0': UNIT1_PUBKEY_1, + 'authorized_keys_max_index': 1} + user_settings = { + 'nova_known_hosts_0': UNIT1_HOST_KEY_1, + 'nova_known_hosts_max_index': 1, + 'nova_authorized_keys_0': UNIT1_PUBKEY_1, + 'nova_authorized_keys_max_index': 1} + if user: + return user_settings + else: + return base_settings + _get_ssh_settings.side_effect = ssh_setiings + expect = { + 'known_hosts_0': UNIT1_HOST_KEY_1, + 'known_hosts_max_index': 1, + 'authorized_keys_0': UNIT1_PUBKEY_1, + 'authorized_keys_max_index': 1, + 'nova_known_hosts_0': UNIT1_HOST_KEY_1, + 'nova_known_hosts_max_index': 1, + 'nova_authorized_keys_0': UNIT1_PUBKEY_1, + 'nova_authorized_keys_max_index': 1} + self.assertEqual( + ssh_migrations.get_all_user_ssh_settings('nova-compute-lxd'), + expect) diff --git a/nrpe/mod/charmhelpers/tests/contrib/openstack/test_vaultlocker.py b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_vaultlocker.py new file mode 100644 index 0000000..496fd56 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/openstack/test_vaultlocker.py @@ -0,0 +1,261 @@ +# Copyright 2018 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import mock +import os +import sys +import unittest + +import charmhelpers.contrib.openstack.vaultlocker as vaultlocker + +from .test_os_contexts import TestDB + + +INCOMPLETE_RELATION = { + 'secrets-storage:1': { + 'vault/0': {} + } +} + +COMPLETE_RELATION = { + 'secrets-storage:1': { + 'vault/0': { + 'vault_url': json.dumps('http://vault:8200'), + 'test-service/0_role_id': json.dumps('test-role-from-vault'), + 'test-service/0_token': + json.dumps('00c9a9ab-c523-459d-a250-2ce8f0877c03'), + } + } +} + +DIRTY_RELATION = { + 'secrets-storage:1': { + 'vault/0': { + 'vault_url': json.dumps('http://vault:8200'), + 'test-service/0_role_id': json.dumps('test-role-from-vault'), + 'test-service/0_token': + json.dumps('00c9a9ab-c523-459d-a250-2ce8f0877c03'), + }, + 'vault/1': { + 'vault_url': json.dumps('http://vault:8200'), + 'test-service/0_role_id': json.dumps('test-role-from-vault'), + 'test-service/0_token': + json.dumps('67b36149-dc86-4b80-96c4-35b91847d16e'), + } + } +} + +COMPLETE_WITH_CA_RELATION = { + 'secrets-storage:1': { + 'vault/0': { + 'vault_url': json.dumps('http://vault:8200'), + 'test-service/0_role_id': json.dumps('test-role-from-vault'), + 'test-service/0_token': + json.dumps('00c9a9ab-c523-459d-a250-2ce8f0877c03'), + 'vault_ca': json.dumps('test-ca-data'), + } + } +} + + +class VaultLockerTestCase(unittest.TestCase): + + to_patch = [ + 'hookenv', + 'templating', + 'alternatives', + 'host', + 'unitdata', + ] + + _target_path = '/var/lib/charm/test-service/vaultlocker.conf' + + def setUp(self): + for m in self.to_patch: + setattr(self, m, self._patch(m)) + self.hookenv.service_name.return_value = 'test-service' + self.hookenv.local_unit.return_value = 'test-service/0' + self.db = TestDB() + self.unitdata.kv.return_value = self.db + fake_exc = mock.MagicMock() + fake_exc.InvalidRequest = Exception + self.fake_hvac = mock.MagicMock() + self.fake_hvac.exceptions = fake_exc + sys.modules['hvac'] = self.fake_hvac + + def fake_retrieve_secret_id(self, url=None, token=None): + if token == self.good_token: + return '31be8e65-20a3-45e0-a4a8-4d5a0554fb60' + else: + raise self.fake_hvac.exceptions.InvalidRequest + + def _patch(self, target): + _m = mock.patch.object(vaultlocker, target) + _mock = _m.start() + self.addCleanup(_m.stop) + return _mock + + def test_write_vl_config(self): + ctxt = {'test': 'data'} + vaultlocker.write_vaultlocker_conf(context=ctxt) + self.hookenv.service_name.assert_called_once_with() + self.host.mkdir.assert_called_once_with( + os.path.dirname(self._target_path), + perms=0o700 + ) + self.templating.render.assert_called_once_with( + source='vaultlocker.conf.j2', + target=self._target_path, + context=ctxt, + perms=0o600, + ) + self.alternatives.install_alternative.assert_called_once_with( + 'vaultlocker.conf', + '/etc/vaultlocker/vaultlocker.conf', + self._target_path, + 100 + ) + + def test_write_vl_config_priority(self): + ctxt = {'test': 'data'} + vaultlocker.write_vaultlocker_conf(context=ctxt, priority=200) + self.hookenv.service_name.assert_called_once_with() + self.host.mkdir.assert_called_once_with( + os.path.dirname(self._target_path), + perms=0o700 + ) + self.templating.render.assert_called_once_with( + source='vaultlocker.conf.j2', + target=self._target_path, + context=ctxt, + perms=0o600, + ) + self.alternatives.install_alternative.assert_called_once_with( + 'vaultlocker.conf', + '/etc/vaultlocker/vaultlocker.conf', + self._target_path, + 200 + ) + + def _setup_relation(self, relation): + self.hookenv.relation_ids.side_effect = ( + lambda _: relation.keys() + ) + self.hookenv.related_units.side_effect = ( + lambda rid: relation[rid].keys() + ) + self.hookenv.relation_get.side_effect = ( + lambda unit, rid: + relation[rid][unit] + ) + + def test_context_incomplete(self): + self._setup_relation(INCOMPLETE_RELATION) + context = vaultlocker.VaultKVContext('charm-test') + self.assertEqual(context(), {}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertFalse(vaultlocker.vault_relation_complete()) + + @mock.patch.object(vaultlocker, 'retrieve_secret_id') + def test_context_complete(self, retrieve_secret_id): + self._setup_relation(COMPLETE_RELATION) + context = vaultlocker.VaultKVContext('charm-test') + retrieve_secret_id.return_value = 'a3551c8d-0147-4cb6-afc6-efb3db2fccb2' + self.assertEqual(context(), + {'role_id': 'test-role-from-vault', + 'secret_backend': 'charm-test', + 'secret_id': 'a3551c8d-0147-4cb6-afc6-efb3db2fccb2', + 'vault_url': 'http://vault:8200'}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertTrue(vaultlocker.vault_relation_complete()) + calls = [mock.call(url='http://vault:8200', + token='00c9a9ab-c523-459d-a250-2ce8f0877c03')] + retrieve_secret_id.assert_has_calls(calls) + + @mock.patch.object(vaultlocker, 'retrieve_secret_id') + def test_context_complete_cached_secret_id(self, retrieve_secret_id): + self._setup_relation(COMPLETE_RELATION) + context = vaultlocker.VaultKVContext('charm-test') + self.db.set('secret-id', '5502fd27-059b-4b0a-91b2-eaff40b6a112') + self.good_token = 'invalid-token' # i.e. cause failure + retrieve_secret_id.side_effect = self.fake_retrieve_secret_id + self.assertEqual(context(), + {'role_id': 'test-role-from-vault', + 'secret_backend': 'charm-test', + 'secret_id': '5502fd27-059b-4b0a-91b2-eaff40b6a112', + 'vault_url': 'http://vault:8200'}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertTrue(vaultlocker.vault_relation_complete()) + calls = [mock.call(url='http://vault:8200', + token='00c9a9ab-c523-459d-a250-2ce8f0877c03')] + retrieve_secret_id.assert_has_calls(calls) + + @mock.patch.object(vaultlocker, 'retrieve_secret_id') + def test_purge_old_tokens(self, retrieve_secret_id): + self._setup_relation(DIRTY_RELATION) + context = vaultlocker.VaultKVContext('charm-test') + self.db.set('secret-id', '5502fd27-059b-4b0a-91b2-eaff40b6a112') + self.good_token = '67b36149-dc86-4b80-96c4-35b91847d16e' + retrieve_secret_id.side_effect = self.fake_retrieve_secret_id + self.assertEqual(context(), + {'role_id': 'test-role-from-vault', + 'secret_backend': 'charm-test', + 'secret_id': '31be8e65-20a3-45e0-a4a8-4d5a0554fb60', + 'vault_url': 'http://vault:8200'}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertTrue(vaultlocker.vault_relation_complete()) + self.assertEquals(self.db.get('secret-id'), + '31be8e65-20a3-45e0-a4a8-4d5a0554fb60') + calls = [mock.call(url='http://vault:8200', + token='67b36149-dc86-4b80-96c4-35b91847d16e')] + retrieve_secret_id.assert_has_calls(calls) + + @mock.patch.object(vaultlocker, 'retrieve_secret_id') + def test_context_complete_cached_dirty_data(self, retrieve_secret_id): + self._setup_relation(DIRTY_RELATION) + context = vaultlocker.VaultKVContext('charm-test') + self.db.set('secret-id', '5502fd27-059b-4b0a-91b2-eaff40b6a112') + self.good_token = '67b36149-dc86-4b80-96c4-35b91847d16e' + retrieve_secret_id.side_effect = self.fake_retrieve_secret_id + self.assertEqual(context(), + {'role_id': 'test-role-from-vault', + 'secret_backend': 'charm-test', + 'secret_id': '31be8e65-20a3-45e0-a4a8-4d5a0554fb60', + 'vault_url': 'http://vault:8200'}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertTrue(vaultlocker.vault_relation_complete()) + self.assertEquals(self.db.get('secret-id'), + '31be8e65-20a3-45e0-a4a8-4d5a0554fb60') + calls = [mock.call(url='http://vault:8200', + token='67b36149-dc86-4b80-96c4-35b91847d16e')] + retrieve_secret_id.assert_has_calls(calls) + + @mock.patch.object(vaultlocker, 'retrieve_secret_id') + def test_context_complete_with_ca(self, retrieve_secret_id): + self._setup_relation(COMPLETE_WITH_CA_RELATION) + retrieve_secret_id.return_value = 'token1234' + context = vaultlocker.VaultKVContext('charm-test') + retrieve_secret_id.return_value = 'a3551c8d-0147-4cb6-afc6-efb3db2fccb2' + self.assertEqual(context(), + {'role_id': 'test-role-from-vault', + 'secret_backend': 'charm-test', + 'secret_id': 'a3551c8d-0147-4cb6-afc6-efb3db2fccb2', + 'vault_url': 'http://vault:8200', + 'vault_ca': 'test-ca-data'}) + self.hookenv.relation_ids.assert_called_with('secrets-storage') + self.assertTrue(vaultlocker.vault_relation_complete()) + calls = [mock.call(url='http://vault:8200', + token='00c9a9ab-c523-459d-a250-2ce8f0877c03')] + retrieve_secret_id.assert_has_calls(calls) diff --git a/nrpe/mod/charmhelpers/tests/contrib/peerstorage/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/peerstorage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/peerstorage/test_peerstorage.py b/nrpe/mod/charmhelpers/tests/contrib/peerstorage/test_peerstorage.py new file mode 100644 index 0000000..63b7136 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/peerstorage/test_peerstorage.py @@ -0,0 +1,337 @@ +import copy +import json + +from tests.helpers import FakeRelation +from testtools import TestCase +from mock import patch, call +from charmhelpers.contrib import peerstorage + + +TO_PATCH = [ + 'current_relation_id', + 'is_relation_made', + 'local_unit', + 'relation_get', + '_relation_get', + 'relation_ids', + 'relation_set', + '_relation_set', + '_leader_get', + 'leader_set', + 'is_leader', +] +FAKE_RELATION_NAME = 'cluster' +FAKE_RELATION = { + 'cluster:0': { + 'cluster/0': { + }, + 'cluster/1': { + }, + 'cluster/2': { + }, + }, + +} +FAKE_RELATION_IDS = ['cluster:0'] +FAKE_LOCAL_UNIT = 'test_host' + + +class TestPeerStorage(TestCase): + def setUp(self): + super(TestPeerStorage, self).setUp() + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + self.fake_relation_name = FAKE_RELATION_NAME + self.fake_relation = FakeRelation(FAKE_RELATION) + self.local_unit.return_value = FAKE_LOCAL_UNIT + self.relation_get.return_value = {'key1': 'value1', + 'key2': 'value2', + 'private-address': '127.0.0.1', + 'public-address': '91.189.90.159'} + + def _patch(self, method): + _m = patch('charmhelpers.contrib.peerstorage.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_peer_retrieve_no_relation(self): + self.relation_ids.return_value = [] + self.assertRaises(ValueError, peerstorage.peer_retrieve, 'key', relation_name=self.fake_relation_name) + + def test_peer_retrieve_with_relation(self): + self.relation_ids.return_value = FAKE_RELATION_IDS + peerstorage.peer_retrieve('key', self.fake_relation_name) + self.relation_get.assert_called_with(attribute='key', rid=FAKE_RELATION_IDS[0], unit=FAKE_LOCAL_UNIT) + + def test_peer_store_no_relation(self): + self.relation_ids.return_value = [] + self.assertRaises(ValueError, peerstorage.peer_store, 'key', 'value', relation_name=self.fake_relation_name) + + def test_peer_store_with_relation(self): + self.relation_ids.return_value = FAKE_RELATION_IDS + peerstorage.peer_store('key', 'value', self.fake_relation_name) + self.relation_set.assert_called_with(relation_id=FAKE_RELATION_IDS[0], + relation_settings={'key': 'value'}) + + def test_peer_echo_no_includes(self): + peerstorage.is_leader.side_effect = NotImplementedError + settings = {'key1': 'value1', 'key2': 'value2'} + self._relation_get.copy.return_value = settings + self._relation_get.return_value = settings + peerstorage.peer_echo() + self._relation_set.assert_called_with(relation_settings=settings) + + def test_peer_echo_includes(self): + peerstorage.is_leader.side_effect = NotImplementedError + settings = {'key1': 'value1'} + self._relation_get.copy.return_value = settings + self._relation_get.return_value = settings + peerstorage.peer_echo(['key1']) + self._relation_set.assert_called_with(relation_settings=settings) + + @patch.object(peerstorage, 'peer_store') + def test_peer_store_and_set_no_relation(self, peer_store): + self.is_relation_made.return_value = False + peerstorage.peer_store_and_set(relation_id='db', kwarg1='kwarg1_v') + self.relation_set.assert_called_with(relation_id='db', + relation_settings={}, + kwarg1='kwarg1_v') + peer_store.assert_not_called() + + @patch.object(peerstorage, 'peer_store') + def test_peer_store_and_set_no_relation_fatal(self, peer_store): + self.is_relation_made.return_value = False + self.assertRaises(ValueError, + peerstorage.peer_store_and_set, + relation_id='db', + kwarg1='kwarg1_v', + peer_store_fatal=True) + + @patch.object(peerstorage, 'peer_store') + def test_peer_store_and_set_kwargs(self, peer_store): + self.is_relation_made.return_value = True + peerstorage.peer_store_and_set(relation_id='db', kwarg1='kwarg1_v') + self.relation_set.assert_called_with(relation_id='db', + relation_settings={}, + kwarg1='kwarg1_v') + calls = [call('db_kwarg1', 'kwarg1_v', relation_name='cluster')] + peer_store.assert_has_calls(calls, any_order=True) + + @patch.object(peerstorage, 'peer_store') + def test_peer_store_and_rel_settings(self, peer_store): + self.is_relation_made.return_value = True + rel_setting = { + 'rel_set1': 'relset1_v' + } + peerstorage.peer_store_and_set(relation_id='db', + relation_settings=rel_setting) + self.relation_set.assert_called_with(relation_id='db', + relation_settings=rel_setting) + calls = [call('db_rel_set1', 'relset1_v', relation_name='cluster')] + peer_store.assert_has_calls(calls, any_order=True) + + @patch.object(peerstorage, 'peer_store') + def test_peer_store_and_set(self, peer_store): + self.is_relation_made.return_value = True + rel_setting = { + 'rel_set1': 'relset1_v' + } + peerstorage.peer_store_and_set(relation_id='db', + relation_settings=rel_setting, + kwarg1='kwarg1_v', + delimiter='+') + self.relation_set.assert_called_with(relation_id='db', + relation_settings=rel_setting, + kwarg1='kwarg1_v') + calls = [call('db+rel_set1', 'relset1_v', relation_name='cluster'), + call('db+kwarg1', 'kwarg1_v', relation_name='cluster')] + peer_store.assert_has_calls(calls, any_order=True) + + @patch.object(peerstorage, 'peer_retrieve') + def test_peer_retrieve_by_prefix(self, peer_retrieve): + rel_id = 'db:2' + settings = { + 'user': 'bob', + 'pass': 'reallyhardpassword', + 'host': 'myhost', + } + peer_settings = {rel_id + '_' + k: v for k, v in settings.items()} + peer_retrieve.return_value = peer_settings + self.assertEquals(peerstorage.peer_retrieve_by_prefix(rel_id), settings) + + @patch.object(peerstorage, 'peer_retrieve') + def test_peer_retrieve_by_prefix_empty_relation(self, peer_retrieve): + # If relation-get returns None, peer_retrieve_by_prefix returns + # an empty dictionary. + peer_retrieve.return_value = None + rel_id = 'db:2' + self.assertEquals(peerstorage.peer_retrieve_by_prefix(rel_id), {}) + + @patch.object(peerstorage, 'peer_retrieve') + def test_peer_retrieve_by_prefix_exc_list(self, peer_retrieve): + rel_id = 'db:2' + settings = { + 'user': 'bob', + 'pass': 'reallyhardpassword', + 'host': 'myhost', + } + peer_settings = {rel_id + '_' + k: v for k, v in settings.items()} + del settings['host'] + peer_retrieve.return_value = peer_settings + self.assertEquals(peerstorage.peer_retrieve_by_prefix(rel_id, + exc_list=['host']), + settings) + + @patch.object(peerstorage, 'peer_retrieve') + def test_peer_retrieve_by_prefix_inc_list(self, peer_retrieve): + rel_id = 'db:2' + settings = { + 'user': 'bob', + 'pass': 'reallyhardpassword', + 'host': 'myhost', + } + peer_settings = {rel_id + '_' + k: v for k, v in settings.items()} + peer_retrieve.return_value = peer_settings + self.assertEquals(peerstorage.peer_retrieve_by_prefix(rel_id, + inc_list=['host']), + {'host': 'myhost'}) + + def test_leader_get_migration_is_leader(self): + self.is_leader.return_value = True + l_settings = {'s3': 3} + r_settings = {'s1': 1, 's2': 2} + + def mock_relation_get(attribute=None, unit=None, rid=None): + if attribute: + if attribute in r_settings: + return r_settings.get(attribute) + else: + return None + + return copy.deepcopy(r_settings) + + def mock_leader_get(attribute=None): + if attribute: + if attribute in l_settings: + return l_settings.get(attribute) + else: + return None + + return copy.deepcopy(l_settings) + + def mock_leader_set(settings=None, **kwargs): + if settings: + l_settings.update(settings) + + l_settings.update(kwargs) + + def check_leader_db(dicta, dictb): + _dicta = copy.deepcopy(dicta) + _dictb = copy.deepcopy(dictb) + miga = json.loads(_dicta[migration_key]).sort() + migb = json.loads(_dictb[migration_key]).sort() + self.assertEqual(miga, migb) + del _dicta[migration_key] + del _dictb[migration_key] + self.assertEqual(_dicta, _dictb) + + migration_key = '__leader_get_migrated_settings__' + self._relation_get.side_effect = mock_relation_get + self._leader_get.side_effect = mock_leader_get + self.leader_set.side_effect = mock_leader_set + + self.assertEqual({'s1': 1, 's2': 2}, peerstorage._relation_get()) + self.assertEqual({'s3': 3}, peerstorage._leader_get()) + self.assertEqual({'s1': 1, 's2': 2, 's3': 3}, peerstorage.leader_get()) + check_leader_db({'s1': 1, 's2': 2, 's3': 3, + migration_key: '["s2", "s1"]'}, l_settings) + self.assertTrue(peerstorage.leader_set.called) + + peerstorage.leader_set.reset_mock() + self.assertEqual({'s1': 1, 's2': 2, 's3': 3}, peerstorage.leader_get()) + check_leader_db({'s1': 1, 's2': 2, 's3': 3, + migration_key: '["s2", "s1"]'}, l_settings) + self.assertFalse(peerstorage.leader_set.called) + + l_settings = {'s3': 3} + peerstorage.leader_set.reset_mock() + self.assertEqual(1, peerstorage.leader_get('s1')) + check_leader_db({'s1': 1, 's3': 3, + migration_key: '["s1"]'}, l_settings) + self.assertTrue(peerstorage.leader_set.called) + + # Test that leader vals take precedence over non-leader vals + r_settings['s3'] = 2 + r_settings['s4'] = 3 + l_settings['s4'] = 4 + + peerstorage.leader_set.reset_mock() + self.assertEqual(4, peerstorage.leader_get('s4')) + check_leader_db({'s1': 1, 's3': 3, 's4': 4, + migration_key: '["s1", "s4"]'}, l_settings) + self.assertTrue(peerstorage.leader_set.called) + + peerstorage.leader_set.reset_mock() + self.assertEqual({'s1': 1, 's2': 2, 's3': 2, 's4': 3}, + peerstorage._relation_get()) + check_leader_db({'s1': 1, 's3': 3, 's4': 4, + migration_key: '["s1", "s4"]'}, + peerstorage._leader_get()) + self.assertEqual({'s1': 1, 's2': 2, 's3': 3, 's4': 4}, + peerstorage.leader_get()) + check_leader_db({'s1': 1, 's2': 2, 's3': 3, 's4': 4, + migration_key: '["s3", "s2", "s1", "s4"]'}, + l_settings) + self.assertTrue(peerstorage.leader_set.called) + + def test_leader_get_migration_is_not_leader(self): + self.is_leader.return_value = False + l_settings = {'s3': 3} + r_settings = {'s1': 1, 's2': 2} + + def mock_relation_get(attribute=None, unit=None, rid=None): + if attribute: + if attribute in r_settings: + return r_settings.get(attribute) + else: + return None + + return copy.deepcopy(r_settings) + + def mock_leader_get(attribute=None): + if attribute: + if attribute in l_settings: + return l_settings.get(attribute) + else: + return None + + return copy.deepcopy(l_settings) + + def mock_leader_set(settings=None, **kwargs): + if settings: + l_settings.update(settings) + + l_settings.update(kwargs) + + self._relation_get.side_effect = mock_relation_get + self._leader_get.side_effect = mock_leader_get + self.leader_set.side_effect = mock_leader_set + self.assertEqual({'s1': 1, 's2': 2}, peerstorage._relation_get()) + self.assertEqual({'s3': 3}, peerstorage._leader_get()) + self.assertEqual({'s3': 3}, peerstorage.leader_get()) + self.assertEqual({'s3': 3}, l_settings) + self.assertFalse(peerstorage.leader_set.called) + + self.assertEqual({'s3': 3}, peerstorage.leader_get()) + self.assertEqual({'s3': 3}, l_settings) + self.assertFalse(peerstorage.leader_set.called) + + # Test that leader vals take precedence over non-leader vals + r_settings['s3'] = 2 + r_settings['s4'] = 3 + l_settings['s4'] = 4 + + self.assertEqual(4, peerstorage.leader_get('s4')) + self.assertEqual({'s3': 3, 's4': 4}, l_settings) + self.assertFalse(peerstorage.leader_set.called) diff --git a/nrpe/mod/charmhelpers/tests/contrib/python/test_python.py b/nrpe/mod/charmhelpers/tests/contrib/python/test_python.py new file mode 100644 index 0000000..59c8b6e --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/python/test_python.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import absolute_import + +from unittest import TestCase + +from charmhelpers.fetch.python import debug +from charmhelpers.fetch.python import packages +from charmhelpers.fetch.python import rpdb +from charmhelpers.fetch.python import version +from charmhelpers.contrib.python import debug as contrib_debug +from charmhelpers.contrib.python import packages as contrib_packages +from charmhelpers.contrib.python import rpdb as contrib_rpdb +from charmhelpers.contrib.python import version as contrib_version + + +class ContribDebugTestCase(TestCase): + def test_aliases(self): + assert contrib_debug is debug + assert contrib_packages is packages + assert contrib_rpdb is rpdb + assert contrib_version is version diff --git a/nrpe/mod/charmhelpers/tests/contrib/saltstack/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/saltstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/saltstack/test_saltstates.py b/nrpe/mod/charmhelpers/tests/contrib/saltstack/test_saltstates.py new file mode 100644 index 0000000..a9ac35e --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/saltstack/test_saltstates.py @@ -0,0 +1,75 @@ +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +import mock +import unittest + +import charmhelpers.contrib.saltstack + + +class InstallSaltSupportTestCase(unittest.TestCase): + + def setUp(self): + super(InstallSaltSupportTestCase, self).setUp() + + patcher = mock.patch('charmhelpers.contrib.saltstack.subprocess') + self.mock_subprocess = patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch('charmhelpers.fetch') + self.mock_charmhelpers_fetch = patcher.start() + self.addCleanup(patcher.stop) + + def test_adds_ppa_by_default(self): + charmhelpers.contrib.saltstack.install_salt_support() + + expected_calls = [((cmd,), {}) for cmd in [ + ['/usr/bin/add-apt-repository', '--yes', 'ppa:saltstack/salt'], + ['/usr/bin/apt-get', 'update'], + ]] + self.assertEqual(self.mock_subprocess.check_call.call_count, 2) + self.assertEqual( + expected_calls, self.mock_subprocess.check_call.call_args_list) + self.mock_charmhelpers_fetch.apt_install.assert_called_once_with( + 'salt-common') + + def test_no_ppa(self): + charmhelpers.contrib.saltstack.install_salt_support( + from_ppa=False) + + self.assertEqual(self.mock_subprocess.check_call.call_count, 0) + self.mock_charmhelpers_fetch.apt_install.assert_called_once_with( + 'salt-common') + + +class UpdateMachineStateTestCase(unittest.TestCase): + + def setUp(self): + super(UpdateMachineStateTestCase, self).setUp() + + patcher = mock.patch('charmhelpers.contrib.saltstack.subprocess') + self.mock_subprocess = patcher.start() + self.addCleanup(patcher.stop) + + patcher = mock.patch('charmhelpers.contrib.templating.contexts.' + 'juju_state_to_yaml') + self.mock_config_2_grains = patcher.start() + self.addCleanup(patcher.stop) + + def test_calls_local_salt_template(self): + charmhelpers.contrib.saltstack.update_machine_state( + 'states/install.yaml') + + self.mock_subprocess.check_call.assert_called_once_with([ + 'salt-call', + '--local', + 'state.template', + 'states/install.yaml', + ]) + + def test_updates_grains(self): + charmhelpers.contrib.saltstack.update_machine_state( + 'states/install.yaml') + + self.mock_config_2_grains.assert_called_once_with('/etc/salt/grains') diff --git a/nrpe/mod/charmhelpers/tests/contrib/ssl/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/ssl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/ssl/test_service.py b/nrpe/mod/charmhelpers/tests/contrib/ssl/test_service.py new file mode 100644 index 0000000..389f6db --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/ssl/test_service.py @@ -0,0 +1,78 @@ +from testtools import TestCase +import tempfile +import shutil +import subprocess +import six +import mock + +from os.path import exists, join, isdir + +from charmhelpers.contrib.ssl import service + + +class ServiceCATest(TestCase): + + def setUp(self): + super(ServiceCATest, self).setUp() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + super(ServiceCATest, self).tearDown() + shutil.rmtree(self.temp_dir, ignore_errors=True) + + @mock.patch("charmhelpers.contrib.ssl.service.log") + def test_init(self, *args): + """ + Tests that a ServiceCA is initialized with the correct directory + layout. + """ + ca_root_dir = join(self.temp_dir, 'ca') + ca = service.ServiceCA('fake-name', ca_root_dir) + ca.init() + + paths_to_verify = [ + 'certs/', + 'crl/', + 'newcerts/', + 'private/', + 'private/cacert.key', + 'cacert.pem', + 'serial', + 'index.txt', + 'ca.cnf', + 'signing.cnf', + ] + + for path in paths_to_verify: + full_path = join(ca_root_dir, path) + self.assertTrue(exists(full_path), + 'Path {} does not exist'.format(full_path)) + + if path.endswith('/'): + self.assertTrue(isdir(full_path), + 'Path {} is not a dir'.format(full_path)) + + @mock.patch("charmhelpers.contrib.ssl.service.log") + def test_create_cert(self, *args): + """ + Tests that a generated certificate is valid against the ca. + """ + ca_root_dir = join(self.temp_dir, 'ca') + ca = service.ServiceCA('fake-name', ca_root_dir) + ca.init() + + ca.get_or_create_cert('fake-cert') + + # Verify that the cert belongs to the ca + self.assertTrue('fake-cert' in ca) + + full_cert_path = join(ca_root_dir, 'certs', 'fake-cert.crt') + cmd = ['openssl', 'verify', '-verbose', + '-CAfile', join(ca_root_dir, 'cacert.pem'), full_cert_path] + + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT).strip() + expected = '{}: OK'.format(full_cert_path) + if six.PY3: + expected = bytes(expected, 'utf-8') + self.assertEqual(expected, output) diff --git a/nrpe/mod/charmhelpers/tests/contrib/ssl/test_ssl.py b/nrpe/mod/charmhelpers/tests/contrib/ssl/test_ssl.py new file mode 100644 index 0000000..047f27a --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/ssl/test_ssl.py @@ -0,0 +1,56 @@ +from mock import patch +from testtools import TestCase + +from charmhelpers.contrib import ssl + + +class HelpersTest(TestCase): + @patch('subprocess.check_call') + def test_generate_selfsigned_dict(self, mock_call): + subject = {"country": "UK", + "locality": "my_locality", + "state": "my_state", + "organization": "my_organization", + "organizational_unit": "my_unit", + "cn": "mysite.example.com", + "email": "me@example.com" + } + + ssl.generate_selfsigned("mykey.key", "mycert.crt", subject=subject) + mock_call.assert_called_with(['/usr/bin/openssl', 'req', '-new', + '-newkey', 'rsa:1024', '-days', '365', + '-nodes', '-x509', '-keyout', + 'mykey.key', '-out', 'mycert.crt', + '-subj', + '/C=UK/ST=my_state/L=my_locality' + '/O=my_organization/OU=my_unit' + '/CN=mysite.example.com' + '/emailAddress=me@example.com'] + ) + + @patch('charmhelpers.core.hookenv.log') + def test_generate_selfsigned_failure(self, mock_log): + # This is NOT enough, function requires cn key + subject = {"country": "UK", + "locality": "my_locality"} + + result = ssl.generate_selfsigned("mykey.key", "mycert.crt", subject=subject) + self.assertFalse(result) + + @patch('subprocess.check_call') + def test_generate_selfsigned_file(self, mock_call): + ssl.generate_selfsigned("mykey.key", "mycert.crt", config="test.cnf") + mock_call.assert_called_with(['/usr/bin/openssl', 'req', '-new', + '-newkey', 'rsa:1024', '-days', '365', + '-nodes', '-x509', '-keyout', + 'mykey.key', '-out', 'mycert.crt', + '-config', 'test.cnf']) + + @patch('subprocess.check_call') + def test_generate_selfsigned_cn_key(self, mock_call): + ssl.generate_selfsigned("mykey.key", "mycert.crt", keysize="2048", cn="mysite.example.com") + mock_call.assert_called_with(['/usr/bin/openssl', 'req', '-new', + '-newkey', 'rsa:2048', '-days', '365', + '-nodes', '-x509', '-keyout', + 'mykey.key', '-out', 'mycert.crt', + '-subj', '/CN=mysite.example.com']) diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/test_bcache.py b/nrpe/mod/charmhelpers/tests/contrib/storage/test_bcache.py new file mode 100644 index 0000000..847828d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/storage/test_bcache.py @@ -0,0 +1,90 @@ +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import json +from mock import patch +from testtools import TestCase +from tempfile import mkdtemp +from charmhelpers.contrib.storage.linux import bcache + +test_stats = { + 'bypassed': '128G\n', + 'cache_bypass_hits': '1132623\n', + 'cache_bypass_misses': '0\n', + 'cache_hit_ratio': '64\n', + 'cache_hits': '12177090\n', + 'cache_miss_collisions': '7091\n', + 'cache_misses': '6717011\n', + 'cache_readaheads': '0\n', +} + +tmpdir = 'bcache-stats-test.' +cacheset = 'abcde' +cachedev = 'sdfoo' + + +class BcacheTestCase(TestCase): + def setUp(self): + super(BcacheTestCase, self).setUp() + self.sysfs = sysfs = mkdtemp(prefix=tmpdir) + self.addCleanup(shutil.rmtree, sysfs) + p = patch('charmhelpers.contrib.storage.linux.bcache.SYSFS', new=sysfs) + p.start() + self.addCleanup(p.stop) + self.cacheset = '{}/fs/bcache/{}'.format(sysfs, cacheset) + os.makedirs(self.cacheset) + self.devcache = '{}/block/{}/bcache'.format(sysfs, cachedev) + for n in ['register', 'register_quiet']: + with open('{}/fs/bcache/{}'.format(sysfs, n), 'w') as f: + f.write('foo') + for kind in self.cacheset, self.devcache: + for sub in bcache.stats_intervals: + intvaldir = '{}/{}'.format(kind, sub) + os.makedirs(intvaldir) + for fn, val in test_stats.items(): + with open(os.path.join(intvaldir, fn), 'w') as f: + f.write(val) + + def test_get_bcache_fs(self): + bcachedirs = bcache.get_bcache_fs() + assert len(bcachedirs) == 1 + assert next(iter(bcachedirs)).cachepath.endswith('/fs/bcache/abcde') + + @patch('charmhelpers.contrib.storage.linux.bcache.log', lambda *args, **kwargs: None) + @patch('charmhelpers.contrib.storage.linux.bcache.os.listdir') + def test_get_bcache_fs_nobcache(self, mock_listdir): + mock_listdir.side_effect = OSError( + '[Errno 2] No such file or directory:...') + bcachedirs = bcache.get_bcache_fs() + assert bcachedirs == [] + + def test_get_stats_global(self): + out = bcache.get_stats_action( + 'global', 'hour') + out = json.loads(out) + assert len(out.keys()) == 1 + k = next(iter(out.keys())) + assert k.endswith(cacheset) + assert out[k]['bypassed'] == '128G' + + def test_get_stats_dev(self): + out = bcache.get_stats_action( + cachedev, 'hour') + out = json.loads(out) + assert len(out.keys()) == 1 + k = next(iter(out.keys())) + assert k.endswith('sdfoo/bcache') + assert out[k]['cache_hit_ratio'] == '64' diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_ceph.py b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_ceph.py new file mode 100644 index 0000000..5ae0278 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_ceph.py @@ -0,0 +1,2244 @@ +from mock import patch, call, mock_open + +import collections +import six +import errno +from shutil import rmtree +from tempfile import mkdtemp +from threading import Timer +from testtools import TestCase +import json +import copy +import shutil + +import charmhelpers.contrib.storage.linux.ceph as ceph_utils + +from charmhelpers.core.unitdata import Storage +from subprocess import CalledProcessError +from tests.helpers import patch_open, FakeRelation +import nose.plugins.attrib +import os +import time + +LS_POOLS = b""" +.rgw.foo +images +volumes +rbd +""" + +LS_RBDS = b""" +rbd1 +rbd2 +rbd3 +""" + +IMG_MAP = b""" +bar +baz +""" +# Vastly abbreviated output from ceph osd dump --format=json +OSD_DUMP = b""" +{ + "pools": [ + { + "pool": 2, + "pool_name": "rbd", + "flags": 1, + "flags_names": "hashpspool", + "type": 1, + "size": 3, + "min_size": 2, + "crush_ruleset": 0, + "object_hash": 2, + "pg_num": 64, + "pg_placement_num": 64, + "crash_replay_interval": 0, + "last_change": "1", + "last_force_op_resend": "0", + "auid": 0, + "snap_mode": "selfmanaged", + "snap_seq": 0, + "snap_epoch": 0, + "pool_snaps": [], + "removed_snaps": "[]", + "quota_max_bytes": 0, + "quota_max_objects": 0, + "tiers": [], + "tier_of": -1, + "read_tier": -1, + "write_tier": -1, + "cache_mode": "writeback", + "target_max_bytes": 0, + "target_max_objects": 0, + "cache_target_dirty_ratio_micro": 0, + "cache_target_full_ratio_micro": 0, + "cache_min_flush_age": 0, + "cache_min_evict_age": 0, + "erasure_code_profile": "", + "hit_set_params": { + "type": "none" + }, + "hit_set_period": 0, + "hit_set_count": 0, + "stripe_width": 0 + } + ] +} +""" + +MONMAP_DUMP = b"""{ + "name": "ip-172-31-13-119", "rank": 0, "state": "leader", + "election_epoch": 18, "quorum": [0, 1, 2], + "outside_quorum": [], + "extra_probe_peers": [], + "sync_provider": [], + "monmap": { + "epoch": 1, + "fsid": "9fdc313c-db30-11e5-9805-0242fda74275", + "modified": "0.000000", + "created": "0.000000", + "mons": [ + { + "rank": 0, + "name": "ip-172-31-13-119", + "addr": "172.31.13.119:6789\\/0"}, + { + "rank": 1, + "name": "ip-172-31-24-50", + "addr": "172.31.24.50:6789\\/0"}, + { + "rank": 2, + "name": "ip-172-31-33-107", + "addr": "172.31.33.107:6789\\/0"} + ]}}""" + +CEPH_CLIENT_RELATION = { + 'ceph:8': { + 'ceph/0': { + 'auth': 'cephx', + 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}', + 'broker-rsp-glance-1': '{"request-id": "0880e22a", "exit-code": 0}', + 'broker-rsp-glance-2': '{"request-id": "0da543b8", "exit-code": 0}', + 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}', + 'ceph-public-address': '10.5.44.103', + 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', + 'private-address': '10.5.44.103', + }, + 'ceph/1': { + 'auth': 'cephx', + 'ceph-public-address': '10.5.44.104', + 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', + 'private-address': '10.5.44.104', + }, + 'ceph/2': { + 'auth': 'cephx', + 'ceph-public-address': '10.5.44.105', + 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', + 'private-address': '10.5.44.105', + }, + 'glance/0': { + 'broker_req': '{"api-version": 1, "request-id": "0bc7dc54", "ops": [{"replicas": 3, "name": "glance", "op": "create-pool", "rbd-mirroring-mode": "pool"}]}', + 'private-address': '10.5.44.109', + }, + } +} + +CEPH_CLIENT_RELATION_LEGACY = copy.deepcopy(CEPH_CLIENT_RELATION) +CEPH_CLIENT_RELATION_LEGACY['ceph:8']['ceph/0'] = { + 'auth': 'cephx', + 'broker_rsp': '{"exit-code": 0}', + 'ceph-public-address': '10.5.44.103', + 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', + 'private-address': '10.5.44.103', +} + + +class TestConfig(): + + def __init__(self): + self.config = {} + + def set(self, key, value): + self.config[key] = value + + def get(self, key): + return self.config.get(key) + + +class CephBasicUtilsTests(TestCase): + def setUp(self): + super(CephBasicUtilsTests, self).setUp() + [self._patch(m) for m in [ + 'check_output', + ]] + + def _patch(self, method): + _m = patch.object(ceph_utils, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def test_enabled_manager_modules(self): + self.check_output.return_value = b'{"enabled_modules": []}' + ceph_utils.enabled_manager_modules() + self.check_output.assert_called_once_with(['ceph', 'mgr', 'module', 'ls']) + + +class CephUtilsTests(TestCase): + def setUp(self): + super(CephUtilsTests, self).setUp() + [self._patch(m) for m in [ + 'check_call', + 'check_output', + 'config', + 'relation_get', + 'related_units', + 'relation_ids', + 'relation_set', + 'log', + 'cmp_pkgrevno', + 'enabled_manager_modules', + ]] + # Ensure the config is setup for mocking properly. + self.test_config = TestConfig() + self.config.side_effect = self.test_config.get + self.cmp_pkgrevno.return_value = 1 + self.enabled_manager_modules.return_value = [] + + def _patch(self, method): + _m = patch.object(ceph_utils, method) + mock = _m.start() + self.addCleanup(_m.stop) + setattr(self, method, mock) + + def _get_osd_settings_test_helper(self, settings, expected=None): + units = { + 'client:1': ['ceph-iscsi/1', 'ceph-iscsi/2'], + 'client:3': ['cinder-ceph/0', 'cinder-ceph/3']} + self.relation_ids.return_value = units.keys() + self.related_units.side_effect = lambda x: units[x] + self.relation_get.side_effect = lambda x, y, z: settings[y] + if expected: + self.assertEqual( + ceph_utils.get_osd_settings('client'), + expected) + else: + ceph_utils.get_osd_settings('client'), + + def test_get_osd_settings_all_unset(self): + settings = { + 'ceph-iscsi/1': None, + 'ceph-iscsi/2': None, + 'cinder-ceph/0': None, + 'cinder-ceph/3': None} + self._get_osd_settings_test_helper(settings, {}) + + def test_get_osd_settings_one_group_set(self): + settings = { + 'ceph-iscsi/1': '{"osd heartbeat grace": 5}', + 'ceph-iscsi/2': '{"osd heartbeat grace": 5}', + 'cinder-ceph/0': '{"osd heartbeat interval": 25}', + 'cinder-ceph/3': '{"osd heartbeat interval": 25}'} + self._get_osd_settings_test_helper( + settings, + {'osd heartbeat interval': 25, + 'osd heartbeat grace': 5}) + + def test_get_osd_settings_invalid_option(self): + settings = { + 'ceph-iscsi/1': '{"osd foobar": 5}', + 'ceph-iscsi/2': None, + 'cinder-ceph/0': None, + 'cinder-ceph/3': None} + self.assertRaises( + ceph_utils.OSDSettingNotAllowed, + self._get_osd_settings_test_helper, + settings) + + def test_get_osd_settings_conflicting_options(self): + settings = { + 'ceph-iscsi/1': '{"osd heartbeat grace": 5}', + 'ceph-iscsi/2': None, + 'cinder-ceph/0': '{"osd heartbeat grace": 6}', + 'cinder-ceph/3': None} + self.assertRaises( + ceph_utils.OSDSettingConflict, + self._get_osd_settings_test_helper, + settings) + + @patch.object(ceph_utils, 'application_name') + def test_send_application_name(self, application_name): + application_name.return_value = 'client' + ceph_utils.send_application_name() + self.relation_set.assert_called_once_with( + relation_settings={'application-name': 'client'}, + relation_id=None) + self.relation_set.reset_mock() + ceph_utils.send_application_name(relid='rid:1') + self.relation_set.assert_called_once_with( + relation_settings={'application-name': 'client'}, + relation_id='rid:1') + + @patch.object(ceph_utils, 'get_osd_settings') + def test_send_osd_settings(self, _get_osd_settings): + self.relation_ids.return_value = ['client:1', 'client:3'] + _get_osd_settings.return_value = { + 'osd heartbeat grace': 5, + 'osd heartbeat interval': 25} + ceph_utils.send_osd_settings() + expected_calls = [ + call( + relation_id='client:1', + relation_settings={ + 'osd-settings': ('{"osd heartbeat grace": 5, ' + '"osd heartbeat interval": 25}')}), + call( + relation_id='client:3', + relation_settings={ + 'osd-settings': ('{"osd heartbeat grace": 5, ' + '"osd heartbeat interval": 25}')})] + self.relation_set.assert_has_calls(expected_calls, any_order=True) + + @patch.object(ceph_utils, 'get_osd_settings') + def test_send_osd_settings_bad_settings(self, _get_osd_settings): + _get_osd_settings.side_effect = ceph_utils.OSDSettingConflict() + ceph_utils.send_osd_settings() + self.assertFalse(self.relation_set.called) + + def test_validator_valid(self): + # 1 is an int + ceph_utils.validator(value=1, + valid_type=int) + + def test_validator_valid_range(self): + # 1 is an int between 0 and 2 + ceph_utils.validator(value=1, + valid_type=int, + valid_range=[0, 2]) + + def test_validator_invalid_range(self): + # 1 is an int that isn't in the valid list of only 0 + self.assertRaises(ValueError, ceph_utils.validator, + value=1, + valid_type=int, + valid_range=[0]) + + def test_validator_invalid_string_list(self): + # foo is a six.string_types that isn't in the valid string list + self.assertRaises(AssertionError, ceph_utils.validator, + value="foo", + valid_type=six.string_types, + valid_range=["valid", "list", "of", "strings"]) + + def test_validator_valid_string(self): + ceph_utils.validator(value="foo", + valid_type=six.string_types, + valid_range=["foo"]) + + def test_validator_valid_string_type(self): + ceph_utils.validator(value="foo", + valid_type=str, + valid_range=["foo"]) + + def test_pool_set_quota(self): + p = ceph_utils.BasePool(service='admin', op={ + 'name': 'fake-pool', + 'max-bytes': 'fake-byte-quota', + }) + p.set_quota() + self.check_call.assert_called_once_with([ + 'ceph', '--id', 'admin', 'osd', + 'pool', 'set-quota', 'fake-pool', 'max_bytes', 'fake-byte-quota']) + self.check_call.reset_mock() + p = ceph_utils.BasePool(service='admin', op={ + 'name': 'fake-pool', + 'max-objects': 'fake-object-count-quota', + }) + p.set_quota() + self.check_call.assert_called_once_with([ + 'ceph', '--id', 'admin', 'osd', + 'pool', 'set-quota', 'fake-pool', 'max_objects', + 'fake-object-count-quota']) + self.check_call.reset_mock() + p = ceph_utils.BasePool(service='admin', op={ + 'name': 'fake-pool', + 'max-bytes': 'fake-byte-quota', + 'max-objects': 'fake-object-count-quota', + }) + p.set_quota() + self.check_call.assert_called_once_with([ + 'ceph', '--id', 'admin', 'osd', + 'pool', 'set-quota', 'fake-pool', 'max_bytes', 'fake-byte-quota', + 'max_objects', 'fake-object-count-quota']) + + def test_pool_set_compression(self): + p = ceph_utils.BasePool(service='admin', op={ + 'name': 'fake-pool', + 'compression-algorithm': 'lz4', + }) + p.set_compression() + self.check_call.assert_called_once_with([ + 'ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_algorithm', 'lz4']) + self.check_call.reset_mock() + p = ceph_utils.BasePool(service='admin', op={ + 'name': 'fake-pool', + 'compression-algorithm': 'lz4', + 'compression-mode': 'fake-mode', + 'compression-required-ratio': 'fake-ratio', + 'compression-min-blob-size': 'fake-min-blob-size', + 'compression-min-blob-size-hdd': 'fake-min-blob-size-hdd', + 'compression-min-blob-size-ssd': 'fake-min-blob-size-ssd', + 'compression-max-blob-size': 'fake-max-blob-size', + 'compression-max-blob-size-hdd': 'fake-max-blob-size-hdd', + 'compression-max-blob-size-ssd': 'fake-max-blob-size-ssd', + }) + p.set_compression() + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_algorithm', 'lz4']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_mode', 'fake-mode']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_required_ratio', 'fake-ratio']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_min_blob_size', 'fake-min-blob-size']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_min_blob_size_hdd', 'fake-min-blob-size-hdd']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_min_blob_size_ssd', 'fake-min-blob-size-ssd']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_max_blob_size', 'fake-max-blob-size']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_max_blob_size_hdd', 'fake-max-blob-size-hdd']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'fake-pool', + 'compression_max_blob_size_ssd', 'fake-max-blob-size-ssd']), + ], any_order=True) + + def test_pool_add_cache_tier(self): + p = ceph_utils.Pool(name='test', service='admin') + p.add_cache_tier('cacher', 'readonly') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'tier', 'add', 'test', 'cacher']), + call(['ceph', '--id', 'admin', 'osd', 'tier', 'cache-mode', 'cacher', 'readonly']), + call(['ceph', '--id', 'admin', 'osd', 'tier', 'set-overlay', 'test', 'cacher']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'cacher', 'hit_set_type', 'bloom']), + ]) + + @patch.object(ceph_utils, 'get_cache_mode') + def test_pool_remove_readonly_cache_tier(self, cache_mode): + cache_mode.return_value = 'readonly' + + p = ceph_utils.Pool(name='test', service='admin') + p.remove_cache_tier(cache_pool='cacher') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'tier', 'cache-mode', 'cacher', 'none']), + call(['ceph', '--id', 'admin', 'osd', 'tier', 'remove', 'test', 'cacher']), + ]) + + @patch.object(ceph_utils, 'get_cache_mode') + def test_pool_remove_writeback_cache_tier(self, cache_mode): + cache_mode.return_value = 'writeback' + self.cmp_pkgrevno.return_value = 1 + + p = ceph_utils.Pool(name='test', service='admin') + p.remove_cache_tier(cache_pool='cacher') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'tier', 'cache-mode', 'cacher', 'forward', + '--yes-i-really-mean-it']), + call(['rados', '--id', 'admin', '-p', 'cacher', 'cache-flush-evict-all']), + call(['ceph', '--id', 'admin', 'osd', 'tier', 'remove-overlay', 'test']), + call(['ceph', '--id', 'admin', 'osd', 'tier', 'remove', 'test', 'cacher']), + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_get_pg_num_pg_calc_values(self, get_osds): + """Tests the calculated pg num in the normal case works""" + # Check the growth case ... e.g. 200 PGs per OSD if the cluster is + # expected to grown in the near future. + get_osds.return_value = range(1, 11) + self.test_config.set('pgs-per-osd', 200) + p = ceph_utils.Pool(name='test', service='admin') + + # For Pool Size of 3, 200 PGs/OSD, and 40% of the overall data, + # the pg num should be 256 + pg_num = p.get_pgs(pool_size=3, percent_data=40) + self.assertEqual(256, pg_num) + + self.test_config.set('pgs-per-osd', 300) + pg_num = p.get_pgs(pool_size=3, percent_data=100) + self.assertEquals(1024, pg_num) + + # Tests the case in which the expected OSD count is provided (and is + # greater than the found OSD count). + self.test_config.set('pgs-per-osd', 100) + self.test_config.set('expected-osd-count', 20) + pg_num = p.get_pgs(pool_size=3, percent_data=100) + self.assertEquals(512, pg_num) + + # Test small % weight with minimal OSD count (3) + get_osds.return_value = range(1, 3) + self.test_config.set('expected-osd-count', None) + self.test_config.set('pgs-per-osd', None) + pg_num = p.get_pgs(pool_size=3, percent_data=0.1) + self.assertEquals(2, pg_num) + + # Check device_class is passed to get_osds + p.get_pgs(pool_size=3, percent_data=90, device_class='nvme') + get_osds.assert_called_with('admin', 'nvme') + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_old_ceph(self, get_osds): + self.cmp_pkgrevno.return_value = -1 + get_osds.return_value = None + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3) + p.create() + + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', 'test', str(200)]), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'test', 'size', str(3)]), + ]) + self.assertEqual(self.check_call.call_count, 2) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_luminous_ceph(self, get_osds): + self.cmp_pkgrevno.side_effect = [-1, 1] + get_osds.return_value = None + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3) + p.create() + + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', + 'pool', 'create', 'test', str(200)]), + call(['ceph', '--id', 'admin', 'osd', + 'pool', 'set', 'test', 'size', str(3)]), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'application', 'enable', 'test', 'unknown']) + ]) + self.assertEqual(self.check_call.call_count, 3) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_small_osds(self, get_osds): + get_osds.return_value = range(1, 5) + self.cmp_pkgrevno.return_value = -1 + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=10) + p.create() + + # Using the PG Calc, for 4 OSDs with a size of 3 and 10% of the data + # at 100 PGs/OSD, the number of expected placement groups will be 16 + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', 'test', + '16']), + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_medium_osds(self, get_osds): + self.cmp_pkgrevno.return_value = -1 + get_osds.return_value = range(1, 9) + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=50) + p.create() + + # Using the PG Calc, for 8 OSDs with a size of 3 and 50% of the data + # at 100 PGs/OSD, the number of expected placement groups will be 128 + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', 'test', + '128']), + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'test', 'size', '3']), + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_autoscaler(self, get_osds): + self.enabled_manager_modules.return_value = ['pg_autoscaler'] + self.cmp_pkgrevno.return_value = 1 + get_osds.return_value = range(1, 9) + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=50) + p.create() + # Using the PG Calc, for 8 OSDs with a size of 3 and 50% of the data + + # at 100 PGs/OSD, the number of expected placement groups will be 128 + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'create', '--pg-num-min=32', 'test', '128']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'size', '3']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'target_size_ratio', '0.5']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'application', 'enable', 'test', 'unknown']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'pg_autoscale_mode', 'on']) + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_autoscaler_small(self, get_osds): + self.enabled_manager_modules.return_value = ['pg_autoscaler'] + self.cmp_pkgrevno.return_value = 1 + get_osds.return_value = range(1, 3) + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=1) + p.create() + # Using the PG Calc, for 8 OSDs with a size of 3 and 50% of the data + + # at 100 PGs/OSD, the number of expected placement groups will be 128 + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'create', '--pg-num-min=2', 'test', '2']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'size', '3']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'target_size_ratio', '0.01']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'application', 'enable', 'test', 'unknown']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'pg_autoscale_mode', 'on']) + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_large_osds(self, get_osds): + get_osds.return_value = range(1, 41) + self.cmp_pkgrevno.return_value = -1 + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=100) + p.create() + + # Using the PG Calc, for 40 OSDs with a size of 3 and 100% of the + # data at 100 PGs/OSD then the number of expected placement groups + # will be 1024. + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', 'test', + '1024']), + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_xlarge_osds(self, get_osds): + get_osds.return_value = range(1, 1001) + self.cmp_pkgrevno.return_value = -1 + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3, + percent_data=100) + p.create() + + # Using the PG Calc, for 1,000 OSDs with a size of 3 and 100% of the + # data at 100 PGs/OSD then the number of expected placement groups + # will be 32768 + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', 'test', + '32768']), + ]) + + @patch.object(ceph_utils, 'get_osds') + def test_replicated_pool_create_failed(self, get_osds): + get_osds.return_value = range(1, 1001) + self.check_call.side_effect = CalledProcessError(returncode=1, + cmd='mock', + output=None) + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3) + self.assertRaises(CalledProcessError, p.create) + + @patch.object(ceph_utils, 'get_osds') + @patch.object(ceph_utils, 'pool_exists') + def test_replicated_pool_skips_creation(self, pool_exists, get_osds): + get_osds.return_value = range(1, 1001) + pool_exists.return_value = True + p = ceph_utils.ReplicatedPool(name='test', service='admin', replicas=3) + p.create() + self.check_call.assert_has_calls([]) + + def test_erasure_pool_create_failed(self): + self.check_output.side_effect = CalledProcessError(returncode=1, + cmd='ceph', + output=None) + p = ceph_utils.ErasurePool('test', 'admin', 'foo') + self.assertRaises(ceph_utils.PoolCreationError, p.create) + + @patch.object(ceph_utils, 'get_erasure_profile') + @patch.object(ceph_utils, 'get_osds') + def test_erasure_pool_create(self, get_osds, erasure_profile): + self.cmp_pkgrevno.return_value = 1 + get_osds.return_value = range(1, 60) + erasure_profile.return_value = { + 'directory': '/usr/lib/x86_64-linux-gnu/ceph/erasure-code', + 'k': '2', + 'technique': 'reed_sol_van', + 'm': '1', + 'plugin': 'jerasure'} + p = ceph_utils.ErasurePool(name='test', service='admin', + percent_data=100) + p.create() + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'create', + '--pg-num-min=32', 'test', + '2048', '2048', 'erasure', 'default']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'target_size_ratio', '1.0']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'application', 'enable', 'test', 'unknown']) + ]) + + @patch.object(ceph_utils, 'get_erasure_profile') + @patch.object(ceph_utils, 'get_osds') + def test_erasure_pool_create_autoscaler(self, + get_osds, + erasure_profile): + self.enabled_manager_modules.return_value = ['pg_autoscaler'] + self.cmp_pkgrevno.return_value = 1 + get_osds.return_value = range(1, 60) + erasure_profile.return_value = { + 'directory': '/usr/lib/x86_64-linux-gnu/ceph/erasure-code', + 'k': '2', + 'technique': 'reed_sol_van', + 'm': '1', + 'plugin': 'jerasure'} + p = ceph_utils.ErasurePool(name='test', service='admin', + percent_data=100, allow_ec_overwrites=True) + p.create() + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'create', '--pg-num-min=32', 'test', + '2048', '2048', 'erasure', 'default']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'target_size_ratio', '1.0']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'application', 'enable', 'test', 'unknown']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'pg_autoscale_mode', 'on']), + call(['ceph', '--id', 'admin', 'osd', 'pool', + 'set', 'test', 'allow_ec_overwrites', 'true']), + ]) + + def test_get_erasure_profile_none(self): + self.check_output.side_effect = CalledProcessError(1, 'ceph') + return_value = ceph_utils.get_erasure_profile('admin', 'unknown') + self.assertEqual(None, return_value) + + def test_pool_set_int(self): + self.check_call.return_value = 0 + ceph_utils.pool_set(service='admin', pool_name='data', key='test', value=2) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'data', 'test', '2']) + ]) + + def test_pool_set_bool(self): + self.check_call.return_value = 0 + ceph_utils.pool_set(service='admin', pool_name='data', key='test', value=True) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'data', 'test', 'true']) + ]) + + def test_pool_set_str(self): + self.check_call.return_value = 0 + ceph_utils.pool_set(service='admin', pool_name='data', key='test', value='two') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set', 'data', 'test', 'two']) + ]) + + def test_pool_set_fails(self): + self.check_call.side_effect = CalledProcessError(returncode=1, cmd='mock', + output=None) + self.assertRaises(CalledProcessError, ceph_utils.pool_set, + service='admin', pool_name='data', key='test', value=2) + + def test_snapshot_pool(self): + self.check_call.return_value = 0 + ceph_utils.snapshot_pool(service='admin', pool_name='data', snapshot_name='test-snap-1') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'mksnap', 'data', 'test-snap-1']) + ]) + + def test_snapshot_pool_fails(self): + self.check_call.side_effect = CalledProcessError(returncode=1, cmd='mock', + output=None) + self.assertRaises(CalledProcessError, ceph_utils.snapshot_pool, + service='admin', pool_name='data', snapshot_name='test-snap-1') + + def test_remove_pool_snapshot(self): + self.check_call.return_value = 0 + ceph_utils.remove_pool_snapshot(service='admin', pool_name='data', snapshot_name='test-snap-1') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'rmsnap', 'data', 'test-snap-1']) + ]) + + def test_set_pool_quota(self): + self.check_call.return_value = 0 + ceph_utils.set_pool_quota(service='admin', pool_name='data', + max_bytes=1024) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set-quota', 'data', + 'max_bytes', '1024']) + ]) + ceph_utils.set_pool_quota(service='admin', pool_name='data', + max_objects=1024) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set-quota', 'data', + 'max_objects', '1024']) + ]) + ceph_utils.set_pool_quota(service='admin', pool_name='data', + max_bytes=1024, max_objects=1024) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set-quota', 'data', + 'max_bytes', '1024', 'max_objects', '1024']) + ]) + + def test_remove_pool_quota(self): + self.check_call.return_value = 0 + ceph_utils.remove_pool_quota(service='admin', pool_name='data') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'admin', 'osd', 'pool', 'set-quota', 'data', 'max_bytes', '0']) + ]) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile(self, existing_profile): + existing_profile.return_value = False + self.test_config.set('customize-failure-domain', False) + self.cmp_pkgrevno.return_value = -1 + ceph_utils.create_erasure_profile( + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + failure_domain='rack', data_chunks=10, coding_chunks=3, + device_class='ssd') + + cmd = ['ceph', '--id', 'admin', 'osd', 'erasure-code-profile', 'set', 'super-profile', + 'plugin=' + 'jerasure', 'k=' + str(10), 'm=' + str(3), + 'ruleset-failure-domain=' + 'rack'] + self.check_call.assert_has_calls([call(cmd)]) + + self.cmp_pkgrevno.return_value = 1 + ceph_utils.create_erasure_profile( + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + failure_domain='rack', data_chunks=10, coding_chunks=3, + device_class='ssd') + + cmd = ['ceph', '--id', 'admin', 'osd', 'erasure-code-profile', 'set', 'super-profile', + 'plugin=' + 'jerasure', 'k=' + str(10), 'm=' + str(3), + 'crush-failure-domain=' + 'rack', + 'crush-device-class=ssd'] + self.check_call.assert_has_calls([call(cmd)]) + + existing_profile.return_value = True + self.check_call.reset_mock() + + ceph_utils.create_erasure_profile( + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + failure_domain='rack', data_chunks=10, coding_chunks=3, + device_class='ssd') + + self.check_call.assert_not_called() + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_failure_domain(self, existing_profile): + existing_profile.return_value = False + self.test_config.set('customize-failure-domain', True) + self.cmp_pkgrevno.return_value = -1 + ceph_utils.create_erasure_profile( + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + failure_domain=None, data_chunks=10, coding_chunks=3, + device_class='ssd') + + cmd = ['ceph', '--id', 'admin', 'osd', 'erasure-code-profile', 'set', 'super-profile', + 'plugin=' + 'jerasure', 'k=' + str(10), 'm=' + str(3), + 'ruleset-failure-domain=' + 'rack'] + self.config.assert_called_once_with('customize-failure-domain') + self.check_call.assert_has_calls([call(cmd)]) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_lrc(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + ceph_utils.create_erasure_profile( + service='admin', profile_name='super-profile', erasure_plugin_name='lrc', + failure_domain='host', data_chunks=10, coding_chunks=3, locality=1, + crush_locality='rack' + ) + + cmd = ['ceph', '--id', 'admin', 'osd', 'erasure-code-profile', 'set', 'super-profile', + 'plugin=' + 'lrc', 'k=' + str(10), 'm=' + str(3), + 'ruleset-failure-domain=' + 'host', 'l=' + str(1), + 'crush-locality=rack'] + self.check_call.assert_has_calls([call(cmd)]) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_lrc_no_locality(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + self.assertRaises( + ValueError, + ceph_utils.create_erasure_profile, + service='admin', profile_name='super-profile', erasure_plugin_name='lrc', + failure_domain='rack', data_chunks=10, coding_chunks=3, locality=None + ) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_invalid_plugin(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + self.assertRaises( + AssertionError, + ceph_utils.create_erasure_profile, + service='admin', profile_name='super-profile', erasure_plugin_name='foobar', + data_chunks=10, coding_chunks=3 + ) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_invalid_technique(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + self.assertRaises( + AssertionError, + ceph_utils.create_erasure_profile, + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + data_chunks=10, coding_chunks=3, + erasure_plugin_technique='foobar' + ) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_invalid_failure_domain(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + self.assertRaises( + AssertionError, + ceph_utils.create_erasure_profile, + service='admin', profile_name='super-profile', erasure_plugin_name='jerasure', + data_chunks=10, coding_chunks=3, + failure_domain='foobar', + ) + + @patch.object(ceph_utils, 'erasure_profile_exists') + def test_create_erasure_profile_shec(self, existing_profile): + self.cmp_pkgrevno.return_value = -1 + self.test_config.set('customize-failure-domain', False) + existing_profile.return_value = False + ceph_utils.create_erasure_profile(service='admin', profile_name='super-profile', erasure_plugin_name='shec', + failure_domain='rack', data_chunks=10, coding_chunks=3, + durability_estimator=1) + + cmd = ['ceph', '--id', 'admin', 'osd', 'erasure-code-profile', 'set', 'super-profile', + 'plugin=' + 'shec', 'k=' + str(10), 'm=' + str(3), + 'ruleset-failure-domain=' + 'rack', 'c=' + str(1)] + self.check_call.assert_has_calls([call(cmd)]) + + def test_rename_pool(self): + ceph_utils.rename_pool(service='admin', old_name='old-pool', new_name='new-pool') + cmd = ['ceph', '--id', 'admin', 'osd', 'pool', 'rename', 'old-pool', 'new-pool'] + self.check_call.assert_called_with(cmd) + + def test_erasure_profile_exists(self): + self.check_call.return_value = 0 + profile_exists = ceph_utils.erasure_profile_exists(service='admin', name='super-profile') + cmd = ['ceph', '--id', 'admin', + 'osd', 'erasure-code-profile', 'get', + 'super-profile'] + self.check_call.assert_called_with(cmd) + self.assertEqual(True, profile_exists) + + def test_set_monitor_key(self): + cmd = ['ceph', '--id', 'admin', + 'config-key', 'put', 'foo', 'bar'] + ceph_utils.monitor_key_set(service='admin', key='foo', value='bar') + self.check_output.assert_called_with(cmd) + + def test_get_monitor_key(self): + cmd = ['ceph', '--id', 'admin', + 'config-key', 'get', 'foo'] + ceph_utils.monitor_key_get(service='admin', key='foo') + self.check_output.assert_called_with(cmd) + + def test_get_monitor_key_failed(self): + self.check_output.side_effect = CalledProcessError( + returncode=2, + cmd='ceph', + output='key foo does not exist') + output = ceph_utils.monitor_key_get(service='admin', key='foo') + self.assertEqual(None, output) + + def test_monitor_key_exists(self): + cmd = ['ceph', '--id', 'admin', + 'config-key', 'exists', 'foo'] + ceph_utils.monitor_key_exists(service='admin', key='foo') + self.check_call.assert_called_with(cmd) + + def test_monitor_key_doesnt_exist(self): + self.check_call.side_effect = CalledProcessError( + returncode=2, + cmd='ceph', + output='key foo does not exist') + output = ceph_utils.monitor_key_exists(service='admin', key='foo') + self.assertEqual(False, output) + + def test_delete_monitor_key(self): + ceph_utils.monitor_key_delete(service='admin', key='foo') + cmd = ['ceph', '--id', 'admin', + 'config-key', 'del', 'foo'] + self.check_output.assert_called_with(cmd) + + def test_delete_monitor_key_failed(self): + self.check_output.side_effect = CalledProcessError( + returncode=2, + cmd='ceph', + output='deletion failed') + self.assertRaises(CalledProcessError, ceph_utils.monitor_key_delete, + service='admin', key='foo') + + def test_get_monmap(self): + self.check_output.return_value = MONMAP_DUMP + cmd = ['ceph', '--id', 'admin', + 'mon_status', '--format=json'] + ceph_utils.get_mon_map(service='admin') + self.check_output.assert_called_with(cmd) + + @patch.object(ceph_utils, 'get_mon_map') + def test_hash_monitor_names(self, monmap): + expected_hash_list = [ + '010d57d581604d411b315dd64112bff832ab92c7323fa06077134b50', + '8e0a9705c1aeafa1ce250cc9f1bb443fc6e5150e5edcbeb6eeb82e3c', + 'c3f8d36ba098c23ee920cb08cfb9beda6b639f8433637c190bdd56ec'] + _monmap_dump = MONMAP_DUMP + if six.PY3: + _monmap_dump = _monmap_dump.decode('UTF-8') + monmap.return_value = json.loads(_monmap_dump) + hashed_mon_list = ceph_utils.hash_monitor_names(service='admin') + self.assertEqual(expected=expected_hash_list, observed=hashed_mon_list) + + def test_get_cache_mode(self): + self.check_output.return_value = OSD_DUMP + cache_mode = ceph_utils.get_cache_mode(service='admin', pool_name='rbd') + self.assertEqual("writeback", cache_mode) + + @patch('os.path.exists') + def test_add_key(self, _exists): + """It creates a new ceph keyring""" + _exists.return_value = False + ceph_utils.add_key('cinder', 'cephkey') + _cmd = ['ceph-authtool', '/etc/ceph/ceph.client.cinder.keyring', + '--create-keyring', '--name=client.cinder', + '--add-key=cephkey'] + self.check_call.assert_called_with(_cmd) + + @patch('os.path.exists') + def test_add_key_already_exists(self, _exists): + """It should insert the key into the existing keyring""" + _exists.return_value = True + try: + with patch("__builtin__.open", mock_open(read_data="foo")): + ceph_utils.add_key('cinder', 'cephkey') + except ImportError: # Python3 + with patch("builtins.open", mock_open(read_data="foo")): + ceph_utils.add_key('cinder', 'cephkey') + self.assertTrue(self.log.called) + _cmd = ['ceph-authtool', '/etc/ceph/ceph.client.cinder.keyring', + '--create-keyring', '--name=client.cinder', + '--add-key=cephkey'] + self.check_call.assert_called_with(_cmd) + + @patch('os.path.exists') + def test_add_key_already_exists_and_key_exists(self, _exists): + """Nothing should happen, apart from a log message""" + _exists.return_value = True + try: + with patch("__builtin__.open", mock_open(read_data="cephkey")): + ceph_utils.add_key('cinder', 'cephkey') + except ImportError: # Python3 + with patch("builtins.open", mock_open(read_data="cephkey")): + ceph_utils.add_key('cinder', 'cephkey') + self.assertTrue(self.log.called) + self.check_call.assert_not_called() + + @patch('os.remove') + @patch('os.path.exists') + def test_delete_keyring(self, _exists, _remove): + """It deletes a ceph keyring.""" + _exists.return_value = True + ceph_utils.delete_keyring('cinder') + _remove.assert_called_with('/etc/ceph/ceph.client.cinder.keyring') + self.assertTrue(self.log.called) + + @patch('os.remove') + @patch('os.path.exists') + def test_delete_keyring_not_exists(self, _exists, _remove): + """It creates a new ceph keyring.""" + _exists.return_value = False + ceph_utils.delete_keyring('cinder') + self.assertTrue(self.log.called) + _remove.assert_not_called() + + @patch('os.path.exists') + def test_create_keyfile(self, _exists): + """It creates a new ceph keyfile""" + _exists.return_value = False + with patch_open() as (_open, _file): + ceph_utils.create_key_file('cinder', 'cephkey') + _file.write.assert_called_with('cephkey') + self.assertTrue(self.log.called) + + @patch('os.path.exists') + def test_create_key_file_already_exists(self, _exists): + """It creates a new ceph keyring""" + _exists.return_value = True + ceph_utils.create_key_file('cinder', 'cephkey') + self.assertTrue(self.log.called) + + @patch('os.mkdir') + @patch.object(ceph_utils, 'apt_install') + @patch('os.path.exists') + def test_install(self, _exists, _install, _mkdir): + _exists.return_value = False + ceph_utils.install() + _mkdir.assert_called_with('/etc/ceph') + _install.assert_called_with('ceph-common', fatal=True) + + def test_get_osds(self): + self.check_output.return_value = json.dumps([1, 2, 3]).encode('UTF-8') + self.assertEquals(ceph_utils.get_osds('test'), [1, 2, 3]) + + def test_get_osds_none(self): + self.check_output.return_value = json.dumps(None).encode('UTF-8') + self.assertEquals(ceph_utils.get_osds('test'), None) + + def test_get_osds_device_class(self): + self.check_output.return_value = json.dumps([1, 2, 3]).encode('UTF-8') + self.assertEquals(ceph_utils.get_osds('test', 'nvme'), [1, 2, 3]) + self.check_output.assert_called_once_with( + ['ceph', '--id', 'test', + 'osd', 'crush', 'class', + 'ls-osd', 'nvme', '--format=json'] + ) + + def test_get_osds_device_class_older(self): + self.check_output.return_value = json.dumps([1, 2, 3]).encode('UTF-8') + self.cmp_pkgrevno.return_value = -1 + self.assertEquals(ceph_utils.get_osds('test', 'nvme'), [1, 2, 3]) + self.check_output.assert_called_once_with( + ['ceph', '--id', 'test', 'osd', 'ls', '--format=json'] + ) + + @patch.object(ceph_utils, 'get_osds') + @patch.object(ceph_utils, 'pool_exists') + def test_create_pool(self, _exists, _get_osds): + """It creates rados pool correctly with default replicas """ + _exists.return_value = False + _get_osds.return_value = [1, 2, 3] + ceph_utils.create_pool(service='cinder', name='foo') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'cinder', 'osd', 'pool', + 'create', 'foo', '100']), + call(['ceph', '--id', 'cinder', 'osd', 'pool', 'set', + 'foo', 'size', '3']) + ]) + + @patch.object(ceph_utils, 'get_osds') + @patch.object(ceph_utils, 'pool_exists') + def test_create_pool_2_replicas(self, _exists, _get_osds): + """It creates rados pool correctly with 3 replicas""" + _exists.return_value = False + _get_osds.return_value = [1, 2, 3] + ceph_utils.create_pool(service='cinder', name='foo', replicas=2) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'cinder', 'osd', 'pool', + 'create', 'foo', '150']), + call(['ceph', '--id', 'cinder', 'osd', 'pool', 'set', + 'foo', 'size', '2']) + ]) + + @patch.object(ceph_utils, 'get_osds') + @patch.object(ceph_utils, 'pool_exists') + def test_create_pool_argonaut(self, _exists, _get_osds): + """It creates rados pool correctly with 3 replicas""" + _exists.return_value = False + _get_osds.return_value = None + ceph_utils.create_pool(service='cinder', name='foo') + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'cinder', 'osd', 'pool', + 'create', 'foo', '200']), + call(['ceph', '--id', 'cinder', 'osd', 'pool', 'set', + 'foo', 'size', '3']) + ]) + + def test_create_pool_already_exists(self): + self._patch('pool_exists') + self.pool_exists.return_value = True + ceph_utils.create_pool(service='cinder', name='foo') + self.assertTrue(self.log.called) + self.check_call.assert_not_called() + + def test_keyring_path(self): + """It correctly derives keyring path from service name""" + result = ceph_utils._keyring_path('cinder') + self.assertEquals('/etc/ceph/ceph.client.cinder.keyring', result) + + def test_keyfile_path(self): + """It correctly derives keyring path from service name""" + result = ceph_utils._keyfile_path('cinder') + self.assertEquals('/etc/ceph/ceph.client.cinder.key', result) + + def test_pool_exists(self): + """It detects an rbd pool exists""" + self.check_output.return_value = LS_POOLS + self.assertTrue(ceph_utils.pool_exists('cinder', 'volumes')) + self.assertTrue(ceph_utils.pool_exists('rgw', '.rgw.foo')) + + def test_pool_does_not_exist(self): + """It detects an rbd pool exists""" + self.check_output.return_value = LS_POOLS + self.assertFalse(ceph_utils.pool_exists('cinder', 'foo')) + self.assertFalse(ceph_utils.pool_exists('rgw', '.rgw')) + + def test_pool_exists_error(self): + """ Ensure subprocess errors and sandboxed with False """ + self.check_output.side_effect = CalledProcessError(1, 'rados') + self.assertFalse(ceph_utils.pool_exists('cinder', 'foo')) + + def test_rbd_exists(self): + self.check_output.return_value = LS_RBDS + self.assertTrue(ceph_utils.rbd_exists('service', 'pool', 'rbd1')) + self.check_output.assert_called_with( + ['rbd', 'list', '--id', 'service', '--pool', 'pool'] + ) + + def test_rbd_does_not_exist(self): + self.check_output.return_value = LS_RBDS + self.assertFalse(ceph_utils.rbd_exists('service', 'pool', 'rbd4')) + self.check_output.assert_called_with( + ['rbd', 'list', '--id', 'service', '--pool', 'pool'] + ) + + def test_rbd_exists_error(self): + """ Ensure subprocess errors and sandboxed with False """ + self.check_output.side_effect = CalledProcessError(1, 'rbd') + self.assertFalse(ceph_utils.rbd_exists('cinder', 'foo', 'rbd')) + + def test_create_rbd_image(self): + ceph_utils.create_rbd_image('service', 'pool', 'image', 128) + _cmd = ['rbd', 'create', 'image', + '--size', '128', + '--id', 'service', + '--pool', 'pool'] + self.check_call.assert_called_with(_cmd) + + def test_delete_pool(self): + ceph_utils.delete_pool('cinder', 'pool') + _cmd = [ + 'ceph', '--id', 'cinder', + 'osd', 'pool', 'delete', + 'pool', '--yes-i-really-really-mean-it' + ] + self.check_call.assert_called_with(_cmd) + + def test_get_ceph_nodes(self): + self._patch('relation_ids') + self._patch('related_units') + self._patch('relation_get') + units = ['ceph/1', 'ceph2', 'ceph/3'] + self.relation_ids.return_value = ['ceph:0'] + self.related_units.return_value = units + self.relation_get.return_value = '192.168.1.1' + self.assertEquals(len(ceph_utils.get_ceph_nodes()), 3) + + def test_get_ceph_nodes_not_related(self): + self._patch('relation_ids') + self.relation_ids.return_value = [] + self.assertEquals(ceph_utils.get_ceph_nodes(), []) + + def test_configure(self): + self._patch('add_key') + self._patch('create_key_file') + self._patch('get_ceph_nodes') + self._patch('modprobe') + _hosts = ['192.168.1.1', '192.168.1.2'] + self.get_ceph_nodes.return_value = _hosts + _conf = ceph_utils.CEPH_CONF.format( + auth='cephx', + keyring=ceph_utils._keyring_path('cinder'), + mon_hosts=",".join(map(str, _hosts)), + use_syslog='true' + ) + with patch_open() as (_open, _file): + ceph_utils.configure('cinder', 'key', 'cephx', 'true') + _file.write.assert_called_with(_conf) + _open.assert_called_with('/etc/ceph/ceph.conf', 'w') + self.modprobe.assert_called_with('rbd') + self.add_key.assert_called_with('cinder', 'key') + self.create_key_file.assert_called_with('cinder', 'key') + + def test_image_mapped(self): + self.check_output.return_value = IMG_MAP + self.assertTrue(ceph_utils.image_mapped('bar')) + + def test_image_not_mapped(self): + self.check_output.return_value = IMG_MAP + self.assertFalse(ceph_utils.image_mapped('foo')) + + def test_image_not_mapped_error(self): + self.check_output.side_effect = CalledProcessError(1, 'rbd') + self.assertFalse(ceph_utils.image_mapped('bar')) + + def test_map_block_storage(self): + _service = 'cinder' + _pool = 'bar' + _img = 'foo' + _cmd = [ + 'rbd', + 'map', + '{}/{}'.format(_pool, _img), + '--user', + _service, + '--secret', + ceph_utils._keyfile_path(_service), + ] + ceph_utils.map_block_storage(_service, _pool, _img) + self.check_call.assert_called_with(_cmd) + + def test_filesystem_mounted(self): + self._patch('mounts') + self.mounts.return_value = [['/afs', '/dev/sdb'], ['/bfs', '/dev/sdd']] + self.assertTrue(ceph_utils.filesystem_mounted('/afs')) + self.assertFalse(ceph_utils.filesystem_mounted('/zfs')) + + @patch('os.path.exists') + def test_make_filesystem(self, _exists): + _exists.return_value = True + ceph_utils.make_filesystem('/dev/sdd') + self.assertTrue(self.log.called) + self.check_call.assert_called_with(['mkfs', '-t', 'ext4', '/dev/sdd']) + + @patch('os.path.exists') + def test_make_filesystem_xfs(self, _exists): + _exists.return_value = True + ceph_utils.make_filesystem('/dev/sdd', 'xfs') + self.assertTrue(self.log.called) + self.check_call.assert_called_with(['mkfs', '-t', 'xfs', '/dev/sdd']) + + @patch('os.chown') + @patch('os.stat') + def test_place_data_on_block_device(self, _stat, _chown): + self._patch('mount') + self._patch('copy_files') + self._patch('umount') + _stat.return_value.st_uid = 100 + _stat.return_value.st_gid = 100 + ceph_utils.place_data_on_block_device('/dev/sdd', '/var/lib/mysql') + self.mount.assert_has_calls([ + call('/dev/sdd', '/mnt'), + call('/dev/sdd', '/var/lib/mysql', persist=True) + ]) + self.copy_files.assert_called_with('/var/lib/mysql', '/mnt') + self.umount.assert_called_with('/mnt') + _chown.assert_called_with('/var/lib/mysql', 100, 100) + + @patch('shutil.copytree') + @patch('os.listdir') + @patch('os.path.isdir') + def test_copy_files_is_dir(self, _isdir, _listdir, _copytree): + _isdir.return_value = True + subdirs = ['a', 'b', 'c'] + _listdir.return_value = subdirs + ceph_utils.copy_files('/source', '/dest') + for d in subdirs: + _copytree.assert_has_calls([ + call('/source/{}'.format(d), '/dest/{}'.format(d), + False, None) + ]) + + @patch('shutil.copytree') + @patch('os.listdir') + @patch('os.path.isdir') + def test_copy_files_include_symlinks(self, _isdir, _listdir, _copytree): + _isdir.return_value = True + subdirs = ['a', 'b', 'c'] + _listdir.return_value = subdirs + ceph_utils.copy_files('/source', '/dest', True) + for d in subdirs: + _copytree.assert_has_calls([ + call('/source/{}'.format(d), '/dest/{}'.format(d), + True, None) + ]) + + @patch('shutil.copytree') + @patch('os.listdir') + @patch('os.path.isdir') + def test_copy_files_ignore(self, _isdir, _listdir, _copytree): + _isdir.return_value = True + subdirs = ['a', 'b', 'c'] + _listdir.return_value = subdirs + ceph_utils.copy_files('/source', '/dest', True, False) + for d in subdirs: + _copytree.assert_has_calls([ + call('/source/{}'.format(d), '/dest/{}'.format(d), + True, False) + ]) + + @patch('shutil.copy2') + @patch('os.listdir') + @patch('os.path.isdir') + def test_copy_files_files(self, _isdir, _listdir, _copy2): + _isdir.return_value = False + files = ['a', 'b', 'c'] + _listdir.return_value = files + ceph_utils.copy_files('/source', '/dest') + for f in files: + _copy2.assert_has_calls([ + call('/source/{}'.format(f), '/dest/{}'.format(f)) + ]) + + def test_ensure_ceph_storage(self): + self._patch('pool_exists') + self.pool_exists.return_value = False + self._patch('create_pool') + self._patch('rbd_exists') + self.rbd_exists.return_value = False + self._patch('create_rbd_image') + self._patch('image_mapped') + self.image_mapped.return_value = False + self._patch('map_block_storage') + self._patch('filesystem_mounted') + self.filesystem_mounted.return_value = False + self._patch('make_filesystem') + self._patch('service_stop') + self._patch('service_start') + self._patch('service_running') + self.service_running.return_value = True + self._patch('place_data_on_block_device') + _service = 'mysql' + _pool = 'bar' + _rbd_img = 'foo' + _mount = '/var/lib/mysql' + _services = ['mysql'] + _blk_dev = '/dev/rbd1' + ceph_utils.ensure_ceph_storage(_service, _pool, + _rbd_img, 1024, _mount, + _blk_dev, 'ext4', _services, 3) + self.create_pool.assert_called_with(_service, _pool, replicas=3) + self.create_rbd_image.assert_called_with(_service, _pool, + _rbd_img, 1024) + self.map_block_storage.assert_called_with(_service, _pool, _rbd_img) + self.make_filesystem.assert_called_with(_blk_dev, 'ext4') + self.service_stop.assert_called_with(_services[0]) + self.place_data_on_block_device.assert_called_with(_blk_dev, _mount) + self.service_start.assert_called_with(_services[0]) + + def test_make_filesystem_default_filesystem(self): + """make_filesystem() uses ext4 as the default filesystem.""" + device = '/dev/zero' + ceph_utils.make_filesystem(device) + self.check_call.assert_called_with(['mkfs', '-t', 'ext4', device]) + + def test_make_filesystem_no_device(self): + """make_filesystem() raises an IOError if the device does not exist.""" + device = '/no/such/device' + e = self.assertRaises(IOError, ceph_utils.make_filesystem, device, + timeout=0) + self.assertEquals(device, e.filename) + self.assertEquals(errno.ENOENT, e.errno) + self.assertEquals(os.strerror(errno.ENOENT), e.strerror) + self.log.assert_called_with( + 'Gave up waiting on block device %s' % device, level='ERROR') + + @nose.plugins.attrib.attr('slow') + def test_make_filesystem_timeout(self): + """ + make_filesystem() allows to specify how long it should wait for the + device to appear before it fails. + """ + device = '/no/such/device' + timeout = 2 + before = time.time() + self.assertRaises(IOError, ceph_utils.make_filesystem, device, + timeout=timeout) + after = time.time() + duration = after - before + self.assertTrue(timeout - duration < 0.1) + self.log.assert_called_with( + 'Gave up waiting on block device %s' % device, level='ERROR') + + @nose.plugins.attrib.attr('slow') + def test_device_is_formatted_if_it_appears(self): + """ + The specified device is formatted if it appears before the timeout + is reached. + """ + + def create_my_device(filename): + with open(filename, "w") as device: + device.write("hello\n") + + temp_dir = mkdtemp() + self.addCleanup(rmtree, temp_dir) + device = "%s/mydevice" % temp_dir + fstype = 'xfs' + timeout = 4 + t = Timer(2, create_my_device, [device]) + t.start() + ceph_utils.make_filesystem(device, fstype, timeout) + self.check_call.assert_called_with(['mkfs', '-t', fstype, device]) + + def test_existing_device_is_formatted(self): + """ + make_filesystem() formats the given device if it exists with the + specified filesystem. + """ + device = '/dev/zero' + fstype = 'xfs' + ceph_utils.make_filesystem(device, fstype) + self.check_call.assert_called_with(['mkfs', '-t', fstype, device]) + self.log.assert_called_with( + 'Formatting block device %s as ' + 'filesystem %s.' % (device, fstype), level='INFO' + ) + + @patch.object(ceph_utils, 'relation_ids') + @patch.object(ceph_utils, 'related_units') + @patch.object(ceph_utils, 'relation_get') + def test_ensure_ceph_keyring_no_relation_no_data(self, rget, runits, rids): + rids.return_value = [] + self.assertEquals(False, ceph_utils.ensure_ceph_keyring(service='foo')) + rids.return_value = ['ceph:0'] + runits.return_value = ['ceph/0'] + rget.return_value = '' + self.assertEquals(False, ceph_utils.ensure_ceph_keyring(service='foo')) + + @patch.object(ceph_utils, '_keyring_path') + @patch.object(ceph_utils, 'add_key') + @patch.object(ceph_utils, 'relation_ids') + def test_ensure_ceph_keyring_no_relation_but_key(self, rids, + create, _path): + rids.return_value = [] + self.assertTrue(ceph_utils.ensure_ceph_keyring(service='foo', + key='testkey')) + create.assert_called_with(service='foo', key='testkey') + _path.assert_called_with('foo') + + @patch.object(ceph_utils, '_keyring_path') + @patch.object(ceph_utils, 'add_key') + @patch.object(ceph_utils, 'relation_ids') + @patch.object(ceph_utils, 'related_units') + @patch.object(ceph_utils, 'relation_get') + def test_ensure_ceph_keyring_with_data(self, rget, runits, + rids, create, _path): + rids.return_value = ['ceph:0'] + runits.return_value = ['ceph/0'] + rget.return_value = 'fookey' + self.assertEquals(True, + ceph_utils.ensure_ceph_keyring(service='foo')) + create.assert_called_with(service='foo', key='fookey') + _path.assert_called_with('foo') + self.assertFalse(self.check_call.called) + + _path.return_value = '/etc/ceph/client.foo.keyring' + self.assertEquals( + True, + ceph_utils.ensure_ceph_keyring( + service='foo', user='adam', group='users')) + create.assert_called_with(service='foo', key='fookey') + _path.assert_called_with('foo') + self.check_call.assert_called_with([ + 'chown', + 'adam.users', + '/etc/ceph/client.foo.keyring' + ]) + + @patch.object(ceph_utils, 'service_name') + @patch.object(ceph_utils, 'uuid') + def test_ceph_broker_rq_class(self, uuid, service_name): + service_name.return_value = 'service_test' + uuid.uuid1.return_value = 'uuid' + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool('pool1', replica_count=1) + rq.add_op_create_pool('pool1', replica_count=1) + rq.add_op_create_pool('pool2') + rq.add_op_create_pool('pool2') + rq.add_op_create_pool('pool3', group='test') + rq.add_op_request_access_to_group(name='test') + rq.add_op_request_access_to_group(name='test') + rq.add_op_request_access_to_group(name='objects', + key_name='test') + rq.add_op_request_access_to_group( + name='others', + object_prefix_permissions={'rwx': ['prefix1']}) + expected = { + 'api-version': 1, + 'request-id': 'uuid', + 'ops': [{'op': 'create-pool', 'name': 'pool1', 'replicas': 1}, + {'op': 'create-pool', 'name': 'pool2', 'replicas': 3}, + {'op': 'create-pool', 'name': 'pool3', 'replicas': 3, 'group': 'test'}, + {'op': 'add-permissions-to-key', 'group': 'test', 'name': 'service_test'}, + {'op': 'add-permissions-to-key', 'group': 'objects', 'name': 'test'}, + { + 'op': 'add-permissions-to-key', + 'group': 'others', + 'name': 'service_test', + 'object-prefix-permissions': {u'rwx': [u'prefix1']}}] + } + request_dict = json.loads(rq.request) + for key in ['api-version', 'request-id']: + self.assertEqual(request_dict[key], expected[key]) + for (op_no, expected_op) in enumerate(expected['ops']): + for key in expected_op.keys(): + self.assertEqual( + request_dict['ops'][op_no][key], + expected_op[key]) + + @patch.object(ceph_utils, 'service_name') + @patch.object(ceph_utils, 'uuid') + def test_ceph_broker_rq_class_test_not_equal(self, uuid, service_name): + service_name.return_value = 'service_test' + uuid.uuid1.return_value = 'uuid' + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_pool('pool1') + rq1.add_op_request_access_to_group(name='test') + rq1.add_op_request_access_to_group(name='objects', + permission='rwx') + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_pool('pool1') + rq2.add_op_request_access_to_group(name='test') + rq2.add_op_request_access_to_group(name='objects', + permission='r') + self.assertFalse(rq1 == rq2) + # now check equality check for common properties + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_replicated_pool('pool1') + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_replicated_pool('pool1') + self.assertTrue(rq1 == rq2) + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_replicated_pool('pool1', compression_mode='none') + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_replicated_pool('pool1', compression_mode='passive') + self.assertFalse(rq1 == rq2) + + def test_ceph_broker_rsp_class(self): + rsp = ceph_utils.CephBrokerRsp(json.dumps({'exit-code': 0, + 'stderr': "Success"})) + self.assertEqual(rsp.exit_code, 0) + self.assertEqual(rsp.exit_msg, "Success") + self.assertEqual(rsp.request_id, None) + + def test_ceph_broker_rsp_class_rqid(self): + rsp = ceph_utils.CephBrokerRsp(json.dumps({'exit-code': 0, + 'stderr': "Success", + 'request-id': 'reqid1'})) + self.assertEqual(rsp.exit_code, 0) + self.assertEqual(rsp.exit_msg, 'Success') + self.assertEqual(rsp.request_id, 'reqid1') + + def setup_client_relation(self, relation): + relation = FakeRelation(relation) + self.relation_get.side_effect = relation.get + self.relation_ids.side_effect = relation.relation_ids + self.related_units.side_effect = relation.related_units + + # @patch.object(ceph_utils, 'uuid') + # @patch.object(ceph_utils, 'local_unit') + # def test_get_request_states(self, mlocal_unit, muuid): + # muuid.uuid1.return_value = '0bc7dc54' + @patch.object(ceph_utils, 'local_unit') + def test_get_request_states(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + expect = {'ceph:8': {'complete': True, 'sent': True}} + self.assertEqual(ceph_utils.get_request_states(rq), expect) + + @patch.object(ceph_utils, 'local_unit') + def test_get_request_states_newrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + expect = {'ceph:8': {'complete': False, 'sent': False}} + self.assertEqual(ceph_utils.get_request_states(rq), expect) + + @patch.object(ceph_utils, 'local_unit') + def test_get_request_states_pendingrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + del rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] + self.setup_client_relation(rel) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + expect = {'ceph:8': {'complete': False, 'sent': True}} + self.assertEqual(ceph_utils.get_request_states(rq), expect) + + @patch.object(ceph_utils, 'local_unit') + def test_get_request_states_failedrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] = '{"request-id": "0bc7dc54", "exit-code": 1}' + self.setup_client_relation(rel) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + expect = {'ceph:8': {'complete': False, 'sent': True}} + self.assertEqual(ceph_utils.get_request_states(rq), expect) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent_newrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + self.assertFalse(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent_pending(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + del rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] + self.setup_client_relation(rel) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent_legacy(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION_LEGACY) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent_legacy_newrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION_LEGACY) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + self.assertFalse(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_sent_legacy_pending(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION_LEGACY) + del rel['ceph:8']['ceph/0']['broker_rsp'] + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_sent(rq)) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete(self, mlocal_unit, muuid): + muuid.uuid1.return_value = '0bc7dc54' + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_complete(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_newrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + self.assertFalse(ceph_utils.is_request_complete(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_pending(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + del rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] + self.setup_client_relation(rel) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertFalse(ceph_utils.is_request_complete(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_legacy(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION_LEGACY) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertTrue(ceph_utils.is_request_complete(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_legacy_newrq(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION_LEGACY) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + self.assertFalse(ceph_utils.is_request_complete(rq)) + + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_legacy_pending(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION_LEGACY) + del rel['ceph:8']['ceph/0']['broker_rsp'] + self.setup_client_relation(rel) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + self.assertFalse(ceph_utils.is_request_complete(rq)) + + def test_equivalent_broker_requests(self): + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_pool(name='glance', replica_count=4) + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_pool(name='glance', replica_count=4) + self.assertTrue(rq1 == rq2) + + def test_equivalent_broker_requests_diff1(self): + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_pool(name='glance', replica_count=3) + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_pool(name='glance', replica_count=4) + self.assertFalse(rq1 == rq2) + + def test_equivalent_broker_requests_diff2(self): + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_pool(name='glance', replica_count=3) + rq2 = ceph_utils.CephBrokerRq() + rq2.add_op_create_pool(name='cinder', replica_count=3) + self.assertFalse(rq1 == rq2) + + def test_equivalent_broker_requests_diff3(self): + rq1 = ceph_utils.CephBrokerRq() + rq1.add_op_create_pool(name='glance', replica_count=3) + rq2 = ceph_utils.CephBrokerRq(api_version=2) + rq2.add_op_create_pool(name='glance', replica_count=3) + self.assertFalse(rq1 == rq2) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_for_rid(self, mlocal_unit, muuid): + muuid.uuid1.return_value = '0bc7dc54' + req = ceph_utils.CephBrokerRq() + req.add_op_create_pool(name='glance', replica_count=3) + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + self.assertTrue(ceph_utils.is_request_complete_for_rid(req, 'ceph:8')) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_for_rid_newrq(self, mlocal_unit, muuid): + muuid.uuid1.return_value = 'a44c0fa6' + req = ceph_utils.CephBrokerRq() + req.add_op_create_pool(name='glance', replica_count=4) + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + self.assertFalse(ceph_utils.is_request_complete_for_rid(req, 'ceph:8')) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_for_rid_failed(self, mlocal_unit, muuid): + muuid.uuid1.return_value = '0bc7dc54' + req = ceph_utils.CephBrokerRq() + req.add_op_create_pool(name='glance', replica_count=4) + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] = '{"request-id": "0bc7dc54", "exit-code": 1}' + self.setup_client_relation(rel) + self.assertFalse(ceph_utils.is_request_complete_for_rid(req, 'ceph:8')) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_for_rid_pending(self, mlocal_unit, muuid): + muuid.uuid1.return_value = '0bc7dc54' + req = ceph_utils.CephBrokerRq() + req.add_op_create_pool(name='glance', replica_count=4) + mlocal_unit.return_value = 'glance/0' + rel = copy.deepcopy(CEPH_CLIENT_RELATION) + del rel['ceph:8']['ceph/0']['broker-rsp-glance-0'] + self.setup_client_relation(rel) + self.assertFalse(ceph_utils.is_request_complete_for_rid(req, 'ceph:8')) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_is_request_complete_for_rid_legacy(self, mlocal_unit, muuid): + muuid.uuid1.return_value = '0bc7dc54' + req = ceph_utils.CephBrokerRq() + req.add_op_create_pool(name='glance', replica_count=3) + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION_LEGACY) + self.assertTrue(ceph_utils.is_request_complete_for_rid(req, 'ceph:8')) + + @patch.object(ceph_utils, 'local_unit') + def test_get_broker_rsp_key(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.assertEqual(ceph_utils.get_broker_rsp_key(), 'broker-rsp-glance-0') + + @patch.object(ceph_utils, 'local_unit') + def test_send_request_if_needed(self, mlocal_unit): + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=3) + ceph_utils.send_request_if_needed(rq) + self.relation_set.assert_has_calls([]) + + @patch.object(ceph_utils, 'uuid') + @patch.object(ceph_utils, 'local_unit') + def test_send_request_if_needed_newrq(self, mlocal_unit, muuid): + muuid.uuid1.return_value = 'de67511e' + mlocal_unit.return_value = 'glance/0' + self.setup_client_relation(CEPH_CLIENT_RELATION) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool(name='glance', replica_count=4) + ceph_utils.send_request_if_needed(rq) + actual = json.loads(self.relation_set.call_args_list[0][1]['broker_req']) + self.assertEqual(actual['api-version'], 1) + self.assertEqual(actual['request-id'], 'de67511e') + self.assertEqual(actual['ops'][0]['replicas'], 4) + self.assertEqual(actual['ops'][0]['op'], 'create-pool') + self.assertEqual(actual['ops'][0]['name'], 'glance') + + @patch.object(ceph_utils, 'config') + def test_ceph_conf_context(self, mock_config): + mock_config.return_value = "{'osd': {'foo': 1}}" + ctxt = ceph_utils.CephConfContext()() + self.assertEqual({'osd': {'foo': 1}}, ctxt) + ctxt = ceph_utils.CephConfContext(['osd', 'mon'])() + mock_config.return_value = ("{'osd': {'foo': 1}," + "'unknown': {'blah': 1}}") + self.assertEqual({'osd': {'foo': 1}}, ctxt) + + @patch.object(ceph_utils, 'get_osd_settings') + @patch.object(ceph_utils, 'config') + def test_ceph_osd_conf_context_conflict(self, mock_config, + mock_get_osd_settings): + mock_config.return_value = "{'osd': {'osd heartbeat grace': 20}}" + mock_get_osd_settings.return_value = { + 'osd heartbeat grace': 25, + 'osd heartbeat interval': 5} + ctxt = ceph_utils.CephOSDConfContext()() + self.assertEqual(ctxt, { + 'osd': collections.OrderedDict([('osd heartbeat grace', 20)]), + 'osd_from_client': collections.OrderedDict( + [('osd heartbeat interval', 5)]), + 'osd_from_client_conflict': collections.OrderedDict( + [('osd heartbeat grace', 25)])}) + + @patch.object(ceph_utils, 'local_unit', lambda: "nova-compute/0") + def test_is_broker_action_done(self): + tmpdir = mkdtemp() + try: + db_path = '{}/kv.db'.format(tmpdir) + with patch('charmhelpers.core.unitdata._KV', Storage(db_path)): + rq_id = "3d03e9f6-4c36-11e7-89ba-fa163e7c7ec6" + broker_key = ceph_utils.get_broker_rsp_key() + self.relation_get.return_value = {broker_key: + json.dumps({"request-id": + rq_id, + "exit-code": 0})} + action = 'restart_nova_compute' + ret = ceph_utils.is_broker_action_done(action, rid="ceph:1", + unit="ceph/0") + self.relation_get.assert_has_calls([call(rid='ceph:1', unit='ceph/0')]) + self.assertFalse(ret) + + ceph_utils.mark_broker_action_done(action) + self.assertTrue(os.path.exists(tmpdir)) + ret = ceph_utils.is_broker_action_done(action, rid="ceph:1", + unit="ceph/0") + self.assertTrue(ret) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + + def test_has_broker_rsp(self): + rq_id = "3d03e9f6-4c36-11e7-89ba-fa163e7c7ec6" + broker_key = ceph_utils.get_broker_rsp_key() + self.relation_get.return_value = {broker_key: + json.dumps({"request-id": + rq_id, + "exit-code": 0})} + ret = ceph_utils.has_broker_rsp(rid="ceph:1", unit="ceph/0") + self.assertTrue(ret) + self.relation_get.assert_has_calls([call(rid='ceph:1', unit='ceph/0')]) + + self.relation_get.return_value = {'something_else': + json.dumps({"request-id": + rq_id, + "exit-code": 0})} + ret = ceph_utils.has_broker_rsp(rid="ceph:1", unit="ceph/0") + self.assertFalse(ret) + + self.relation_get.return_value = None + ret = ceph_utils.has_broker_rsp(rid="ceph:1", unit="ceph/0") + self.assertFalse(ret) + + @patch.object(ceph_utils, 'local_unit', lambda: "nova-compute/0") + def test_mark_broker_action_done(self): + tmpdir = mkdtemp() + try: + db_path = '{}/kv.db'.format(tmpdir) + with patch('charmhelpers.core.unitdata._KV', Storage(db_path)): + rq_id = "3d03e9f6-4c36-11e7-89ba-fa163e7c7ec6" + broker_key = ceph_utils.get_broker_rsp_key() + self.relation_get.return_value = {broker_key: + json.dumps({"request-id": + rq_id})} + action = 'restart_nova_compute' + ceph_utils.mark_broker_action_done(action, rid="ceph:1", + unit="ceph/0") + key = 'unit_0_ceph_broker_action.{}'.format(action) + self.relation_get.assert_has_calls([call(rid='ceph:1', unit='ceph/0')]) + kvstore = Storage(db_path) + self.assertEqual(kvstore.get(key=key), rq_id) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + + def test__partial_build_common_op_create(self): + self.maxDiff = None + rq = ceph_utils.CephBrokerRq() + expect = { + 'app-name': None, + 'compression-algorithm': 'lz4', + 'compression-mode': 'passive', + 'compression-required-ratio': 0.85, + 'compression-min-blob-size': 131072, + 'compression-min-blob-size-hdd': 131072, + 'compression-min-blob-size-ssd': 8192, + 'compression-max-blob-size': 524288, + 'compression-max-blob-size-hdd': 524288, + 'compression-max-blob-size-ssd': 65536, + 'group': None, + 'max-bytes': None, + 'max-objects': None, + 'group-namespace': None, + 'rbd-mirroring-mode': 'pool', + 'weight': None, + } + self.assertDictEqual( + rq._partial_build_common_op_create( + compression_algorithm='lz4', + compression_mode='passive', + compression_required_ratio=0.85, + compression_min_blob_size=131072, + compression_min_blob_size_hdd=131072, + compression_min_blob_size_ssd=8192, + compression_max_blob_size=524288, + compression_max_blob_size_hdd=524288, + compression_max_blob_size_ssd=65536), + expect) + + def test_add_op_create_replicated_pool(self): + base_op = {'app-name': None, + 'compression-algorithm': None, + 'compression-max-blob-size': None, + 'compression-max-blob-size-hdd': None, + 'compression-max-blob-size-ssd': None, + 'compression-min-blob-size': None, + 'compression-min-blob-size-hdd': None, + 'compression-min-blob-size-ssd': None, + 'compression-mode': None, + 'compression-required-ratio': None, + 'group': None, + 'group-namespace': None, + 'max-bytes': None, + 'max-objects': None, + 'name': 'apool', + 'op': 'create-pool', + 'pg_num': None, + 'rbd-mirroring-mode': 'pool', + 'replicas': 3, + 'weight': None} + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool') + self.assertDictEqual(rq.ops[0], base_op) + self.assertRaises( + ValueError, rq.add_op_create_pool, 'apool', pg_num=51, weight=100) + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_algorithm='invalid') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_mode='invalid') + # these parameters should be float / int + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_required_ratio='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_min_blob_size='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_min_blob_size_hdd='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_min_blob_size_ssd='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_max_blob_size='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_max_blob_size_hdd='1') + rq = ceph_utils.CephBrokerRq() + self.assertRaises( + ValueError, + rq.add_op_create_replicated_pool, 'apool', + compression_max_blob_size_ssd='1') + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_pool('apool', replica_count=42) + op = base_op.copy() + op['replicas'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', pg_num=42) + op = base_op.copy() + op['pg_num'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', weight=42) + op = base_op.copy() + op['weight'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', group=51) + op = base_op.copy() + op['group'] = 51 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', namespace='sol-iii') + op = base_op.copy() + op['group-namespace'] = 'sol-iii' + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', app_name='earth') + op = base_op.copy() + op['app-name'] = 'earth' + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', max_bytes=42) + op = base_op.copy() + op['max-bytes'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', max_objects=42) + op = base_op.copy() + op['max-objects'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', rbd_mirroring_mode='pool') + op = base_op.copy() + op['rbd-mirroring-mode'] = 'pool' + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_replicated_pool('apool', rbd_mirroring_mode='image') + op = base_op.copy() + op['rbd-mirroring-mode'] = 'image' + self.assertEqual(rq.ops, [op]) + + def test_add_op_create_erasure_pool(self): + base_op = {'app-name': None, + 'allow-ec-overwrites': False, + 'compression-algorithm': None, + 'compression-max-blob-size': None, + 'compression-max-blob-size-hdd': None, + 'compression-max-blob-size-ssd': None, + 'compression-min-blob-size': None, + 'compression-min-blob-size-hdd': None, + 'compression-min-blob-size-ssd': None, + 'compression-mode': None, + 'compression-required-ratio': None, + 'erasure-profile': None, + 'group': None, + 'group-namespace': None, + 'max-bytes': None, + 'max-objects': None, + 'name': 'apool', + 'op': 'create-pool', + 'pool-type': 'erasure', + 'rbd-mirroring-mode': 'pool', + 'weight': None} + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool') + self.assertDictEqual(rq.ops[0], base_op) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool', weight=42) + op = base_op.copy() + op['weight'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool', group=51) + op = base_op.copy() + op['group'] = 51 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool', app_name='earth') + op = base_op.copy() + op['app-name'] = 'earth' + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool', max_bytes=42) + op = base_op.copy() + op['max-bytes'] = 42 + self.assertEqual(rq.ops, [op]) + rq = ceph_utils.CephBrokerRq() + rq.add_op_create_erasure_pool('apool', max_objects=42) + op = base_op.copy() + op['max-objects'] = 42 + self.assertEqual(rq.ops, [op]) + + @patch.object(ceph_utils, 'local_unit') + def test_get_previous_request(self, _local_unit): + raw_request = CEPH_CLIENT_RELATION['ceph:8']['glance/0']['broker_req'] + self.relation_get.return_value = raw_request + req = ceph_utils.get_previous_request('aRid') + self.assertDictEqual(json.loads(req.request), json.loads(raw_request)) + self.relation_get.assert_called_once_with( + attribute='broker_req', rid='aRid', unit=_local_unit()) + + @patch.object(ceph_utils, 'uuid') + def test_CephBrokerRq__init__(self, _uuid): + raw_request = CEPH_CLIENT_RELATION['ceph:8']['glance/0']['broker_req'] + request = json.loads(raw_request) + req1 = ceph_utils.CephBrokerRq(api_version=request['api-version'], + request_id=request['request-id']) + req1.set_ops(request['ops']) + req2 = ceph_utils.CephBrokerRq(raw_request_data=raw_request) + self.assertDictEqual( + json.loads(req1.request), + json.loads(req2.request)) + _uuid.uuid1.return_value = 'fake-uuid' + new_req = ceph_utils.CephBrokerRq() + expect = { + 'api-version': 1, + 'request-id': 'fake-uuid', + 'ops': [], + } + self.assertDictEqual( + json.loads(new_req.request), expect) + + def test_update_pool(self): + ceph_utils.update_pool('aUser', 'aPool', {'aKey': 'aValue'}) + self.check_call.assert_called_once_with([ + 'ceph', '--id', 'aUser', 'osd', 'pool', 'set', + 'aPool', 'aKey', 'aValue']) + self.check_call.reset_mock() + ceph_utils.update_pool('aUser', 'aPool', { + 'aKey': 'aValue', + 'anotherKey': 'anotherValue'}) + self.check_call.assert_has_calls([ + call(['ceph', '--id', 'aUser', 'osd', 'pool', 'set', + 'aPool', 'aKey', 'aValue']), + call(['ceph', '--id', 'aUser', 'osd', 'pool', 'set', + 'aPool', 'anotherKey', 'anotherValue']), + ], any_order=True) diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_loopback.py b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_loopback.py new file mode 100644 index 0000000..85a130d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_loopback.py @@ -0,0 +1,82 @@ +import unittest + +from mock import patch + +import charmhelpers.contrib.storage.linux.loopback as loopback + +LOOPBACK_DEVICES = b""" +/dev/loop0: [0805]:2244465 (/tmp/foo.img) +/dev/loop1: [0805]:2244466 (/tmp/bar.img) +/dev/loop2: [0805]:2244467 (/tmp/baz.img (deleted)) +""" + +# It's a mouthful. +STORAGE_LINUX_LOOPBACK = 'charmhelpers.contrib.storage.linux.loopback' + + +class LoopbackStorageUtilsTests(unittest.TestCase): + @patch(STORAGE_LINUX_LOOPBACK + '.check_output') + def test_loopback_devices(self, output): + """It translates current loopback mapping to a dict""" + output.return_value = LOOPBACK_DEVICES + ex = { + '/dev/loop1': '/tmp/bar.img', + '/dev/loop0': '/tmp/foo.img', + '/dev/loop2': '/tmp/baz.img (deleted)' + } + self.assertEquals(loopback.loopback_devices(), ex) + + @patch(STORAGE_LINUX_LOOPBACK + '.create_loopback') + @patch('subprocess.check_call') + @patch(STORAGE_LINUX_LOOPBACK + '.loopback_devices') + def test_loopback_create_already_exists(self, loopbacks, check_call, + create): + """It finds existing loopback device for requested file""" + loopbacks.return_value = {'/dev/loop1': '/tmp/bar.img'} + res = loopback.ensure_loopback_device('/tmp/bar.img', '5G') + self.assertEquals(res, '/dev/loop1') + self.assertFalse(create.called) + self.assertFalse(check_call.called) + + @patch(STORAGE_LINUX_LOOPBACK + '.loopback_devices') + @patch(STORAGE_LINUX_LOOPBACK + '.create_loopback') + @patch('os.path.exists') + def test_loop_creation_no_truncate(self, path_exists, create_loopback, + loopbacks): + """It does not create a new sparse image for loopback if one exists""" + loopbacks.return_value = {} + path_exists.return_value = True + with patch('subprocess.check_call') as check_call: + loopback.ensure_loopback_device('/tmp/foo.img', '15G') + self.assertFalse(check_call.called) + + @patch(STORAGE_LINUX_LOOPBACK + '.loopback_devices') + @patch(STORAGE_LINUX_LOOPBACK + '.create_loopback') + @patch('os.path.exists') + def test_ensure_loopback_creation(self, path_exists, create_loopback, + loopbacks): + """It creates a new sparse image for loopback if one does not exists""" + loopbacks.return_value = {} + path_exists.return_value = False + create_loopback.return_value = '/dev/loop0' + with patch(STORAGE_LINUX_LOOPBACK + '.check_call') as check_call: + loopback.ensure_loopback_device('/tmp/foo.img', '15G') + check_call.assert_called_with(['truncate', '--size', '15G', + '/tmp/foo.img']) + + @patch.object(loopback, 'loopback_devices') + def test_create_loopback(self, _devs): + """It correctly calls losetup to create a loopback device""" + _devs.return_value = {'/dev/loop0': '/tmp/foo'} + with patch(STORAGE_LINUX_LOOPBACK + '.check_call') as check_call: + check_call.return_value = '' + result = loopback.create_loopback('/tmp/foo') + check_call.assert_called_with(['losetup', '--find', '/tmp/foo']) + self.assertEquals(result, '/dev/loop0') + + @patch.object(loopback, 'loopback_devices') + def test_create_is_mapped_loopback_device(self, devs): + devs.return_value = {'/dev/loop0': "/tmp/manco"} + self.assertEquals(loopback.is_mapped_loopback_device("/dev/loop0"), + "/tmp/manco") + self.assertFalse(loopback.is_mapped_loopback_device("/dev/loop1")) diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_lvm.py b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_lvm.py new file mode 100644 index 0000000..015d451 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_lvm.py @@ -0,0 +1,212 @@ +import unittest +import subprocess + +from mock import patch + +import charmhelpers.contrib.storage.linux.lvm as lvm + +PVDISPLAY = b""" + --- Physical volume --- + PV Name /dev/loop0 + VG Name foo + PV Size 10.00 MiB / not usable 2.00 MiB + Allocatable yes + PE Size 4.00 MiB + Total PE 2 + Free PE 2 + Allocated PE 0 + PV UUID fyVqlr-pyrL-89On-f6MD-U91T-dEfc-SL0V2V + +""" + +EMPTY_VG_IN_PVDISPLAY = b""" + --- Physical volume --- + PV Name /dev/loop0 + VG Name + PV Size 10.00 MiB / not usable 2.00 MiB + Allocatable yes + PE Size 4.00 MiB + Total PE 2 + Free PE 2 + Allocated PE 0 + PV UUID fyVqlr-pyrL-89On-f6MD-U91T-dEfc-SL0V2V + +""" +LVS_DEFAULT = b""" + cinder-volumes-pool + testvol + volume-48be6ba0-84c3-4b8d-9be5-e68e47fc7682 + volume-f8c1d2fd-1fa1-4d84-b4e0-431dba7d582e +""" +LVS_WITH_VG = b""" + cinder-volumes cinder-volumes-pool + cinder-volumes testvol + cinder-volumes volume-48be6ba0-84c3-4b8d-9be5-e68e47fc7682 + cinder-volumes volume-f8c1d2fd-1fa1-4d84-b4e0-431dba7d582e +""" +LVS_THIN_POOLS = b""" + cinder-volumes-pool +""" +LVS_THIN_POOLS_WITH_VG = b""" + cinder-volumes cinder-volumes-pool +""" + +# It's a mouthful. +STORAGE_LINUX_LVM = 'charmhelpers.contrib.storage.linux.lvm' + + +class LVMStorageUtilsTests(unittest.TestCase): + def test_find_volume_group_on_pv(self): + """It determines any volume group assigned to a LVM PV""" + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = PVDISPLAY + vg = lvm.list_lvm_volume_group('/dev/loop0') + self.assertEquals(vg, 'foo') + + def test_find_empty_volume_group_on_pv(self): + """Return empty string when no volume group is assigned to the PV""" + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = EMPTY_VG_IN_PVDISPLAY + vg = lvm.list_lvm_volume_group('/dev/loop0') + self.assertEquals(vg, '') + + @patch(STORAGE_LINUX_LVM + '.list_lvm_volume_group') + def test_deactivate_lvm_volume_groups(self, ls_vg): + """It deactivates active volume groups on LVM PV""" + ls_vg.return_value = 'foo-vg' + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.deactivate_lvm_volume_group('/dev/loop0') + check_call.assert_called_with(['vgchange', '-an', 'foo-vg']) + + def test_remove_lvm_physical_volume(self): + """It removes LVM physical volume signatures from block device""" + with patch(STORAGE_LINUX_LVM + '.Popen') as popen: + lvm.remove_lvm_physical_volume('/dev/foo') + popen.assert_called_with(['pvremove', '-ff', '/dev/foo'], stdin=-1) + + def test_is_physical_volume(self): + """It properly reports block dev is an LVM PV""" + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = PVDISPLAY + self.assertTrue(lvm.is_lvm_physical_volume('/dev/loop0')) + + def test_is_not_physical_volume(self): + """It properly reports block dev is an LVM PV""" + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.side_effect = subprocess.CalledProcessError('cmd', 2) + self.assertFalse(lvm.is_lvm_physical_volume('/dev/loop0')) + + def test_pvcreate(self): + """It correctly calls pvcreate for a given block dev""" + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.create_lvm_physical_volume('/dev/foo') + check_call.assert_called_with(['pvcreate', '/dev/foo']) + + def test_vgcreate(self): + """It correctly calls vgcreate for given block dev and vol group""" + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.create_lvm_volume_group('foo-vg', '/dev/foo') + check_call.assert_called_with(['vgcreate', 'foo-vg', '/dev/foo']) + + def test_list_logical_volumes(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = LVS_DEFAULT + self.assertEqual(lvm.list_logical_volumes(), [ + 'cinder-volumes-pool', + 'testvol', + 'volume-48be6ba0-84c3-4b8d-9be5-e68e47fc7682', + 'volume-f8c1d2fd-1fa1-4d84-b4e0-431dba7d582e']) + check_output.assert_called_with([ + 'lvs', + '--options', + 'lv_name', + '--noheadings']) + + def test_list_logical_volumes_empty(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = b'' + self.assertEqual(lvm.list_logical_volumes(), []) + + def test_list_logical_volumes_path_mode(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = LVS_WITH_VG + self.assertEqual(lvm.list_logical_volumes(path_mode=True), [ + 'cinder-volumes/cinder-volumes-pool', + 'cinder-volumes/testvol', + 'cinder-volumes/volume-48be6ba0-84c3-4b8d-9be5-e68e47fc7682', + 'cinder-volumes/volume-f8c1d2fd-1fa1-4d84-b4e0-431dba7d582e']) + check_output.assert_called_with([ + 'lvs', + '--options', + 'vg_name,lv_name', + '--noheadings']) + + def test_list_logical_volumes_select_criteria(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = LVS_THIN_POOLS + self.assertEqual( + lvm.list_logical_volumes(select_criteria='lv_attr =~ ^t'), + ['cinder-volumes-pool']) + check_output.assert_called_with([ + 'lvs', + '--options', + 'lv_name', + '--noheadings', + '--select', + 'lv_attr =~ ^t']) + + def test_list_thin_logical_volume_pools(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = LVS_THIN_POOLS + self.assertEqual( + lvm.list_thin_logical_volume_pools(), + ['cinder-volumes-pool']) + check_output.assert_called_with([ + 'lvs', + '--options', + 'lv_name', + '--noheadings', + '--select', + 'lv_attr =~ ^t']) + + def test_list_thin_logical_volume_pools_path_mode(self): + with patch(STORAGE_LINUX_LVM + '.check_output') as check_output: + check_output.return_value = LVS_THIN_POOLS_WITH_VG + self.assertEqual( + lvm.list_thin_logical_volume_pools(path_mode=True), + ['cinder-volumes/cinder-volumes-pool']) + check_output.assert_called_with([ + 'lvs', + '--options', + 'vg_name,lv_name', + '--noheadings', + '--select', + 'lv_attr =~ ^t']) + + def test_extend_logical_volume_by_device(self): + """It correctly calls pvcreate for a given block dev""" + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.extend_logical_volume_by_device('mylv', '/dev/foo') + check_call.assert_called_with(['lvextend', 'mylv', '/dev/foo']) + + def test_create_logical_volume_nosize(self): + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.create_logical_volume('testlv', 'testvg') + check_call.assert_called_with([ + 'lvcreate', + '--yes', + '-l', + '100%FREE', + '-n', 'testlv', 'testvg' + ]) + + def test_create_logical_volume_size(self): + with patch(STORAGE_LINUX_LVM + '.check_call') as check_call: + lvm.create_logical_volume('testlv', 'testvg', '10G') + check_call.assert_called_with([ + 'lvcreate', + '--yes', + '-L', + '10G', + '-n', 'testlv', 'testvg' + ]) diff --git a/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_utils.py b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_utils.py new file mode 100644 index 0000000..bf4da91 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/storage/test_linux_storage_utils.py @@ -0,0 +1,206 @@ +from mock import patch +import unittest + +import charmhelpers.contrib.storage.linux.utils as storage_utils + +# It's a mouthful. +STORAGE_LINUX_UTILS = 'charmhelpers.contrib.storage.linux.utils' + + +class MiscStorageUtilsTests(unittest.TestCase): + + @patch(STORAGE_LINUX_UTILS + '.check_output') + @patch(STORAGE_LINUX_UTILS + '.call') + @patch(STORAGE_LINUX_UTILS + '.check_call') + def test_zap_disk(self, check_call, call, check_output): + """It calls sgdisk correctly to zap disk""" + check_output.return_value = b'200\n' + storage_utils.zap_disk('/dev/foo') + call.assert_any_call(['sgdisk', '--zap-all', '--', '/dev/foo']) + call.assert_any_call(['sgdisk', '--clear', '--mbrtogpt', + '--', '/dev/foo']) + check_output.assert_any_call(['blockdev', '--getsz', '/dev/foo']) + check_call.assert_any_call(['dd', 'if=/dev/zero', 'of=/dev/foo', + 'bs=1M', 'count=1']) + check_call.assert_any_call(['dd', 'if=/dev/zero', 'of=/dev/foo', + 'bs=512', 'count=100', 'seek=100']) + + @patch(STORAGE_LINUX_UTILS + '.S_ISBLK') + @patch('os.path.exists') + @patch('os.stat') + def test_is_block_device(self, S_ISBLK, exists, stat): + """It detects device node is block device""" + class fake_stat: + st_mode = True + S_ISBLK.return_value = fake_stat() + exists.return_value = True + self.assertTrue(storage_utils.is_block_device('/dev/foo')) + + @patch(STORAGE_LINUX_UTILS + '.S_ISBLK') + @patch('os.path.exists') + @patch('os.stat') + def test_is_block_device_does_not_exist(self, S_ISBLK, exists, stat): + """It detects device node is block device""" + class fake_stat: + st_mode = True + S_ISBLK.return_value = fake_stat() + exists.return_value = False + self.assertFalse(storage_utils.is_block_device('/dev/foo')) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted(self, check_output): + """It detects mounted devices as mounted.""" + check_output.return_value = ( + b'NAME="sda" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT="/tmp"\n') + result = storage_utils.is_device_mounted('/dev/sda') + self.assertTrue(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_partition(self, check_output): + """It detects mounted partitions as mounted.""" + check_output.return_value = ( + b'NAME="sda1" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT="/tmp"\n') + result = storage_utils.is_device_mounted('/dev/sda1') + self.assertTrue(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_partition_with_device(self, check_output): + """It detects mounted devices as mounted if "mount" shows only a + partition as mounted.""" + check_output.return_value = ( + b'NAME="sda1" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT="/tmp"\n') + result = storage_utils.is_device_mounted('/dev/sda') + self.assertTrue(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_not_mounted(self, check_output): + """It detects unmounted devices as not mounted.""" + check_output.return_value = ( + b'NAME="sda" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT=""\n') + result = storage_utils.is_device_mounted('/dev/sda') + self.assertFalse(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_not_mounted_partition(self, check_output): + """It detects unmounted partitions as not mounted.""" + check_output.return_value = ( + b'NAME="sda" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT=""\n') + result = storage_utils.is_device_mounted('/dev/sda1') + self.assertFalse(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_full_disks(self, check_output): + """It detects mounted full disks as mounted.""" + check_output.return_value = ( + b'NAME="sda" MAJ:MIN="8:16" RM="0" SIZE="238.5G" RO="0" TYPE="disk" MOUNTPOINT="/tmp"\n') + result = storage_utils.is_device_mounted('/dev/sda') + self.assertTrue(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_cciss(self, check_output): + """It detects mounted cciss partitions as mounted.""" + check_output.return_value = ( + b'NAME="cciss!c0d0" MAJ:MIN="104:0" RM="0" SIZE="273.3G" RO="0" TYPE="disk" MOUNTPOINT="/root"\n') + result = storage_utils.is_device_mounted('/dev/cciss/c0d0') + self.assertTrue(result) + + @patch(STORAGE_LINUX_UTILS + '.check_output') + def test_is_device_mounted_cciss_not_mounted(self, check_output): + """It detects unmounted cciss partitions as not mounted.""" + check_output.return_value = ( + b'NAME="cciss!c0d0" MAJ:MIN="104:0" RM="0" SIZE="273.3G" RO="0" TYPE="disk" MOUNTPOINT=""\n') + result = storage_utils.is_device_mounted('/dev/cciss/c0d0') + self.assertFalse(result) + + @patch(STORAGE_LINUX_UTILS + '.check_call') + def test_mkfs_xfs(self, check_call): + storage_utils.mkfs_xfs('/dev/sdb') + check_call.assert_called_with( + ['mkfs.xfs', '-i', 'size=1024', '/dev/sdb'] + ) + + @patch(STORAGE_LINUX_UTILS + '.check_call') + def test_mkfs_xfs_force(self, check_call): + storage_utils.mkfs_xfs('/dev/sdb', force=True) + check_call.assert_called_with( + ['mkfs.xfs', '-f', '-i', 'size=1024', '/dev/sdb'] + ) + + @patch(STORAGE_LINUX_UTILS + '.check_call') + def test_mkfs_xfs_inode_size(self, check_call): + storage_utils.mkfs_xfs('/dev/sdb', inode_size=512) + check_call.assert_called_with( + ['mkfs.xfs', '-i', 'size=512', '/dev/sdb'] + ) + + +class CephLUKSDeviceTestCase(unittest.TestCase): + + @patch.object(storage_utils, '_luks_uuid') + def test_no_luks_header(self, _luks_uuid): + _luks_uuid.return_value = None + self.assertEqual(storage_utils.is_luks_device('/dev/sdb'), False) + + @patch.object(storage_utils, '_luks_uuid') + def test_luks_header(self, _luks_uuid): + _luks_uuid.return_value = '5e1e4c89-4f68-4b9a-bd93-e25eec34e80f' + self.assertEqual(storage_utils.is_luks_device('/dev/sdb'), True) + + +class CephMappedLUKSDeviceTestCase(unittest.TestCase): + + @patch.object(storage_utils.os, 'walk') + @patch.object(storage_utils, '_luks_uuid') + def test_no_luks_header_not_mapped(self, _luks_uuid, _walk): + _luks_uuid.return_value = None + + def os_walk_side_effect(path): + return { + '/sys/class/block/sdb/holders/': iter([('', [], [])]), + }[path] + _walk.side_effect = os_walk_side_effect + + self.assertEqual(storage_utils.is_mapped_luks_device('/dev/sdb'), False) + + @patch.object(storage_utils.os, 'walk') + @patch.object(storage_utils, '_luks_uuid') + def test_luks_header_mapped(self, _luks_uuid, _walk): + _luks_uuid.return_value = 'db76d142-4782-42f2-84c6-914f9db889a0' + + def os_walk_side_effect(path): + return { + '/sys/class/block/sdb/holders/': iter([('', ['dm-0'], [])]), + }[path] + _walk.side_effect = os_walk_side_effect + + self.assertEqual(storage_utils.is_mapped_luks_device('/dev/sdb'), True) + + @patch.object(storage_utils.os, 'walk') + @patch.object(storage_utils, '_luks_uuid') + def test_luks_header_not_mapped(self, _luks_uuid, _walk): + _luks_uuid.return_value = 'db76d142-4782-42f2-84c6-914f9db889a0' + + def os_walk_side_effect(path): + return { + '/sys/class/block/sdb/holders/': iter([('', [], [])]), + }[path] + _walk.side_effect = os_walk_side_effect + + self.assertEqual(storage_utils.is_mapped_luks_device('/dev/sdb'), False) + + @patch.object(storage_utils.os, 'walk') + @patch.object(storage_utils, '_luks_uuid') + def test_no_luks_header_mapped(self, _luks_uuid, _walk): + """ + This is an edge case where a device is mapped (i.e. used for something + else) but has no LUKS header. Should be handled by other checks. + """ + _luks_uuid.return_value = None + + def os_walk_side_effect(path): + return { + '/sys/class/block/sdb/holders/': iter([('', ['dm-0'], [])]), + }[path] + _walk.side_effect = os_walk_side_effect + + self.assertEqual(storage_utils.is_mapped_luks_device('/dev/sdb'), False) diff --git a/nrpe/mod/charmhelpers/tests/contrib/sysctl/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/sysctl/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/sysctl/test_watermark_scale_factor.py b/nrpe/mod/charmhelpers/tests/contrib/sysctl/test_watermark_scale_factor.py new file mode 100644 index 0000000..add0973 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/sysctl/test_watermark_scale_factor.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +from charmhelpers.contrib.sysctl.watermark_scale_factor import ( + watermark_scale_factor, + calculate_watermark_scale_factor, + get_normal_managed_pages, +) + +from mock import patch +import unittest + +from tests.helpers import patch_open + +TO_PATCH = [ + "log", + "ERROR", + "DEBUG" +] + +PROC_ZONEINFO = """ +Node 0, zone Normal + pages free 1253032 + min 16588 + low 40833 + high 65078 + spanned 24674304 + present 24674304 + managed 24247810 + protection: (0, 0, 0, 0, 0) +""" + + +class TestWatermarkScaleFactor(unittest.TestCase): + + def setUp(self): + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.contrib.sysctl.watermark_scale_factor.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + @patch('charmhelpers.contrib.sysctl.watermark_scale_factor.get_normal_managed_pages') + @patch('charmhelpers.core.host.get_total_ram') + def test_calculate_watermark_scale_factor(self, get_total_ram, get_normal_managed_pages): + get_total_ram.return_value = 101254156288 + get_normal_managed_pages.return_value = [24247810] + wmark = calculate_watermark_scale_factor() + self.assertTrue(wmark >= 10, "ret {}".format(wmark)) + self.assertTrue(wmark <= 1000, "ret {}".format(wmark)) + + def test_get_normal_managed_pages(self): + with patch_open() as (mock_open, mock_file): + mock_file.readlines.return_value = PROC_ZONEINFO.splitlines() + self.assertEqual(get_normal_managed_pages(), [24247810]) + mock_open.assert_called_with('/proc/zoneinfo', 'r') + + def test_watermark_scale_factor(self): + mem_totals = [17179803648, 34359607296, 549753716736] + managed_pages = [4194288, 24247815, 8388576, 134217216] + arglists = [[mem, managed] for mem in mem_totals for managed in managed_pages] + + for arglist in arglists: + wmark = watermark_scale_factor(*arglist) + self.assertTrue(wmark >= 10, "assert failed for args: {}, ret {}".format(arglist, wmark)) + self.assertTrue(wmark <= 1000, "assert failed for args: {}, ret {}".format(arglist, wmark)) diff --git a/nrpe/mod/charmhelpers/tests/contrib/templating/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/templating/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/templating/test_contexts.py b/nrpe/mod/charmhelpers/tests/contrib/templating/test_contexts.py new file mode 100644 index 0000000..1abc3d5 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/templating/test_contexts.py @@ -0,0 +1,252 @@ +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +import mock +import os +import shutil +import tempfile +import unittest +import yaml + +import six + +import charmhelpers.contrib.templating.contexts + + +class JujuState2YamlTestCase(unittest.TestCase): + maxDiff = None + + unit_data = { + 'private-address': '10.0.3.2', + 'public-address': '123.123.123.123', + } + + def setUp(self): + super(JujuState2YamlTestCase, self).setUp() + + # Hookenv patches (a single patch to hookenv doesn't work): + patcher = mock.patch('charmhelpers.core.hookenv.config') + self.mock_config = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relation_get') + self.mock_relation_get = patcher.start() + self.mock_relation_get.return_value = {} + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relations') + self.mock_relations = patcher.start() + self.mock_relations.return_value = { + 'wsgi-file': {}, + 'website': {}, + 'nrpe-external-master': {}, + } + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relation_type') + self.mock_relation_type = patcher.start() + self.mock_relation_type.return_value = None + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.local_unit') + self.mock_local_unit = patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch('charmhelpers.core.hookenv.relations_of_type') + self.mock_relations_of_type = patcher.start() + self.addCleanup(patcher.stop) + self.mock_relations_of_type.return_value = [] + + def unit_get_data(argument): + "dummy unit_get that accesses dummy unit data" + return self.unit_data[argument] + + patcher = mock.patch( + 'charmhelpers.core.hookenv.unit_get', unit_get_data) + self.mock_unit_get = patcher.start() + self.addCleanup(patcher.stop) + + # patches specific to this test class. + etc_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, etc_dir) + self.context_path = os.path.join(etc_dir, 'some', 'context') + + patcher = mock.patch.object(charmhelpers.contrib.templating.contexts, + 'charm_dir', '/tmp/charm_dir') + patcher.start() + self.addCleanup(patcher.stop) + + def default_context(self): + return { + "charm_dir": "/tmp/charm_dir", + "group_code_owner": "webops_deploy", + "user_code_runner": "ubunet", + "current_relation": {}, + "relations_full": { + 'wsgi-file': {}, + 'website': {}, + 'nrpe-external-master': {}, + }, + "relations": { + 'wsgi-file': [], + 'website': [], + 'nrpe-external-master': [], + }, + "local_unit": "click-index/3", + "unit_private_address": "10.0.3.2", + "unit_public_address": "123.123.123.123", + } + + def test_output_with_empty_relation(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + } + self.mock_local_unit.return_value = "click-index/3" + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path) + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + self.assertEqual(expected, result) + + def test_output_with_no_relation(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + } + self.mock_local_unit.return_value = "click-index/3" + self.mock_relation_get.return_value = None + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path) + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + self.assertEqual(expected, result) + + def test_output_with_relation(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + } + self.mock_relation_type.return_value = 'wsgi-file' + self.mock_relation_get.return_value = { + 'relation_key1': 'relation_value1', + 'relation_key2': 'relation_value2', + } + self.mock_relations.return_value = { + 'wsgi-file': { + six.u('wsgi-file:0'): { + six.u('gunicorn/1'): { + six.u('private-address'): six.u('10.0.3.99'), + }, + 'click-index/3': { + six.u('wsgi_group'): six.u('ubunet'), + }, + }, + }, + 'website': {}, + 'nrpe-external-master': {}, + } + self.mock_local_unit.return_value = "click-index/3" + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path) + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + expected['current_relation'] = { + "relation_key1": "relation_value1", + "relation_key2": "relation_value2", + } + expected["wsgi_file:relation_key1"] = "relation_value1" + expected["wsgi_file:relation_key2"] = "relation_value2" + expected["relations_full"]['wsgi-file'] = { + 'wsgi-file:0': { + 'gunicorn/1': { + six.u('private-address'): six.u('10.0.3.99')}, + 'click-index/3': {six.u('wsgi_group'): six.u('ubunet')}, + }, + } + expected["relations"]["wsgi-file"] = [ + { + '__relid__': 'wsgi-file:0', + '__unit__': 'gunicorn/1', + 'private-address': '10.0.3.99', + } + ] + self.assertEqual(expected, result) + + def test_relation_with_separator(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + } + self.mock_relation_type.return_value = 'wsgi-file' + self.mock_relation_get.return_value = { + 'relation_key1': 'relation_value1', + 'relation_key2': 'relation_value2', + } + self.mock_local_unit.return_value = "click-index/3" + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path, namespace_separator='__') + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + expected['current_relation'] = { + "relation_key1": "relation_value1", + "relation_key2": "relation_value2", + } + expected["wsgi_file__relation_key1"] = "relation_value1" + expected["wsgi_file__relation_key2"] = "relation_value2" + self.assertEqual(expected, result) + + def test_keys_with_hyphens(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + 'private-address': '10.1.1.10', + } + self.mock_local_unit.return_value = "click-index/3" + self.mock_relation_get.return_value = None + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path) + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + expected["private-address"] = "10.1.1.10" + self.assertEqual(expected, result) + + def test_keys_with_hypens_not_allowed_in_keys(self): + self.mock_config.return_value = { + 'group_code_owner': 'webops_deploy', + 'user_code_runner': 'ubunet', + 'private-address': '10.1.1.10', + } + self.mock_local_unit.return_value = "click-index/3" + self.mock_relation_type.return_value = 'wsgi-file' + self.mock_relation_get.return_value = { + 'relation-key1': 'relation_value1', + 'relation-key2': 'relation_value2', + } + + charmhelpers.contrib.templating.contexts.juju_state_to_yaml( + self.context_path, allow_hyphens_in_keys=False, + namespace_separator='__') + + with open(self.context_path, 'r') as context_file: + result = yaml.safe_load(context_file.read()) + expected = self.default_context() + expected["private_address"] = "10.1.1.10" + expected["wsgi_file__relation_key1"] = "relation_value1" + expected["wsgi_file__relation_key2"] = "relation_value2" + expected['current_relation'] = { + "relation-key1": "relation_value1", + "relation-key2": "relation_value2", + } + self.assertEqual(expected, result) diff --git a/nrpe/mod/charmhelpers/tests/contrib/templating/test_jinja.py b/nrpe/mod/charmhelpers/tests/contrib/templating/test_jinja.py new file mode 100644 index 0000000..82bafc0 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/templating/test_jinja.py @@ -0,0 +1,63 @@ +import tempfile +import os + +from shutil import rmtree +from testtools import TestCase + +from charmhelpers.contrib.templating.jinja import render + + +SIMPLE_TEMPLATE = "{{ somevar }}" + + +LOOP_TEMPLATE = "{% for i in somevar %}{{ i }}{% endfor %}" + +CUSTOM_DELIM_TEMPLATE = "{{ not_var }} << template_var >>" + + +class Jinja2Test(TestCase): + + def setUp(self): + super(Jinja2Test, self).setUp() + # Create a "templates directory" in temp + self.templates_dir = tempfile.mkdtemp() + + def tearDown(self): + super(Jinja2Test, self).tearDown() + # Remove the temporary directory so as not to pollute /tmp + rmtree(self.templates_dir) + + def _write_template_to_file(self, name, contents): + path = os.path.join(self.templates_dir, name) + with open(path, "w") as thefile: + thefile.write(contents) + + def test_render_simple_template(self): + name = "simple" + self._write_template_to_file(name, SIMPLE_TEMPLATE) + expected = "hello" + result = render( + name, {"somevar": expected}, template_dir=self.templates_dir) + self.assertEqual(expected, result) + + def test_render_loop_template(self): + name = "loop" + self._write_template_to_file(name, LOOP_TEMPLATE) + expected = "12345" + result = render( + name, {"somevar": ["1", "2", "3", "4", "5"]}, + template_dir=self.templates_dir) + self.assertEqual(expected, result) + + def test_custom_delimiters(self): + name = "custom_delimiters" + template_var = "foo" + jinja_env_args = {"variable_start_string": "<<", + "variable_end_string": ">>"} + expected = "{{ not_var }} %s" % template_var + self._write_template_to_file(name, CUSTOM_DELIM_TEMPLATE) + + result = render(name, {'template_var': template_var}, + template_dir=self.templates_dir, + jinja_env_args=jinja_env_args) + self.assertEqual(expected, result) diff --git a/nrpe/mod/charmhelpers/tests/contrib/templating/test_pyformat.py b/nrpe/mod/charmhelpers/tests/contrib/templating/test_pyformat.py new file mode 100644 index 0000000..0873a4a --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/templating/test_pyformat.py @@ -0,0 +1,36 @@ +from mock import patch +from testtools import TestCase + +from charmhelpers.contrib.templating.pyformat import render +from charmhelpers.core import hookenv + + +class PyFormatTest(TestCase): + @patch.object(hookenv, 'execution_environment') + def test_renders_using_environment(self, execution_environment): + execution_environment.return_value = { + 'foo': 'FOO', + } + + self.assertEqual(render('foo is {foo}'), 'foo is FOO') + + @patch.object(hookenv, 'execution_environment') + def test_extra_overrides(self, execution_environment): + execution_environment.return_value = { + 'foo': 'FOO', + } + + extra = {'foo': 'BAR'} + + self.assertEqual(render('foo is {foo}', extra=extra), 'foo is BAR') + + @patch.object(hookenv, 'execution_environment') + def test_kwargs_overrides(self, execution_environment): + execution_environment.return_value = { + 'foo': 'FOO', + } + + extra = {'foo': 'BAR'} + + self.assertEqual( + render('foo is {foo}', extra=extra, foo='BAZ'), 'foo is BAZ') diff --git a/nrpe/mod/charmhelpers/tests/contrib/unison/__init__.py b/nrpe/mod/charmhelpers/tests/contrib/unison/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/contrib/unison/test_unison.py b/nrpe/mod/charmhelpers/tests/contrib/unison/test_unison.py new file mode 100644 index 0000000..bc407d8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/contrib/unison/test_unison.py @@ -0,0 +1,482 @@ + +from mock import call, patch, MagicMock, sentinel +from testtools import TestCase + +from tests.helpers import patch_open, FakeRelation +from charmhelpers.contrib import unison + + +FAKE_RELATION_ENV = { + 'cluster:0': ['cluster/0', 'cluster/1'] +} + + +TO_PATCH = [ + 'log', 'check_call', 'check_output', 'relation_ids', + 'related_units', 'relation_get', 'relation_set', + 'hook_name', 'unit_private_ip', +] + +FAKE_LOCAL_UNIT = 'test_host' +FAKE_RELATION = { + 'cluster:0': { + 'cluster/0': { + 'private-address': 'cluster0.local', + 'ssh_authorized_hosts': 'someotherhost:test_host' + }, + 'clsuter/1': { + 'private-address': 'cluster1.local', + 'ssh_authorized_hosts': 'someotherhost' + }, + 'clsuter/3': { + 'private-address': 'cluster2.local', + 'ssh_authorized_hosts': 'someotherthirdhost' + }, + + }, + +} + + +class UnisonHelperTests(TestCase): + def setUp(self): + super(UnisonHelperTests, self).setUp() + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + self.fake_relation = FakeRelation(FAKE_RELATION) + self.unit_private_ip.return_value = FAKE_LOCAL_UNIT + self.relation_get.side_effect = self.fake_relation.get + self.relation_ids.side_effect = self.fake_relation.relation_ids + self.related_units.side_effect = self.fake_relation.related_units + + def _patch(self, method): + _m = patch('charmhelpers.contrib.unison.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + @patch('pwd.getpwnam') + def test_get_homedir(self, pwnam): + fake_user = MagicMock() + fake_user.pw_dir = '/home/foo' + pwnam.return_value = fake_user + self.assertEquals(unison.get_homedir('foo'), + '/home/foo') + + @patch('pwd.getpwnam') + def test_get_homedir_no_user(self, pwnam): + e = KeyError + pwnam.side_effect = e + self.assertRaises(Exception, unison.get_homedir, user='foo') + + def _ensure_calls_in(self, calls): + for _call in calls: + self.assertIn(call(_call), self.check_call.call_args_list) + + @patch('os.chmod') + @patch('os.chown') + @patch('os.path.isfile') + @patch('pwd.getpwnam') + def test_create_private_key_rsa(self, pwnam, isfile, chown, chmod): + fake_user = MagicMock() + fake_user.pw_uid = 3133 + pwnam.return_value = fake_user + create_cmd = [ + 'ssh-keygen', '-q', '-N', '', '-t', 'rsa', '-b', '2048', + '-f', '/home/foo/.ssh/id_rsa'] + + def _ensure_perms(): + chown.assert_called_with('/home/foo/.ssh/id_rsa', 3133, -1) + chmod.assert_called_with('/home/foo/.ssh/id_rsa', 0o600) + + isfile.return_value = False + unison.create_private_key( + user='foo', priv_key_path='/home/foo/.ssh/id_rsa') + self.assertIn(call(create_cmd), self.check_call.call_args_list) + _ensure_perms() + self.check_call.call_args_list = [] + + chown.reset_mock() + chmod.reset_mock() + isfile.return_value = True + unison.create_private_key( + user='foo', priv_key_path='/home/foo/.ssh/id_rsa') + self.assertNotIn(call(create_cmd), self.check_call.call_args_list) + _ensure_perms() + + @patch('os.chmod') + @patch('os.chown') + @patch('os.path.isfile') + @patch('pwd.getpwnam') + def test_create_private_key_ecdsa(self, pwnam, isfile, chown, chmod): + fake_user = MagicMock() + fake_user.pw_uid = 3133 + pwnam.return_value = fake_user + create_cmd = [ + 'ssh-keygen', '-q', '-N', '', '-t', 'ecdsa', '-b', '521', + '-f', '/home/foo/.ssh/id_ecdsa'] + + def _ensure_perms(): + chown.assert_called_with('/home/foo/.ssh/id_ecdsa', 3133, -1) + chmod.assert_called_with('/home/foo/.ssh/id_ecdsa', 0o600) + + isfile.return_value = False + unison.create_private_key( + user='foo', + priv_key_path='/home/foo/.ssh/id_ecdsa', + key_type='ecdsa') + self.assertIn(call(create_cmd), self.check_call.call_args_list) + _ensure_perms() + self.check_call.call_args_list = [] + + chown.reset_mock() + chmod.reset_mock() + isfile.return_value = True + unison.create_private_key( + user='foo', + priv_key_path='/home/foo/.ssh/id_ecdsa', + key_type='ecdsa') + self.assertNotIn(call(create_cmd), self.check_call.call_args_list) + _ensure_perms() + + @patch('os.chown') + @patch('os.path.isfile') + @patch('pwd.getpwnam') + def test_create_public_key(self, pwnam, isfile, chown): + fake_user = MagicMock() + fake_user.pw_uid = 3133 + pwnam.return_value = fake_user + create_cmd = ['ssh-keygen', '-y', '-f', '/home/foo/.ssh/id_rsa'] + + def _ensure_perms(): + chown.assert_called_with('/home/foo/.ssh/id_rsa.pub', 3133, -1) + + isfile.return_value = True + unison.create_public_key( + user='foo', priv_key_path='/home/foo/.ssh/id_rsa', + pub_key_path='/home/foo/.ssh/id_rsa.pub') + self.assertNotIn(call(create_cmd), self.check_output.call_args_list) + _ensure_perms() + + isfile.return_value = False + with patch_open() as (_open, _file): + self.check_output.return_value = b'fookey' + unison.create_public_key( + user='foo', priv_key_path='/home/foo/.ssh/id_rsa', + pub_key_path='/home/foo/.ssh/id_rsa.pub') + self.assertIn(call(create_cmd), self.check_output.call_args_list) + _ensure_perms() + _open.assert_called_with('/home/foo/.ssh/id_rsa.pub', 'wb') + _file.write.assert_called_with(b'fookey') + + @patch('os.mkdir') + @patch('os.path.isdir') + @patch.object(unison, 'get_homedir') + @patch.multiple(unison, create_private_key=MagicMock(), + create_public_key=MagicMock()) + def test_get_keypair(self, get_homedir, isdir, mkdir): + get_homedir.return_value = '/home/foo' + isdir.return_value = False + with patch_open() as (_open, _file): + _file.read.side_effect = [ + 'foopriv', 'foopub' + ] + priv, pub = unison.get_keypair('adam') + for f in ['/home/foo/.ssh/id_rsa', + '/home/foo/.ssh/id_rsa.pub']: + self.assertIn(call(f, 'r'), _open.call_args_list) + self.assertEquals(priv, 'foopriv') + self.assertEquals(pub, 'foopub') + + @patch.object(unison, 'get_homedir') + @patch('os.chown') + @patch('pwd.getpwnam') + def test_write_auth_keys(self, pwnam, chown, get_homedir): + fake_user = MagicMock() + fake_user.pw_uid = 3133 + pwnam.return_value = fake_user + get_homedir.return_value = '/home/foo' + keys = [ + 'ssh-rsa AAAB3Nz adam', + 'ssh-rsa ALKJFz adam@whereschuck.org', + ] + + def _ensure_perms(): + chown.assert_called_with('/home/foo/.ssh/authorized_keys', 3133, -1) + + with patch_open() as (_open, _file): + unison.write_authorized_keys('foo', keys) + _open.assert_called_with('/home/foo/.ssh/authorized_keys', 'w') + for k in keys: + self.assertIn(call('%s\n' % k), _file.write.call_args_list) + _ensure_perms() + + @patch.object(unison, 'get_homedir') + @patch('os.chown') + @patch('pwd.getpwnam') + def test_write_known_hosts(self, pwnam, chown, get_homedir): + fake_user = MagicMock() + fake_user.pw_uid = 3133 + pwnam.return_value = fake_user + get_homedir.return_value = '/home/foo' + keys = [ + '10.0.0.1 ssh-rsa KJDSJF=', + '10.0.0.2 ssh-rsa KJDSJF=', + ] + self.check_output.side_effect = keys + + def _ensure_perms(): + chown.assert_called_with('/home/foo/.ssh/known_hosts', 3133, -1) + + with patch_open() as (_open, _file): + unison.write_known_hosts('foo', ['10.0.0.1', '10.0.0.2']) + _open.assert_called_with('/home/foo/.ssh/known_hosts', 'w') + for k in keys: + self.assertIn(call('%s\n' % k), _file.write.call_args_list) + _ensure_perms() + + @patch.object(unison, 'remove_password_expiry') + @patch.object(unison, 'pwgen') + @patch.object(unison, 'add_user_to_group') + @patch.object(unison, 'adduser') + def test_ensure_user(self, adduser, to_group, pwgen, + remove_password_expiry): + pwgen.return_value = sentinel.password + unison.ensure_user('foo', group='foobar') + adduser.assert_called_with('foo', sentinel.password) + to_group.assert_called_with('foo', 'foobar') + remove_password_expiry.assert_called_with('foo') + + @patch.object(unison, '_run_as_user') + def test_run_as_user(self, _run): + with patch.object(unison, '_run_as_user') as _run: + fake_preexec = MagicMock() + _run.return_value = fake_preexec + unison.run_as_user('foo', ['echo', 'foo']) + self.check_output.assert_called_with( + ['echo', 'foo'], preexec_fn=fake_preexec, cwd='/') + + @patch('pwd.getpwnam') + def test_run_user_not_found(self, getpwnam): + e = KeyError + getpwnam.side_effect = e + self.assertRaises(Exception, unison._run_as_user, 'nouser') + + @patch('os.setuid') + @patch('os.setgid') + @patch('os.environ', spec=dict) + @patch('pwd.getpwnam') + def test_run_as_user_preexec(self, pwnam, environ, setgid, setuid): + fake_env = {'HOME': '/root'} + environ.__getitem__ = MagicMock() + environ.__setitem__ = MagicMock() + environ.__setitem__.side_effect = fake_env.__setitem__ + environ.__getitem__.side_effect = fake_env.__getitem__ + + fake_user = MagicMock() + fake_user.pw_uid = 1010 + fake_user.pw_gid = 1011 + fake_user.pw_dir = '/home/foo' + pwnam.return_value = fake_user + inner = unison._run_as_user('foo') + self.assertEquals(fake_env['HOME'], '/home/foo') + inner() + setgid.assert_called_with(1011) + setuid.assert_called_with(1010) + + @patch('os.setuid') + @patch('os.setgid') + @patch('os.environ', spec=dict) + @patch('pwd.getpwnam') + def test_run_as_user_preexec_with_group(self, pwnam, environ, setgid, setuid): + fake_env = {'HOME': '/root'} + environ.__getitem__ = MagicMock() + environ.__setitem__ = MagicMock() + environ.__setitem__.side_effect = fake_env.__setitem__ + environ.__getitem__.side_effect = fake_env.__getitem__ + + fake_user = MagicMock() + fake_user.pw_uid = 1010 + fake_user.pw_gid = 1011 + fake_user.pw_dir = '/home/foo' + fake_group_id = 2000 + pwnam.return_value = fake_user + inner = unison._run_as_user('foo', gid=fake_group_id) + self.assertEquals(fake_env['HOME'], '/home/foo') + inner() + setgid.assert_called_with(2000) + setuid.assert_called_with(1010) + + @patch.object(unison, 'get_keypair') + @patch.object(unison, 'ensure_user') + def test_ssh_auth_peer_joined(self, ensure_user, get_keypair): + get_keypair.return_value = ('privkey', 'pubkey') + self.hook_name.return_value = 'cluster-relation-joined' + unison.ssh_authorized_peers(peer_interface='cluster', + user='foo', group='foo', + ensure_local_user=True) + self.relation_set.assert_called_with(ssh_pub_key='pubkey') + self.assertFalse(self.relation_get.called) + ensure_user.assert_called_with('foo', 'foo') + get_keypair.assert_called_with('foo') + + @patch.object(unison, 'write_known_hosts') + @patch.object(unison, 'write_authorized_keys') + @patch.object(unison, 'get_keypair') + @patch.object(unison, 'ensure_user') + def test_ssh_auth_peer_changed(self, ensure_user, get_keypair, + write_keys, write_hosts): + get_keypair.return_value = ('privkey', 'pubkey') + + self.hook_name.return_value = 'cluster-relation-changed' + + self.relation_get.side_effect = [ + 'key1', + 'host1', + 'key2', + 'host2', + '', '' + ] + unison.ssh_authorized_peers(peer_interface='cluster', + user='foo', group='foo', + ensure_local_user=True) + + ensure_user.assert_called_with('foo', 'foo') + get_keypair.assert_called_with('foo') + write_keys.assert_called_with('foo', ['key1', 'key2']) + write_hosts.assert_called_with('foo', ['host1', 'host2']) + self.relation_set.assert_called_with(ssh_authorized_hosts='host1:host2') + + @patch.object(unison, 'write_known_hosts') + @patch.object(unison, 'write_authorized_keys') + @patch.object(unison, 'get_keypair') + @patch.object(unison, 'ensure_user') + def test_ssh_auth_peer_departed(self, ensure_user, get_keypair, + write_keys, write_hosts): + get_keypair.return_value = ('privkey', 'pubkey') + + self.hook_name.return_value = 'cluster-relation-departed' + + self.relation_get.side_effect = [ + 'key1', + 'host1', + 'key2', + 'host2', + '', '' + ] + unison.ssh_authorized_peers(peer_interface='cluster', + user='foo', group='foo', + ensure_local_user=True) + + ensure_user.assert_called_with('foo', 'foo') + get_keypair.assert_called_with('foo') + write_keys.assert_called_with('foo', ['key1', 'key2']) + write_hosts.assert_called_with('foo', ['host1', 'host2']) + self.relation_set.assert_called_with(ssh_authorized_hosts='host1:host2') + + def test_collect_authed_hosts(self): + # only one of the hosts in fake environment has auth'd + # the local peer + hosts = unison.collect_authed_hosts('cluster') + self.assertEquals(hosts, ['cluster0.local']) + + def test_collect_authed_hosts_none_authed(self): + with patch.object(unison, 'relation_get') as relation_get: + relation_get.return_value = '' + hosts = unison.collect_authed_hosts('cluster') + self.assertEquals(hosts, []) + + @patch.object(unison, 'run_as_user') + def test_sync_path_to_host(self, run_as_user, verbose=True, gid=None): + for path in ['/tmp/foo', '/tmp/foo/']: + unison.sync_path_to_host(path=path, host='clusterhost1', + user='foo', verbose=verbose, gid=gid) + ex_cmd = ['unison', '-auto', '-batch=true', + '-confirmbigdel=false', '-fastcheck=true', + '-group=false', '-owner=false', + '-prefer=newer', '-times=true'] + if not verbose: + ex_cmd.append('-silent') + ex_cmd += ['/tmp/foo', 'ssh://foo@clusterhost1//tmp/foo'] + run_as_user.assert_called_with('foo', ex_cmd, gid) + + @patch.object(unison, 'run_as_user') + def test_sync_path_to_host_error(self, run_as_user): + for i, path in enumerate(['/tmp/foo', '/tmp/foo/']): + run_as_user.side_effect = Exception + if i == 0: + unison.sync_path_to_host(path=path, host='clusterhost1', + user='foo', verbose=True, gid=None) + else: + self.assertRaises(Exception, unison.sync_path_to_host, + path=path, host='clusterhost1', + user='foo', verbose=True, gid=None, + fatal=True) + + ex_cmd = ['unison', '-auto', '-batch=true', + '-confirmbigdel=false', '-fastcheck=true', + '-group=false', '-owner=false', + '-prefer=newer', '-times=true', + '/tmp/foo', 'ssh://foo@clusterhost1//tmp/foo'] + run_as_user.assert_called_with('foo', ex_cmd, None) + + def test_sync_path_to_host_non_verbose(self): + return self.test_sync_path_to_host(verbose=False) + + def test_sync_path_to_host_with_gid(self): + return self.test_sync_path_to_host(gid=111) + + @patch.object(unison, 'sync_path_to_host') + def test_sync_to_peer(self, sync_path_to_host): + paths = ['/tmp/foo1', '/tmp/foo2'] + host = 'host1' + unison.sync_to_peer(host, 'foouser', paths, True) + calls = [call('/tmp/foo1', host, 'foouser', True, None, None, False), + call('/tmp/foo2', host, 'foouser', True, None, None, False)] + sync_path_to_host.assert_has_calls(calls) + + @patch.object(unison, 'sync_path_to_host') + def test_sync_to_peer_with_gid(self, sync_path_to_host): + paths = ['/tmp/foo1', '/tmp/foo2'] + host = 'host1' + unison.sync_to_peer(host, 'foouser', paths, True, gid=111) + calls = [call('/tmp/foo1', host, 'foouser', True, None, 111, False), + call('/tmp/foo2', host, 'foouser', True, None, 111, False)] + sync_path_to_host.assert_has_calls(calls) + + @patch.object(unison, 'collect_authed_hosts') + @patch.object(unison, 'sync_to_peer') + def test_sync_to_peers(self, sync_to_peer, collect_hosts): + collect_hosts.return_value = ['host1', 'host2', 'host3'] + paths = ['/tmp/foo'] + unison.sync_to_peers(peer_interface='cluster', user='foouser', + paths=paths, verbose=True) + calls = [call('host1', 'foouser', ['/tmp/foo'], True, None, None, False), + call('host2', 'foouser', ['/tmp/foo'], True, None, None, False), + call('host3', 'foouser', ['/tmp/foo'], True, None, None, False)] + sync_to_peer.assert_has_calls(calls) + + @patch.object(unison, 'collect_authed_hosts') + @patch.object(unison, 'sync_to_peer') + def test_sync_to_peers_with_gid(self, sync_to_peer, collect_hosts): + collect_hosts.return_value = ['host1', 'host2', 'host3'] + paths = ['/tmp/foo'] + unison.sync_to_peers(peer_interface='cluster', user='foouser', + paths=paths, verbose=True, gid=111) + calls = [call('host1', 'foouser', ['/tmp/foo'], True, None, 111, False), + call('host2', 'foouser', ['/tmp/foo'], True, None, 111, False), + call('host3', 'foouser', ['/tmp/foo'], True, None, 111, False)] + sync_to_peer.assert_has_calls(calls) + + @patch.object(unison, 'collect_authed_hosts') + @patch.object(unison, 'sync_to_peer') + def test_sync_to_peers_with_cmd(self, sync_to_peer, collect_hosts): + collect_hosts.return_value = ['host1', 'host2', 'host3'] + paths = ['/tmp/foo'] + cmd = ['dummy_cmd'] + unison.sync_to_peers(peer_interface='cluster', user='foouser', + paths=paths, verbose=True, cmd=cmd, gid=111) + calls = [call('host1', 'foouser', ['/tmp/foo'], True, cmd, 111, False), + call('host2', 'foouser', ['/tmp/foo'], True, cmd, 111, False), + call('host3', 'foouser', ['/tmp/foo'], True, cmd, 111, False)] + sync_to_peer.assert_has_calls(calls) diff --git a/nrpe/mod/charmhelpers/tests/coordinator/__init__.py b/nrpe/mod/charmhelpers/tests/coordinator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/coordinator/test_coordinator.py b/nrpe/mod/charmhelpers/tests/coordinator/test_coordinator.py new file mode 100644 index 0000000..a57850a --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/coordinator/test_coordinator.py @@ -0,0 +1,533 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime, timedelta +import json +import tempfile +import unittest +from mock import call, MagicMock, patch, sentinel + +from charmhelpers import coordinator +from charmhelpers.core import hookenv + + +class TestCoordinator(unittest.TestCase): + + def setUp(self): + del hookenv._atstart[:] + del hookenv._atexit[:] + hookenv.cache.clear() + coordinator.Singleton._instances.clear() + + def install(patch): + patch.start() + self.addCleanup(patch.stop) + + install(patch.object(hookenv, 'local_unit', return_value='foo/1')) + install(patch.object(hookenv, 'is_leader', return_value=False)) + install(patch.object(hookenv, 'metadata', + return_value={'peers': {'cluster': None}})) + install(patch.object(hookenv, 'log')) + + # Ensure _timestamp always increases. + install(patch.object(coordinator, '_utcnow', + side_effect=self._utcnow)) + + _last_utcnow = datetime(2015, 1, 1, 00, 00) + + def _utcnow(self, ts=coordinator._timestamp): + self._last_utcnow += timedelta(minutes=1) + return self._last_utcnow + + def test_is_singleton(self): + # BaseCoordinator and subclasses are singletons. Placing this + # burden on charm authors is impractical, particularly if + # libraries start wanting to use coordinator instances. + # With singletons, we don't need to worry about sharing state + # between instances or have them stomping on each other when they + # need to serialize their state. + self.assertTrue(coordinator.BaseCoordinator() + is coordinator.BaseCoordinator()) + self.assertTrue(coordinator.Serial() is coordinator.Serial()) + self.assertFalse(coordinator.BaseCoordinator() is coordinator.Serial()) + + @patch.object(hookenv, 'atstart') + def test_implicit_initialize_and_handle(self, atstart): + # When you construct a BaseCoordinator(), its initialize() and + # handle() method are invoked automatically every hook. This + # is done using hookenv.atstart + c = coordinator.BaseCoordinator() + atstart.assert_has_calls([call(c.initialize), call(c.handle)]) + + @patch.object(hookenv, 'has_juju_version', return_value=False) + def test_initialize_enforces_juju_version(self, has_juju_version): + c = coordinator.BaseCoordinator() + with self.assertRaises(AssertionError): + c.initialize() + has_juju_version.assert_called_once_with('1.23') + + @patch.object(hookenv, 'atexit') + @patch.object(hookenv, 'has_juju_version', return_value=True) + @patch.object(hookenv, 'relation_ids') + def test_initialize(self, relation_ids, ver, atexit): + # First initialization are done before there is a peer relation. + relation_ids.return_value = [] + c = coordinator.BaseCoordinator() + + with patch.object(c, '_load_state') as _load_state, \ + patch.object(c, '_emit_state') as _emit_state: # IGNORE: E127 + c.initialize() + _load_state.assert_called_once_with() + _emit_state.assert_called_once_with() + + self.assertEqual(c.relname, 'cluster') + self.assertIsNone(c.relid) + relation_ids.assert_called_once_with('cluster') + + # Methods installed to save state and release locks if the + # hook is successful. + atexit.assert_has_calls([call(c._save_state), + call(c._release_granted)]) + + # If we have a peer relation, the id is stored. + relation_ids.return_value = ['cluster:1'] + c = coordinator.BaseCoordinator() + with patch.object(c, '_load_state'), patch.object(c, '_emit_state'): + c.initialize() + self.assertEqual(c.relid, 'cluster:1') + + # If we are already initialized, nothing happens. + c.grants = {} + c.requests = {} + c.initialize() + + def test_acquire(self): + c = coordinator.BaseCoordinator() + lock = 'mylock' + c.grants = {} + c.requests = {hookenv.local_unit(): {}} + + # We are not the leader, so first acquire will return False. + self.assertFalse(c.acquire(lock)) + + # But the request is in the queue. + self.assertTrue(c.requested(lock)) + ts = c.request_timestamp(lock) + + # A further attempts at acquiring the lock do nothing, + # and the timestamp of the request remains unchanged. + self.assertFalse(c.acquire(lock)) + self.assertEqual(ts, c.request_timestamp(lock)) + + # Once the leader has granted the lock, acquire returns True. + with patch.object(c, 'granted') as granted: + granted.return_value = True + self.assertTrue(c.acquire(lock)) + granted.assert_called_once_with(lock) + + def test_acquire_leader(self): + # When acquire() is called by the leader, it needs + # to make a grant decision immediately. It can't defer + # making the decision until a future hook, as no future + # hooks will be triggered. + hookenv.is_leader.return_value = True + c = coordinator.Serial() # Not Base. Test hooks into default_grant. + lock = 'mylock' + unit = hookenv.local_unit() + c.grants = {} + c.requests = {unit: {}} + with patch.object(c, 'default_grant') as default_grant: + default_grant.side_effect = iter([False, True]) + + self.assertFalse(c.acquire(lock)) + ts = c.request_timestamp(lock) + + self.assertTrue(c.acquire(lock)) + self.assertEqual(ts, c.request_timestamp(lock)) + + # If it it granted, the leader doesn't make a decision again. + self.assertTrue(c.acquire(lock)) + self.assertEqual(ts, c.request_timestamp(lock)) + + self.assertEqual(default_grant.call_count, 2) + + def test_granted(self): + c = coordinator.BaseCoordinator() + unit = hookenv.local_unit() + lock = 'mylock' + ts = coordinator._timestamp() + c.grants = {} + + # Unit makes a request, but it isn't granted + c.requests = {unit: {lock: ts}} + self.assertFalse(c.granted(lock)) + + # Once the leader has granted the request, all good. + # It does this by mirroring the request timestamp. + c.grants = {unit: {lock: ts}} + self.assertTrue(c.granted(lock)) + + # The unit releases the lock by removing the request. + c.requests = {unit: {}} + self.assertFalse(c.granted(lock)) + + # If the unit makes a new request before the leader + # has had a chance to do its housekeeping, the timestamps + # do not match and the lock not considered granted. + ts = coordinator._timestamp() + c.requests = {unit: {lock: ts}} + self.assertFalse(c.granted(lock)) + + # Until the leader gets around to its duties. + c.grants = {unit: {lock: ts}} + self.assertTrue(c.granted(lock)) + + def test_requested(self): + c = coordinator.BaseCoordinator() + lock = 'mylock' + c.requests = {hookenv.local_unit(): {}} + c.grants = {} + + self.assertFalse(c.requested(lock)) + c.acquire(lock) + self.assertTrue(c.requested(lock)) + + def test_request_timestamp(self): + c = coordinator.BaseCoordinator() + lock = 'mylock' + unit = hookenv.local_unit() + + c.requests = {unit: {}} + c.grants = {} + self.assertIsNone(c.request_timestamp(lock)) + + now = datetime.utcnow() + fmt = coordinator._timestamp_format + c.requests = {hookenv.local_unit(): {lock: now.strftime(fmt)}} + + self.assertEqual(c.request_timestamp(lock), now) + + def test_handle_not_leader(self): + c = coordinator.BaseCoordinator() + # If we are not the leader, handle does nothing. We know this, + # because without mocks or initialization it would otherwise crash. + c.handle() + + def test_handle(self): + hookenv.is_leader.return_value = True + lock = 'mylock' + c = coordinator.BaseCoordinator() + c.relid = 'cluster:1' + + ts = coordinator._timestamp + ts1, ts2, ts3 = ts(), ts(), ts() + + # Grant one of these requests. + requests = {'foo/1': {lock: ts1}, + 'foo/2': {lock: ts2}, + 'foo/3': {lock: ts3}} + c.requests = requests.copy() + # Because the existing grant should be released. + c.grants = {'foo/2': {lock: ts()}} # No request, release. + + with patch.object(c, 'grant') as grant: + c.handle() + + # The requests are unchanged. This is normally state on the + # peer relation, and only the units themselves can change it. + self.assertDictEqual(requests, c.requests) + + # The grant without a corresponding requests was released. + self.assertDictEqual({'foo/2': {}}, c.grants) + + # A potential grant was made for each of the outstanding requests. + grant.assert_has_calls([call(lock, 'foo/1'), + call(lock, 'foo/2'), + call(lock, 'foo/3')], any_order=True) + + def test_grant_not_leader(self): + c = coordinator.BaseCoordinator() + c.grant(sentinel.whatever, sentinel.whatever) # Nothing happens. + + def test_grant(self): + hookenv.is_leader.return_value = True + c = coordinator.BaseCoordinator() + c.default_grant = MagicMock() + c.grant_other = MagicMock() + + ts = coordinator._timestamp + ts1, ts2 = ts(), ts() + + c.requests = {'foo/1': {'mylock': ts1, 'other': ts()}, + 'foo/2': {'mylock': ts2}, + 'foo/3': {'mylock': ts()}} + grants = {'foo/1': {'mylock': ts1}} + c.grants = grants.copy() + + # foo/1 already has a granted mylock, so returns True. + self.assertTrue(c.grant('mylock', 'foo/1')) + + # foo/2 does not have a granted mylock. default_grant will + # be called to make a decision (no) + c.default_grant.return_value = False + self.assertFalse(c.grant('mylock', 'foo/2')) + self.assertDictEqual(grants, c.grants) + c.default_grant.assert_called_once_with('mylock', 'foo/2', + set(['foo/1']), + ['foo/2', 'foo/3']) + c.default_grant.reset_mock() + + # Lets say yes. + c.default_grant.return_value = True + self.assertTrue(c.grant('mylock', 'foo/2')) + grants = {'foo/1': {'mylock': ts1}, 'foo/2': {'mylock': ts2}} + self.assertDictEqual(grants, c.grants) + c.default_grant.assert_called_once_with('mylock', 'foo/2', + set(['foo/1']), + ['foo/2', 'foo/3']) + + # The other lock has custom logic, in the form of the overridden + # grant_other method. + c.grant_other.return_value = False + self.assertFalse(c.grant('other', 'foo/1')) + c.grant_other.assert_called_once_with('other', 'foo/1', + set(), ['foo/1']) + + # If there is no request, grant returns False + c.grant_other.return_value = True + self.assertFalse(c.grant('other', 'foo/2')) + + def test_released(self): + c = coordinator.BaseCoordinator() + with patch.object(c, 'msg') as msg: + c.released('foo/2', 'mylock', coordinator._utcnow()) + expected = 'Leader released mylock from foo/2, held 0:01:00' + msg.assert_called_once_with(expected) + + def test_require(self): + c = coordinator.BaseCoordinator() + c.acquire = MagicMock() + c.granted = MagicMock() + guard = MagicMock() + + wrapped = MagicMock() + + @c.require('mylock', guard) + def func(*args, **kw): + wrapped(*args, **kw) + + # If the lock is granted, the wrapped function is called. + c.granted.return_value = True + func(arg=True) + wrapped.assert_called_once_with(arg=True) + wrapped.reset_mock() + + # If the lock is not granted, and the guard returns False, + # the lock is not acquired. + c.acquire.return_value = False + c.granted.return_value = False + guard.return_value = False + func() + self.assertFalse(wrapped.called) + self.assertFalse(c.acquire.called) + + # If the lock is not granted, and the guard returns True, + # the lock is acquired. But the function still isn't called if + # it cannot be acquired immediately. + guard.return_value = True + func() + self.assertFalse(wrapped.called) + c.acquire.assert_called_once_with('mylock') + + # Finally, if the lock is not granted, and the guard returns True, + # and the lock acquired immediately, the function is called. + c.acquire.return_value = True + func(sentinel.arg) + wrapped.assert_called_once_with(sentinel.arg) + + def test_msg(self): + c = coordinator.BaseCoordinator() + # Just a wrapper around hookenv.log + c.msg('hi') + hookenv.log.assert_called_once_with('coordinator.BaseCoordinator hi', + level=hookenv.INFO) + + def test_name(self): + # We use the class name in a few places to avoid conflicts. + # We assume we won't be using multiple BaseCoordinator subclasses + # with the same name at the same time. + c = coordinator.BaseCoordinator() + self.assertEqual(c._name(), 'BaseCoordinator') + c = coordinator.Serial() + self.assertEqual(c._name(), 'Serial') + + @patch.object(hookenv, 'leader_get') + def test_load_state(self, leader_get): + c = coordinator.BaseCoordinator() + unit = hookenv.local_unit() + + # c.granted is just the leader_get decoded. + leader_get.return_value = '{"json": true}' + c._load_state() + self.assertDictEqual(c.grants, {'json': True}) + + # With no relid, there is no peer relation so request state + # is pulled from a local stash. + with patch.object(c, '_load_local_state') as loc_state: + loc_state.return_value = {'local': True} + c._load_state() + self.assertDictEqual(c.requests, {unit: {'local': True}}) + + # With a relid, request details are pulled from the peer relation. + # If there is no data in the peer relation from the local unit, + # we still pull it from the local stash as it means this is the + # first time we have joined. + c.relid = 'cluster:1' + with patch.object(c, '_load_local_state') as loc_state, \ + patch.object(c, '_load_peer_state') as peer_state: + loc_state.return_value = {'local': True} + peer_state.return_value = {'foo/2': {'mylock': 'whatever'}} + c._load_state() + self.assertDictEqual(c.requests, {unit: {'local': True}, + 'foo/2': {'mylock': 'whatever'}}) + + # If there are local details in the peer relation, the local + # stash is ignored. + with patch.object(c, '_load_local_state') as loc_state, \ + patch.object(c, '_load_peer_state') as peer_state: + loc_state.return_value = {'local': True} + peer_state.return_value = {unit: {}, + 'foo/2': {'mylock': 'whatever'}} + c._load_state() + self.assertDictEqual(c.requests, {unit: {}, + 'foo/2': {'mylock': 'whatever'}}) + + def test_emit_state(self): + c = coordinator.BaseCoordinator() + unit = hookenv.local_unit() + c.requests = {unit: {'lock_a': sentinel.ts, + 'lock_b': sentinel.ts, + 'lock_c': sentinel.ts}} + c.grants = {unit: {'lock_a': sentinel.ts, + 'lock_b': sentinel.ts2}} + with patch.object(c, 'msg') as msg: + c._emit_state() + msg.assert_has_calls([call('Granted lock_a'), + call('Waiting on lock_b'), + call('Waiting on lock_c')], + any_order=True) + + @patch.object(hookenv, 'relation_set') + @patch.object(hookenv, 'leader_set') + def test_save_state(self, leader_set, relation_set): + c = coordinator.BaseCoordinator() + unit = hookenv.local_unit() + c.grants = {'directdump': True} + c.requests = {unit: 'data1', 'foo/2': 'data2'} + + # grants is dumped to leadership settings, if the unit is leader. + with patch.object(c, '_save_local_state') as save_loc: + c._save_state() + self.assertFalse(leader_set.called) + hookenv.is_leader.return_value = True + c._save_state() + leader_set.assert_called_once_with({c.key: '{"directdump": true}'}) + + # If there is no relation id, the local units requests is dumped + # to a local stash. + with patch.object(c, '_save_local_state') as save_loc: + c._save_state() + save_loc.assert_called_once_with('data1') + + # If there is a relation id, the local units requests is dumped + # to the peer relation. + with patch.object(c, '_save_local_state') as save_loc: + c.relid = 'cluster:1' + c._save_state() + self.assertFalse(save_loc.called) + relation_set.assert_called_once_with( + c.relid, relation_settings={c.key: '"data1"'}) # JSON encoded + + @patch.object(hookenv, 'relation_get') + @patch.object(hookenv, 'related_units') + def test_load_peer_state(self, related_units, relation_get): + # Standard relation-get loops, decoding results from JSON. + c = coordinator.BaseCoordinator() + c.key = sentinel.key + c.relid = sentinel.relid + related_units.return_value = ['foo/2', 'foo/3'] + d = {'foo/1': {'foo/1': True}, + 'foo/2': {'foo/2': True}, + 'foo/3': {'foo/3': True}} + + def _get(key, unit, relid): + assert key == sentinel.key + assert relid == sentinel.relid + return json.dumps(d[unit]) + relation_get.side_effect = _get + + self.assertDictEqual(c._load_peer_state(), d) + + def test_local_state_filename(self): + c = coordinator.BaseCoordinator() + self.assertEqual(c._local_state_filename(), + '.charmhelpers.coordinator.BaseCoordinator') + + def test_load_local_state(self): + c = coordinator.BaseCoordinator() + with tempfile.NamedTemporaryFile(mode='w') as f: + with patch.object(c, '_local_state_filename') as fn: + fn.return_value = f.name + d = 'some data' + json.dump(d, f) + f.flush() + d2 = c._load_local_state() + self.assertEqual(d, d2) + + def test_save_local_state(self): + c = coordinator.BaseCoordinator() + with tempfile.NamedTemporaryFile(mode='r') as f: + with patch.object(c, '_local_state_filename') as fn: + fn.return_value = f.name + c._save_local_state('some data') + self.assertEqual(json.load(f), 'some data') + + def test_release_granted(self): + c = coordinator.BaseCoordinator() + unit = hookenv.local_unit() + c.requests = {unit: {'lock1': sentinel.ts, 'lock2': sentinel.ts}, + 'foo/2': {'lock1': sentinel.ts}} + c.grants = {unit: {'lock1': sentinel.ts}, + 'foo/2': {'lock1': sentinel.ts}} + # The granted lock for the local unit is released. + c._release_granted() + self.assertDictEqual(c.requests, {unit: {'lock2': sentinel.ts}, + 'foo/2': {'lock1': sentinel.ts}}) + + def test_implicit_peer_relation_name(self): + self.assertEqual(coordinator._implicit_peer_relation_name(), + 'cluster') + + def test_default_grant(self): + c = coordinator.Serial() + # Lock not granted. First in the queue. + self.assertTrue(c.default_grant(sentinel.lock, sentinel.u1, + set(), [sentinel.u1, sentinel.u2])) + + # Lock not granted. Later in the queue. + self.assertFalse(c.default_grant(sentinel.lock, sentinel.u1, + set(), [sentinel.u2, sentinel.u1])) + + # Lock already granted + self.assertFalse(c.default_grant(sentinel.lock, sentinel.u1, + set([sentinel.u2]), [sentinel.u1])) diff --git a/nrpe/mod/charmhelpers/tests/core/__init__.py b/nrpe/mod/charmhelpers/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/core/templates/cloud_controller_ng.yml b/nrpe/mod/charmhelpers/tests/core/templates/cloud_controller_ng.yml new file mode 100644 index 0000000..7f72f89 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/templates/cloud_controller_ng.yml @@ -0,0 +1,173 @@ +--- +# TODO cc_ip cc public ip +local_route: {{ domain }} +port: {{ cc_port }} +pid_filename: /var/vcap/sys/run/cloud_controller_ng/cloud_controller_ng.pid +development_mode: false + +message_bus_servers: + - nats://{{ nats['user'] }}:{{ nats['password'] }}@{{ nats['address'] }}:{{ nats['port'] }} + +external_domain: + - api.{{ domain }} + +system_domain_organization: {{ default_organization }} +system_domain: {{ domain }} +app_domains: [ {{ domain }} ] +srv_api_uri: http://api.{{ domain }} + +default_app_memory: 1024 + +cc_partition: default + +bootstrap_admin_email: admin@{{ default_organization }} + +bulk_api: + auth_user: bulk_api + auth_password: "Password" + +nginx: + use_nginx: false + instance_socket: "/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock" + +index: 1 +name: cloud_controller_ng + +info: + name: vcap + build: "2222" + version: 2 + support_address: http://support.cloudfoundry.com + description: Cloud Foundry sponsored by Pivotal + api_version: 2.0.0 + + +directories: + tmpdir: /var/vcap/data/cloud_controller_ng/tmp + + +logging: + file: /var/vcap/sys/log/cloud_controller_ng/cloud_controller_ng.log + + syslog: vcap.cloud_controller_ng + + level: debug2 + max_retries: 1 + + + + + +db: &db + database: sqlite:///var/lib/cloudfoundry/cfcloudcontroller/db/cc.db + max_connections: 25 + pool_timeout: 10 + log_level: debug2 + + +login: + url: http://uaa.{{ domain }} + +uaa: + url: http://uaa.{{ domain }} + resource_id: cloud_controller + #symmetric_secret: cc-secret + verification_key: | + -----BEGIN PUBLIC KEY----- + MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHFr+KICms+tuT1OXJwhCUmR2d + KVy7psa8xzElSyzqx7oJyfJ1JZyOzToj9T5SfTIq396agbHJWVfYphNahvZ/7uMX + qHxf+ZH9BL1gk9Y6kCnbM5R60gfwjyW1/dQPjOzn9N394zd2FJoFHwdq9Qs0wBug + spULZVNRxq7veq/fzwIDAQAB + -----END PUBLIC KEY----- + +# App staging parameters +staging: + max_staging_runtime: 900 + auth: + user: + password: "Password" + +maximum_health_check_timeout: 180 + +runtimes_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/runtimes.yml +stacks_file: /var/lib/cloudfoundry/cfcloudcontroller/jobs/config/stacks.yml + +quota_definitions: + free: + non_basic_services_allowed: false + total_services: 2 + total_routes: 1000 + memory_limit: 1024 + paid: + non_basic_services_allowed: true + total_services: 32 + total_routes: 1000 + memory_limit: 204800 + runaway: + non_basic_services_allowed: true + total_services: 500 + total_routes: 1000 + memory_limit: 204800 + trial: + non_basic_services_allowed: false + total_services: 10 + memory_limit: 2048 + total_routes: 1000 + trial_db_allowed: true + +default_quota_definition: free + +resource_pool: + minimum_size: 65536 + maximum_size: 536870912 + resource_directory_key: cc-resources + + cdn: + uri: + key_pair_id: + private_key: "" + + fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"} + +packages: + app_package_directory_key: cc-packages + + cdn: + uri: + key_pair_id: + private_key: "" + + fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"} + +droplets: + droplet_directory_key: cc-droplets + + cdn: + uri: + key_pair_id: + private_key: "" + + fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"} + +buildpacks: + buildpack_directory_key: cc-buildpacks + + cdn: + uri: + key_pair_id: + private_key: "" + + fog_connection: {"provider":"Local","local_root":"/var/vcap/nfs/store"} + +db_encryption_key: Password + +trial_db: + guid: "78ad16cf-3c22-4427-a982-b9d35d746914" + +tasks_disabled: false +hm9000_noop: true +flapping_crash_count_threshold: 3 + +disable_custom_buildpacks: false + +broker_client_timeout_seconds: 60 diff --git a/nrpe/mod/charmhelpers/tests/core/templates/fake_cc.yml b/nrpe/mod/charmhelpers/tests/core/templates/fake_cc.yml new file mode 100644 index 0000000..5e3f8b6 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/templates/fake_cc.yml @@ -0,0 +1,3 @@ +host: {{nats['host']}} +port: {{nats['port']}} +domain: {{router['domain']}} diff --git a/nrpe/mod/charmhelpers/tests/core/templates/nginx.conf b/nrpe/mod/charmhelpers/tests/core/templates/nginx.conf new file mode 100644 index 0000000..1c967db --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/templates/nginx.conf @@ -0,0 +1,154 @@ +# deployment cloudcontroller nginx.conf +#user vcap vcap; + +error_log /var/vcap/sys/log/nginx_ccng/nginx.error.log; +pid /var/vcap/sys/run/nginx_ccng/nginx.pid; + +events { + worker_connections 8192; + use epoll; +} + +http { + include mime.types; + default_type text/html; + server_tokens off; + variables_hash_max_size 1024; + + log_format main '$host - [$time_local] ' + '"$request" $status $bytes_sent ' + '"$http_referer" "$http_#user_agent" ' + '$proxy_add_x_forwarded_for response_time:$upstream_response_time'; + + access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main; + + sendfile on; #enable use of sendfile() + tcp_nopush on; + tcp_nodelay on; #disable nagel's algorithm + + keepalive_timeout 75 20; #inherited from router + + client_max_body_size 256M; #already enforced upstream/but doesn't hurt. + + upstream cloud_controller { + server unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock; + } + + server { + listen {{ nginx_port }}; + server_name _; + server_name_in_redirect off; + proxy_send_timeout 300; + proxy_read_timeout 300; + + # proxy and log all CC traffic + location / { + access_log /var/vcap/sys/log/nginx_ccng/nginx.access.log main; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real_IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + proxy_connect_timeout 10; + proxy_pass http://cloud_controller; + } + + + # used for x-accel-redirect uri://location/foo.txt + # nginx will serve the file root || location || foo.txt + location /droplets/ { + internal; + root /var/vcap/nfs/store; + } + + + + # used for x-accel-redirect uri://location/foo.txt + # nginx will serve the file root || location || foo.txt + location /cc-packages/ { + internal; + root /var/vcap/nfs/store; + } + + + # used for x-accel-redirect uri://location/foo.txt + # nginx will serve the file root || location || foo.txt + location /cc-droplets/ { + internal; + root /var/vcap/nfs/store; + } + + + location ~ (/apps/.*/application|/v2/apps/.*/bits|/services/v\d+/configurations/.*/serialized/data|/v2/buildpacks/.*/bits) { + # Pass altered request body to this location + upload_pass @cc_uploads; + upload_pass_args on; + + # Store files to this directory + upload_store /var/vcap/data/cloud_controller_ng/tmp/uploads; + + # No limit for output body forwarded to CC + upload_max_output_body_len 0; + + # Allow uploaded files to be read only by #user + #upload_store_access #user:r; + + # Set specified fields in request body + upload_set_form_field "${upload_field_name}_name" $upload_file_name; + upload_set_form_field "${upload_field_name}_path" $upload_tmp_path; + + #forward the following fields from existing body + upload_pass_form_field "^resources$"; + upload_pass_form_field "^_method$"; + + #on any error, delete uploaded files. + upload_cleanup 400-505; + } + + location ~ /staging/(buildpack_cache|droplets)/.*/upload { + + # Allow download the droplets and buildpacks + if ($request_method = GET){ + proxy_pass http://cloud_controller; + } + + # Pass along auth header + set $auth_header $upstream_http_x_auth; + proxy_set_header Authorization $auth_header; + + # Pass altered request body to this location + upload_pass @cc_uploads; + + # Store files to this directory + upload_store /var/vcap/data/cloud_controller_ng/tmp/staged_droplet_uploads; + + # Allow uploaded files to be read only by #user + upload_store_access user:r; + + # Set specified fields in request body + upload_set_form_field "droplet_path" $upload_tmp_path; + + #on any error, delete uploaded files. + upload_cleanup 400-505; + } + + # Pass altered request body to a backend + location @cc_uploads { + proxy_pass http://unix:/var/vcap/sys/run/cloud_controller_ng/cloud_controller.sock; + } + + location ~ ^/internal_redirect/(.*){ + # only allow internal redirects + internal; + + set $download_url $1; + + #have to manually pass along auth header + set $auth_header $upstream_http_x_auth; + proxy_set_header Authorization $auth_header; + + # Download the file and send it to client + proxy_pass $download_url; + } + } +} diff --git a/nrpe/mod/charmhelpers/tests/core/templates/test.conf b/nrpe/mod/charmhelpers/tests/core/templates/test.conf new file mode 100644 index 0000000..bb02adc --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/templates/test.conf @@ -0,0 +1,3 @@ +something +listen {{nginx_port}} +something else diff --git a/nrpe/mod/charmhelpers/tests/core/test_files.py b/nrpe/mod/charmhelpers/tests/core/test_files.py new file mode 100644 index 0000000..8b443a2 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_files.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from charmhelpers.core import files + +import mock +import unittest +import tempfile +import os + + +class FileTests(unittest.TestCase): + + @mock.patch("subprocess.check_call") + def test_sed(self, check_call): + files.sed("/tmp/test-sed-file", "replace", "this") + check_call.assert_called_once_with( + ['sed', '-i', '-r', '-e', 's/replace/this/g', + '/tmp/test-sed-file'] + ) + + def test_sed_file(self): + tmp = tempfile.NamedTemporaryFile(mode='w', delete=False) + tmp.write("IPV6=yes") + tmp.close() + + files.sed(tmp.name, "IPV6=.*", "IPV6=no") + + with open(tmp.name) as tmp: + self.assertEquals(tmp.read(), "IPV6=no") + + os.unlink(tmp.name) diff --git a/nrpe/mod/charmhelpers/tests/core/test_fstab.py b/nrpe/mod/charmhelpers/tests/core/test_fstab.py new file mode 100644 index 0000000..846c8c5 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_fstab.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from charmhelpers.core.fstab import Fstab +from nose.tools import (assert_is, + assert_is_not, + assert_equal) +import unittest +import tempfile +import os + +__author__ = 'Jorge Niedbalski R. ' + +DEFAULT_FSTAB_FILE = """/dev/sda /mnt/sda ext3 defaults 0 0 + # This is an indented comment, and the next line is entirely blank. + +/dev/sdb /mnt/sdb ext3 defaults 0 0 +/dev/sdc /mnt/sdc ext3 defaults 0 0 +UUID=3af44368-c50b-4768-8e58-aff003cef8be / ext4 errors=remount-ro 0 1 +""" + +GENERATED_FSTAB_FILE = '\n'.join( + # Helper will writeback with spaces instead of tabs + line.replace('\t', ' ') for line in DEFAULT_FSTAB_FILE.splitlines() + if line.strip() and not line.strip().startswith('#')) + + +class FstabTest(unittest.TestCase): + + def setUp(self): + self.tempfile = tempfile.NamedTemporaryFile('w+', delete=False) + self.tempfile.write(DEFAULT_FSTAB_FILE) + self.tempfile.close() + self.fstab = Fstab(path=self.tempfile.name) + + def tearDown(self): + os.unlink(self.tempfile.name) + + def test_entries(self): + """Test if entries are correctly read from fstab file""" + assert_equal(sorted(GENERATED_FSTAB_FILE.splitlines()), + sorted(str(entry) for entry in self.fstab.entries)) + + def test_get_entry_by_device_attr(self): + """Test if the get_entry_by_attr method works for device attr""" + for device in ('sda', 'sdb', 'sdc', ): + assert_is_not(self.fstab.get_entry_by_attr('device', + '/dev/%s' % device), + None) + + def test_get_entry_by_mountpoint_attr(self): + """Test if the get_entry_by_attr method works for mountpoint attr""" + for mnt in ('sda', 'sdb', 'sdc', ): + assert_is_not(self.fstab.get_entry_by_attr('mountpoint', + '/mnt/%s' % mnt), None) + + def test_add_entry(self): + """Test if add_entry works for a new entry""" + for device in ('sdf', 'sdg', 'sdh'): + entry = Fstab.Entry('/dev/%s' % device, '/mnt/%s' % device, 'ext3', + None) + assert_is_not(self.fstab.add_entry(entry), None) + assert_is_not(self.fstab.get_entry_by_attr( + 'device', '/dev/%s' % device), None) + + assert_is(self.fstab.add_entry(entry), False, + "Check if adding an existing entry returns false") + + def test_remove_entry(self): + """Test if remove entry works for already existing entries""" + for entry in self.fstab.entries: + assert_is(self.fstab.remove_entry(entry), True) + + assert_equal(len([entry for entry in self.fstab.entries]), 0) + assert_equal(self.fstab.add_entry(entry), entry) + assert_equal(len([entry for entry in self.fstab.entries]), 1) + + def test_assert_remove_add_all(self): + """Test if removing/adding all the entries works""" + for entry in self.fstab.entries: + assert_is(self.fstab.remove_entry(entry), True) + + for device in ('sda', 'sdb', 'sdc', ): + self.fstab.add_entry( + Fstab.Entry('/dev/%s' % device, '/mnt/%s' % device, 'ext3', + None)) + + self.fstab.add_entry(Fstab.Entry( + 'UUID=3af44368-c50b-4768-8e58-aff003cef8be', + '/', 'ext4', 'errors=remount-ro', 0, 1)) + + assert_equal(sorted(GENERATED_FSTAB_FILE.splitlines()), + sorted(str(entry) for entry in self.fstab.entries)) diff --git a/nrpe/mod/charmhelpers/tests/core/test_hookenv.py b/nrpe/mod/charmhelpers/tests/core/test_hookenv.py new file mode 100644 index 0000000..1b65410 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_hookenv.py @@ -0,0 +1,2411 @@ +import os +import json +from subprocess import CalledProcessError +import shutil +import tempfile +import types +from mock import call, MagicMock, mock_open, patch, sentinel +from testtools import TestCase +from enum import Enum +import yaml + +import six +import io + +from charmhelpers.core import hookenv + +if six.PY3: + import pickle +else: + import cPickle as pickle + + +CHARM_METADATA = b"""name: testmock +summary: test mock summary +description: test mock description +requires: + testreqs: + interface: mock +provides: + testprov: + interface: mock +peers: + testpeer: + interface: mock +""" + + +def _clean_globals(): + hookenv.cache.clear() + del hookenv._atstart[:] + del hookenv._atexit[:] + + +class ConfigTest(TestCase): + def setUp(self): + super(ConfigTest, self).setUp() + + _clean_globals() + self.addCleanup(_clean_globals) + + self.charm_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.charm_dir)) + + patcher = patch.object(hookenv, 'charm_dir', lambda: self.charm_dir) + self.addCleanup(patcher.stop) + patcher.start() + + def test_init(self): + d = dict(foo='bar') + c = hookenv.Config(d) + + self.assertEqual(c['foo'], 'bar') + self.assertEqual(c._prev_dict, None) + + def test_init_empty_state_file(self): + d = dict(foo='bar') + c = hookenv.Config(d) + + state_path = os.path.join(self.charm_dir, hookenv.Config.CONFIG_FILE_NAME) + with open(state_path, 'w') as f: + f.close() + + self.assertEqual(c['foo'], 'bar') + self.assertEqual(c._prev_dict, None) + self.assertEqual(c.path, state_path) + + def test_init_invalid_state_file(self): + d = dict(foo='bar') + + state_path = os.path.join(self.charm_dir, hookenv.Config.CONFIG_FILE_NAME) + with open(state_path, 'w') as f: + f.write('blah') + + c = hookenv.Config(d) + + self.assertEqual(c['foo'], 'bar') + self.assertEqual(c._prev_dict, None) + self.assertEqual(c.path, state_path) + + def test_load_previous(self): + d = dict(foo='bar') + c = hookenv.Config() + + with open(c.path, 'w') as f: + json.dump(d, f) + + c.load_previous() + self.assertEqual(c._prev_dict, d) + + def test_load_previous_alternate_path(self): + d = dict(foo='bar') + c = hookenv.Config() + + alt_path = os.path.join(self.charm_dir, '.alt-config') + with open(alt_path, 'w') as f: + json.dump(d, f) + + c.load_previous(path=alt_path) + self.assertEqual(c._prev_dict, d) + self.assertEqual(c.path, alt_path) + + def test_changed_without_prev_dict(self): + d = dict(foo='bar') + c = hookenv.Config(d) + + self.assertTrue(c.changed('foo')) + + def test_changed_with_prev_dict(self): + c = hookenv.Config(dict(foo='bar', a='b')) + c.save() + c = hookenv.Config(dict(foo='baz', a='b')) + + self.assertTrue(c.changed('foo')) + self.assertFalse(c.changed('a')) + + def test_previous_without_prev_dict(self): + c = hookenv.Config() + + self.assertEqual(c.previous('foo'), None) + + def test_previous_with_prev_dict(self): + c = hookenv.Config(dict(foo='bar')) + c.save() + c = hookenv.Config(dict(foo='baz', a='b')) + + self.assertEqual(c.previous('foo'), 'bar') + self.assertEqual(c.previous('a'), None) + + def test_save_without_prev_dict(self): + c = hookenv.Config(dict(foo='bar')) + c.save() + + with open(c.path, 'r') as f: + self.assertEqual(c, json.load(f)) + self.assertEqual(c, dict(foo='bar')) + self.assertEqual(os.stat(c.path).st_mode & 0o777, 0o600) + + def test_save_with_prev_dict(self): + c = hookenv.Config(dict(foo='bar')) + c.save() + c = hookenv.Config(dict(a='b')) + c.save() + + with open(c.path, 'r') as f: + self.assertEqual(c, json.load(f)) + self.assertEqual(c, dict(foo='bar', a='b')) + self.assertEqual(os.stat(c.path).st_mode & 0o777, 0o600) + + def test_deep_change(self): + # After loading stored data into our previous dictionary, + # it gets copied into our current dictionary. If this is not + # a deep copy, then mappings and lists will be shared instances + # and changes will not be detected. + c = hookenv.Config(dict(ll=[])) + c.save() + c = hookenv.Config() + c['ll'].append(42) + self.assertTrue(c.changed('ll'), 'load_previous() did not deepcopy') + + def test_getitem(self): + c = hookenv.Config(dict(foo='bar')) + c.save() + c = hookenv.Config(dict(baz='bam')) + + self.assertRaises(KeyError, lambda: c['missing']) + self.assertEqual(c['foo'], 'bar') + self.assertEqual(c['baz'], 'bam') + + def test_get(self): + c = hookenv.Config(dict(foo='bar')) + c.save() + c = hookenv.Config(dict(baz='bam')) + + self.assertIsNone(c.get('missing')) + self.assertIs(c.get('missing', sentinel.missing), sentinel.missing) + self.assertEqual(c.get('foo'), 'bar') + self.assertEqual(c.get('baz'), 'bam') + + def test_keys(self): + c = hookenv.Config(dict(foo='bar')) + c["baz"] = "bar" + self.assertEqual(sorted([six.u("foo"), "baz"]), sorted(c.keys())) + + def test_in(self): + # Test behavior of the in operator. + + prev_path = os.path.join(hookenv.charm_dir(), + hookenv.Config.CONFIG_FILE_NAME) + with open(prev_path, 'w') as f: + json.dump(dict(user='one'), f) + c = hookenv.Config(dict(charm='one')) + + # Items that exist in the dict exist. Items that don't don't. + self.assertTrue('user' in c) + self.assertTrue('charm' in c) + self.assertFalse('bar' in c) + + # Adding items works as expected. + c['user'] = 'two' + c['charm'] = 'two' + c['bar'] = 'two' + self.assertTrue('user' in c) + self.assertTrue('charm' in c) + self.assertTrue('bar' in c) + c.save() + self.assertTrue('user' in c) + self.assertTrue('charm' in c) + self.assertTrue('bar' in c) + + # Removing items works as expected. + del c['user'] + del c['charm'] + self.assertTrue('user' not in c) + self.assertTrue('charm' not in c) + c.save() + self.assertTrue('user' not in c) + self.assertTrue('charm' not in c) + + +class SerializableTest(TestCase): + def test_serializes_object_to_json(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + self.assertEqual(wrapped.json(), json.dumps(foo)) + + def test_serializes_object_to_yaml(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + self.assertEqual(wrapped.yaml(), yaml.dump(foo)) + + def test_gets_attribute_from_inner_object_as_dict(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + + self.assertEqual(wrapped.bar, 'baz') + + def test_raises_error_from_inner_object_as_dict(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + + self.assertRaises(AttributeError, getattr, wrapped, 'baz') + + def test_dict_methods_from_inner_object(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + for meth in ('keys', 'values', 'items'): + self.assertEqual(sorted(list(getattr(wrapped, meth)())), + sorted(list(getattr(foo, meth)()))) + + self.assertEqual(wrapped.get('bar'), foo.get('bar')) + self.assertEqual(wrapped.get('baz', 42), foo.get('baz', 42)) + self.assertIn('bar', wrapped) + + def test_get_gets_from_inner_object(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + + self.assertEqual(wrapped.get('foo'), None) + self.assertEqual(wrapped.get('bar'), 'baz') + self.assertEqual(wrapped.get('zoo', 'bla'), 'bla') + + def test_gets_inner_object(self): + foo = { + 'bar': 'baz', + } + wrapped = hookenv.Serializable(foo) + + self.assertIs(wrapped.data, foo) + + def test_pickle(self): + foo = {'bar': 'baz'} + wrapped = hookenv.Serializable(foo) + pickled = pickle.dumps(wrapped) + unpickled = pickle.loads(pickled) + + self.assert_(isinstance(unpickled, hookenv.Serializable)) + self.assertEqual(unpickled, foo) + + def test_boolean(self): + true_dict = {'foo': 'bar'} + false_dict = {} + + self.assertIs(bool(hookenv.Serializable(true_dict)), True) + self.assertIs(bool(hookenv.Serializable(false_dict)), False) + + def test_equality(self): + foo = {'bar': 'baz'} + bar = {'baz': 'bar'} + wrapped_foo = hookenv.Serializable(foo) + + self.assertEqual(wrapped_foo, foo) + self.assertEqual(wrapped_foo, wrapped_foo) + self.assertNotEqual(wrapped_foo, bar) + + +class HelpersTest(TestCase): + def setUp(self): + super(HelpersTest, self).setUp() + _clean_globals() + self.addCleanup(_clean_globals) + + @patch('subprocess.call') + def test_logs_messages_to_juju_with_default_level(self, mock_call): + hookenv.log('foo') + + mock_call.assert_called_with(['juju-log', 'foo']) + + @patch('subprocess.call') + def test_logs_messages_object(self, mock_call): + hookenv.log(object) + mock_call.assert_called_with(['juju-log', repr(object)]) + + @patch('subprocess.call') + def test_logs_messages_with_alternative_levels(self, mock_call): + alternative_levels = [ + hookenv.CRITICAL, + hookenv.ERROR, + hookenv.WARNING, + hookenv.INFO, + ] + + for level in alternative_levels: + hookenv.log('foo', level) + mock_call.assert_called_with(['juju-log', '-l', level, 'foo']) + + @patch('subprocess.call') + def test_function_log_message(self, mock_call): + hookenv.function_log('foo') + mock_call.assert_called_with(['function-log', 'foo']) + + @patch('subprocess.call') + def test_function_log_message_object(self, mock_call): + hookenv.function_log(object) + mock_call.assert_called_with(['function-log', repr(object)]) + + @patch('charmhelpers.core.hookenv._cache_config', None) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_charm_config_with_scope(self, check_output, charm_dir): + check_output.return_value = json.dumps(dict(baz='bar')).encode('UTF-8') + charm_dir.return_value = '/nonexistent' + + result = hookenv.config(scope='baz') + + self.assertEqual(result, 'bar') + check_output.assert_called_with(['config-get', '--all', + '--format=json']) + + # The result can be used like a string + self.assertEqual(result[1], 'a') + + # ... because the result is actually a string + self.assert_(isinstance(result, six.string_types)) + + @patch('charmhelpers.core.hookenv.log', lambda *args, **kwargs: None) + @patch('charmhelpers.core.hookenv._cache_config', None) + @patch('subprocess.check_output') + def test_gets_missing_charm_config_with_scope(self, check_output): + check_output.return_value = b'' + + result = hookenv.config(scope='baz') + + self.assertEqual(result, None) + check_output.assert_called_with(['config-get', '--all', + '--format=json']) + + @patch('charmhelpers.core.hookenv._cache_config', None) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_config_without_scope(self, check_output, charm_dir): + check_output.return_value = json.dumps(dict(baz='bar')).encode('UTF-8') + charm_dir.return_value = '/nonexistent' + + result = hookenv.config() + + self.assertIsInstance(result, hookenv.Config) + self.assertEqual(result['baz'], 'bar') + check_output.assert_called_with(['config-get', '--all', + '--format=json']) + + @patch('charmhelpers.core.hookenv.log') + @patch('charmhelpers.core.hookenv._cache_config', None) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_charm_config_invalid_json_with_scope(self, + check_output, + charm_dir, + log): + check_output.return_value = '{"invalid: "json"}'.encode('UTF-8') + charm_dir.return_value = '/nonexistent' + + result = hookenv.config(scope='invalid') + + self.assertEqual(result, None) + cmd_line = ['config-get', '--all', '--format=json'] + check_output.assert_called_with(cmd_line) + log.assert_called_with( + 'Unable to parse output from config-get: ' + 'config_cmd_line="{}" message="{}"' + .format(str(cmd_line), + "Expecting ':' delimiter: line 1 column 13 (char 12)"), + level=hookenv.ERROR, + ) + + @patch('charmhelpers.core.hookenv.log') + @patch('charmhelpers.core.hookenv._cache_config', None) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_charm_config_invalid_utf8_with_scope(self, + check_output, + charm_dir, + log): + check_output.return_value = b'{"invalid: "json"}\x9D' + charm_dir.return_value = '/nonexistent' + + result = hookenv.config(scope='invalid') + + self.assertEqual(result, None) + cmd_line = ['config-get', '--all', '--format=json'] + check_output.assert_called_with(cmd_line) + try: + # Python3 + log.assert_called_with( + 'Unable to parse output from config-get: ' + 'config_cmd_line="{}" message="{}"' + .format(str(cmd_line), + "'utf8' codec can't decode byte 0x9d in position " + "18: invalid start byte"), + level=hookenv.ERROR, + ) + except AssertionError: + # Python2.7 + log.assert_called_with( + 'Unable to parse output from config-get: ' + 'config_cmd_line="{}" message="{}"' + .format(str(cmd_line), + "'utf-8' codec can't decode byte 0x9d in position " + "18: invalid start byte"), + level=hookenv.ERROR, + ) + + @patch('charmhelpers.core.hookenv._cache_config', {'baz': 'bar'}) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_config_from_cache_without_scope(self, + check_output, + charm_dir): + charm_dir.return_value = '/nonexistent' + + result = hookenv.config() + + self.assertEqual(result['baz'], 'bar') + self.assertFalse(check_output.called) + + @patch('charmhelpers.core.hookenv._cache_config', {'baz': 'bar'}) + @patch('charmhelpers.core.hookenv.charm_dir') + @patch('subprocess.check_output') + def test_gets_config_from_cache_with_scope(self, + check_output, + charm_dir): + charm_dir.return_value = '/nonexistent' + + result = hookenv.config('baz') + + self.assertEqual(result, 'bar') + + # The result can be used like a string + self.assertEqual(result[1], 'a') + + # ... because the result is actually a string + self.assert_(isinstance(result, six.string_types)) + + self.assertFalse(check_output.called) + + @patch('charmhelpers.core.hookenv._cache_config', {'foo': 'bar'}) + @patch('subprocess.check_output') + def test_gets_missing_charm_config_from_cache_with_scope(self, + check_output): + + result = hookenv.config(scope='baz') + + self.assertEqual(result, None) + self.assertFalse(check_output.called) + + @patch('charmhelpers.core.hookenv.os') + def test_gets_the_local_unit(self, os_): + os_.environ = { + 'JUJU_UNIT_NAME': 'foo', + } + + self.assertEqual(hookenv.local_unit(), 'foo') + + @patch('charmhelpers.core.hookenv.unit_get') + def test_gets_unit_public_ip(self, _unitget): + _unitget.return_value = sentinel.public_ip + self.assertEqual(sentinel.public_ip, hookenv.unit_public_ip()) + _unitget.assert_called_once_with('public-address') + + @patch('charmhelpers.core.hookenv.unit_get') + def test_gets_unit_private_ip(self, _unitget): + _unitget.return_value = sentinel.private_ip + self.assertEqual(sentinel.private_ip, hookenv.unit_private_ip()) + _unitget.assert_called_once_with('private-address') + + @patch('charmhelpers.core.hookenv.os') + def test_checks_that_is_running_in_relation_hook(self, os_): + os_.environ = { + 'JUJU_RELATION': 'foo', + } + + self.assertTrue(hookenv.in_relation_hook()) + + @patch('charmhelpers.core.hookenv.os') + def test_checks_that_is_not_running_in_relation_hook(self, os_): + os_.environ = { + 'bar': 'foo', + } + + self.assertFalse(hookenv.in_relation_hook()) + + @patch('charmhelpers.core.hookenv.os') + def test_gets_the_relation_type(self, os_): + os_.environ = { + 'JUJU_RELATION': 'foo', + } + + self.assertEqual(hookenv.relation_type(), 'foo') + + @patch('charmhelpers.core.hookenv.os') + def test_relation_type_none_if_not_in_environment(self, os_): + os_.environ = {} + self.assertEqual(hookenv.relation_type(), None) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_type') + def test_gets_relation_ids(self, relation_type, check_output): + ids = [1, 2, 3] + check_output.return_value = json.dumps(ids).encode('UTF-8') + reltype = 'foo' + relation_type.return_value = reltype + + result = hookenv.relation_ids() + + self.assertEqual(result, ids) + check_output.assert_called_with(['relation-ids', '--format=json', + reltype]) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_type') + def test_gets_relation_ids_empty_array(self, relation_type, check_output): + ids = [] + check_output.return_value = json.dumps(None).encode('UTF-8') + reltype = 'foo' + relation_type.return_value = reltype + + result = hookenv.relation_ids() + + self.assertEqual(result, ids) + check_output.assert_called_with(['relation-ids', '--format=json', + reltype]) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_type') + def test_relation_ids_no_relation_type(self, relation_type, check_output): + ids = [1, 2, 3] + check_output.return_value = json.dumps(ids).encode('UTF-8') + relation_type.return_value = None + + result = hookenv.relation_ids() + + self.assertEqual(result, []) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_type') + def test_gets_relation_ids_for_type(self, relation_type, check_output): + ids = [1, 2, 3] + check_output.return_value = json.dumps(ids).encode('UTF-8') + reltype = 'foo' + + result = hookenv.relation_ids(reltype) + + self.assertEqual(result, ids) + check_output.assert_called_with(['relation-ids', '--format=json', + reltype]) + self.assertFalse(relation_type.called) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_id') + def test_gets_related_units(self, relation_id, check_output): + relid = 123 + units = ['foo', 'bar'] + relation_id.return_value = relid + check_output.return_value = json.dumps(units).encode('UTF-8') + + result = hookenv.related_units() + + self.assertEqual(result, units) + check_output.assert_called_with(['relation-list', '--format=json', + '-r', relid]) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_id') + def test_gets_related_units_empty_array(self, relation_id, check_output): + relid = str(123) + units = [] + relation_id.return_value = relid + check_output.return_value = json.dumps(None).encode('UTF-8') + + result = hookenv.related_units() + + self.assertEqual(result, units) + check_output.assert_called_with(['relation-list', '--format=json', + '-r', relid]) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_id') + def test_related_units_no_relation(self, relation_id, check_output): + units = ['foo', 'bar'] + relation_id.return_value = None + check_output.return_value = json.dumps(units).encode('UTF-8') + + result = hookenv.related_units() + + self.assertEqual(result, units) + check_output.assert_called_with(['relation-list', '--format=json']) + + @patch('subprocess.check_output') + @patch('charmhelpers.core.hookenv.relation_id') + def test_gets_related_units_for_id(self, relation_id, check_output): + relid = 123 + units = ['foo', 'bar'] + check_output.return_value = json.dumps(units).encode('UTF-8') + + result = hookenv.related_units(relid) + + self.assertEqual(result, units) + check_output.assert_called_with(['relation-list', '--format=json', + '-r', relid]) + self.assertFalse(relation_id.called) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('charmhelpers.core.hookenv.goal_state') + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_gets_expected_peer_units(self, has_juju_version, goal_state, + local_unit): + has_juju_version.return_value = True + goal_state.return_value = { + 'units': { + 'keystone/0': { + 'status': 'active', + 'since': '2018-09-27 11:38:28Z', + }, + 'keystone/1': { + 'status': 'active', + 'since': '2018-09-27 11:39:23Z', + }, + }, + } + local_unit.return_value = 'keystone/0' + + result = hookenv.expected_peer_units() + + self.assertIsInstance(result, types.GeneratorType) + self.assertEqual(sorted(result), ['keystone/1']) + has_juju_version.assertCalledOnceWith("2.4.0") + local_unit.assertCalledOnceWith() + + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_gets_expected_peer_units_wrong_version(self, has_juju_version): + has_juju_version.return_value = False + + def x(): + # local helper function to make testtools.TestCase.assertRaises + # work with generator + list(hookenv.expected_peer_units()) + + self.assertRaises(NotImplementedError, x) + has_juju_version.assertCalledOnceWith("2.4.0") + + @patch('charmhelpers.core.hookenv.goal_state') + @patch('charmhelpers.core.hookenv.relation_type') + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_gets_expected_related_units(self, has_juju_version, relation_type, + goal_state): + has_juju_version.return_value = True + relation_type.return_value = 'identity-service' + goal_state.return_value = { + 'relations': { + 'identity-service': { + 'glance': { + 'status': 'joined', + 'since': '2018-09-27 11:37:16Z' + }, + 'glance/0': { + 'status': 'active', + 'since': '2018-09-27 11:27:19Z' + }, + 'glance/1': { + 'status': 'active', + 'since': '2018-09-27 11:27:34Z' + }, + }, + }, + } + + result = hookenv.expected_related_units() + + self.assertIsInstance(result, types.GeneratorType) + self.assertEqual(sorted(result), ['glance/0', 'glance/1']) + + @patch('charmhelpers.core.hookenv.goal_state') + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_gets_expected_related_units_for_type(self, has_juju_version, + goal_state): + has_juju_version.return_value = True + goal_state.return_value = { + 'relations': { + 'identity-service': { + 'glance': { + 'status': 'joined', + 'since': '2018-09-27 11:37:16Z' + }, + 'glance/0': { + 'status': 'active', + 'since': '2018-09-27 11:27:19Z' + }, + 'glance/1': { + 'status': 'active', + 'since': '2018-09-27 11:27:34Z' + }, + }, + }, + } + + result = hookenv.expected_related_units('identity-service') + + self.assertIsInstance(result, types.GeneratorType) + self.assertEqual(sorted(result), ['glance/0', 'glance/1']) + + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_gets_expected_related_units_wrong_version(self, has_juju_version): + has_juju_version.return_value = False + + def x(): + # local helper function to make testtools.TestCase.assertRaises + # work with generator + list(hookenv.expected_related_units()) + + self.assertRaises(NotImplementedError, x) + has_juju_version.assertCalledOnceWith("2.4.4") + + @patch('charmhelpers.core.hookenv.os') + def test_gets_the_departing_unit(self, os_): + os_.environ = { + 'JUJU_DEPARTING_UNIT': 'foo/0', + } + + self.assertEqual(hookenv.departing_unit(), 'foo/0') + + @patch('charmhelpers.core.hookenv.os') + def test_no_departing_unit(self, os_): + os_.environ = {} + self.assertEqual(hookenv.departing_unit(), None) + + @patch('charmhelpers.core.hookenv.os') + def test_gets_the_remote_unit(self, os_): + os_.environ = { + 'JUJU_REMOTE_UNIT': 'foo', + } + + self.assertEqual(hookenv.remote_unit(), 'foo') + + @patch('charmhelpers.core.hookenv.os') + def test_no_remote_unit(self, os_): + os_.environ = {} + self.assertEqual(hookenv.remote_unit(), None) + + @patch('charmhelpers.core.hookenv.remote_unit') + @patch('charmhelpers.core.hookenv.relation_get') + def test_gets_relation_for_unit(self, relation_get, remote_unit): + unit = 'foo-unit' + raw_relation = { + 'foo': 'bar', + } + remote_unit.return_value = unit + relation_get.return_value = raw_relation + + result = hookenv.relation_for_unit() + + self.assertEqual(result['__unit__'], unit) + self.assertEqual(result['foo'], 'bar') + relation_get.assert_called_with(unit=unit, rid=None) + + @patch('charmhelpers.core.hookenv.remote_unit') + @patch('charmhelpers.core.hookenv.relation_get') + def test_gets_relation_for_unit_with_list(self, relation_get, remote_unit): + unit = 'foo-unit' + raw_relation = { + 'foo-list': 'one two three', + } + remote_unit.return_value = unit + relation_get.return_value = raw_relation + + result = hookenv.relation_for_unit() + + self.assertEqual(result['__unit__'], unit) + self.assertEqual(result['foo-list'], ['one', 'two', 'three']) + relation_get.assert_called_with(unit=unit, rid=None) + + @patch('charmhelpers.core.hookenv.remote_unit') + @patch('charmhelpers.core.hookenv.relation_get') + def test_gets_relation_for_specific_unit(self, relation_get, remote_unit): + unit = 'foo-unit' + raw_relation = { + 'foo': 'bar', + } + relation_get.return_value = raw_relation + + result = hookenv.relation_for_unit(unit) + + self.assertEqual(result['__unit__'], unit) + self.assertEqual(result['foo'], 'bar') + relation_get.assert_called_with(unit=unit, rid=None) + self.assertFalse(remote_unit.called) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_for_unit') + def test_gets_relations_for_id(self, relation_for_unit, related_units, + relation_ids): + relid = 123 + units = ['foo', 'bar'] + unit_data = [ + {'foo-item': 'bar-item'}, + {'foo-item2': 'bar-item2'}, + ] + relation_ids.return_value = relid + related_units.return_value = units + relation_for_unit.side_effect = unit_data + + result = hookenv.relations_for_id() + + self.assertEqual(result[0]['__relid__'], relid) + self.assertEqual(result[0]['foo-item'], 'bar-item') + self.assertEqual(result[1]['__relid__'], relid) + self.assertEqual(result[1]['foo-item2'], 'bar-item2') + related_units.assert_called_with(relid) + self.assertEqual(relation_for_unit.mock_calls, [ + call('foo', relid), + call('bar', relid), + ]) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_for_unit') + def test_gets_relations_for_specific_id(self, relation_for_unit, + related_units, relation_ids): + relid = 123 + units = ['foo', 'bar'] + unit_data = [ + {'foo-item': 'bar-item'}, + {'foo-item2': 'bar-item2'}, + ] + related_units.return_value = units + relation_for_unit.side_effect = unit_data + + result = hookenv.relations_for_id(relid) + + self.assertEqual(result[0]['__relid__'], relid) + self.assertEqual(result[0]['foo-item'], 'bar-item') + self.assertEqual(result[1]['__relid__'], relid) + self.assertEqual(result[1]['foo-item2'], 'bar-item2') + related_units.assert_called_with(relid) + self.assertEqual(relation_for_unit.mock_calls, [ + call('foo', relid), + call('bar', relid), + ]) + self.assertFalse(relation_ids.called) + + @patch('charmhelpers.core.hookenv.in_relation_hook') + @patch('charmhelpers.core.hookenv.relation_type') + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.relations_for_id') + def test_gets_relations_for_type(self, relations_for_id, relation_ids, + relation_type, in_relation_hook): + reltype = 'foo-type' + relids = [123, 234] + relations = [ + [ + {'foo': 'bar'}, + {'foo2': 'bar2'}, + ], + [ + {'FOO': 'BAR'}, + {'FOO2': 'BAR2'}, + ], + ] + is_in_relation = True + + relation_type.return_value = reltype + relation_ids.return_value = relids + relations_for_id.side_effect = relations + in_relation_hook.return_value = is_in_relation + + result = hookenv.relations_of_type() + + self.assertEqual(result[0]['__relid__'], 123) + self.assertEqual(result[0]['foo'], 'bar') + self.assertEqual(result[1]['__relid__'], 123) + self.assertEqual(result[1]['foo2'], 'bar2') + self.assertEqual(result[2]['__relid__'], 234) + self.assertEqual(result[2]['FOO'], 'BAR') + self.assertEqual(result[3]['__relid__'], 234) + self.assertEqual(result[3]['FOO2'], 'BAR2') + relation_ids.assert_called_with(reltype) + self.assertEqual(relations_for_id.mock_calls, [ + call(123), + call(234), + ]) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('charmhelpers.core.hookenv.relation_types') + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_gets_relations(self, relation_get, related_units, + relation_ids, relation_types, local_unit): + local_unit.return_value = 'u0' + relation_types.return_value = ['t1', 't2'] + relation_ids.return_value = ['i1'] + related_units.return_value = ['u1', 'u2'] + relation_get.return_value = {'key': 'val'} + + result = hookenv.relations() + + self.assertEqual(result, { + 't1': { + 'i1': { + 'u0': {'key': 'val'}, + 'u1': {'key': 'val'}, + 'u2': {'key': 'val'}, + }, + }, + 't2': { + 'i1': { + 'u0': {'key': 'val'}, + 'u1': {'key': 'val'}, + 'u2': {'key': 'val'}, + }, + }, + }) + + @patch('charmhelpers.core.hookenv.relation_set') + @patch('charmhelpers.core.hookenv.relation_get') + @patch('charmhelpers.core.hookenv.local_unit') + def test_relation_clear(self, local_unit, + relation_get, + relation_set): + local_unit.return_value = 'local-unit' + relation_get.return_value = { + 'private-address': '10.5.0.1', + 'foo': 'bar', + 'public-address': '146.192.45.6' + } + hookenv.relation_clear('relation:1') + relation_get.assert_called_with(rid='relation:1', + unit='local-unit') + relation_set.assert_called_with( + relation_id='relation:1', + **{'private-address': '10.5.0.1', + 'foo': None, + 'public-address': '146.192.45.6'}) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made(self, relation_get, related_units, + relation_ids): + relation_get.return_value = 'hostname' + related_units.return_value = ['test/1'] + relation_ids.return_value = ['test:0'] + self.assertTrue(hookenv.is_relation_made('test')) + relation_get.assert_called_with('private-address', + rid='test:0', unit='test/1') + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made_multi_unit(self, relation_get, related_units, + relation_ids): + relation_get.side_effect = [None, 'hostname'] + related_units.return_value = ['test/1', 'test/2'] + relation_ids.return_value = ['test:0'] + self.assertTrue(hookenv.is_relation_made('test')) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made_different_key(self, + relation_get, related_units, + relation_ids): + relation_get.return_value = 'hostname' + related_units.return_value = ['test/1'] + relation_ids.return_value = ['test:0'] + self.assertTrue(hookenv.is_relation_made('test', keys='auth')) + relation_get.assert_called_with('auth', + rid='test:0', unit='test/1') + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made_multiple_keys(self, + relation_get, related_units, + relation_ids): + relation_get.side_effect = ['password', 'hostname'] + related_units.return_value = ['test/1'] + relation_ids.return_value = ['test:0'] + self.assertTrue(hookenv.is_relation_made('test', + keys=['auth', 'host'])) + relation_get.assert_has_calls( + [call('auth', rid='test:0', unit='test/1'), + call('host', rid='test:0', unit='test/1')] + ) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made_not_made(self, + relation_get, related_units, + relation_ids): + relation_get.return_value = None + related_units.return_value = ['test/1'] + relation_ids.return_value = ['test:0'] + self.assertFalse(hookenv.is_relation_made('test')) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.relation_get') + def test_is_relation_made_not_made_multiple_keys(self, + relation_get, + related_units, + relation_ids): + relation_get.side_effect = ['password', None] + related_units.return_value = ['test/1'] + relation_ids.return_value = ['test:0'] + self.assertFalse(hookenv.is_relation_made('test', + keys=['auth', 'host'])) + relation_get.assert_has_calls( + [call('auth', rid='test:0', unit='test/1'), + call('host', rid='test:0', unit='test/1')] + ) + + @patch('charmhelpers.core.hookenv.config') + @patch('charmhelpers.core.hookenv.relation_type') + @patch('charmhelpers.core.hookenv.local_unit') + @patch('charmhelpers.core.hookenv.relation_id') + @patch('charmhelpers.core.hookenv.relations') + @patch('charmhelpers.core.hookenv.relation_get') + @patch('charmhelpers.core.hookenv.os') + def test_gets_execution_environment(self, os_, relations_get, + relations, relation_id, local_unit, + relation_type, config): + config.return_value = 'some-config' + relation_type.return_value = 'some-type' + local_unit.return_value = 'some-unit' + relation_id.return_value = 'some-id' + relations.return_value = 'all-relations' + relations_get.return_value = 'some-relations' + os_.environ = 'some-environment' + + result = hookenv.execution_environment() + + self.assertEqual(result, { + 'conf': 'some-config', + 'reltype': 'some-type', + 'unit': 'some-unit', + 'relid': 'some-id', + 'rel': 'some-relations', + 'rels': 'all-relations', + 'env': 'some-environment', + }) + + @patch('charmhelpers.core.hookenv.config') + @patch('charmhelpers.core.hookenv.relation_type') + @patch('charmhelpers.core.hookenv.local_unit') + @patch('charmhelpers.core.hookenv.relation_id') + @patch('charmhelpers.core.hookenv.relations') + @patch('charmhelpers.core.hookenv.relation_get') + @patch('charmhelpers.core.hookenv.os') + def test_gets_execution_environment_no_relation( + self, os_, relations_get, relations, relation_id, + local_unit, relation_type, config): + config.return_value = 'some-config' + relation_type.return_value = 'some-type' + local_unit.return_value = 'some-unit' + relation_id.return_value = None + relations.return_value = 'all-relations' + relations_get.return_value = 'some-relations' + os_.environ = 'some-environment' + + result = hookenv.execution_environment() + + self.assertEqual(result, { + 'conf': 'some-config', + 'unit': 'some-unit', + 'rels': 'all-relations', + 'env': 'some-environment', + }) + + @patch('charmhelpers.core.hookenv.remote_service_name') + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.os') + def test_gets_the_relation_id(self, os_, relation_ids, remote_service_name): + os_.environ = { + 'JUJU_RELATION_ID': 'foo', + } + + self.assertEqual(hookenv.relation_id(), 'foo') + + relation_ids.return_value = ['r:1', 'r:2'] + remote_service_name.side_effect = ['other', 'service'] + self.assertEqual(hookenv.relation_id('rel', 'service/0'), 'r:2') + relation_ids.assert_called_once_with('rel') + self.assertEqual(remote_service_name.call_args_list, [ + call('r:1'), + call('r:2'), + ]) + remote_service_name.side_effect = ['other', 'service'] + self.assertEqual(hookenv.relation_id('rel', 'service'), 'r:2') + + @patch('charmhelpers.core.hookenv.os') + def test_relation_id_none_if_no_env(self, os_): + os_.environ = {} + self.assertEqual(hookenv.relation_id(), None) + + @patch('subprocess.check_output') + def test_gets_relation(self, check_output): + data = {"foo": "BAR"} + check_output.return_value = json.dumps(data).encode('UTF-8') + result = hookenv.relation_get() + + self.assertEqual(result['foo'], 'BAR') + check_output.assert_called_with(['relation-get', '--format=json', '-']) + + hookenv.relation_get(unit='foo/0') + check_output.assert_called_with(['relation-get', '--format=json', '-', 'foo/0']) + + hookenv.relation_get(app='foo') + check_output.assert_called_with(['relation-get', '--format=json', '--app', '-', 'foo']) + + self.assertRaises(ValueError, hookenv.relation_get, app='foo', unit='foo/0') + + @patch('charmhelpers.core.hookenv.subprocess') + def test_relation_get_none(self, mock_subprocess): + mock_subprocess.check_output.return_value = b'null' + + result = hookenv.relation_get() + + self.assertIsNone(result) + + @patch('charmhelpers.core.hookenv.subprocess') + def test_relation_get_calledprocesserror(self, mock_subprocess): + """relation-get called outside a relation will errors without id.""" + mock_subprocess.check_output.side_effect = CalledProcessError( + 2, '/foo/bin/relation-get' + 'no relation id specified') + + result = hookenv.relation_get() + + self.assertIsNone(result) + + @patch('charmhelpers.core.hookenv.subprocess') + def test_relation_get_calledprocesserror_other(self, mock_subprocess): + """relation-get can fail for other more serious errors.""" + mock_subprocess.check_output.side_effect = CalledProcessError( + 1, '/foo/bin/relation-get' + 'connection refused') + + self.assertRaises(CalledProcessError, hookenv.relation_get) + + @patch('subprocess.check_output') + def test_gets_relation_with_scope(self, check_output): + check_output.return_value = json.dumps('bar').encode('UTF-8') + + result = hookenv.relation_get(attribute='baz-scope') + + self.assertEqual(result, 'bar') + check_output.assert_called_with(['relation-get', '--format=json', + 'baz-scope']) + + @patch('subprocess.check_output') + def test_gets_missing_relation_with_scope(self, check_output): + check_output.return_value = b"" + + result = hookenv.relation_get(attribute='baz-scope') + + self.assertEqual(result, None) + check_output.assert_called_with(['relation-get', '--format=json', + 'baz-scope']) + + @patch('subprocess.check_output') + def test_gets_relation_with_unit_name(self, check_output): + check_output.return_value = json.dumps('BAR').encode('UTF-8') + + result = hookenv.relation_get(attribute='baz-scope', unit='baz-unit') + + self.assertEqual(result, 'BAR') + check_output.assert_called_with(['relation-get', '--format=json', + 'baz-scope', 'baz-unit']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('subprocess.check_call') + @patch('subprocess.check_output') + def test_relation_set_flushes_local_unit_cache(self, check_output, + check_call, local_unit): + check_output.return_value = json.dumps('BAR').encode('UTF-8') + local_unit.return_value = 'baz_unit' + hookenv.relation_get(attribute='baz_scope', unit='baz_unit') + hookenv.relation_get(attribute='bar_scope') + self.assertTrue(len(hookenv.cache) == 2) + check_output.return_value = "" + hookenv.relation_set(baz_scope='hello') + # relation_set should flush any entries for local_unit + self.assertTrue(len(hookenv.cache) == 1) + + @patch('subprocess.check_output') + def test_gets_relation_with_relation_id(self, check_output): + check_output.return_value = json.dumps('BAR').encode('UTF-8') + + result = hookenv.relation_get(attribute='baz-scope', unit='baz-unit', + rid=123) + + self.assertEqual(result, 'BAR') + check_output.assert_called_with(['relation-get', '--format=json', '-r', + 123, 'baz-scope', 'baz-unit']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_sets_relation_with_kwargs(self, check_call_, check_output, + local_unit): + hookenv.relation_set(foo="bar") + check_call_.assert_called_with(['relation-set', 'foo=bar']) + + hookenv.relation_set(foo="bar", app=True) + check_call_.assert_called_with(['relation-set', '--app', 'foo=bar']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_sets_relation_with_dict(self, check_call_, check_output, + local_unit): + hookenv.relation_set(relation_settings={"foo": "bar"}) + check_call_.assert_called_with(['relation-set', 'foo=bar']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_sets_relation_with_relation_id(self, check_call_, check_output, + local_unit): + hookenv.relation_set(relation_id="foo", bar="baz") + check_call_.assert_called_with(['relation-set', '-r', 'foo', + 'bar=baz']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_sets_relation_with_missing_value(self, check_call_, check_output, + local_unit): + hookenv.relation_set(foo=None) + check_call_.assert_called_with(['relation-set', 'foo=']) + + @patch('charmhelpers.core.hookenv.local_unit', MagicMock()) + @patch('os.remove') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_relation_set_file(self, check_call, check_output, remove): + """If relation-set accepts a --file parameter, it's used. + + Juju 1.23.2 introduced a --file parameter, which means you can + pass the data through a file. Not using --file would make + relation_set break if the relation data is too big. + """ + # check_output(["relation-set", "--help"]) is used to determine + # whether we can pass --file to it. + check_output.return_value = "--file" + hookenv.relation_set(foo="bar") + check_output.assert_called_with( + ["relation-set", "--help"], universal_newlines=True) + # relation-set is called with relation-set --file + # with data as YAML and the temp_file is then removed. + self.assertEqual(1, len(check_call.call_args[0])) + command = check_call.call_args[0][0] + self.assertEqual(3, len(command)) + self.assertEqual("relation-set", command[0]) + self.assertEqual("--file", command[1]) + temp_file = command[2] + with open(temp_file, "r") as f: + self.assertEqual("foo: bar", f.read().strip()) + remove.assert_called_with(temp_file) + + @patch('charmhelpers.core.hookenv.local_unit', MagicMock()) + @patch('os.remove') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_relation_set_file_non_str(self, check_call, check_output, remove): + """If relation-set accepts a --file parameter, it's used. + + Any value that is not a string is converted to a string before encoding + the settings to YAML. + """ + # check_output(["relation-set", "--help"]) is used to determine + # whether we can pass --file to it. + check_output.return_value = "--file" + hookenv.relation_set(foo={"bar": 1}) + check_output.assert_called_with( + ["relation-set", "--help"], universal_newlines=True) + # relation-set is called with relation-set --file + # with data as YAML and the temp_file is then removed. + self.assertEqual(1, len(check_call.call_args[0])) + command = check_call.call_args[0][0] + self.assertEqual(3, len(command)) + self.assertEqual("relation-set", command[0]) + self.assertEqual("--file", command[1]) + temp_file = command[2] + with open(temp_file, "r") as f: + self.assertEqual("foo: '{''bar'': 1}'", f.read().strip()) + remove.assert_called_with(temp_file) + + def test_lists_relation_types(self): + open_ = mock_open() + open_.return_value = io.BytesIO(CHARM_METADATA) + + with patch('charmhelpers.core.hookenv.open', open_, create=True): + with patch.dict('os.environ', {'CHARM_DIR': '/var/empty'}): + reltypes = set(hookenv.relation_types()) + open_.assert_called_once_with('/var/empty/metadata.yaml') + self.assertEqual(set(('testreqs', 'testprov', 'testpeer')), reltypes) + + def test_metadata(self): + open_ = mock_open() + open_.return_value = io.BytesIO(CHARM_METADATA) + + with patch('charmhelpers.core.hookenv.open', open_, create=True): + with patch.dict('os.environ', {'CHARM_DIR': '/var/empty'}): + metadata = hookenv.metadata() + self.assertEqual(metadata, yaml.safe_load(CHARM_METADATA)) + + @patch('charmhelpers.core.hookenv.relation_ids') + @patch('charmhelpers.core.hookenv.metadata') + def test_peer_relation_id(self, metadata, relation_ids): + metadata.return_value = {'peers': {sentinel.peer_relname: {}}} + relation_ids.return_value = [sentinel.pid1, sentinel.pid2] + self.assertEqual(hookenv.peer_relation_id(), sentinel.pid1) + relation_ids.assert_called_once_with(sentinel.peer_relname) + + def test_charm_name(self): + open_ = mock_open() + open_.return_value = io.BytesIO(CHARM_METADATA) + + with patch('charmhelpers.core.hookenv.open', open_, create=True): + with patch.dict('os.environ', {'CHARM_DIR': '/var/empty'}): + charm_name = hookenv.charm_name() + self.assertEqual("testmock", charm_name) + + @patch('subprocess.check_call') + def test_opens_port(self, check_call_): + hookenv.open_port(443, "TCP") + hookenv.open_port(80) + hookenv.open_port(100, "UDP") + hookenv.open_port(0, "ICMP") + calls = [ + call(['open-port', '443/TCP']), + call(['open-port', '80/TCP']), + call(['open-port', '100/UDP']), + call(['open-port', 'ICMP']), + ] + check_call_.assert_has_calls(calls) + + @patch('subprocess.check_call') + def test_closes_port(self, check_call_): + hookenv.close_port(443, "TCP") + hookenv.close_port(80) + hookenv.close_port(100, "UDP") + hookenv.close_port(0, "ICMP") + calls = [ + call(['close-port', '443/TCP']), + call(['close-port', '80/TCP']), + call(['close-port', '100/UDP']), + call(['close-port', 'ICMP']), + ] + check_call_.assert_has_calls(calls) + + @patch('subprocess.check_call') + def test_opens_ports(self, check_call_): + hookenv.open_ports(443, 447, "TCP") + hookenv.open_ports(80, 91) + hookenv.open_ports(100, 200, "UDP") + calls = [ + call(['open-port', '443-447/TCP']), + call(['open-port', '80-91/TCP']), + call(['open-port', '100-200/UDP']), + ] + check_call_.assert_has_calls(calls) + + @patch('subprocess.check_call') + def test_closes_ports(self, check_call_): + hookenv.close_ports(443, 447, "TCP") + hookenv.close_ports(80, 91) + hookenv.close_ports(100, 200, "UDP") + calls = [ + call(['close-port', '443-447/TCP']), + call(['close-port', '80-91/TCP']), + call(['close-port', '100-200/UDP']), + ] + check_call_.assert_has_calls(calls) + + @patch('subprocess.check_output') + def test_gets_opened_ports(self, check_output_): + prts = ['8080/tcp', '8081-8083/tcp'] + check_output_.return_value = json.dumps(prts).encode('UTF-8') + self.assertEqual(hookenv.opened_ports(), prts) + check_output_.assert_called_with(['opened-ports', '--format=json']) + + @patch('subprocess.check_output') + def test_gets_unit_attribute(self, check_output_): + check_output_.return_value = json.dumps('bar').encode('UTF-8') + self.assertEqual(hookenv.unit_get('foo'), 'bar') + check_output_.assert_called_with(['unit-get', '--format=json', 'foo']) + + @patch('subprocess.check_output') + def test_gets_missing_unit_attribute(self, check_output_): + check_output_.return_value = b"" + self.assertEqual(hookenv.unit_get('foo'), None) + check_output_.assert_called_with(['unit-get', '--format=json', 'foo']) + + def test_cached_decorator(self): + class Unserializable(object): + def __str__(self): + return 'unserializable' + + unserializable = Unserializable() + calls = [] + values = { + 'hello': 'world', + 'foo': 'bar', + 'baz': None, + unserializable: 'qux', + } + + @hookenv.cached + def cache_function(attribute): + calls.append(attribute) + return values[attribute] + + self.assertEquals(cache_function('hello'), 'world') + self.assertEquals(cache_function('hello'), 'world') + self.assertEquals(cache_function('foo'), 'bar') + self.assertEquals(cache_function('baz'), None) + self.assertEquals(cache_function('baz'), None) + self.assertEquals(cache_function(unserializable), 'qux') + self.assertEquals(calls, ['hello', 'foo', 'baz', unserializable]) + + def test_gets_charm_dir(self): + with patch.dict('os.environ', {}): + self.assertEqual(hookenv.charm_dir(), None) + with patch.dict('os.environ', {'CHARM_DIR': '/var/empty'}): + self.assertEqual(hookenv.charm_dir(), '/var/empty') + with patch.dict('os.environ', {'JUJU_CHARM_DIR': '/var/another'}): + self.assertEqual(hookenv.charm_dir(), '/var/another') + + @patch('subprocess.check_output') + def test_resource_get_unsupported(self, check_output_): + check_output_.side_effect = OSError(2, 'resource-get') + self.assertRaises(NotImplementedError, hookenv.resource_get, 'foo') + + @patch('subprocess.check_output') + def test_resource_get(self, check_output_): + check_output_.return_value = b'/tmp/file' + self.assertEqual(hookenv.resource_get('file'), '/tmp/file') + check_output_.side_effect = CalledProcessError( + 1, '/foo/bin/resource-get', + 'error: could not download resource: resource#foo/file not found') + self.assertFalse(hookenv.resource_get('no-file')) + self.assertFalse(hookenv.resource_get(None)) + + @patch('subprocess.check_output') + def test_goal_state_unsupported(self, check_output_): + check_output_.side_effect = OSError(2, 'goal-state') + self.assertRaises(NotImplementedError, hookenv.goal_state) + + @patch('subprocess.check_output') + def test_goal_state(self, check_output_): + expect = { + 'units': {}, + 'relations': {}, + } + check_output_.return_value = json.dumps(expect).encode('UTF-8') + result = hookenv.goal_state() + + self.assertEqual(result, expect) + check_output_.assert_called_with(['goal-state', '--format=json']) + + @patch('subprocess.check_output') + def test_is_leader_unsupported(self, check_output_): + check_output_.side_effect = OSError(2, 'is-leader') + self.assertRaises(NotImplementedError, hookenv.is_leader) + + @patch('subprocess.check_output') + def test_is_leader(self, check_output_): + check_output_.return_value = b'false' + self.assertFalse(hookenv.is_leader()) + check_output_.return_value = b'true' + self.assertTrue(hookenv.is_leader()) + + @patch('subprocess.check_call') + def test_payload_register(self, check_call_): + hookenv.payload_register('monitoring', 'docker', 'abc123') + check_call_.assert_called_with(['payload-register', 'monitoring', + 'docker', 'abc123']) + + @patch('subprocess.check_call') + def test_payload_unregister(self, check_call_): + hookenv.payload_unregister('monitoring', 'abc123') + check_call_.assert_called_with(['payload-unregister', 'monitoring', + 'abc123']) + + @patch('subprocess.check_call') + def test_payload_status_set(self, check_call_): + hookenv.payload_status_set('monitoring', 'abc123', 'Running') + check_call_.assert_called_with(['payload-status-set', 'monitoring', + 'abc123', 'Running']) + + @patch('subprocess.check_call') + def test_application_version_set(self, check_call_): + hookenv.application_version_set('v1.2.3') + check_call_.assert_called_with(['application-version-set', 'v1.2.3']) + + @patch.object(os, 'getenv') + @patch.object(hookenv, 'log') + def test_env_proxy_settings_juju_charm_all_selected(self, faux_log, + get_env): + expected_settings = { + 'HTTP_PROXY': 'http://squid.internal:3128', + 'http_proxy': 'http://squid.internal:3128', + 'HTTPS_PROXY': 'https://squid.internals:3128', + 'https_proxy': 'https://squid.internals:3128', + 'NO_PROXY': '192.0.2.0/24,198.51.100.0/24,.bar.com', + 'no_proxy': '192.0.2.0/24,198.51.100.0/24,.bar.com', + 'FTP_PROXY': 'ftp://ftp.internal:21', + 'ftp_proxy': 'ftp://ftp.internal:21', + } + + def get_env_side_effect(var): + return { + 'HTTP_PROXY': None, + 'HTTPS_PROXY': None, + 'NO_PROXY': None, + 'FTP_PROXY': None, + 'JUJU_CHARM_HTTP_PROXY': 'http://squid.internal:3128', + 'JUJU_CHARM_HTTPS_PROXY': 'https://squid.internals:3128', + 'JUJU_CHARM_FTP_PROXY': 'ftp://ftp.internal:21', + 'JUJU_CHARM_NO_PROXY': '192.0.2.0/24,198.51.100.0/24,.bar.com' + }[var] + get_env.side_effect = get_env_side_effect + + proxy_settings = hookenv.env_proxy_settings() + get_env.assert_has_calls([call("HTTP_PROXY"), + call("HTTPS_PROXY"), + call("NO_PROXY"), + call("FTP_PROXY"), + call("JUJU_CHARM_HTTP_PROXY"), + call("JUJU_CHARM_HTTPS_PROXY"), + call("JUJU_CHARM_FTP_PROXY"), + call("JUJU_CHARM_NO_PROXY")], + any_order=True) + self.assertEqual(expected_settings, proxy_settings) + # Verify that we logged a warning about the cidr in NO_PROXY. + faux_log.assert_called_with(hookenv.RANGE_WARNING, + level=hookenv.WARNING) + + @patch.object(os, 'getenv') + def test_env_proxy_settings_legacy_https(self, get_env): + expected_settings = { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'https_proxy': 'http://squid.internal:3128', + } + + def get_env_side_effect(var): + return { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'JUJU_CHARM_HTTPS_PROXY': None, + }[var] + get_env.side_effect = get_env_side_effect + + proxy_settings = hookenv.env_proxy_settings(['https']) + get_env.assert_has_calls([call("HTTPS_PROXY"), + call("JUJU_CHARM_HTTPS_PROXY")], + any_order=True) + self.assertEqual(expected_settings, proxy_settings) + + @patch.object(os, 'getenv') + def test_env_proxy_settings_juju_charm_https(self, get_env): + expected_settings = { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'https_proxy': 'http://squid.internal:3128', + } + + def get_env_side_effect(var): + return { + 'HTTPS_PROXY': None, + 'JUJU_CHARM_HTTPS_PROXY': 'http://squid.internal:3128', + }[var] + get_env.side_effect = get_env_side_effect + + proxy_settings = hookenv.env_proxy_settings(['https']) + get_env.assert_has_calls([call("HTTPS_PROXY"), + call("JUJU_CHARM_HTTPS_PROXY")], + any_order=True) + self.assertEqual(expected_settings, proxy_settings) + + @patch.object(os, 'getenv') + def test_env_proxy_settings_legacy_http(self, get_env): + expected_settings = { + 'HTTP_PROXY': 'http://squid.internal:3128', + 'http_proxy': 'http://squid.internal:3128', + } + + def get_env_side_effect(var): + return { + 'HTTP_PROXY': 'http://squid.internal:3128', + 'JUJU_CHARM_HTTP_PROXY': None, + }[var] + get_env.side_effect = get_env_side_effect + + proxy_settings = hookenv.env_proxy_settings(['http']) + get_env.assert_has_calls([call("HTTP_PROXY"), + call("JUJU_CHARM_HTTP_PROXY")], + any_order=True) + self.assertEqual(expected_settings, proxy_settings) + + @patch.object(os, 'getenv') + def test_env_proxy_settings_juju_charm_http(self, get_env): + expected_settings = { + 'HTTP_PROXY': 'http://squid.internal:3128', + 'http_proxy': 'http://squid.internal:3128', + } + + def get_env_side_effect(var): + return { + 'HTTP_PROXY': None, + 'JUJU_CHARM_HTTP_PROXY': 'http://squid.internal:3128', + }[var] + get_env.side_effect = get_env_side_effect + + proxy_settings = hookenv.env_proxy_settings(['http']) + get_env.assert_has_calls([call("HTTP_PROXY"), + call("JUJU_CHARM_HTTP_PROXY")], + any_order=True) + self.assertEqual(expected_settings, proxy_settings) + + @patch.object(hookenv, 'metadata') + def test_is_subordinate(self, mock_metadata): + mock_metadata.return_value = {} + self.assertFalse(hookenv.is_subordinate()) + mock_metadata.return_value = {'subordinate': False} + self.assertFalse(hookenv.is_subordinate()) + mock_metadata.return_value = {'subordinate': True} + self.assertTrue(hookenv.is_subordinate()) + + +class HooksTest(TestCase): + def setUp(self): + super(HooksTest, self).setUp() + + _clean_globals() + self.addCleanup(_clean_globals) + + charm_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(charm_dir)) + patcher = patch.object(hookenv, 'charm_dir', lambda: charm_dir) + self.addCleanup(patcher.stop) + patcher.start() + + config = hookenv.Config({}) + + def _mock_config(scope=None): + return config if scope is None else config[scope] + patcher = patch.object(hookenv, 'config', _mock_config) + self.addCleanup(patcher.stop) + patcher.start() + + def test_config_saved_after_execute(self): + config = hookenv.config() + config.implicit_save = True + + foo = MagicMock() + hooks = hookenv.Hooks() + hooks.register('foo', foo) + hooks.execute(['foo', 'some', 'other', 'args']) + self.assertTrue(os.path.exists(config.path)) + + def test_config_not_saved_after_execute(self): + config = hookenv.config() + config.implicit_save = False + + foo = MagicMock() + hooks = hookenv.Hooks() + hooks.register('foo', foo) + hooks.execute(['foo', 'some', 'other', 'args']) + self.assertFalse(os.path.exists(config.path)) + + def test_config_save_disabled(self): + config = hookenv.config() + config.implicit_save = True + + foo = MagicMock() + hooks = hookenv.Hooks(config_save=False) + hooks.register('foo', foo) + hooks.execute(['foo', 'some', 'other', 'args']) + self.assertFalse(os.path.exists(config.path)) + + def test_runs_a_registered_function(self): + foo = MagicMock() + hooks = hookenv.Hooks() + hooks.register('foo', foo) + + hooks.execute(['foo', 'some', 'other', 'args']) + + foo.assert_called_with() + + def test_cannot_run_unregistered_function(self): + foo = MagicMock() + hooks = hookenv.Hooks() + hooks.register('foo', foo) + + self.assertRaises(hookenv.UnregisteredHookError, hooks.execute, + ['bar']) + + def test_can_run_a_decorated_function_as_one_or_more_hooks(self): + execs = [] + hooks = hookenv.Hooks() + + @hooks.hook('bar', 'baz') + def func(): + execs.append(True) + + hooks.execute(['bar']) + hooks.execute(['baz']) + self.assertRaises(hookenv.UnregisteredHookError, hooks.execute, + ['brew']) + self.assertEqual(execs, [True, True]) + + def test_can_run_a_decorated_function_as_itself(self): + execs = [] + hooks = hookenv.Hooks() + + @hooks.hook() + def func(): + execs.append(True) + + hooks.execute(['func']) + self.assertRaises(hookenv.UnregisteredHookError, hooks.execute, + ['brew']) + self.assertEqual(execs, [True]) + + def test_magic_underscores(self): + # Juju hook names use hyphens as separators. Python functions use + # underscores. If explicit names have not been provided, hooks + # are registered with both the function name and the function + # name with underscores replaced with hyphens for convenience. + execs = [] + hooks = hookenv.Hooks() + + @hooks.hook() + def call_me_maybe(): + execs.append(True) + + hooks.execute(['call-me-maybe']) + hooks.execute(['call_me_maybe']) + self.assertEqual(execs, [True, True]) + + @patch('charmhelpers.core.hookenv.local_unit') + def test_gets_service_name(self, _unit): + _unit.return_value = 'mysql/3' + self.assertEqual(hookenv.service_name(), 'mysql') + + @patch('charmhelpers.core.hookenv.related_units') + @patch('charmhelpers.core.hookenv.remote_unit') + def test_gets_remote_service_name(self, remote_unit, related_units): + remote_unit.return_value = 'mysql/3' + related_units.return_value = ['pgsql/0', 'pgsql/1'] + self.assertEqual(hookenv.remote_service_name(), 'mysql') + self.assertEqual(hookenv.remote_service_name('pgsql:1'), 'pgsql') + + def test_gets_hook_name(self): + with patch.dict(os.environ, JUJU_HOOK_NAME='hook'): + self.assertEqual(hookenv.hook_name(), 'hook') + with patch('sys.argv', ['other-hook']): + self.assertEqual(hookenv.hook_name(), 'other-hook') + + @patch('subprocess.check_output') + def test_action_get_with_key(self, check_output): + action_data = 'bar' + check_output.return_value = json.dumps(action_data).encode('UTF-8') + + result = hookenv.action_get(key='foo') + + self.assertEqual(result, 'bar') + check_output.assert_called_with(['action-get', 'foo', '--format=json']) + + @patch('subprocess.check_output') + def test_action_get_without_key(self, check_output): + check_output.return_value = json.dumps(dict(foo='bar')).encode('UTF-8') + + result = hookenv.action_get() + + self.assertEqual(result['foo'], 'bar') + check_output.assert_called_with(['action-get', '--format=json']) + + @patch('subprocess.check_call') + def test_action_set(self, check_call): + values = {'foo': 'bar', 'fooz': 'barz'} + hookenv.action_set(values) + # The order of the key/value pairs can change, so sort them before test + called_args = check_call.call_args_list[0][0][0] + called_args.pop(0) + called_args.sort() + self.assertEqual(called_args, ['foo=bar', 'fooz=barz']) + + @patch('subprocess.check_call') + def test_action_fail(self, check_call): + message = "Ooops, the action failed" + hookenv.action_fail(message) + check_call.assert_called_with(['action-fail', message]) + + @patch('charmhelpers.core.hookenv.cmd_exists') + @patch('subprocess.check_output') + def test_function_get_with_key(self, check_output, cmd_exists): + function_data = 'bar' + check_output.return_value = json.dumps(function_data).encode('UTF-8') + cmd_exists.return_value = True + + result = hookenv.function_get(key='foo') + + self.assertEqual(result, 'bar') + check_output.assert_called_with(['function-get', 'foo', '--format=json']) + + @patch('charmhelpers.core.hookenv.cmd_exists') + @patch('subprocess.check_output') + def test_function_get_without_key(self, check_output, cmd_exists): + check_output.return_value = json.dumps(dict(foo='bar')).encode('UTF-8') + cmd_exists.return_value = True + + result = hookenv.function_get() + + self.assertEqual(result['foo'], 'bar') + check_output.assert_called_with(['function-get', '--format=json']) + + @patch('subprocess.check_call') + def test_function_set(self, check_call): + values = {'foo': 'bar', 'fooz': 'barz'} + hookenv.function_set(values) + # The order of the key/value pairs can change, so sort them before test + called_args = check_call.call_args_list[0][0][0] + called_args.pop(0) + called_args.sort() + self.assertEqual(called_args, ['foo=bar', 'fooz=barz']) + + @patch('charmhelpers.core.hookenv.cmd_exists') + @patch('subprocess.check_call') + def test_function_fail(self, check_call, cmd_exists): + cmd_exists.return_value = True + + message = "Ooops, the function failed" + hookenv.function_fail(message) + check_call.assert_called_with(['function-fail', message]) + + def test_status_set_invalid_state(self): + self.assertRaises(ValueError, hookenv.status_set, 'random', 'message') + + def test_status_set_invalid_state_enum(self): + + class RandomEnum(Enum): + FOO = 1 + self.assertRaises( + ValueError, + hookenv.status_set, + RandomEnum.FOO, + 'message') + + @patch('subprocess.call') + def test_status(self, call): + call.return_value = 0 + hookenv.status_set('active', 'Everything is Awesome!') + call.assert_called_with(['status-set', 'active', 'Everything is Awesome!']) + + @patch('subprocess.call') + def test_status_enum(self, call): + call.return_value = 0 + hookenv.status_set( + hookenv.WORKLOAD_STATES.ACTIVE, + 'Everything is Awesome!') + call.assert_called_with(['status-set', 'active', 'Everything is Awesome!']) + + @patch('subprocess.call') + def test_status_app(self, call): + call.return_value = 0 + hookenv.status_set( + 'active', + 'Everything is Awesome!', + application=True) + call.assert_called_with([ + 'status-set', + '--application', + 'active', + 'Everything is Awesome!']) + + @patch('subprocess.call') + @patch.object(hookenv, 'log') + def test_status_enoent(self, log, call): + call.side_effect = OSError(2, 'fail') + hookenv.status_set('active', 'Everything is Awesome!') + log.assert_called_with('status-set failed: active Everything is Awesome!', level='INFO') + + @patch('subprocess.call') + @patch.object(hookenv, 'log') + def test_status_statuscmd_fail(self, log, call): + call.side_effect = OSError(3, 'fail') + self.assertRaises(OSError, hookenv.status_set, 'active', 'msg') + call.assert_called_with(['status-set', 'active', 'msg']) + + @patch('subprocess.check_output') + def test_status_get(self, check_output): + check_output.return_value = json.dumps( + {"message": "foo", + "status": "active", + "status-data": {}}).encode("UTF-8") + result = hookenv.status_get() + self.assertEqual(result, ('active', 'foo')) + check_output.assert_called_with( + ['status-get', "--format=json", "--include-data"]) + + @patch('subprocess.check_output') + def test_status_get_nostatus(self, check_output): + check_output.side_effect = OSError(2, 'fail') + result = hookenv.status_get() + self.assertEqual(result, ('unknown', '')) + + @patch('subprocess.check_output') + def test_status_get_status_error(self, check_output): + check_output.side_effect = OSError(3, 'fail') + self.assertRaises(OSError, hookenv.status_get) + + @patch('subprocess.check_output') + @patch('glob.glob') + def test_juju_version(self, glob, check_output): + glob.return_value = [sentinel.jujud] + check_output.return_value = '1.23.3.1-trusty-amd64\n' + self.assertEqual(hookenv.juju_version(), '1.23.3.1-trusty-amd64') + # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1 + glob.assert_called_once_with('/var/lib/juju/tools/machine-*/jujud') + check_output.assert_called_once_with([sentinel.jujud, 'version'], + universal_newlines=True) + + @patch('charmhelpers.core.hookenv.juju_version') + def test_has_juju_version(self, juju_version): + juju_version.return_value = '1.23.1.2.3.4.5-with-a-cherry-on-top.amd64' + self.assertTrue(hookenv.has_juju_version('1.23')) + self.assertTrue(hookenv.has_juju_version('1.23.1')) + self.assertTrue(hookenv.has_juju_version('1.23.1.1')) + self.assertFalse(hookenv.has_juju_version('1.23.2.1')) + self.assertFalse(hookenv.has_juju_version('1.24')) + + juju_version.return_value = '1.24-beta5.1-trusty-amd64' + self.assertTrue(hookenv.has_juju_version('1.23')) + self.assertTrue(hookenv.has_juju_version('1.24')) # Better if this was false! + self.assertTrue(hookenv.has_juju_version('1.24-beta5')) + self.assertTrue(hookenv.has_juju_version('1.24-beta5.1')) + self.assertFalse(hookenv.has_juju_version('1.25')) + self.assertTrue(hookenv.has_juju_version('1.18-backport6')) + + @patch.object(hookenv, 'relation_to_role_and_interface') + def test_relation_to_interface(self, rtri): + rtri.return_value = (None, 'foo') + self.assertEqual(hookenv.relation_to_interface('rel'), 'foo') + + @patch.object(hookenv, 'metadata') + def test_relation_to_role_and_interface(self, metadata): + metadata.return_value = { + 'provides': { + 'pro-rel': { + 'interface': 'pro-int', + }, + 'pro-rel2': { + 'interface': 'pro-int', + }, + }, + 'requires': { + 'req-rel': { + 'interface': 'req-int', + }, + }, + 'peers': { + 'pee-rel': { + 'interface': 'pee-int', + }, + }, + } + rtri = hookenv.relation_to_role_and_interface + self.assertEqual(rtri('pro-rel'), ('provides', 'pro-int')) + self.assertEqual(rtri('req-rel'), ('requires', 'req-int')) + self.assertEqual(rtri('pee-rel'), ('peers', 'pee-int')) + + @patch.object(hookenv, 'metadata') + def test_role_and_interface_to_relations(self, metadata): + metadata.return_value = { + 'provides': { + 'pro-rel': { + 'interface': 'pro-int', + }, + 'pro-rel2': { + 'interface': 'pro-int', + }, + }, + 'requires': { + 'req-rel': { + 'interface': 'int', + }, + }, + 'peers': { + 'pee-rel': { + 'interface': 'int', + }, + }, + } + ritr = hookenv.role_and_interface_to_relations + assertItemsEqual = getattr(self, 'assertItemsEqual', getattr(self, 'assertCountEqual', None)) + assertItemsEqual(ritr('provides', 'pro-int'), ['pro-rel', 'pro-rel2']) + assertItemsEqual(ritr('requires', 'int'), ['req-rel']) + assertItemsEqual(ritr('peers', 'int'), ['pee-rel']) + + @patch.object(hookenv, 'metadata') + def test_interface_to_relations(self, metadata): + metadata.return_value = { + 'provides': { + 'pro-rel': { + 'interface': 'pro-int', + }, + 'pro-rel2': { + 'interface': 'pro-int', + }, + }, + 'requires': { + 'req-rel': { + 'interface': 'req-int', + }, + }, + 'peers': { + 'pee-rel': { + 'interface': 'pee-int', + }, + }, + } + itr = hookenv.interface_to_relations + assertItemsEqual = getattr(self, 'assertItemsEqual', getattr(self, 'assertCountEqual', None)) + assertItemsEqual(itr('pro-int'), ['pro-rel', 'pro-rel2']) + assertItemsEqual(itr('req-int'), ['req-rel']) + assertItemsEqual(itr('pee-int'), ['pee-rel']) + + def test_action_name(self): + with patch.dict('os.environ', JUJU_ACTION_NAME='action-jack'): + self.assertEqual(hookenv.action_name(), 'action-jack') + + def test_action_uuid(self): + with patch.dict('os.environ', JUJU_ACTION_UUID='action-jack'): + self.assertEqual(hookenv.action_uuid(), 'action-jack') + + def test_action_tag(self): + with patch.dict('os.environ', JUJU_ACTION_TAG='action-jack'): + self.assertEqual(hookenv.action_tag(), 'action-jack') + + @patch('subprocess.check_output') + def test_storage_list(self, check_output): + ids = ['data/0', 'data/1', 'data/2'] + check_output.return_value = json.dumps(ids).encode('UTF-8') + + storage_name = 'arbitrary' + result = hookenv.storage_list(storage_name) + + self.assertEqual(result, ids) + check_output.assert_called_with(['storage-list', '--format=json', + storage_name]) + + @patch('subprocess.check_output') + def test_storage_list_notexist(self, check_output): + import errno + e = OSError() + e.errno = errno.ENOENT + check_output.side_effect = e + + result = hookenv.storage_list() + + self.assertEqual(result, []) + check_output.assert_called_with(['storage-list', '--format=json']) + + @patch('subprocess.check_output') + def test_storage_get_notexist(self, check_output): + # storage_get does not catch ENOENT, because there's no reason why you + # should be calling storage_get except from a storage hook, or with + # the result of storage_list (which will return [] as shown above). + + import errno + e = OSError() + e.errno = errno.ENOENT + check_output.side_effect = e + self.assertRaises(OSError, hookenv.storage_get) + + @patch('subprocess.check_output') + def test_storage_get(self, check_output): + expect = { + 'location': '/dev/sda', + 'kind': 'block', + } + check_output.return_value = json.dumps(expect).encode('UTF-8') + + result = hookenv.storage_get() + + self.assertEqual(result, expect) + check_output.assert_called_with(['storage-get', '--format=json']) + + @patch('subprocess.check_output') + def test_storage_get_attr(self, check_output): + expect = '/dev/sda' + check_output.return_value = json.dumps(expect).encode('UTF-8') + + attribute = 'location' + result = hookenv.storage_get(attribute) + + self.assertEqual(result, expect) + check_output.assert_called_with(['storage-get', '--format=json', + attribute]) + + @patch('subprocess.check_output') + def test_storage_get_with_id(self, check_output): + expect = { + 'location': '/dev/sda', + 'kind': 'block', + } + check_output.return_value = json.dumps(expect).encode('UTF-8') + + storage_id = 'data/0' + result = hookenv.storage_get(storage_id=storage_id) + + self.assertEqual(result, expect) + check_output.assert_called_with(['storage-get', '--format=json', + '-s', storage_id]) + + @patch('subprocess.check_output') + def test_network_get_primary(self, check_output): + """Ensure that network-get is called correctly and output is returned""" + check_output.return_value = b'192.168.22.1' + ip = hookenv.network_get_primary_address('mybinding') + check_output.assert_called_with( + ['network-get', '--primary-address', 'mybinding'], stderr=-2) + self.assertEqual(ip, '192.168.22.1') + + @patch('subprocess.check_output') + def test_network_get_primary_unsupported(self, check_output): + """Ensure that NotImplementedError is thrown when run on Juju < 2.0""" + check_output.side_effect = OSError(2, 'network-get') + self.assertRaises(NotImplementedError, hookenv.network_get_primary_address, + 'mybinding') + + @patch('subprocess.check_output') + def test_network_get_primary_no_binding_found(self, check_output): + """Ensure that NotImplementedError when no binding is found""" + check_output.side_effect = CalledProcessError( + 1, 'network-get', + output='no network config found for binding'.encode('UTF-8')) + self.assertRaises(hookenv.NoNetworkBinding, + hookenv.network_get_primary_address, + 'doesnotexist') + check_output.assert_called_with( + ['network-get', '--primary-address', 'doesnotexist'], stderr=-2) + + @patch('subprocess.check_output') + def test_network_get_primary_other_exception(self, check_output): + """Ensure that CalledProcessError still thrown when not + a missing binding""" + check_output.side_effect = CalledProcessError( + 1, 'network-get', + output='any other message'.encode('UTF-8')) + self.assertRaises(CalledProcessError, + hookenv.network_get_primary_address, + 'mybinding') + + @patch('charmhelpers.core.hookenv.juju_version') + @patch('subprocess.check_output') + def test_network_get(self, check_output, juju_version): + """Ensure that network-get is called correctly""" + juju_version.return_value = '2.2.0' + check_output.return_value = b'result' + hookenv.network_get('endpoint') + check_output.assert_called_with( + ['network-get', 'endpoint', '--format', 'yaml'], stderr=-2) + + @patch('charmhelpers.core.hookenv.juju_version') + @patch('subprocess.check_output') + def test_network_get_primary_required(self, check_output, juju_version): + """Ensure that NotImplementedError is thrown with Juju < 2.2.0""" + check_output.return_value = b'result' + + juju_version.return_value = '2.1.4' + self.assertRaises(NotImplementedError, hookenv.network_get, 'binding') + juju_version.return_value = '2.2.0' + self.assertEquals(hookenv.network_get('endpoint'), 'result') + + @patch('charmhelpers.core.hookenv.juju_version') + @patch('subprocess.check_output') + def test_network_get_relation_bound(self, check_output, juju_version): + """Ensure that network-get supports relation context, requires Juju 2.3""" + juju_version.return_value = '2.3.0' + check_output.return_value = b'result' + hookenv.network_get('endpoint', 'db') + check_output.assert_called_with( + ['network-get', 'endpoint', '--format', 'yaml', '-r', 'db'], + stderr=-2) + juju_version.return_value = '2.2.8' + self.assertRaises(NotImplementedError, hookenv.network_get, 'endpoint', 'db') + + @patch('charmhelpers.core.hookenv.juju_version') + @patch('subprocess.check_output') + def test_network_get_parses_yaml(self, check_output, juju_version): + """network-get returns loaded YAML output.""" + juju_version.return_value = '2.3.0' + check_output.return_value = b""" +bind-addresses: +- macaddress: "" + interfacename: "" + addresses: + - address: 10.136.107.33 + cidr: "" +ingress-addresses: +- 10.136.107.33 + """ + ip = hookenv.network_get('mybinding') + self.assertEqual(len(ip['bind-addresses']), 1) + self.assertEqual(ip['ingress-addresses'], ['10.136.107.33']) + + @patch('subprocess.check_call') + def test_add_metric(self, check_call_): + hookenv.add_metric(flips='1.5', flops='2.1') + hookenv.add_metric('juju-units=6') + hookenv.add_metric('foo-bar=3.333', 'baz-quux=8', users='2') + calls = [ + call(['add-metric', 'flips=1.5', 'flops=2.1']), + call(['add-metric', 'juju-units=6']), + call(['add-metric', 'baz-quux=8', 'foo-bar=3.333', 'users=2']), + ] + check_call_.assert_has_calls(calls) + + @patch('subprocess.check_call') + @patch.object(hookenv, 'log') + def test_add_metric_enoent(self, log, _check_call): + _check_call.side_effect = OSError(2, 'fail') + hookenv.add_metric(flips='1') + log.assert_called_with('add-metric failed: flips=1', level='INFO') + + @patch('charmhelpers.core.hookenv.os') + def test_meter_status(self, os_): + os_.environ = { + 'JUJU_METER_STATUS': 'GREEN', + 'JUJU_METER_INFO': 'all good', + } + self.assertEqual(hookenv.meter_status(), 'GREEN') + self.assertEqual(hookenv.meter_info(), 'all good') + + @patch.object(hookenv, 'related_units') + @patch.object(hookenv, 'relation_ids') + def test_iter_units_for_relation_name(self, relation_ids, related_units): + relation_ids.return_value = ['rel:1'] + related_units.return_value = ['unit/0', 'unit/1', 'unit/2'] + expected = [('rel:1', 'unit/0'), + ('rel:1', 'unit/1'), + ('rel:1', 'unit/2')] + related_units_data = [ + (u.rid, u.unit) + for u in hookenv.iter_units_for_relation_name('rel')] + self.assertEqual(expected, related_units_data) + + @patch.object(hookenv, 'relation_get') + def test_ingress_address(self, relation_get): + """Ensure ingress_address returns the ingress-address when available + and returns the private-address when not. + """ + _with_ingress = {'egress-subnets': '10.5.0.23/32', + 'ingress-address': '10.5.0.23', + 'private-address': '172.16.5.10'} + + _without_ingress = {'private-address': '172.16.5.10'} + + # Return the ingress-address + relation_get.return_value = _with_ingress + self.assertEqual(hookenv.ingress_address(rid='test:1', unit='unit/1'), + '10.5.0.23') + relation_get.assert_called_with(rid='test:1', unit='unit/1') + # Return the private-address + relation_get.return_value = _without_ingress + self.assertEqual(hookenv.ingress_address(rid='test:1'), + '172.16.5.10') + + @patch.object(hookenv, 'relation_get') + def test_egress_subnets(self, relation_get): + """Ensure egress_subnets returns the decoded egress-subnets when available + and falls back correctly when not. + """ + d = {'egress-subnets': '10.5.0.23/32,2001::F00F/64', + 'ingress-address': '10.5.0.23', + 'private-address': '2001::D0:F00D'} + + # Return the egress-subnets + relation_get.return_value = d + self.assertEqual(hookenv.egress_subnets(rid='test:1', unit='unit/1'), + ['10.5.0.23/32', '2001::F00F/64']) + relation_get.assert_called_with(rid='test:1', unit='unit/1') + + # Return the ingress-address + del d['egress-subnets'] + self.assertEqual(hookenv.egress_subnets(), ['10.5.0.23/32']) + + # Return the private-address + del d['ingress-address'] + self.assertEqual(hookenv.egress_subnets(), ['2001::D0:F00D/128']) + + @patch('charmhelpers.core.hookenv.local_unit') + @patch('charmhelpers.core.hookenv.goal_state') + @patch('charmhelpers.core.hookenv.has_juju_version') + def test_unit_doomed(self, has_juju_version, goal_state, local_unit): + # We need to test for a minimum patch level, or we risk + # data loss by returning bogus results with Juju 2.4.0 + has_juju_version.return_value = False + self.assertRaises(NotImplementedError, hookenv.unit_doomed) + has_juju_version.assertCalledOnceWith("2.4.1") + has_juju_version.return_value = True + + goal_state.return_value = json.loads(''' + { + "units": { + "postgresql/0": { + "status": "dying", + "since": "2018-07-30 10:01:06Z" + }, + "postgresql/1": { + "status": "active", + "since": "2018-07-30 10:22:39Z" + } + }, + "relations": {} + } + ''') + self.assertTrue(hookenv.unit_doomed('postgresql/0')) # unit removed, status "dying" + self.assertFalse(hookenv.unit_doomed('postgresql/1')) # unit exists, status "active", maybe other states + self.assertTrue(hookenv.unit_doomed('postgresql/2')) # unit does not exist + + local_unit.return_value = 'postgresql/0' + self.assertTrue(hookenv.unit_doomed()) + + def test_contains_addr_range(self): + # Contains cidr + self.assertTrue(hookenv._contains_range("192.168.1/20")) + self.assertTrue(hookenv._contains_range("192.168.0/24")) + self.assertTrue( + hookenv._contains_range("10.40.50.1,192.168.1/20,10.56.78.9")) + self.assertTrue(hookenv._contains_range("192.168.22/24")) + self.assertTrue(hookenv._contains_range("2001:db8::/32")) + self.assertTrue(hookenv._contains_range("*.foo.com")) + self.assertTrue(hookenv._contains_range(".foo.com")) + self.assertTrue( + hookenv._contains_range("192.168.1.20,.foo.com")) + self.assertTrue( + hookenv._contains_range("192.168.1.20, .foo.com")) + self.assertTrue( + hookenv._contains_range("192.168.1.20,*.foo.com")) + + # Doesn't contain cidr + self.assertFalse(hookenv._contains_range("192.168.1")) + self.assertFalse(hookenv._contains_range("192.168.145")) + self.assertFalse(hookenv._contains_range("192.16.14")) diff --git a/nrpe/mod/charmhelpers/tests/core/test_host.py b/nrpe/mod/charmhelpers/tests/core/test_host.py new file mode 100644 index 0000000..c917b9b --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_host.py @@ -0,0 +1,2063 @@ +import os.path +from collections import OrderedDict +import subprocess +from tempfile import mkdtemp +from shutil import rmtree +from textwrap import dedent + +import imp + +from charmhelpers import osplatform +from mock import patch, call, mock_open +from testtools import TestCase +from tests.helpers import patch_open +from tests.helpers import mock_open as mocked_open +import six + +from charmhelpers.core import host +from charmhelpers.fetch import ubuntu_apt_pkg + + +MOUNT_LINES = (""" +rootfs / rootfs rw 0 0 +sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +udev /dev devtmpfs rw,relatime,size=8196788k,nr_inodes=2049197,mode=755 0 0 +devpts /dev/pts devpts """ + """rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 +""").strip().split('\n') + +LSB_RELEASE = '''DISTRIB_ID=Ubuntu +DISTRIB_RELEASE=13.10 +DISTRIB_CODENAME=saucy +DISTRIB_DESCRIPTION="Ubuntu Saucy Salamander (development branch)" +''' + +OS_RELEASE = '''NAME="CentOS Linux" +ANSI_COLOR="0;31" +ID_LIKE="rhel fedora" +VERSION_ID="7" +BUG_REPORT_URL="https://bugs.centos.org/" +CENTOS_MANTISBT_PROJECT="CentOS-7" +PRETTY_NAME="CentOS Linux 7 (Core)" +VERSION="7 (Core)" +REDHAT_SUPPORT_PRODUCT_VERSION="7" +CENTOS_MANTISBT_PROJECT_VERSION="7" +REDHAT_SUPPORT_PRODUCT="centos" +HOME_URL="https://www.centos.org/" +CPE_NAME="cpe:/o:centos:centos:7" +ID="centos" +''' + +IP_LINE_ETH0 = b""" +2: eth0: mtu 1500 qdisc mq master bond0 state UP qlen 1000 + link/ether e4:11:5b:ab:a7:3c brd ff:ff:ff:ff:ff:ff +""" + +IP_LINE_ETH100 = b""" +2: eth100: mtu 1500 qdisc mq master bond0 state UP qlen 1000 + link/ether e4:11:5b:ab:a7:3d brd ff:ff:ff:ff:ff:ff +""" + +IP_LINE_ETH0_VLAN = b""" +6: eth0.10@eth0: mtu 1500 qdisc noqueue state UP group default + link/ether 08:00:27:16:b9:5f brd ff:ff:ff:ff:ff:ff +""" + +IP_LINE_ETH1 = b""" +3: eth1: mtu 1546 qdisc noop state DOWN qlen 1000 + link/ether e4:11:5b:ab:a7:3c brd ff:ff:ff:ff:ff:ff +""" + +IP_LINE_HWADDR = b"""2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000\\ link/ether e4:11:5b:ab:a7:3c brd ff:ff:ff:ff:ff:ff""" + +IP_LINES = IP_LINE_ETH0 + IP_LINE_ETH1 + IP_LINE_ETH0_VLAN + IP_LINE_ETH100 + +IP_LINE_BONDS = b""" +6: bond0.10@bond0: mtu 1500 qdisc noqueue state UP group default +link/ether 08:00:27:16:b9:5f brd ff:ff:ff:ff:ff:ff +""" + + +class HelpersTest(TestCase): + @patch('charmhelpers.core.host.lsb_release') + @patch('os.path') + def test_init_is_systemd_service_snap(self, path, lsb_release): + # If Service begins with 'snap.' it should be True + service_name = "snap.package.service" + self.assertTrue(host.init_is_systemd(service_name=service_name)) + + # If service doesn't begin with snap. use normal evaluation. + service_name = "package.service" + lsb_release.return_value = {'DISTRIB_CODENAME': 'whatever'} + path.isdir.return_value = True + self.assertTrue(host.init_is_systemd(service_name=service_name)) + path.isdir.assert_called_with('/run/systemd/system') + + @patch('charmhelpers.core.host.lsb_release') + @patch('os.path') + def test_init_is_systemd_upstart(self, path, lsb_release): + """Upstart based init is correctly detected""" + lsb_release.return_value = {'DISTRIB_CODENAME': 'whatever'} + path.isdir.return_value = False + self.assertFalse(host.init_is_systemd()) + path.isdir.assert_called_with('/run/systemd/system') + + @patch('charmhelpers.core.host.lsb_release') + @patch('os.path') + def test_init_is_systemd_system(self, path, lsb_release): + """Systemd based init is correctly detected""" + lsb_release.return_value = {'DISTRIB_CODENAME': 'whatever'} + path.isdir.return_value = True + self.assertTrue(host.init_is_systemd()) + path.isdir.assert_called_with('/run/systemd/system') + + @patch('charmhelpers.core.host.lsb_release') + @patch('os.path') + def test_init_is_systemd_trusty(self, path, lsb_release): + # Never returns true under trusty, even if the systemd + # packages have been installed. lp:1670944 + lsb_release.return_value = {'DISTRIB_CODENAME': 'trusty'} + path.isdir.return_value = True + self.assertFalse(host.init_is_systemd()) + self.assertFalse(path.isdir.called) + + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_runs_service_action(self, mock_call, systemd): + systemd.return_value = False + mock_call.return_value = 0 + action = 'some-action' + service_name = 'foo-service' + + result = host.service(action, service_name) + + self.assertTrue(result) + mock_call.assert_called_with(['service', service_name, action]) + + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_runs_systemctl_action(self, mock_call, systemd): + """Ensure that service calls under systemd call 'systemctl'.""" + systemd.return_value = True + mock_call.return_value = 0 + action = 'some-action' + service_name = 'foo-service' + + result = host.service(action, service_name) + + self.assertTrue(result) + mock_call.assert_called_with(['systemctl', action, service_name]) + + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_returns_false_when_service_fails(self, mock_call, systemd): + systemd.return_value = False + mock_call.return_value = 1 + action = 'some-action' + service_name = 'foo-service' + + result = host.service(action, service_name) + + self.assertFalse(result) + mock_call.assert_called_with(['service', service_name, action]) + + @patch.object(host, 'service') + def test_starts_a_service(self, service): + service_name = 'foo-service' + service.side_effect = [True] + self.assertTrue(host.service_start(service_name)) + + service.assert_called_with('start', service_name) + + @patch.object(host, 'service') + def test_starts_a_service_with_parms(self, service): + service_name = 'foo-service' + service.side_effect = [True] + self.assertTrue(host.service_start(service_name, id=4)) + + service.assert_called_with('start', service_name, id=4) + + @patch.object(host, 'service') + def test_stops_a_service(self, service): + service_name = 'foo-service' + service.side_effect = [True] + self.assertTrue(host.service_stop(service_name)) + + service.assert_called_with('stop', service_name) + + @patch.object(host, 'service') + def test_restarts_a_service(self, service): + service_name = 'foo-service' + service.side_effect = [True] + self.assertTrue(host.service_restart(service_name)) + + service.assert_called_with('restart', service_name) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_pauses_a_running_systemd_unit(self, service, systemd, + service_running): + """Pause on a running systemd unit will be stopped and disabled.""" + service_name = 'foo-service' + service_running.return_value = True + systemd.return_value = True + self.assertTrue(host.service_pause(service_name)) + service.assert_has_calls([ + call('stop', service_name), + call('disable', service_name), + call('mask', service_name)]) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_resumes_a_stopped_systemd_unit(self, service, systemd, + service_running): + """Resume on a stopped systemd unit will be started and enabled.""" + service_name = 'foo-service' + service_running.return_value = False + systemd.return_value = True + self.assertTrue(host.service_resume(service_name)) + service.assert_has_calls([ + call('unmask', service_name), + # Ensures a package starts up if disabled but not masked, + # per lp:1692178 + call('enable', service_name), + call('start', service_name)]) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_pauses_a_running_upstart_service(self, service, systemd, + service_running): + """Pause on a running service will call service stop.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = True + tempdir = mkdtemp(prefix="test_pauses_an_upstart_service") + conf_path = os.path.join(tempdir, "{}.conf".format(service_name)) + # Just needs to exist + with open(conf_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_pause(service_name, init_dir=tempdir)) + + service.assert_called_with('stop', service_name) + override_path = os.path.join( + tempdir, "{}.override".format(service_name)) + with open(override_path, "r") as fh: + override_contents = fh.read() + self.assertEqual("manual\n", override_contents) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_pauses_a_stopped_upstart_service(self, service, systemd, + service_running): + """Pause on a stopped service will not call service stop.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = False + tempdir = mkdtemp(prefix="test_pauses_an_upstart_service") + conf_path = os.path.join(tempdir, "{}.conf".format(service_name)) + # Just needs to exist + with open(conf_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_pause(service_name, init_dir=tempdir)) + + # Stop isn't called because service is already stopped + self.assertRaises( + AssertionError, service.assert_called_with, 'stop', service_name) + override_path = os.path.join( + tempdir, "{}.override".format(service_name)) + with open(override_path, "r") as fh: + override_contents = fh.read() + self.assertEqual("manual\n", override_contents) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_call') + @patch.object(host, 'service') + def test_pauses_a_running_sysv_service(self, service, check_call, + systemd, service_running): + """Pause calls service stop on a running sysv service.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = True + tempdir = mkdtemp(prefix="test_pauses_a_sysv_service") + sysv_path = os.path.join(tempdir, service_name) + # Just needs to exist + with open(sysv_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_pause( + service_name, init_dir=tempdir, initd_dir=tempdir)) + + service.assert_called_with('stop', service_name) + check_call.assert_called_with(["update-rc.d", service_name, "disable"]) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_call') + @patch.object(host, 'service') + def test_pauses_a_stopped_sysv_service(self, service, check_call, + systemd, service_running): + """Pause does not call service stop on a stopped sysv service.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = False + tempdir = mkdtemp(prefix="test_pauses_a_sysv_service") + sysv_path = os.path.join(tempdir, service_name) + # Just needs to exist + with open(sysv_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_pause( + service_name, init_dir=tempdir, initd_dir=tempdir)) + + # Stop isn't called because service is already stopped + self.assertRaises( + AssertionError, service.assert_called_with, 'stop', service_name) + check_call.assert_called_with(["update-rc.d", service_name, "disable"]) + + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_pause_with_unknown_service(self, service, systemd): + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + tempdir = mkdtemp(prefix="test_pauses_with_unknown_service") + self.addCleanup(rmtree, tempdir) + exception = self.assertRaises( + ValueError, host.service_pause, + service_name, init_dir=tempdir, initd_dir=tempdir) + self.assertIn( + "Unable to detect {0}".format(service_name), str(exception)) + self.assertIn(tempdir, str(exception)) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_output') + @patch.object(host, 'service') + def test_resumes_a_running_upstart_service(self, service, check_output, + systemd, service_running): + """When the service is already running, service start isn't called.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = True + tempdir = mkdtemp(prefix="test_resumes_an_upstart_service") + conf_path = os.path.join(tempdir, "{}.conf".format(service_name)) + with open(conf_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_resume(service_name, init_dir=tempdir)) + + # Start isn't called because service is already running + self.assertFalse(service.called) + override_path = os.path.join( + tempdir, "{}.override".format(service_name)) + self.assertFalse(os.path.exists(override_path)) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_output') + @patch.object(host, 'service') + def test_resumes_a_stopped_upstart_service(self, service, check_output, + systemd, service_running): + """When the service is stopped, service start is called.""" + check_output.return_value = b'foo-service stop/waiting' + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = False + tempdir = mkdtemp(prefix="test_resumes_an_upstart_service") + conf_path = os.path.join(tempdir, "{}.conf".format(service_name)) + with open(conf_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_resume(service_name, init_dir=tempdir)) + + service.assert_called_with('start', service_name) + override_path = os.path.join( + tempdir, "{}.override".format(service_name)) + self.assertFalse(os.path.exists(override_path)) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_call') + @patch.object(host, 'service') + def test_resumes_a_sysv_service(self, service, check_call, systemd, + service_running): + """When process is in a stop/waiting state, service start is called.""" + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = False + tempdir = mkdtemp(prefix="test_resumes_a_sysv_service") + sysv_path = os.path.join(tempdir, service_name) + # Just needs to exist + with open(sysv_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_resume( + service_name, init_dir=tempdir, initd_dir=tempdir)) + + service.assert_called_with('start', service_name) + check_call.assert_called_with(["update-rc.d", service_name, "enable"]) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_call') + @patch.object(host, 'service') + def test_resume_a_running_sysv_service(self, service, check_call, + systemd, service_running): + """When process is already running, service start isn't called.""" + service_name = 'foo-service' + systemd.return_value = False + service_running.return_value = True + tempdir = mkdtemp(prefix="test_resumes_a_sysv_service") + sysv_path = os.path.join(tempdir, service_name) + # Just needs to exist + with open(sysv_path, "w") as fh: + fh.write("") + self.addCleanup(rmtree, tempdir) + self.assertTrue(host.service_resume( + service_name, init_dir=tempdir, initd_dir=tempdir)) + + # Start isn't called because service is already running + self.assertFalse(service.called) + check_call.assert_called_with(["update-rc.d", service_name, "enable"]) + + @patch.object(host, 'service_running') + @patch.object(host, 'init_is_systemd') + @patch.object(host, 'service') + def test_resume_with_unknown_service(self, service, systemd, + service_running): + service_name = 'foo-service' + service.side_effect = [True] + systemd.return_value = False + service_running.return_value = False + tempdir = mkdtemp(prefix="test_resumes_with_unknown_service") + self.addCleanup(rmtree, tempdir) + exception = self.assertRaises( + ValueError, host.service_resume, + service_name, init_dir=tempdir, initd_dir=tempdir) + self.assertIn( + "Unable to detect {0}".format(service_name), str(exception)) + self.assertIn(tempdir, str(exception)) + + @patch.object(host, 'service') + def test_reloads_a_service(self, service): + service_name = 'foo-service' + service.side_effect = [True] + self.assertTrue(host.service_reload(service_name)) + + service.assert_called_with('reload', service_name) + + @patch.object(host, 'service') + def test_failed_reload_restarts_a_service(self, service): + service_name = 'foo-service' + service.side_effect = [False, True] + self.assertTrue( + host.service_reload(service_name, restart_on_failure=True)) + + service.assert_has_calls([ + call('reload', service_name), + call('restart', service_name) + ]) + + @patch.object(host, 'service') + def test_failed_reload_without_restart(self, service): + service_name = 'foo-service' + service.side_effect = [False] + self.assertFalse(host.service_reload(service_name)) + + service.assert_called_with('reload', service_name) + + @patch.object(host, 'service') + def test_start_a_service_fails(self, service): + service_name = 'foo-service' + service.side_effect = [False] + self.assertFalse(host.service_start(service_name)) + + service.assert_called_with('start', service_name) + + @patch.object(host, 'service') + def test_stop_a_service_fails(self, service): + service_name = 'foo-service' + service.side_effect = [False] + self.assertFalse(host.service_stop(service_name)) + + service.assert_called_with('stop', service_name) + + @patch.object(host, 'service') + def test_restart_a_service_fails(self, service): + service_name = 'foo-service' + service.side_effect = [False] + self.assertFalse(host.service_restart(service_name)) + + service.assert_called_with('restart', service_name) + + @patch.object(host, 'service') + def test_reload_a_service_fails(self, service): + service_name = 'foo-service' + service.side_effect = [False] + self.assertFalse(host.service_reload(service_name)) + + service.assert_called_with('reload', service_name) + + @patch.object(host, 'service') + def test_failed_reload_restarts_a_service_fails(self, service): + service_name = 'foo-service' + service.side_effect = [False, False] + self.assertFalse( + host.service_reload(service_name, restart_on_failure=True)) + + service.assert_has_calls([ + call('reload', service_name), + call('restart', service_name) + ]) + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_output') + def test_service_running_on_stopped_service(self, check_output, systemd, + os): + systemd.return_value = False + os.path.exists.return_value = True + check_output.return_value = b'foo stop/waiting' + self.assertFalse(host.service_running('foo')) + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_output') + def test_service_running_on_running_service(self, check_output, systemd, + os): + systemd.return_value = False + os.path.exists.return_value = True + check_output.return_value = b'foo start/running, process 23871' + self.assertTrue(host.service_running('foo')) + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.check_output') + def test_service_running_on_unknown_service(self, check_output, systemd, + os): + systemd.return_value = False + os.path.exists.return_value = True + exc = subprocess.CalledProcessError(1, ['status']) + check_output.side_effect = exc + self.assertFalse(host.service_running('foo')) + + @patch.object(host, 'os') + @patch.object(host, 'service') + @patch.object(host, 'init_is_systemd') + def test_service_systemv_running(self, systemd, service, os): + systemd.return_value = False + service.return_value = True + os.path.exists.side_effect = [False, True] + self.assertTrue(host.service_running('rabbitmq-server')) + service.assert_called_with('status', 'rabbitmq-server') + + @patch.object(host, 'os') + @patch.object(host, 'service') + @patch.object(host, 'init_is_systemd') + def test_service_systemv_not_running(self, systemd, service, + os): + systemd.return_value = False + service.return_value = False + os.path.exists.side_effect = [False, True] + self.assertFalse(host.service_running('keystone')) + service.assert_called_with('status', 'keystone') + + @patch('subprocess.call') + @patch.object(host, 'init_is_systemd') + def test_service_start_with_params(self, systemd, call): + systemd.return_value = False + call.return_value = 0 + self.assertTrue(host.service_start('ceph-osd', id=4)) + call.assert_called_with(['service', 'ceph-osd', 'start', 'id=4']) + + @patch('subprocess.call') + @patch.object(host, 'init_is_systemd') + def test_service_stop_with_params(self, systemd, call): + systemd.return_value = False + call.return_value = 0 + self.assertTrue(host.service_stop('ceph-osd', id=4)) + call.assert_called_with(['service', 'ceph-osd', 'stop', 'id=4']) + + @patch('subprocess.call') + @patch.object(host, 'init_is_systemd') + def test_service_start_systemd_with_params(self, systemd, call): + systemd.return_value = True + call.return_value = 0 + self.assertTrue(host.service_start('ceph-osd', id=4)) + call.assert_called_with(['systemctl', 'start', 'ceph-osd']) + + @patch('grp.getgrnam') + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adds_a_user_if_it_doesnt_exist(self, log, check_call, + getpwnam, getgrnam): + username = 'johndoe' + password = 'eodnhoj' + shell = '/bin/bash' + existing_user_pwnam = KeyError('user not found') + new_user_pwnam = 'some user pwnam' + + getpwnam.side_effect = [existing_user_pwnam, new_user_pwnam] + + result = host.adduser(username, password=password) + + self.assertEqual(result, new_user_pwnam) + check_call.assert_called_with([ + 'useradd', + '--create-home', + '--shell', shell, + '--password', password, + '-g', username, + username + ]) + getpwnam.assert_called_with(username) + + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_doesnt_add_user_if_it_already_exists(self, log, check_call, + getpwnam): + username = 'johndoe' + password = 'eodnhoj' + existing_user_pwnam = 'some user pwnam' + + getpwnam.return_value = existing_user_pwnam + + result = host.adduser(username, password=password) + + self.assertEqual(result, existing_user_pwnam) + self.assertFalse(check_call.called) + getpwnam.assert_called_with(username) + + @patch('grp.getgrnam') + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adds_a_user_with_different_shell(self, log, check_call, getpwnam, + getgrnam): + username = 'johndoe' + password = 'eodnhoj' + shell = '/bin/zsh' + existing_user_pwnam = KeyError('user not found') + new_user_pwnam = 'some user pwnam' + + getpwnam.side_effect = [existing_user_pwnam, new_user_pwnam] + getgrnam.side_effect = KeyError('group not found') + + result = host.adduser(username, password=password, shell=shell) + + self.assertEqual(result, new_user_pwnam) + check_call.assert_called_with([ + 'useradd', + '--create-home', + '--shell', shell, + '--password', password, + username + ]) + getpwnam.assert_called_with(username) + + @patch('grp.getgrnam') + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adduser_with_groups(self, log, check_call, getpwnam, getgrnam): + username = 'johndoe' + password = 'eodnhoj' + shell = '/bin/bash' + existing_user_pwnam = KeyError('user not found') + new_user_pwnam = 'some user pwnam' + + getpwnam.side_effect = [existing_user_pwnam, new_user_pwnam] + + result = host.adduser(username, password=password, + primary_group='foo', secondary_groups=[ + 'bar', 'qux', + ]) + + self.assertEqual(result, new_user_pwnam) + check_call.assert_called_with([ + 'useradd', + '--create-home', + '--shell', shell, + '--password', password, + '-g', 'foo', + '-G', 'bar,qux', + username + ]) + getpwnam.assert_called_with(username) + assert not getgrnam.called + + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adds_a_systemuser(self, log, check_call, getpwnam): + username = 'johndoe' + existing_user_pwnam = KeyError('user not found') + new_user_pwnam = 'some user pwnam' + + getpwnam.side_effect = [existing_user_pwnam, new_user_pwnam] + + result = host.adduser(username, system_user=True) + + self.assertEqual(result, new_user_pwnam) + check_call.assert_called_with([ + 'useradd', + '--system', + username + ]) + getpwnam.assert_called_with(username) + + @patch('pwd.getpwnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adds_a_systemuser_with_home_dir(self, log, check_call, getpwnam): + username = 'johndoe' + existing_user_pwnam = KeyError('user not found') + new_user_pwnam = 'some user pwnam' + + getpwnam.side_effect = [existing_user_pwnam, new_user_pwnam] + + result = host.adduser(username, system_user=True, + home_dir='/var/lib/johndoe') + + self.assertEqual(result, new_user_pwnam) + check_call.assert_called_with([ + 'useradd', + '--home', + '/var/lib/johndoe', + '--system', + username + ]) + getpwnam.assert_called_with(username) + + @patch('pwd.getpwnam') + @patch('pwd.getpwuid') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_add_user_uid(self, log, check_call, getgrnam, getpwuid, getpwnam): + user_name = 'james' + user_id = 1111 + uid_key_error = KeyError('user not found') + getpwuid.side_effect = uid_key_error + host.adduser(user_name, uid=user_id) + + check_call.assert_called_with([ + 'useradd', + '--uid', + str(user_id), + '--system', + '-g', + user_name, + user_name + ]) + getpwnam.assert_called_with(user_name) + getpwuid.assert_called_with(user_id) + + @patch('grp.getgrnam') + @patch('grp.getgrgid') + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_add_group_gid(self, log, check_call, getgrgid, getgrnam): + group_name = 'darkhorse' + group_id = 1005 + existing_group_gid = KeyError('group not found') + new_group_gid = 1006 + getgrgid.side_effect = [existing_group_gid, new_group_gid] + + host.add_group(group_name, gid=group_id) + check_call.assert_called_with([ + 'addgroup', + '--gid', + str(group_id), + '--group', + group_name + ]) + getgrgid.assert_called_with(group_id) + getgrnam.assert_called_with(group_name) + + @patch('pwd.getpwnam') + def test_user_exists_true(self, getpwnam): + getpwnam.side_effect = 'pw info' + self.assertTrue(host.user_exists('bob')) + + @patch('pwd.getpwnam') + def test_user_exists_false(self, getpwnam): + getpwnam.side_effect = KeyError('user not found') + self.assertFalse(host.user_exists('bob')) + + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_adds_a_user_to_a_group(self, log, check_call): + username = 'foo' + group = 'bar' + + host.add_user_to_group(username, group) + + check_call.assert_called_with([ + 'gpasswd', '-a', + username, + group + ]) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_add_a_group_if_it_doesnt_exist_ubuntu(self, check_call, + getgrnam, platform): + platform.return_value = 'ubuntu' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = KeyError('group not found') + new_group_grnam = 'some group grnam' + + getgrnam.side_effect = [existing_group_grnam, new_group_grnam] + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name) + + self.assertEqual(result, new_group_grnam) + check_call.assert_called_with(['addgroup', '--group', group_name]) + getgrnam.assert_called_with(group_name) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_add_a_group_if_it_doesnt_exist_centos(self, check_call, + getgrnam, platform): + platform.return_value = 'centos' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = KeyError('group not found') + new_group_grnam = 'some group grnam' + + getgrnam.side_effect = [existing_group_grnam, new_group_grnam] + + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name) + + self.assertEqual(result, new_group_grnam) + check_call.assert_called_with(['groupadd', group_name]) + getgrnam.assert_called_with(group_name) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_doesnt_add_group_if_it_already_exists_ubuntu(self, check_call, + getgrnam, platform): + platform.return_value = 'ubuntu' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = 'some group grnam' + + getgrnam.return_value = existing_group_grnam + + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name) + + self.assertEqual(result, existing_group_grnam) + self.assertFalse(check_call.called) + getgrnam.assert_called_with(group_name) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_doesnt_add_group_if_it_already_exists_centos(self, check_call, + getgrnam, platform): + platform.return_value = 'centos' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = 'some group grnam' + + getgrnam.return_value = existing_group_grnam + + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name) + + self.assertEqual(result, existing_group_grnam) + self.assertFalse(check_call.called) + getgrnam.assert_called_with(group_name) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_add_a_system_group_ubuntu(self, check_call, getgrnam, platform): + platform.return_value = 'ubuntu' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = KeyError('group not found') + new_group_grnam = 'some group grnam' + + getgrnam.side_effect = [existing_group_grnam, new_group_grnam] + + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name, system_group=True) + + self.assertEqual(result, new_group_grnam) + check_call.assert_called_with([ + 'addgroup', + '--system', + group_name + ]) + getgrnam.assert_called_with(group_name) + + @patch.object(osplatform, 'get_platform') + @patch('grp.getgrnam') + @patch('subprocess.check_call') + def test_add_a_system_group_centos(self, check_call, getgrnam, platform): + platform.return_value = 'centos' + imp.reload(host) + + group_name = 'testgroup' + existing_group_grnam = KeyError('group not found') + new_group_grnam = 'some group grnam' + + getgrnam.side_effect = [existing_group_grnam, new_group_grnam] + + with patch("charmhelpers.core.host.log"): + result = host.add_group(group_name, system_group=True) + + self.assertEqual(result, new_group_grnam) + check_call.assert_called_with([ + 'groupadd', + '-r', + group_name + ]) + getgrnam.assert_called_with(group_name) + + @patch('subprocess.check_call') + def test_chage_no_chroot(self, check_call): + host.chage('usera', expiredate='2019-09-28', maxdays='11') + check_call.assert_called_with([ + 'chage', + '--expiredate', '2019-09-28', + '--maxdays', '11', + 'usera' + ]) + + @patch('subprocess.check_call') + def test_chage_chroot(self, check_call): + host.chage('usera', expiredate='2019-09-28', maxdays='11', + root='mychroot') + check_call.assert_called_with([ + 'chage', + '--root', 'mychroot', + '--expiredate', '2019-09-28', + '--maxdays', '11', + 'usera' + ]) + + @patch('subprocess.check_call') + def test_remove_password_expiry(self, check_call): + host.remove_password_expiry('usera') + check_call.assert_called_with([ + 'chage', + '--expiredate', '-1', + '--inactive', '-1', + '--mindays', '0', + '--maxdays', '-1', + 'usera' + ]) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_rsyncs_a_path(self, log, check_output): + from_path = '/from/this/path/foo' + to_path = '/to/this/path/bar' + check_output.return_value = b' some output ' # Spaces will be stripped + + result = host.rsync(from_path, to_path) + + self.assertEqual(result, 'some output') + check_output.assert_called_with(['/usr/bin/rsync', '-r', '--delete', + '--executability', + '/from/this/path/foo', + '/to/this/path/bar'], stderr=subprocess.STDOUT) + + @patch('subprocess.check_call') + @patch.object(host, 'log') + def test_creates_a_symlink(self, log, check_call): + source = '/from/this/path/foo' + destination = '/to/this/path/bar' + + host.symlink(source, destination) + + check_call.assert_called_with(['ln', '-sf', + '/from/this/path/foo', + '/to/this/path/bar']) + + @patch('pwd.getpwnam') + @patch('grp.getgrnam') + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_creates_a_directory_if_it_doesnt_exist(self, os_, log, + getgrnam, getpwnam): + uid = 123 + gid = 234 + owner = 'some-user' + group = 'some-group' + path = '/some/other/path/from/link' + realpath = '/some/path' + path_exists = False + perms = 0o644 + + getpwnam.return_value.pw_uid = uid + getgrnam.return_value.gr_gid = gid + os_.path.abspath.return_value = realpath + os_.path.exists.return_value = path_exists + + host.mkdir(path, owner=owner, group=group, perms=perms) + + getpwnam.assert_called_with('some-user') + getgrnam.assert_called_with('some-group') + os_.path.abspath.assert_called_with(path) + os_.path.exists.assert_called_with(realpath) + os_.makedirs.assert_called_with(realpath, perms) + os_.chown.assert_called_with(realpath, uid, gid) + + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_creates_a_directory_with_defaults(self, os_, log): + uid = 0 + gid = 0 + path = '/some/other/path/from/link' + realpath = '/some/path' + path_exists = False + perms = 0o555 + + os_.path.abspath.return_value = realpath + os_.path.exists.return_value = path_exists + + host.mkdir(path) + + os_.path.abspath.assert_called_with(path) + os_.path.exists.assert_called_with(realpath) + os_.makedirs.assert_called_with(realpath, perms) + os_.chown.assert_called_with(realpath, uid, gid) + + @patch('pwd.getpwnam') + @patch('grp.getgrnam') + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_removes_file_with_same_path_before_mkdir(self, os_, log, + getgrnam, getpwnam): + uid = 123 + gid = 234 + owner = 'some-user' + group = 'some-group' + path = '/some/other/path/from/link' + realpath = '/some/path' + path_exists = True + force = True + is_dir = False + perms = 0o644 + + getpwnam.return_value.pw_uid = uid + getgrnam.return_value.gr_gid = gid + os_.path.abspath.return_value = realpath + os_.path.exists.return_value = path_exists + os_.path.isdir.return_value = is_dir + + host.mkdir(path, owner=owner, group=group, perms=perms, force=force) + + getpwnam.assert_called_with('some-user') + getgrnam.assert_called_with('some-group') + os_.path.abspath.assert_called_with(path) + os_.path.exists.assert_called_with(realpath) + os_.unlink.assert_called_with(realpath) + os_.makedirs.assert_called_with(realpath, perms) + os_.chown.assert_called_with(realpath, uid, gid) + + @patch('pwd.getpwnam') + @patch('grp.getgrnam') + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_writes_content_to_a_file(self, os_, log, getgrnam, getpwnam): + # Curly brackets here demonstrate that we are *not* rendering + # these strings with Python's string formatting. This is a + # change from the original behavior per Bug #1195634. + uid = 123 + gid = 234 + owner = 'some-user-{foo}' + group = 'some-group-{bar}' + path = '/some/path/{baz}' + contents = b'what is {juju}' + perms = 0o644 + fileno = 'some-fileno' + + getpwnam.return_value.pw_uid = uid + getgrnam.return_value.gr_gid = gid + + with patch_open() as (mock_open, mock_file): + mock_file.fileno.return_value = fileno + + host.write_file(path, contents, owner=owner, group=group, + perms=perms) + + getpwnam.assert_called_with('some-user-{foo}') + getgrnam.assert_called_with('some-group-{bar}') + mock_open.assert_called_with('/some/path/{baz}', 'wb') + os_.fchown.assert_called_with(fileno, uid, gid) + os_.fchmod.assert_called_with(fileno, perms) + mock_file.write.assert_called_with(b'what is {juju}') + + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_writes_content_with_default(self, os_, log): + uid = 0 + gid = 0 + path = '/some/path/{baz}' + fmtstr = b'what is {juju}' + perms = 0o444 + fileno = 'some-fileno' + + with patch_open() as (mock_open, mock_file): + mock_file.fileno.return_value = fileno + + host.write_file(path, fmtstr) + + mock_open.assert_called_with('/some/path/{baz}', 'wb') + os_.fchown.assert_called_with(fileno, uid, gid) + os_.fchmod.assert_called_with(fileno, perms) + mock_file.write.assert_called_with(b'what is {juju}') + + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_does_not_write_duplicate_content(self, os_, log): + uid = 0 + gid = 0 + path = '/some/path/{baz}' + fmtstr = b'what is {juju}' + perms = 0o444 + fileno = 'some-fileno' + + os_.stat.return_value.st_uid = 1 + os_.stat.return_value.st_gid = 1 + os_.stat.return_value.st_mode = 0o777 + + with patch_open() as (mock_open, mock_file): + mock_file.fileno.return_value = fileno + mock_file.read.return_value = fmtstr + + host.write_file(path, fmtstr) + + self.assertEqual(mock_open.call_count, 1) # Called to read + os_.chown.assert_has_calls([ + call(path, uid, -1), + call(path, -1, gid), + ]) + os_.chmod.assert_called_with(path, perms) + + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_only_changes_incorrect_ownership(self, os_, log): + uid = 0 + gid = 0 + path = '/some/path/{baz}' + fmtstr = b'what is {juju}' + perms = 0o444 + fileno = 'some-fileno' + + os_.stat.return_value.st_uid = uid + os_.stat.return_value.st_gid = gid + os_.stat.return_value.st_mode = perms + + with patch_open() as (mock_open, mock_file): + mock_file.fileno.return_value = fileno + mock_file.read.return_value = fmtstr + + host.write_file(path, fmtstr) + + self.assertEqual(mock_open.call_count, 1) # Called to read + self.assertEqual(os_.chown.call_count, 0) + + @patch.object(host, 'log') + @patch.object(host, 'os') + def test_writes_binary_contents(self, os_, log): + path = '/some/path/{baz}' + fmtstr = six.u('what is {juju}\N{TRADE MARK SIGN}').encode('UTF-8') + fileno = 'some-fileno' + + with patch_open() as (mock_open, mock_file): + mock_file.fileno.return_value = fileno + + host.write_file(path, fmtstr) + + mock_open.assert_called_with('/some/path/{baz}', 'wb') + mock_file.write.assert_called_with(fmtstr) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_mounts_a_device(self, log, check_output): + device = '/dev/guido' + mountpoint = '/mnt/guido' + options = 'foo,bar' + + result = host.mount(device, mountpoint, options) + + self.assertTrue(result) + check_output.assert_called_with(['mount', '-o', 'foo,bar', + '/dev/guido', '/mnt/guido']) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_doesnt_mount_on_error(self, log, check_output): + device = '/dev/guido' + mountpoint = '/mnt/guido' + options = 'foo,bar' + + error = subprocess.CalledProcessError(123, 'mount it', 'Oops...') + check_output.side_effect = error + + result = host.mount(device, mountpoint, options) + + self.assertFalse(result) + check_output.assert_called_with(['mount', '-o', 'foo,bar', + '/dev/guido', '/mnt/guido']) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_mounts_a_device_without_options(self, log, check_output): + device = '/dev/guido' + mountpoint = '/mnt/guido' + + result = host.mount(device, mountpoint) + + self.assertTrue(result) + check_output.assert_called_with(['mount', '/dev/guido', '/mnt/guido']) + + @patch.object(host, 'Fstab') + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_mounts_and_persist_a_device(self, log, check_output, fstab): + """Check if a mount works with the persist flag set to True + """ + device = '/dev/guido' + mountpoint = '/mnt/guido' + options = 'foo,bar' + + result = host.mount(device, mountpoint, options, persist=True) + + self.assertTrue(result) + check_output.assert_called_with(['mount', '-o', 'foo,bar', + '/dev/guido', '/mnt/guido']) + + fstab.add.assert_called_with('/dev/guido', '/mnt/guido', 'ext3', + options='foo,bar') + + result = host.mount(device, mountpoint, options, persist=True, + filesystem="xfs") + + self.assertTrue(result) + fstab.add.assert_called_with('/dev/guido', '/mnt/guido', 'xfs', + options='foo,bar') + + @patch.object(host, 'Fstab') + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_umounts_a_device(self, log, check_output, fstab): + mountpoint = '/mnt/guido' + + result = host.umount(mountpoint, persist=True) + + self.assertTrue(result) + check_output.assert_called_with(['umount', mountpoint]) + fstab.remove_by_mountpoint_called_with(mountpoint) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_umounts_and_persist_device(self, log, check_output): + mountpoint = '/mnt/guido' + + result = host.umount(mountpoint) + + self.assertTrue(result) + check_output.assert_called_with(['umount', '/mnt/guido']) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_doesnt_umount_on_error(self, log, check_output): + mountpoint = '/mnt/guido' + + error = subprocess.CalledProcessError(123, 'mount it', 'Oops...') + check_output.side_effect = error + + result = host.umount(mountpoint) + + self.assertFalse(result) + check_output.assert_called_with(['umount', '/mnt/guido']) + + def test_lists_the_mount_points(self): + with patch_open() as (mock_open, mock_file): + mock_file.readlines.return_value = MOUNT_LINES + result = host.mounts() + + self.assertEqual(result, [ + ['/', 'rootfs'], + ['/sys', 'sysfs'], + ['/proc', 'proc'], + ['/dev', 'udev'], + ['/dev/pts', 'devpts'] + ]) + mock_open.assert_called_with('/proc/mounts') + + _hash_files = { + '/etc/exists.conf': 'lots of nice ceph configuration', + '/etc/missing.conf': None + } + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_fstab_mount(self, log, check_output): + self.assertTrue(host.fstab_mount('/mnt/mymntpnt')) + check_output.assert_called_with(['mount', '/mnt/mymntpnt']) + + @patch('subprocess.check_output') + @patch.object(host, 'log') + def test_fstab_mount_fail(self, log, check_output): + error = subprocess.CalledProcessError(123, 'mount it', 'Oops...') + check_output.side_effect = error + self.assertFalse(host.fstab_mount('/mnt/mymntpnt')) + check_output.assert_called_with(['mount', '/mnt/mymntpnt']) + + @patch('hashlib.md5') + @patch('os.path.exists') + def test_file_hash_exists(self, exists, md5): + filename = '/etc/exists.conf' + exists.side_effect = [True] + m = md5() + m.hexdigest.return_value = self._hash_files[filename] + with patch_open() as (mock_open, mock_file): + mock_file.read.return_value = self._hash_files[filename] + result = host.file_hash(filename) + self.assertEqual(result, self._hash_files[filename]) + + @patch('os.path.exists') + def test_file_hash_missing(self, exists): + filename = '/etc/missing.conf' + exists.side_effect = [False] + with patch_open() as (mock_open, mock_file): + mock_file.read.return_value = self._hash_files[filename] + result = host.file_hash(filename) + self.assertEqual(result, None) + + @patch('hashlib.sha1') + @patch('os.path.exists') + def test_file_hash_sha1(self, exists, sha1): + filename = '/etc/exists.conf' + exists.side_effect = [True] + m = sha1() + m.hexdigest.return_value = self._hash_files[filename] + with patch_open() as (mock_open, mock_file): + mock_file.read.return_value = self._hash_files[filename] + result = host.file_hash(filename, hash_type='sha1') + self.assertEqual(result, self._hash_files[filename]) + + @patch.object(host, 'file_hash') + def test_check_hash(self, file_hash): + file_hash.return_value = 'good-hash' + self.assertRaises(host.ChecksumError, host.check_hash, + 'file', 'bad-hash') + host.check_hash('file', 'good-hash', 'sha256') + self.assertEqual(file_hash.call_args_list, [ + call('file', 'md5'), + call('file', 'sha256'), + ]) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_restart_no_changes(self, iglob, exists, service): + file_name = '/etc/missing.conf' + restart_map = { + file_name: ['test-service'] + } + iglob.return_value = [] + + @host.restart_on_change(restart_map) + def make_no_changes(): + pass + + make_no_changes() + + assert not service.called + assert not exists.called + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_restart_on_change(self, iglob, exists, service): + file_name = '/etc/missing.conf' + restart_map = { + file_name: ['test-service'] + } + iglob.side_effect = [[], [file_name]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(mock_file): + mock_file.read.return_value = b"newstuff" + + with patch_open() as (mock_open, mock_file): + make_some_changes(mock_file) + + for service_name in restart_map[file_name]: + service.assert_called_with('restart', service_name) + + exists.assert_has_calls([ + call(file_name), + ]) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_restart_on_change_context_manager(self, iglob, exists, service): + file_name = '/etc/missing.conf' + restart_map = { + file_name: ['test-service'] + } + iglob.side_effect = [[], [file_name]] + exists.return_value = True + + with patch_open() as (mock_open, mock_file): + with host.restart_on_change(restart_map): + mock_file.read.return_value = b"newstuff" + + for service_name in restart_map[file_name]: + service.assert_called_with('restart', service_name) + + exists.assert_has_calls([ + call(file_name), + ]) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_multiservice_restart_on_change(self, iglob, exists, service): + file_name_one = '/etc/missing.conf' + file_name_two = '/etc/exists.conf' + restart_map = { + file_name_one: ['test-service'], + file_name_two: ['test-service', 'test-service2'] + } + iglob.side_effect = [[], [file_name_two], + [file_name_one], [file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'exists', b'missing', b'exists2'] + make_some_changes() + + # Restart should only happen once per service + for svc in ['test-service2', 'test-service']: + c = call('restart', svc) + self.assertEquals(1, service.call_args_list.count(c)) + + exists.assert_has_calls([ + call(file_name_one), + call(file_name_two) + ]) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_multiservice_restart_on_change_in_order(self, iglob, exists, + service): + file_name_one = '/etc/cinder/cinder.conf' + file_name_two = '/etc/haproxy/haproxy.conf' + restart_map = OrderedDict([ + (file_name_one, ['some-api']), + (file_name_two, ['haproxy']) + ]) + iglob.side_effect = [[], [file_name_two], + [file_name_one], [file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'exists', b'missing', b'exists2'] + make_some_changes() + + # Restarts should happen in the order they are described in the + # restart map. + expected = [ + call('restart', 'some-api'), + call('restart', 'haproxy') + ] + self.assertEquals(expected, service.call_args_list) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_glob_no_restart(self, iglob, exists, service): + glob_path = '/etc/service/*.conf' + file_name_one = '/etc/service/exists.conf' + file_name_two = '/etc/service/exists2.conf' + restart_map = { + glob_path: ['service'] + } + iglob.side_effect = [[file_name_one, file_name_two], + [file_name_one, file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'content', b'content2', + b'content', b'content2'] + make_some_changes() + + self.assertEquals([], service.call_args_list) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_glob_restart_on_change(self, iglob, exists, service): + glob_path = '/etc/service/*.conf' + file_name_one = '/etc/service/exists.conf' + file_name_two = '/etc/service/exists2.conf' + restart_map = { + glob_path: ['service'] + } + iglob.side_effect = [[file_name_one, file_name_two], + [file_name_one, file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'content', b'content2', + b'changed', b'content2'] + make_some_changes() + + self.assertEquals([call('restart', 'service')], service.call_args_list) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_glob_restart_on_create(self, iglob, exists, service): + glob_path = '/etc/service/*.conf' + file_name_one = '/etc/service/exists.conf' + file_name_two = '/etc/service/missing.conf' + restart_map = { + glob_path: ['service'] + } + iglob.side_effect = [[file_name_one], + [file_name_one, file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'exists', + b'exists', b'created'] + make_some_changes() + + self.assertEquals([call('restart', 'service')], service.call_args_list) + + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_glob_restart_on_delete(self, iglob, exists, service): + glob_path = '/etc/service/*.conf' + file_name_one = '/etc/service/exists.conf' + file_name_two = '/etc/service/exists2.conf' + restart_map = { + glob_path: ['service'] + } + iglob.side_effect = [[file_name_one, file_name_two], + [file_name_two]] + exists.return_value = True + + @host.restart_on_change(restart_map) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'exists', b'exists2', + b'exists2'] + make_some_changes() + + self.assertEquals([call('restart', 'service')], service.call_args_list) + + @patch.object(host, 'service_reload') + @patch.object(host, 'service') + @patch('os.path.exists') + @patch('glob.iglob') + def test_restart_on_change_restart_functs(self, iglob, exists, service, + service_reload): + file_name_one = '/etc/cinder/cinder.conf' + file_name_two = '/etc/haproxy/haproxy.conf' + restart_map = OrderedDict([ + (file_name_one, ['some-api']), + (file_name_two, ['haproxy']) + ]) + iglob.side_effect = [[], [file_name_two], + [file_name_one], [file_name_two]] + exists.return_value = True + + restart_funcs = { + 'some-api': service_reload, + } + + @host.restart_on_change(restart_map, restart_functions=restart_funcs) + def make_some_changes(): + pass + + with patch_open() as (mock_open, mock_file): + mock_file.read.side_effect = [b'exists', b'missing', b'exists2'] + make_some_changes() + + self.assertEquals([call('restart', 'haproxy')], service.call_args_list) + self.assertEquals([call('some-api')], service_reload.call_args_list) + + @patch.object(osplatform, 'get_platform') + def test_lsb_release_ubuntu(self, platform): + platform.return_value = 'ubuntu' + imp.reload(host) + + result = { + "DISTRIB_ID": "Ubuntu", + "DISTRIB_RELEASE": "13.10", + "DISTRIB_CODENAME": "saucy", + "DISTRIB_DESCRIPTION": "\"Ubuntu Saucy Salamander " + "(development branch)\"" + } + with mocked_open('/etc/lsb-release', LSB_RELEASE): + lsb_release = host.lsb_release() + for key in result: + self.assertEqual(result[key], lsb_release[key]) + + @patch.object(osplatform, 'get_platform') + def test_lsb_release_centos(self, platform): + platform.return_value = 'centos' + imp.reload(host) + + result = { + 'NAME': '"CentOS Linux"', + 'ANSI_COLOR': '"0;31"', + 'ID_LIKE': '"rhel fedora"', + 'VERSION_ID': '"7"', + 'BUG_REPORT_URL': '"https://bugs.centos.org/"', + 'CENTOS_MANTISBT_PROJECT': '"CentOS-7"', + 'PRETTY_NAME': '"CentOS Linux 7 (Core)"', + 'VERSION': '"7 (Core)"', + 'REDHAT_SUPPORT_PRODUCT_VERSION': '"7"', + 'CENTOS_MANTISBT_PROJECT_VERSION': '"7"', + 'REDHAT_SUPPORT_PRODUCT': '"centos"', + 'HOME_URL': '"https://www.centos.org/"', + 'CPE_NAME': '"cpe:/o:centos:centos:7"', + 'ID': '"centos"' + } + with mocked_open('/etc/os-release', OS_RELEASE): + lsb_release = host.lsb_release() + for key in result: + self.assertEqual(result[key], lsb_release[key]) + + def test_pwgen(self): + pw = host.pwgen() + self.assert_(len(pw) >= 35, 'Password is too short') + + pw = host.pwgen(10) + self.assertEqual(len(pw), 10, 'Password incorrect length') + + pw2 = host.pwgen(10) + self.assertNotEqual(pw, pw2, 'Duplicated password') + + @patch.object(host, 'glob') + @patch('os.path.realpath') + @patch('os.path.isdir') + def test_is_phy_iface(self, mock_isdir, mock_realpath, mock_glob): + mock_isdir.return_value = True + mock_glob.glob.return_value = ['/sys/class/net/eth0', + '/sys/class/net/veth0'] + + def fake_realpath(soft): + if soft.endswith('/eth0'): + hard = ('/sys/devices/pci0000:00/0000:00:1c.4' + '/0000:02:00.1/net/eth0') + else: + hard = '/sys/devices/virtual/net/veth0' + + return hard + + mock_realpath.side_effect = fake_realpath + self.assertTrue(host.is_phy_iface('eth0')) + self.assertFalse(host.is_phy_iface('veth0')) + + @patch('os.path.exists') + @patch('os.path.realpath') + @patch('os.path.isdir') + def test_get_bond_master(self, mock_isdir, mock_realpath, mock_exists): + mock_isdir.return_value = True + + def fake_realpath(soft): + if soft.endswith('/eth0'): + return ('/sys/devices/pci0000:00/0000:00:1c.4' + '/0000:02:00.1/net/eth0') + elif soft.endswith('/br0'): + return '/sys/devices/virtual/net/br0' + elif soft.endswith('/master'): + return '/sys/devices/virtual/net/bond0' + + return None + + def fake_exists(path): + return True + + mock_exists.side_effect = fake_exists + mock_realpath.side_effect = fake_realpath + self.assertEqual(host.get_bond_master('eth0'), 'bond0') + self.assertIsNone(host.get_bond_master('br0')) + + @patch('subprocess.check_output') + def test_list_nics(self, check_output): + check_output.return_value = IP_LINES + nics = host.list_nics() + self.assertEqual(nics, ['eth0', 'eth1', 'eth0.10', 'eth100']) + nics = host.list_nics('eth') + self.assertEqual(nics, ['eth0', 'eth1', 'eth0.10', 'eth100']) + nics = host.list_nics(['eth']) + self.assertEqual(nics, ['eth0', 'eth1', 'eth0.10', 'eth100']) + + @patch('subprocess.check_output') + def test_list_nics_with_bonds(self, check_output): + check_output.return_value = IP_LINE_BONDS + nics = host.list_nics('bond') + self.assertEqual(nics, ['bond0.10', ]) + + @patch('subprocess.check_output') + def test_get_nic_mtu_with_bonds(self, check_output): + check_output.return_value = IP_LINE_BONDS + nic = "bond0.10" + mtu = host.get_nic_mtu(nic) + self.assertEqual(mtu, '1500') + + @patch('subprocess.check_call') + def test_set_nic_mtu(self, mock_call): + mock_call.return_value = 0 + nic = 'eth7' + mtu = '1546' + host.set_nic_mtu(nic, mtu) + mock_call.assert_called_with(['ip', 'link', 'set', nic, 'mtu', mtu]) + + @patch('subprocess.check_output') + def test_get_nic_mtu(self, check_output): + check_output.return_value = IP_LINE_ETH0 + nic = "eth0" + mtu = host.get_nic_mtu(nic) + self.assertEqual(mtu, '1500') + + @patch('subprocess.check_output') + def test_get_nic_mtu_vlan(self, check_output): + check_output.return_value = IP_LINE_ETH0_VLAN + nic = "eth0.10" + mtu = host.get_nic_mtu(nic) + self.assertEqual(mtu, '1500') + + @patch('subprocess.check_output') + def test_get_nic_hwaddr(self, check_output): + check_output.return_value = IP_LINE_HWADDR + nic = "eth0" + hwaddr = host.get_nic_hwaddr(nic) + self.assertEqual(hwaddr, 'e4:11:5b:ab:a7:3c') + + @patch('charmhelpers.core.host_factory.ubuntu.lsb_release') + def test_get_distrib_codename(self, lsb_release): + lsb_release.return_value = {'DISTRIB_CODENAME': 'bionic'} + self.assertEqual(host.get_distrib_codename(), 'bionic') + + @patch('charmhelpers.fetch.get_installed_version') + @patch.object(osplatform, 'get_platform') + @patch.object(ubuntu_apt_pkg, 'Cache') + def test_cmp_pkgrevno_revnos_ubuntu(self, pkg_cache, platform, + get_installed_version): + platform.return_value = 'ubuntu' + imp.reload(host) + current_ver = '2.4' + + class MockPackage: + class MockPackageRevno: + def __init__(self, ver_str): + self.ver_str = ver_str + + def __init__(self, current_ver): + self.current_ver = self.MockPackageRevno(current_ver) + + pkg_dict = { + 'python': MockPackage(current_ver) + } + pkg_cache.return_value = pkg_dict + get_installed_version.return_value = MockPackage.MockPackageRevno( + current_ver) + self.assertEqual(host.cmp_pkgrevno('python', '2.3'), 1) + self.assertEqual(host.cmp_pkgrevno('python', '2.4'), 0) + self.assertEqual(host.cmp_pkgrevno('python', '2.5'), -1) + self.assertEqual( + host.cmp_pkgrevno('python', '2.3', pkgcache=pkg_dict), + 1 + ) + self.assertEqual( + host.cmp_pkgrevno('python', '2.4', pkgcache=pkg_dict), + 0 + ) + self.assertEqual( + host.cmp_pkgrevno('python', '2.5', pkgcache=pkg_dict), + -1 + ) + + @patch.object(osplatform, 'get_platform') + def test_cmp_pkgrevno_revnos_centos(self, platform): + platform.return_value = 'centos' + imp.reload(host) + + class MockPackage: + def __init__(self, name, version): + self.Name = name + self.version = version + + yum_dict = { + 'installed': { + MockPackage('python', '2.4') + } + } + + import yum + yum.YumBase.return_value.doPackageLists.return_value = ( + yum_dict) + + self.assertEqual(host.cmp_pkgrevno('python', '2.3'), 1) + self.assertEqual(host.cmp_pkgrevno('python', '2.4'), 0) + self.assertEqual(host.cmp_pkgrevno('python', '2.5'), -1) + + @patch.object(host.os, 'stat') + @patch.object(host.pwd, 'getpwuid') + @patch.object(host.grp, 'getgrgid') + @patch('posix.stat_result') + def test_owner(self, stat_result_, getgrgid_, getpwuid_, stat_): + getgrgid_.return_value = ['testgrp'] + getpwuid_.return_value = ['testuser'] + stat_.return_value = stat_result_() + + user, group = host.owner('/some/path') + stat_.assert_called_once_with('/some/path') + self.assertEqual('testuser', user) + self.assertEqual('testgrp', group) + + def test_get_total_ram(self): + raw = dedent('''\ + MemFree: 183868 kB + MemTotal: 7096108 kB + MemAvailable: 5645240 kB + ''').strip() + with patch_open() as (mock_open, mock_file): + mock_file.readlines.return_value = raw.splitlines() + self.assertEqual(host.get_total_ram(), 7266414592) # 7GB + mock_open.assert_called_once_with('/proc/meminfo', 'r') + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_is_container_with_systemd_container(self, + call, + init_is_systemd, + mock_os): + init_is_systemd.return_value = True + call.return_value = 0 + self.assertTrue(host.is_container()) + call.assert_called_with(['systemd-detect-virt', '--container']) + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_is_container_with_systemd_non_container(self, + call, + init_is_systemd, + mock_os): + init_is_systemd.return_value = True + call.return_value = 1 + self.assertFalse(host.is_container()) + call.assert_called_with(['systemd-detect-virt', '--container']) + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_is_container_with_upstart_container(self, + call, + init_is_systemd, + mock_os): + init_is_systemd.return_value = False + mock_os.path.exists.return_value = True + self.assertTrue(host.is_container()) + mock_os.path.exists.assert_called_with('/run/container_type') + + @patch.object(host, 'os') + @patch.object(host, 'init_is_systemd') + @patch('subprocess.call') + def test_is_container_with_upstart_not_container(self, + call, + init_is_systemd, + mock_os): + init_is_systemd.return_value = False + mock_os.path.exists.return_value = False + self.assertFalse(host.is_container()) + mock_os.path.exists.assert_called_with('/run/container_type') + + def test_updatedb(self): + updatedb_text = 'PRUNEPATHS="/tmp"' + self.assertEqual(host.updatedb(updatedb_text, '/srv/node'), + 'PRUNEPATHS="/tmp /srv/node"') + + def test_no_change_updatedb(self): + updatedb_text = 'PRUNEPATHS="/tmp /srv/node"' + self.assertEqual(host.updatedb(updatedb_text, '/srv/node'), + updatedb_text) + + def test_no_prunepaths(self): + updatedb_text = 'PRUNE_BIND_MOUNTS="yes"' + self.assertEqual(host.updatedb(updatedb_text, '/srv/node'), + updatedb_text) + + @patch('os.path') + def test_write_updatedb(self, mock_path): + mock_path.exists.return_value = True + mock_path.isdir.return_value = False + _open = mock_open(read_data='PRUNEPATHS="/tmp /srv/node"') + with patch('charmhelpers.core.host.open', _open, create=True): + host.add_to_updatedb_prunepath("/tmp/test") + handle = _open() + + self.assertTrue(handle.read.call_count == 1) + self.assertTrue(handle.seek.call_count == 1) + handle.write.assert_called_once_with( + 'PRUNEPATHS="/tmp /srv/node /tmp/test"') + + @patch.object(host, 'os') + def test_prunepaths_no_updatedb_conf_file(self, mock_os): + mock_os.path.exists.return_value = False + _open = mock_open(read_data='PRUNEPATHS="/tmp /srv/node"') + with patch('charmhelpers.core.host.open', _open, create=True): + host.add_to_updatedb_prunepath("/tmp/test") + handle = _open() + + self.assertTrue(handle.call_count == 0) + + @patch.object(host, 'os') + def test_prunepaths_updatedb_conf_file_isdir(self, mock_os): + mock_os.path.exists.return_value = True + mock_os.path.isdir.return_value = True + _open = mock_open(read_data='PRUNEPATHS="/tmp /srv/node"') + with patch('charmhelpers.core.host.open', _open, create=True): + host.add_to_updatedb_prunepath("/tmp/test") + handle = _open() + + self.assertTrue(handle.call_count == 0) + + @patch.object(host, 'local_unit') + def test_modulo_distribution(self, local_unit): + local_unit.return_value = 'test/7' + + # unit % modulo * wait + self.assertEqual(host.modulo_distribution(modulo=6, wait=10), 10) + + # Zero wait when unit % modulo == 0 + self.assertEqual(host.modulo_distribution(modulo=7, wait=10), 0) + + # modulo * wait when unit % modulo == 0 and non_zero_wait=True + self.assertEqual(host.modulo_distribution(modulo=7, wait=10, + non_zero_wait=True), + 70) + + @patch.object(host, 'log') + @patch.object(host, 'charm_name') + @patch.object(host, 'write_file') + @patch.object(subprocess, 'check_call') + @patch.object(host, 'file_hash') + @patch('hashlib.md5') + def test_install_ca_cert_new_cert(self, md5, file_hash, check_call, + write_file, charm_name, log): + file_hash.return_value = 'old_hash' + charm_name.return_value = 'charm-name' + + md5().hexdigest.return_value = 'old_hash' + host.install_ca_cert('cert_data') + assert not check_call.called + + md5().hexdigest.return_value = 'new_hash' + host.install_ca_cert(None) + assert not check_call.called + host.install_ca_cert('') + assert not check_call.called + + host.install_ca_cert('cert_data', 'name') + write_file.assert_called_with( + '/usr/local/share/ca-certificates/name.crt', + b'cert_data') + check_call.assert_called_with(['update-ca-certificates', '--fresh']) + + host.install_ca_cert('cert_data') + write_file.assert_called_with( + '/usr/local/share/ca-certificates/juju-charm-name.crt', + b'cert_data') + check_call.assert_called_with(['update-ca-certificates', '--fresh']) + + @patch('subprocess.check_output') + def test_arch(self, check_output): + _ = host.arch() + check_output.assert_called_with( + ['dpkg', '--print-architecture'] + ) + + @patch('subprocess.check_output') + def test_get_system_env(self, check_output): + check_output.return_value = '' + self.assertEquals( + host.get_system_env('aKey', 'aDefault'), 'aDefault') + self.assertEquals(host.get_system_env('aKey'), None) + check_output.return_value = 'aKey=aValue\n' + self.assertEquals( + host.get_system_env('aKey', 'aDefault'), 'aValue') + check_output.return_value = 'otherKey=shell=wicked\n' + self.assertEquals( + host.get_system_env('otherKey', 'aDefault'), 'shell=wicked') + + +class TestHostCompator(TestCase): + + def test_compare_ubuntu_releases(self): + from charmhelpers.osplatform import get_platform + if get_platform() == 'ubuntu': + self.assertTrue(host.CompareHostReleases('yakkety') < 'zesty') diff --git a/nrpe/mod/charmhelpers/tests/core/test_hugepage.py b/nrpe/mod/charmhelpers/tests/core/test_hugepage.py new file mode 100644 index 0000000..69ab9b2 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_hugepage.py @@ -0,0 +1,118 @@ +from testtools import TestCase +from mock import patch +from charmhelpers.core import hugepage +import yaml + +TO_PATCH = [ + 'fstab', + 'add_group', + 'add_user_to_group', + 'sysctl', + 'fstab_mount', + 'mkdir', + 'check_output', +] + + +class Group(object): + def __init__(self): + self.gr_gid = '1010' + + +class HugepageTests(TestCase): + + def setUp(self): + super(HugepageTests, self).setUp() + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.core.hugepage.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def test_hugepage_support(self): + self.add_group.return_value = Group() + self.fstab.Fstab().get_entry_by_attr.return_value = 'old fstab entry' + self.fstab.Fstab().Entry.return_value = 'new fstab entry' + hugepage.hugepage_support('nova') + sysctl_expect = (""" +vm.hugetlb_shm_group: '1010' +vm.max_map_count: 65536 +vm.nr_hugepages: 256 +""".lstrip()) + self.sysctl.create.assert_called_with(sysctl_expect, + '/etc/sysctl.d/10-hugepage.conf') + self.mkdir.assert_called_with('/run/hugepages/kvm', owner='root', + group='root', perms=0o755, force=False) + self.fstab.Fstab().remove_entry.assert_called_with('old fstab entry') + self.fstab.Fstab().Entry.assert_called_with( + 'nodev', '/run/hugepages/kvm', 'hugetlbfs', + 'mode=1770,gid=1010,pagesize=2MB', 0, 0) + self.fstab.Fstab().add_entry.assert_called_with('new fstab entry') + self.fstab_mount.assert_called_with('/run/hugepages/kvm') + + def test_hugepage_support_new_mnt(self): + self.add_group.return_value = Group() + self.fstab.Fstab().get_entry_by_attr.return_value = None + self.fstab.Fstab().Entry.return_value = 'new fstab entry' + hugepage.hugepage_support('nova') + self.assertEqual(self.fstab.Fstab().remove_entry.call_args_list, []) + + def test_hugepage_support_no_automount(self): + self.add_group.return_value = Group() + self.fstab.Fstab().get_entry_by_attr.return_value = None + self.fstab.Fstab().Entry.return_value = 'new fstab entry' + hugepage.hugepage_support('nova', mount=False) + self.assertEqual(self.fstab_mount.call_args_list, []) + + def test_hugepage_support_nodefaults(self): + self.add_group.return_value = Group() + self.fstab.Fstab().get_entry_by_attr.return_value = 'old fstab entry' + self.fstab.Fstab().Entry.return_value = 'new fstab entry' + hugepage.hugepage_support( + 'nova', group='neutron', nr_hugepages=512, max_map_count=70000, + mnt_point='/hugepages', pagesize='1G', mount=False) + sysctl_expect = { + 'vm.hugetlb_shm_group': '1010', + 'vm.max_map_count': 70000, + 'vm.nr_hugepages': 512, + } + sysctl_setting_arg = self.sysctl.create.call_args_list[0][0][0] + self.assertEqual(yaml.safe_load(sysctl_setting_arg), sysctl_expect) + self.mkdir.assert_called_with('/hugepages', owner='root', + group='root', perms=0o755, force=False) + self.fstab.Fstab().remove_entry.assert_called_with('old fstab entry') + self.fstab.Fstab().Entry.assert_called_with( + 'nodev', '/hugepages', 'hugetlbfs', + 'mode=1770,gid=1010,pagesize=1G', 0, 0) + self.fstab.Fstab().add_entry.assert_called_with('new fstab entry') + + def test_hugepage_support_set_shmmax(self): + self.add_group.return_value = Group() + self.fstab.Fstab().get_entry_by_attr.return_value = None + self.fstab.Fstab().Entry.return_value = 'new fstab entry' + self.check_output.return_value = 2000 + hugepage.hugepage_support('nova', mount=False, set_shmmax=True) + sysctl_expect = { + 'kernel.shmmax': 536870912, + 'vm.hugetlb_shm_group': '1010', + 'vm.max_map_count': 65536, + 'vm.nr_hugepages': 256 + } + sysctl_setting_arg = self.sysctl.create.call_args_list[0][0][0] + self.assertEqual(yaml.safe_load(sysctl_setting_arg), sysctl_expect) + + def test_hugepage_support_auto_increase_max_map_count(self): + self.add_group.return_value = Group() + hugepage.hugepage_support( + 'nova', group='neutron', nr_hugepages=512, max_map_count=200, + mnt_point='/hugepages', pagesize='1G', mount=False) + sysctl_expect = { + 'vm.hugetlb_shm_group': '1010', + 'vm.max_map_count': 1024, + 'vm.nr_hugepages': 512, + } + sysctl_setting_arg = self.sysctl.create.call_args_list[0][0][0] + self.assertEqual(yaml.safe_load(sysctl_setting_arg), sysctl_expect) diff --git a/nrpe/mod/charmhelpers/tests/core/test_kernel.py b/nrpe/mod/charmhelpers/tests/core/test_kernel.py new file mode 100644 index 0000000..9da1998 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_kernel.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import unittest +import imp + +from charmhelpers import osplatform +from mock import patch +from tests.helpers import patch_open +from charmhelpers.core import kernel + + +class TestKernel(unittest.TestCase): + + @patch('subprocess.check_call') + @patch.object(osplatform, 'get_platform') + def test_modprobe_persistent_ubuntu(self, platform, check_call): + platform.return_value = 'ubuntu' + imp.reload(kernel) + + with patch_open() as (_open, _file): + _file.read.return_value = 'anothermod\n' + with patch("charmhelpers.core.kernel.log"): + kernel.modprobe('mymod') + _open.assert_called_with('/etc/modules', 'r+') + _file.read.assert_called_with() + _file.write.assert_called_with('mymod\n') + check_call.assert_called_with(['modprobe', 'mymod']) + + @patch('os.chmod') + @patch('subprocess.check_call') + @patch.object(osplatform, 'get_platform') + def test_modprobe_persistent_centos(self, platform, check_call, os): + platform.return_value = 'centos' + imp.reload(kernel) + + with patch_open() as (_open, _file): + _file.read.return_value = 'anothermod\n' + with patch("charmhelpers.core.kernel.log"): + kernel.modprobe('mymod') + _open.assert_called_with('/etc/rc.modules', 'r+') + os.assert_called_with('/etc/rc.modules', 111) + _file.read.assert_called_with() + _file.write.assert_called_with('modprobe mymod\n') + check_call.assert_called_with(['modprobe', 'mymod']) + + @patch('subprocess.check_call') + @patch.object(osplatform, 'get_platform') + def test_modprobe_not_persistent_ubuntu(self, platform, check_call): + platform.return_value = 'ubuntu' + imp.reload(kernel) + + with patch_open() as (_open, _file): + _file.read.return_value = 'anothermod\n' + with patch("charmhelpers.core.kernel.log"): + kernel.modprobe('mymod', persist=False) + assert not _open.called + check_call.assert_called_with(['modprobe', 'mymod']) + + @patch('subprocess.check_call') + @patch.object(osplatform, 'get_platform') + def test_modprobe_not_persistent_centos(self, platform, check_call): + platform.return_value = 'centos' + imp.reload(kernel) + + with patch_open() as (_open, _file): + _file.read.return_value = 'anothermod\n' + with patch("charmhelpers.core.kernel.log"): + kernel.modprobe('mymod', persist=False) + assert not _open.called + check_call.assert_called_with(['modprobe', 'mymod']) + + @patch.object(kernel, 'log') + @patch('subprocess.check_call') + def test_rmmod_not_forced(self, check_call, log): + kernel.rmmod('mymod') + check_call.assert_called_with(['rmmod', 'mymod']) + + @patch.object(kernel, 'log') + @patch('subprocess.check_call') + def test_rmmod_forced(self, check_call, log): + kernel.rmmod('mymod', force=True) + check_call.assert_called_with(['rmmod', '-f', 'mymod']) + + @patch.object(kernel, 'log') + @patch('subprocess.check_output') + def test_lsmod(self, check_output, log): + kernel.lsmod() + check_output.assert_called_with(['lsmod'], + universal_newlines=True) + + @patch('charmhelpers.core.kernel.lsmod') + def test_is_module_loaded(self, lsmod): + lsmod.return_value = "ip6_tables 28672 1 ip6table_filter" + self.assertTrue(kernel.is_module_loaded("ip6_tables")) + + @patch.object(osplatform, 'get_platform') + @patch('subprocess.check_call') + def test_update_initramfs_ubuntu(self, check_call, platform): + platform.return_value = 'ubuntu' + imp.reload(kernel) + + kernel.update_initramfs() + check_call.assert_called_with(["update-initramfs", "-k", "all", "-u"]) + + @patch.object(osplatform, 'get_platform') + @patch('subprocess.check_call') + def test_update_initramfs_centos(self, check_call, platform): + platform.return_value = 'centos' + imp.reload(kernel) + + kernel.update_initramfs() + check_call.assert_called_with(['dracut', '-f', 'all']) diff --git a/nrpe/mod/charmhelpers/tests/core/test_services.py b/nrpe/mod/charmhelpers/tests/core/test_services.py new file mode 100644 index 0000000..8f69353 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_services.py @@ -0,0 +1,868 @@ +import os +import mock +import unittest +import uuid +from charmhelpers.core import hookenv +from charmhelpers.core import host +from charmhelpers.core import services +from functools import partial + + +class TestServiceManager(unittest.TestCase): + def setUp(self): + self.pcharm_dir = mock.patch.object(hookenv, 'charm_dir') + self.mcharm_dir = self.pcharm_dir.start() + self.mcharm_dir.return_value = 'charm_dir' + + def tearDown(self): + self.pcharm_dir.stop() + + def test_register(self): + manager = services.ServiceManager([ + {'service': 'service1', + 'foo': 'bar'}, + {'service': 'service2', + 'qux': 'baz'}, + ]) + self.assertEqual(manager.services, { + 'service1': {'service': 'service1', + 'foo': 'bar'}, + 'service2': {'service': 'service2', + 'qux': 'baz'}, + }) + + def test_register_preserves_order(self): + service_list = [dict(service='a'), dict(service='b')] + + # Test that the services list order is preserved by checking + # both forwards and backwards - only one of these will be + # dictionary order, and if both work we know order is being + # preserved. + manager = services.ServiceManager(service_list) + self.assertEqual(list(manager.services.keys()), ['a', 'b']) + manager = services.ServiceManager(reversed(service_list)) + self.assertEqual(list(manager.services.keys()), ['b', 'a']) + + @mock.patch.object(services.ServiceManager, 'reconfigure_services') + @mock.patch.object(services.ServiceManager, 'stop_services') + @mock.patch.object(hookenv, 'hook_name') + @mock.patch.object(hookenv, 'config') + def test_manage_stop(self, config, hook_name, stop_services, reconfigure_services): + manager = services.ServiceManager() + hook_name.return_value = 'stop' + manager.manage() + stop_services.assert_called_once_with() + assert not reconfigure_services.called + + @mock.patch.object(services.ServiceManager, 'provide_data') + @mock.patch.object(services.ServiceManager, 'reconfigure_services') + @mock.patch.object(services.ServiceManager, 'stop_services') + @mock.patch.object(hookenv, 'hook_name') + @mock.patch.object(hookenv, 'config') + def test_manage_other(self, config, hook_name, stop_services, reconfigure_services, provide_data): + manager = services.ServiceManager() + hook_name.return_value = 'config-changed' + manager.manage() + assert not stop_services.called + reconfigure_services.assert_called_once_with() + provide_data.assert_called_once_with() + + def test_manage_calls_atstart(self): + cb = mock.MagicMock() + hookenv.atstart(cb) + manager = services.ServiceManager() + manager.manage() + self.assertTrue(cb.called) + + def test_manage_calls_atexit(self): + cb = mock.MagicMock() + hookenv.atexit(cb) + manager = services.ServiceManager() + manager.manage() + self.assertTrue(cb.called) + + @mock.patch.object(hookenv, 'config') + def test_manage_config_not_saved(self, config): + config = config.return_value + config.implicit_save = False + manager = services.ServiceManager() + manager.manage() + self.assertFalse(config.save.called) + + @mock.patch.object(services.ServiceManager, 'save_ready') + @mock.patch.object(services.ServiceManager, 'fire_event') + @mock.patch.object(services.ServiceManager, 'is_ready') + def test_reconfigure_ready(self, is_ready, fire_event, save_ready): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + is_ready.return_value = True + manager.reconfigure_services() + is_ready.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + fire_event.assert_has_calls([ + mock.call('data_ready', 'service1'), + mock.call('start', 'service1', default=[ + services.service_restart, + services.manage_ports]), + ], any_order=False) + fire_event.assert_has_calls([ + mock.call('data_ready', 'service2'), + mock.call('start', 'service2', default=[ + services.service_restart, + services.manage_ports]), + ], any_order=False) + save_ready.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + + @mock.patch.object(services.ServiceManager, 'save_ready') + @mock.patch.object(services.ServiceManager, 'fire_event') + @mock.patch.object(services.ServiceManager, 'is_ready') + def test_reconfigure_ready_list(self, is_ready, fire_event, save_ready): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + is_ready.return_value = True + manager.reconfigure_services('service3', 'service4') + self.assertEqual(is_ready.call_args_list, [ + mock.call('service3'), + mock.call('service4'), + ]) + self.assertEqual(fire_event.call_args_list, [ + mock.call('data_ready', 'service3'), + mock.call('start', 'service3', default=[ + services.service_restart, + services.open_ports]), + mock.call('data_ready', 'service4'), + mock.call('start', 'service4', default=[ + services.service_restart, + services.open_ports]), + ]) + self.assertEqual(save_ready.call_args_list, [ + mock.call('service3'), + mock.call('service4'), + ]) + + @mock.patch.object(services.ServiceManager, 'save_lost') + @mock.patch.object(services.ServiceManager, 'fire_event') + @mock.patch.object(services.ServiceManager, 'was_ready') + @mock.patch.object(services.ServiceManager, 'is_ready') + def test_reconfigure_not_ready(self, is_ready, was_ready, fire_event, save_lost): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + is_ready.return_value = False + was_ready.return_value = False + manager.reconfigure_services() + is_ready.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + fire_event.assert_has_calls([ + mock.call('stop', 'service1', default=[ + services.close_ports, + services.service_stop]), + mock.call('stop', 'service2', default=[ + services.close_ports, + services.service_stop]), + ], any_order=True) + save_lost.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + + @mock.patch.object(services.ServiceManager, 'save_lost') + @mock.patch.object(services.ServiceManager, 'fire_event') + @mock.patch.object(services.ServiceManager, 'was_ready') + @mock.patch.object(services.ServiceManager, 'is_ready') + def test_reconfigure_no_longer_ready(self, is_ready, was_ready, fire_event, save_lost): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + is_ready.return_value = False + was_ready.return_value = True + manager.reconfigure_services() + is_ready.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + fire_event.assert_has_calls([ + mock.call('data_lost', 'service1'), + mock.call('stop', 'service1', default=[ + services.close_ports, + services.service_stop]), + ], any_order=False) + fire_event.assert_has_calls([ + mock.call('data_lost', 'service2'), + mock.call('stop', 'service2', default=[ + services.close_ports, + services.service_stop]), + ], any_order=False) + save_lost.assert_has_calls([ + mock.call('service1'), + mock.call('service2'), + ], any_order=True) + + @mock.patch.object(services.ServiceManager, 'fire_event') + def test_stop_services(self, fire_event): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + manager.stop_services() + fire_event.assert_has_calls([ + mock.call('stop', 'service1', default=[ + services.close_ports, + services.service_stop]), + mock.call('stop', 'service2', default=[ + services.close_ports, + services.service_stop]), + ], any_order=True) + + @mock.patch.object(services.ServiceManager, 'fire_event') + def test_stop_services_list(self, fire_event): + manager = services.ServiceManager([ + {'service': 'service1'}, {'service': 'service2'}]) + manager.stop_services('service3', 'service4') + self.assertEqual(fire_event.call_args_list, [ + mock.call('stop', 'service3', default=[ + services.close_ports, + services.service_stop]), + mock.call('stop', 'service4', default=[ + services.close_ports, + services.service_stop]), + ]) + + def test_get_service(self): + service = {'service': 'test', 'test': 'test_service'} + manager = services.ServiceManager([service]) + self.assertEqual(manager.get_service('test'), service) + + def test_get_service_not_registered(self): + service = {'service': 'test', 'test': 'test_service'} + manager = services.ServiceManager([service]) + self.assertRaises(KeyError, manager.get_service, 'foo') + + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_default(self, get_service): + get_service.return_value = {} + cb = mock.Mock() + manager = services.ServiceManager() + manager.fire_event('event', 'service', cb) + cb.assert_called_once_with('service') + + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_default_list(self, get_service): + get_service.return_value = {} + cb = mock.Mock() + manager = services.ServiceManager() + manager.fire_event('event', 'service', [cb]) + cb.assert_called_once_with('service') + + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_simple_callback(self, get_service): + cb = mock.Mock() + dcb = mock.Mock() + get_service.return_value = {'event': cb} + manager = services.ServiceManager() + manager.fire_event('event', 'service', dcb) + assert not dcb.called + cb.assert_called_once_with('service') + + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_simple_callback_list(self, get_service): + cb = mock.Mock() + dcb = mock.Mock() + get_service.return_value = {'event': [cb]} + manager = services.ServiceManager() + manager.fire_event('event', 'service', dcb) + assert not dcb.called + cb.assert_called_once_with('service') + + @mock.patch.object(services.ManagerCallback, '__call__') + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_manager_callback(self, get_service, mcall): + cb = services.ManagerCallback() + dcb = mock.Mock() + get_service.return_value = {'event': cb} + manager = services.ServiceManager() + manager.fire_event('event', 'service', dcb) + assert not dcb.called + mcall.assert_called_once_with(manager, 'service', 'event') + + @mock.patch.object(services.ManagerCallback, '__call__') + @mock.patch.object(services.ServiceManager, 'get_service') + def test_fire_event_manager_callback_list(self, get_service, mcall): + cb = services.ManagerCallback() + dcb = mock.Mock() + get_service.return_value = {'event': [cb]} + manager = services.ServiceManager() + manager.fire_event('event', 'service', dcb) + assert not dcb.called + mcall.assert_called_once_with(manager, 'service', 'event') + + @mock.patch.object(services.ServiceManager, 'get_service') + def test_is_ready(self, get_service): + get_service.side_effect = [ + {}, + {'required_data': [True]}, + {'required_data': [False]}, + {'required_data': [True, False]}, + ] + manager = services.ServiceManager() + assert manager.is_ready('foo') + assert manager.is_ready('bar') + assert not manager.is_ready('foo') + assert not manager.is_ready('foo') + get_service.assert_has_calls([mock.call('foo'), mock.call('bar')]) + + def test_load_ready_file_short_circuit(self): + manager = services.ServiceManager() + manager._ready = 'foo' + manager._load_ready_file() + self.assertEqual(manager._ready, 'foo') + + @mock.patch('os.path.exists') + @mock.patch.object(services.base, 'open', create=True) + def test_load_ready_file_new(self, mopen, exists): + manager = services.ServiceManager() + exists.return_value = False + manager._load_ready_file() + self.assertEqual(manager._ready, set()) + assert not mopen.called + + @mock.patch('json.load') + @mock.patch('os.path.exists') + @mock.patch.object(services.base, 'open', create=True) + def test_load_ready_file(self, mopen, exists, jload): + manager = services.ServiceManager() + exists.return_value = True + jload.return_value = ['bar'] + manager._load_ready_file() + self.assertEqual(manager._ready, set(['bar'])) + exists.assert_called_once_with('charm_dir/READY-SERVICES.json') + mopen.assert_called_once_with('charm_dir/READY-SERVICES.json') + + @mock.patch('json.dump') + @mock.patch.object(services.base, 'open', create=True) + def test_save_ready_file(self, mopen, jdump): + manager = services.ServiceManager() + manager._save_ready_file() + assert not mopen.called + manager._ready = set(['foo']) + manager._save_ready_file() + mopen.assert_called_once_with('charm_dir/READY-SERVICES.json', 'w') + jdump.assert_called_once_with(['foo'], mopen.return_value.__enter__()) + + @mock.patch.object(services.base.ServiceManager, '_save_ready_file') + @mock.patch.object(services.base.ServiceManager, '_load_ready_file') + def test_save_ready(self, _lrf, _srf): + manager = services.ServiceManager() + manager._ready = set(['foo']) + manager.save_ready('bar') + _lrf.assert_called_once_with() + self.assertEqual(manager._ready, set(['foo', 'bar'])) + _srf.assert_called_once_with() + + @mock.patch.object(services.base.ServiceManager, '_save_ready_file') + @mock.patch.object(services.base.ServiceManager, '_load_ready_file') + def test_save_lost(self, _lrf, _srf): + manager = services.ServiceManager() + manager._ready = set(['foo', 'bar']) + manager.save_lost('bar') + _lrf.assert_called_once_with() + self.assertEqual(manager._ready, set(['foo'])) + _srf.assert_called_once_with() + manager.save_lost('bar') + self.assertEqual(manager._ready, set(['foo'])) + + @mock.patch.object(services.base.ServiceManager, '_save_ready_file') + @mock.patch.object(services.base.ServiceManager, '_load_ready_file') + def test_was_ready(self, _lrf, _srf): + manager = services.ServiceManager() + manager._ready = set() + manager.save_ready('foo') + manager.save_ready('bar') + assert manager.was_ready('foo') + assert manager.was_ready('bar') + manager.save_lost('bar') + assert manager.was_ready('foo') + assert not manager.was_ready('bar') + + @mock.patch.object(services.base.hookenv, 'relation_set') + @mock.patch.object(services.base.hookenv, 'related_units') + @mock.patch.object(services.base.hookenv, 'relation_ids') + def test_provide_data_no_match(self, relation_ids, related_units, relation_set): + provider = mock.Mock() + provider.name = 'provided' + manager = services.ServiceManager([ + {'service': 'service', 'provided_data': [provider]} + ]) + relation_ids.return_value = [] + manager.provide_data() + assert not provider.provide_data.called + relation_ids.assert_called_once_with('provided') + + @mock.patch.object(services.base.hookenv, 'relation_set') + @mock.patch.object(services.base.hookenv, 'related_units') + @mock.patch.object(services.base.hookenv, 'relation_ids') + def test_provide_data_not_ready(self, relation_ids, related_units, relation_set): + provider = mock.Mock() + provider.name = 'provided' + pd = mock.Mock() + data = pd.return_value = {'data': True} + provider.provide_data = lambda remote_service, service_ready: pd(remote_service, service_ready) + manager = services.ServiceManager([ + {'service': 'service', 'provided_data': [provider]} + ]) + manager.is_ready = mock.Mock(return_value=False) + relation_ids.return_value = ['relid'] + related_units.return_value = ['service/0'] + manager.provide_data() + relation_set.assert_called_once_with('relid', data) + pd.assert_called_once_with('service', False) + + @mock.patch.object(services.base.hookenv, 'relation_set') + @mock.patch.object(services.base.hookenv, 'related_units') + @mock.patch.object(services.base.hookenv, 'relation_ids') + def test_provide_data_ready(self, relation_ids, related_units, relation_set): + provider = mock.Mock() + provider.name = 'provided' + pd = mock.Mock() + data = pd.return_value = {'data': True} + provider.provide_data = lambda remote_service, service_ready: pd(remote_service, service_ready) + manager = services.ServiceManager([ + {'service': 'service', 'provided_data': [provider]} + ]) + manager.is_ready = mock.Mock(return_value=True) + relation_ids.return_value = ['relid'] + related_units.return_value = ['service/0'] + manager.provide_data() + relation_set.assert_called_once_with('relid', data) + pd.assert_called_once_with('service', True) + + +class TestRelationContext(unittest.TestCase): + def setUp(self): + self.phookenv = mock.patch.object(services.helpers, 'hookenv') + self.mhookenv = self.phookenv.start() + self.mhookenv.relation_ids.return_value = [] + self.context = services.RelationContext() + self.context.name = 'http' + self.context.interface = 'http' + self.context.required_keys = ['foo', 'bar'] + self.mhookenv.reset_mock() + + def tearDown(self): + self.phookenv.stop() + + def test_no_relations(self): + self.context.get_data() + self.assertFalse(self.context.is_ready()) + self.assertEqual(self.context, {}) + self.mhookenv.relation_ids.assert_called_once_with('http') + + def test_no_units(self): + self.mhookenv.relation_ids.return_value = ['nginx'] + self.mhookenv.related_units.return_value = [] + self.context.get_data() + self.assertFalse(self.context.is_ready()) + self.assertEqual(self.context, {'http': []}) + + def test_incomplete(self): + self.mhookenv.relation_ids.return_value = ['nginx', 'apache'] + self.mhookenv.related_units.side_effect = lambda i: [i + '/0'] + self.mhookenv.relation_get.side_effect = [{}, {'foo': '1'}] + self.context.get_data() + self.assertFalse(bool(self.context)) + self.assertEqual(self.mhookenv.relation_get.call_args_list, [ + mock.call(rid='apache', unit='apache/0'), + mock.call(rid='nginx', unit='nginx/0'), + ]) + + def test_complete(self): + self.mhookenv.relation_ids.return_value = ['nginx', 'apache', 'tomcat'] + self.mhookenv.related_units.side_effect = lambda i: [i + '/0'] + self.mhookenv.relation_get.side_effect = [{'foo': '1'}, {'foo': '2', 'bar': '3'}, {}] + self.context.get_data() + self.assertTrue(self.context.is_ready()) + self.assertEqual(self.context, {'http': [ + { + 'foo': '2', + 'bar': '3', + }, + ]}) + self.mhookenv.relation_ids.assert_called_with('http') + self.assertEqual(self.mhookenv.relation_get.call_args_list, [ + mock.call(rid='apache', unit='apache/0'), + mock.call(rid='nginx', unit='nginx/0'), + mock.call(rid='tomcat', unit='tomcat/0'), + ]) + + def test_provide(self): + self.assertEqual(self.context.provide_data(), {}) + + +class TestHttpRelation(unittest.TestCase): + def setUp(self): + self.phookenv = mock.patch.object(services.helpers, 'hookenv') + self.mhookenv = self.phookenv.start() + + self.context = services.helpers.HttpRelation() + + def tearDown(self): + self.phookenv.stop() + + def test_provide_data(self): + self.mhookenv.unit_get.return_value = "127.0.0.1" + self.assertEqual(self.context.provide_data(), { + 'host': "127.0.0.1", + 'port': 80, + }) + + def test_complete(self): + self.mhookenv.relation_ids.return_value = ['website'] + self.mhookenv.related_units.side_effect = lambda i: [i + '/0'] + self.mhookenv.relation_get.side_effect = [{'host': '127.0.0.2', + 'port': 8080}] + self.context.get_data() + self.assertTrue(self.context.is_ready()) + self.assertEqual(self.context, {'website': [ + { + 'host': '127.0.0.2', + 'port': 8080, + }, + ]}) + + self.mhookenv.relation_ids.assert_called_with('website') + self.assertEqual(self.mhookenv.relation_get.call_args_list, [ + mock.call(rid='website', unit='website/0'), + ]) + + +class TestMysqlRelation(unittest.TestCase): + + def setUp(self): + self.phookenv = mock.patch.object(services.helpers, 'hookenv') + self.mhookenv = self.phookenv.start() + + self.context = services.helpers.MysqlRelation() + + def tearDown(self): + self.phookenv.stop() + + def test_complete(self): + self.mhookenv.relation_ids.return_value = ['db'] + self.mhookenv.related_units.side_effect = lambda i: [i + '/0'] + self.mhookenv.relation_get.side_effect = [{'host': '127.0.0.2', + 'user': 'mysql', + 'password': 'mysql', + 'database': 'mysql', + }] + self.context.get_data() + self.assertTrue(self.context.is_ready()) + self.assertEqual(self.context, {'db': [ + { + 'host': '127.0.0.2', + 'user': 'mysql', + 'password': 'mysql', + 'database': 'mysql', + }, + ]}) + + self.mhookenv.relation_ids.assert_called_with('db') + self.assertEqual(self.mhookenv.relation_get.call_args_list, [ + mock.call(rid='db', unit='db/0'), + ]) + + +class TestRequiredConfig(unittest.TestCase): + def setUp(self): + self.options = { + 'options': { + 'option1': { + 'type': 'string', + 'description': 'First option', + }, + 'option2': { + 'type': 'int', + 'default': 0, + 'description': 'Second option', + }, + }, + } + self.config = { + 'option1': None, + 'option2': 0, + } + self._pyaml = mock.patch.object(services.helpers, 'yaml') + self.myaml = self._pyaml.start() + self.myaml.load.side_effect = lambda fp: self.options + self._pconfig = mock.patch.object(hookenv, 'config') + self.mconfig = self._pconfig.start() + self.mconfig.side_effect = lambda: self.config + self._pcharm_dir = mock.patch.object(hookenv, 'charm_dir') + self.mcharm_dir = self._pcharm_dir.start() + self.mcharm_dir.return_value = 'charm_dir' + + def tearDown(self): + self._pyaml.stop() + self._pconfig.stop() + self._pcharm_dir.stop() + + def test_none_changed(self): + with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True): + context = services.helpers.RequiredConfig('option1', 'option2') + self.assertFalse(bool(context)) + self.assertEqual(context['config']['option1'], None) + self.assertEqual(context['config']['option2'], 0) + + def test_partial(self): + self.config['option1'] = 'value' + with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True): + context = services.helpers.RequiredConfig('option1', 'option2') + self.assertFalse(bool(context)) + self.assertEqual(context['config']['option1'], 'value') + self.assertEqual(context['config']['option2'], 0) + + def test_ready(self): + self.config['option1'] = 'value' + self.config['option2'] = 1 + with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True): + context = services.helpers.RequiredConfig('option1', 'option2') + self.assertTrue(bool(context)) + self.assertEqual(context['config']['option1'], 'value') + self.assertEqual(context['config']['option2'], 1) + + def test_none_empty(self): + self.config['option1'] = '' + self.config['option2'] = 1 + with mock.patch.object(services.helpers, 'open', mock.mock_open(), create=True): + context = services.helpers.RequiredConfig('option1', 'option2') + self.assertFalse(bool(context)) + self.assertEqual(context['config']['option1'], '') + self.assertEqual(context['config']['option2'], 1) + + +class TestStoredContext(unittest.TestCase): + @mock.patch.object(services.helpers.StoredContext, 'read_context') + @mock.patch.object(services.helpers.StoredContext, 'store_context') + @mock.patch('os.path.exists') + def test_new(self, exists, store_context, read_context): + exists.return_value = False + context = services.helpers.StoredContext('foo.yaml', {'key': 'val'}) + assert not read_context.called + store_context.assert_called_once_with('foo.yaml', {'key': 'val'}) + self.assertEqual(context, {'key': 'val'}) + + @mock.patch.object(services.helpers.StoredContext, 'read_context') + @mock.patch.object(services.helpers.StoredContext, 'store_context') + @mock.patch('os.path.exists') + def test_existing(self, exists, store_context, read_context): + exists.return_value = True + read_context.return_value = {'key': 'other'} + context = services.helpers.StoredContext('foo.yaml', {'key': 'val'}) + read_context.assert_called_once_with('foo.yaml') + assert not store_context.called + self.assertEqual(context, {'key': 'other'}) + + @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir') + @mock.patch.object(services.helpers.StoredContext, 'read_context') + @mock.patch.object(services.helpers, 'yaml') + @mock.patch('os.fchmod') + @mock.patch('os.path.exists') + def test_store_context(self, exists, fchmod, yaml, read_context): + exists.return_value = False + mopen = mock.mock_open() + with mock.patch.object(services.helpers, 'open', mopen, create=True): + services.helpers.StoredContext('foo.yaml', {'key': 'val'}) + mopen.assert_called_once_with('charm_dir/foo.yaml', 'w') + fchmod.assert_called_once_with(mopen.return_value.fileno(), 0o600) + yaml.dump.assert_called_once_with({'key': 'val'}, mopen.return_value) + + @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir') + @mock.patch.object(services.helpers.StoredContext, 'read_context') + @mock.patch.object(services.helpers, 'yaml') + @mock.patch('os.fchmod') + @mock.patch('os.path.exists') + def test_store_context_abs(self, exists, fchmod, yaml, read_context): + exists.return_value = False + mopen = mock.mock_open() + with mock.patch.object(services.helpers, 'open', mopen, create=True): + services.helpers.StoredContext('/foo.yaml', {'key': 'val'}) + mopen.assert_called_once_with('/foo.yaml', 'w') + + @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir') + @mock.patch.object(services.helpers, 'yaml') + @mock.patch('os.path.exists') + def test_read_context(self, exists, yaml): + exists.return_value = True + yaml.load.return_value = {'key': 'other'} + mopen = mock.mock_open() + with mock.patch.object(services.helpers, 'open', mopen, create=True): + context = services.helpers.StoredContext('foo.yaml', {'key': 'val'}) + mopen.assert_called_once_with('charm_dir/foo.yaml', 'r') + yaml.load.assert_called_once_with(mopen.return_value) + self.assertEqual(context, {'key': 'other'}) + + @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir') + @mock.patch.object(services.helpers, 'yaml') + @mock.patch('os.path.exists') + def test_read_context_abs(self, exists, yaml): + exists.return_value = True + yaml.load.return_value = {'key': 'other'} + mopen = mock.mock_open() + with mock.patch.object(services.helpers, 'open', mopen, create=True): + context = services.helpers.StoredContext('/foo.yaml', {'key': 'val'}) + mopen.assert_called_once_with('/foo.yaml', 'r') + yaml.load.assert_called_once_with(mopen.return_value) + self.assertEqual(context, {'key': 'other'}) + + @mock.patch.object(hookenv, 'charm_dir', lambda: 'charm_dir') + @mock.patch.object(services.helpers, 'yaml') + @mock.patch('os.path.exists') + def test_read_context_empty(self, exists, yaml): + exists.return_value = True + yaml.load.return_value = None + mopen = mock.mock_open() + with mock.patch.object(services.helpers, 'open', mopen, create=True): + self.assertRaises(OSError, services.helpers.StoredContext, '/foo.yaml', {}) + + +class TestTemplateCallback(unittest.TestCase): + @mock.patch.object(services.helpers, 'templating') + def test_template_defaults(self, mtemplating): + manager = mock.Mock(**{'get_service.return_value': { + 'required_data': [{'foo': 'bar'}]}}) + self.assertRaises(TypeError, services.template, source='foo.yml') + callback = services.template(source='foo.yml', target='bar.yml') + assert isinstance(callback, services.ManagerCallback) + assert not mtemplating.render.called + callback(manager, 'test', 'event') + mtemplating.render.assert_called_once_with( + 'foo.yml', 'bar.yml', {'foo': 'bar', 'ctx': {'foo': 'bar'}}, + 'root', 'root', 0o444, template_loader=None) + + @mock.patch.object(services.helpers, 'templating') + def test_template_explicit(self, mtemplating): + manager = mock.Mock(**{'get_service.return_value': { + 'required_data': [{'foo': 'bar'}]}}) + callback = services.template( + source='foo.yml', target='bar.yml', + owner='user', group='group', perms=0o555 + ) + assert isinstance(callback, services.ManagerCallback) + assert not mtemplating.render.called + callback(manager, 'test', 'event') + mtemplating.render.assert_called_once_with( + 'foo.yml', 'bar.yml', {'foo': 'bar', 'ctx': {'foo': 'bar'}}, + 'user', 'group', 0o555, template_loader=None) + + @mock.patch.object(services.helpers, 'templating') + def test_template_loader(self, mtemplating): + manager = mock.Mock(**{'get_service.return_value': { + 'required_data': [{'foo': 'bar'}]}}) + callback = services.template( + source='foo.yml', target='bar.yml', + owner='user', group='group', perms=0o555, + template_loader='myloader' + ) + assert isinstance(callback, services.ManagerCallback) + assert not mtemplating.render.called + callback(manager, 'test', 'event') + mtemplating.render.assert_called_once_with( + 'foo.yml', 'bar.yml', {'foo': 'bar', 'ctx': {'foo': 'bar'}}, + 'user', 'group', 0o555, template_loader='myloader') + + @mock.patch.object(os.path, 'isfile') + @mock.patch.object(host, 'file_hash') + @mock.patch.object(host, 'service_restart') + @mock.patch.object(services.helpers, 'templating') + def test_template_onchange_restart(self, mtemplating, mrestart, mfile_hash, misfile): + def random_string(arg): + return uuid.uuid4() + mfile_hash.side_effect = random_string + misfile.return_value = True + manager = mock.Mock(**{'get_service.return_value': { + 'required_data': [{'foo': 'bar'}]}}) + callback = services.template( + source='foo.yml', target='bar.yml', + owner='user', group='group', perms=0o555, + on_change_action=(partial(mrestart, "mysuperservice")), + ) + assert isinstance(callback, services.ManagerCallback) + assert not mtemplating.render.called + callback(manager, 'test', 'event') + mtemplating.render.assert_called_once_with( + 'foo.yml', 'bar.yml', {'foo': 'bar', 'ctx': {'foo': 'bar'}}, + 'user', 'group', 0o555, template_loader=None) + mrestart.assert_called_with('mysuperservice') + + @mock.patch.object(hookenv, 'log') + @mock.patch.object(os.path, 'isfile') + @mock.patch.object(host, 'file_hash') + @mock.patch.object(host, 'service_restart') + @mock.patch.object(services.helpers, 'templating') + def test_template_onchange_restart_nochange(self, mtemplating, mrestart, + mfile_hash, misfile, mlog): + mfile_hash.return_value = "myhash" + misfile.return_value = True + manager = mock.Mock(**{'get_service.return_value': { + 'required_data': [{'foo': 'bar'}]}}) + callback = services.template( + source='foo.yml', target='bar.yml', + owner='user', group='group', perms=0o555, + on_change_action=(partial(mrestart, "mysuperservice")), + ) + assert isinstance(callback, services.ManagerCallback) + assert not mtemplating.render.called + callback(manager, 'test', 'event') + mtemplating.render.assert_called_once_with( + 'foo.yml', 'bar.yml', {'foo': 'bar', 'ctx': {'foo': 'bar'}}, + 'user', 'group', 0o555, template_loader=None) + self.assertEqual(mrestart.call_args_list, []) + + +class TestPortsCallback(unittest.TestCase): + def setUp(self): + self.phookenv = mock.patch.object(services.base, 'hookenv') + self.mhookenv = self.phookenv.start() + self.mhookenv.relation_ids.return_value = [] + self.mhookenv.charm_dir.return_value = 'charm_dir' + self.popen = mock.patch.object(services.base, 'open', create=True) + self.mopen = self.popen.start() + + def tearDown(self): + self.phookenv.stop() + self.popen.stop() + + def test_no_ports(self): + manager = mock.Mock(**{'get_service.return_value': {}}) + services.PortManagerCallback()(manager, 'service', 'event') + assert not self.mhookenv.open_port.called + assert not self.mhookenv.close_port.called + + def test_open_ports(self): + manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}}) + services.open_ports(manager, 'service', 'start') + self.mhookenv.open_port.has_calls([mock.call(1), mock.call(2)]) + assert not self.mhookenv.close_port.called + + def test_close_ports(self): + manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}}) + services.close_ports(manager, 'service', 'stop') + assert not self.mhookenv.open_port.called + self.mhookenv.close_port.has_calls([mock.call(1), mock.call(2)]) + + def test_close_old_ports(self): + self.mopen.return_value.read.return_value = '10,20' + manager = mock.Mock(**{'get_service.return_value': {'ports': [1, 2]}}) + services.close_ports(manager, 'service', 'stop') + assert not self.mhookenv.open_port.called + self.mhookenv.close_port.has_calls([ + mock.call(10), + mock.call(20), + mock.call(1), + mock.call(2)]) + + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/mod/charmhelpers/tests/core/test_strutils.py b/nrpe/mod/charmhelpers/tests/core/test_strutils.py new file mode 100644 index 0000000..9110e56 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_strutils.py @@ -0,0 +1,108 @@ +import unittest + +import charmhelpers.core.strutils as strutils + + +class TestStrUtils(unittest.TestCase): + def setUp(self): + super(TestStrUtils, self).setUp() + + def tearDown(self): + super(TestStrUtils, self).tearDown() + + def test_bool_from_string(self): + self.assertTrue(strutils.bool_from_string('true')) + self.assertTrue(strutils.bool_from_string('True')) + self.assertTrue(strutils.bool_from_string('yes')) + self.assertTrue(strutils.bool_from_string('Yes')) + self.assertTrue(strutils.bool_from_string('y')) + self.assertTrue(strutils.bool_from_string('Y')) + self.assertTrue(strutils.bool_from_string('on')) + + # unicode should also work + self.assertTrue(strutils.bool_from_string(u'true')) + + self.assertFalse(strutils.bool_from_string('False')) + self.assertFalse(strutils.bool_from_string('false')) + self.assertFalse(strutils.bool_from_string('no')) + self.assertFalse(strutils.bool_from_string('No')) + self.assertFalse(strutils.bool_from_string('n')) + self.assertFalse(strutils.bool_from_string('N')) + self.assertFalse(strutils.bool_from_string('off')) + + self.assertRaises(ValueError, strutils.bool_from_string, None) + self.assertRaises(ValueError, strutils.bool_from_string, 'foo') + + def test_bytes_from_string(self): + self.assertEqual(strutils.bytes_from_string('10'), 10) + self.assertEqual(strutils.bytes_from_string('3K'), 3072) + self.assertEqual(strutils.bytes_from_string('3KB'), 3072) + self.assertEqual(strutils.bytes_from_string('3M'), 3145728) + self.assertEqual(strutils.bytes_from_string('3MB'), 3145728) + self.assertEqual(strutils.bytes_from_string('3G'), 3221225472) + self.assertEqual(strutils.bytes_from_string('3GB'), 3221225472) + self.assertEqual(strutils.bytes_from_string('3T'), 3298534883328) + self.assertEqual(strutils.bytes_from_string('3TB'), 3298534883328) + self.assertEqual(strutils.bytes_from_string('3P'), 3377699720527872) + self.assertEqual(strutils.bytes_from_string('3PB'), 3377699720527872) + + self.assertRaises(ValueError, strutils.bytes_from_string, None) + self.assertRaises(ValueError, strutils.bytes_from_string, 'foo') + + def test_basic_string_comparator_class_fails_instantiation(self): + try: + strutils.BasicStringComparator('hello') + raise Exception("instantiating BasicStringComparator should fail") + except Exception as e: + assert (str(e) == "Must define the _list in the class definition!") + + def test_basic_string_comparator_class(self): + + class MyComparator(strutils.BasicStringComparator): + + _list = ('zomg', 'bartlet', 'over', 'and') + + x = MyComparator('zomg') + self.assertEquals(x.index, 0) + y = MyComparator('over') + self.assertEquals(y.index, 2) + self.assertTrue(x == 'zomg') + self.assertTrue(x != 'bartlet') + self.assertTrue(x == x) + self.assertTrue(x != y) + self.assertTrue(x < y) + self.assertTrue(y > x) + self.assertTrue(x < 'bartlet') + self.assertTrue(y > 'bartlet') + self.assertTrue(x >= 'zomg') + self.assertTrue(x <= 'zomg') + self.assertTrue(x >= x) + self.assertTrue(x <= x) + self.assertTrue(y >= 'zomg') + self.assertTrue(y <= 'over') + self.assertTrue(y >= x) + self.assertTrue(x <= y) + # ensure that something not in the list dies + try: + MyComparator('nope') + raise Exception("MyComparator('nope') should have failed") + except Exception as e: + self.assertTrue(isinstance(e, KeyError)) + + def test_basic_string_comparator_fails_different_comparators(self): + + class MyComparator1(strutils.BasicStringComparator): + + _list = ('the truth is out there'.split(' ')) + + class MyComparator2(strutils.BasicStringComparator): + + _list = ('no one in space can hear you scream'.split(' ')) + + x = MyComparator1('is') + y = MyComparator2('you') + try: + x > y + raise Exception("Comparing different comparators should fail") + except Exception as e: + self.assertTrue(isinstance(e, AssertionError)) diff --git a/nrpe/mod/charmhelpers/tests/core/test_sysctl.py b/nrpe/mod/charmhelpers/tests/core/test_sysctl.py new file mode 100644 index 0000000..58aec64 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_sysctl.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from charmhelpers.core.sysctl import create +import io +from mock import patch, MagicMock +from subprocess import CalledProcessError +import unittest +import tempfile + +import six +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + +__author__ = 'Jorge Niedbalski R. ' + + +TO_PATCH = [ + 'log', + 'check_call', + 'is_container', +] + + +class SysctlTests(unittest.TestCase): + def setUp(self): + self.tempfile = tempfile.NamedTemporaryFile(delete=False) + for m in TO_PATCH: + setattr(self, m, self._patch(m)) + + def _patch(self, method): + _m = patch('charmhelpers.core.sysctl.' + method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + @patch(builtin_open) + def test_create(self, mock_open): + """Test create sysctl method""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + create('{"kernel.max_pid": 1337}', "/etc/sysctl.d/test-sysctl.conf") + + _file.__enter__().write.assert_called_with("kernel.max_pid=1337\n") + + self.log.assert_called_with( + "Updating sysctl_file: /etc/sysctl.d/test-sysctl.conf" + " values: {'kernel.max_pid': 1337}", + level='DEBUG') + + self.check_call.assert_called_with([ + "sysctl", "-p", + "/etc/sysctl.d/test-sysctl.conf"]) + + @patch(builtin_open) + def test_create_with_ignore(self, mock_open): + """Test create sysctl method""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + create('{"kernel.max_pid": 1337}', + "/etc/sysctl.d/test-sysctl.conf", + ignore=True) + + _file.__enter__().write.assert_called_with("kernel.max_pid=1337\n") + + self.log.assert_called_with( + "Updating sysctl_file: /etc/sysctl.d/test-sysctl.conf" + " values: {'kernel.max_pid': 1337}", + level='DEBUG') + + self.check_call.assert_called_with([ + "sysctl", "-p", + "/etc/sysctl.d/test-sysctl.conf", "-e"]) + + @patch(builtin_open) + def test_create_with_dict(self, mock_open): + """Test create sysctl method""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + create({"kernel.max_pid": 1337}, "/etc/sysctl.d/test-sysctl.conf") + + _file.__enter__().write.assert_called_with("kernel.max_pid=1337\n") + + self.log.assert_called_with( + "Updating sysctl_file: /etc/sysctl.d/test-sysctl.conf" + " values: {'kernel.max_pid': 1337}", + level='DEBUG') + + self.check_call.assert_called_with([ + "sysctl", "-p", + "/etc/sysctl.d/test-sysctl.conf"]) + + @patch(builtin_open) + def test_create_invalid_argument(self, mock_open): + """Test create sysctl with an invalid argument""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + create('{"kernel.max_pid": 1337 xxxx', "/etc/sysctl.d/test-sysctl.conf") + + self.log.assert_called_with( + 'Error parsing YAML sysctl_dict: {"kernel.max_pid": 1337 xxxx', + level='ERROR') + + @patch(builtin_open) + def test_create_raises(self, mock_open): + """CalledProcessErrors are propagated for non-container machines.""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + self.is_container.return_value = False + self.check_call.side_effect = CalledProcessError(1, 'sysctl') + + with self.assertRaises(CalledProcessError): + create('{"kernel.max_pid": 1337}', "/etc/sysctl.d/test-sysctl.conf") + + @patch(builtin_open) + def test_create_raises_container(self, mock_open): + """CalledProcessErrors are logged for containers.""" + _file = MagicMock(spec=io.FileIO) + mock_open.return_value = _file + + self.is_container.return_value = True + self.check_call.side_effect = CalledProcessError(1, 'sysctl', 'foo') + + create('{"kernel.max_pid": 1337}', "/etc/sysctl.d/test-sysctl.conf") + self.log.assert_called_with( + 'Error setting some sysctl keys in this container: foo', + level='WARNING') diff --git a/nrpe/mod/charmhelpers/tests/core/test_templating.py b/nrpe/mod/charmhelpers/tests/core/test_templating.py new file mode 100644 index 0000000..df47677 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_templating.py @@ -0,0 +1,166 @@ +import pkg_resources +import shutil +import tempfile +import unittest +import jinja2 +import os.path +import pwd +import grp + +import mock +from charmhelpers.core import templating + + +TEMPLATES_DIR = pkg_resources.resource_filename(__name__, 'templates') + + +class TestTemplating(unittest.TestCase): + def setUp(self): + self.charm_dir = pkg_resources.resource_filename(__name__, '') + self._charm_dir_patch = mock.patch.object(templating.hookenv, + 'charm_dir') + self._charm_dir_mock = self._charm_dir_patch.start() + self._charm_dir_mock.side_effect = lambda: self.charm_dir + + def tearDown(self): + self._charm_dir_patch.stop() + + @mock.patch.object(templating.host.os, 'fchown') + @mock.patch.object(templating.host, 'mkdir') + @mock.patch.object(templating.host, 'log') + def test_render(self, log, mkdir, fchown): + with tempfile.NamedTemporaryFile() as fn1, \ + tempfile.NamedTemporaryFile() as fn2: + context = { + 'nats': { + 'port': '1234', + 'host': 'example.com', + }, + 'router': { + 'domain': 'api.foo.com' + }, + 'nginx_port': 80, + } + templating.render('fake_cc.yml', fn1.name, + context, templates_dir=TEMPLATES_DIR) + contents = open(fn1.name).read() + self.assertRegexpMatches(contents, 'port: 1234') + self.assertRegexpMatches(contents, 'host: example.com') + self.assertRegexpMatches(contents, 'domain: api.foo.com') + + templating.render('test.conf', fn2.name, context, + templates_dir=TEMPLATES_DIR) + contents = open(fn2.name).read() + self.assertRegexpMatches(contents, 'listen 80') + self.assertEqual(fchown.call_count, 2) + # Not called, because the target directory exists. Calling + # it would make the target directory world readable and + # expose your secrets (!). + self.assertEqual(mkdir.call_count, 0) + + @mock.patch.object(templating.host.os, 'fchown') + @mock.patch.object(templating.host, 'mkdir') + @mock.patch.object(templating.host, 'log') + def test_render_from_string(self, log, mkdir, fchown): + with tempfile.NamedTemporaryFile() as fn: + context = { + 'foo': 'bar' + } + + config_template = '{{ foo }}' + templating.render('somefile.txt', fn.name, + context, templates_dir=TEMPLATES_DIR, + config_template=config_template) + contents = open(fn.name).read() + self.assertRegexpMatches(contents, 'bar') + + self.assertEqual(fchown.call_count, 1) + # Not called, because the target directory exists. Calling + # it would make the target directory world readable and + # expose your secrets (!). + self.assertEqual(mkdir.call_count, 0) + + @mock.patch.object(templating.host.os, 'fchown') + @mock.patch.object(templating.host, 'mkdir') + @mock.patch.object(templating.host, 'log') + def test_render_loader(self, log, mkdir, fchown): + with tempfile.NamedTemporaryFile() as fn1: + context = { + 'nats': { + 'port': '1234', + 'host': 'example.com', + }, + 'router': { + 'domain': 'api.foo.com' + }, + 'nginx_port': 80, + } + template_loader = jinja2.ChoiceLoader([jinja2.FileSystemLoader(TEMPLATES_DIR)]) + templating.render('fake_cc.yml', fn1.name, + context, template_loader=template_loader) + contents = open(fn1.name).read() + self.assertRegexpMatches(contents, 'port: 1234') + self.assertRegexpMatches(contents, 'host: example.com') + self.assertRegexpMatches(contents, 'domain: api.foo.com') + + @mock.patch.object(templating.os.path, 'exists') + @mock.patch.object(templating.host.os, 'fchown') + @mock.patch.object(templating.host, 'mkdir') + @mock.patch.object(templating.host, 'log') + def test_render_no_dir(self, log, mkdir, fchown, exists): + exists.return_value = False + with tempfile.NamedTemporaryFile() as fn1, \ + tempfile.NamedTemporaryFile() as fn2: + context = { + 'nats': { + 'port': '1234', + 'host': 'example.com', + }, + 'router': { + 'domain': 'api.foo.com' + }, + 'nginx_port': 80, + } + templating.render('fake_cc.yml', fn1.name, + context, templates_dir=TEMPLATES_DIR) + contents = open(fn1.name).read() + self.assertRegexpMatches(contents, 'port: 1234') + self.assertRegexpMatches(contents, 'host: example.com') + self.assertRegexpMatches(contents, 'domain: api.foo.com') + + templating.render('test.conf', fn2.name, context, + templates_dir=TEMPLATES_DIR) + contents = open(fn2.name).read() + self.assertRegexpMatches(contents, 'listen 80') + self.assertEqual(fchown.call_count, 2) + # Target directory was created, world readable (!). + self.assertEqual(mkdir.call_count, 2) + + @mock.patch.object(templating.host.os, 'fchown') + @mock.patch.object(templating.host, 'log') + def test_render_2(self, log, fchown): + tmpdir = tempfile.mkdtemp() + fn1 = os.path.join(tmpdir, 'test.conf') + try: + context = {'nginx_port': 80} + templating.render('test.conf', fn1, context, + owner=pwd.getpwuid(os.getuid()).pw_name, + group=grp.getgrgid(os.getgid()).gr_name, + templates_dir=TEMPLATES_DIR) + with open(fn1) as f: + contents = f.read() + + self.assertRegexpMatches(contents, 'something') + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + @mock.patch.object(templating, 'hookenv') + @mock.patch('jinja2.Environment') + def test_load_error(self, Env, hookenv): + Env().get_template.side_effect = jinja2.exceptions.TemplateNotFound( + 'fake_cc.yml') + self.assertRaises( + jinja2.exceptions.TemplateNotFound, templating.render, + 'fake.src', 'fake.tgt', {}, templates_dir='tmpl') + hookenv.log.assert_called_once_with( + 'Could not load template fake.src from tmpl.', level=hookenv.ERROR) diff --git a/nrpe/mod/charmhelpers/tests/core/test_unitdata.py b/nrpe/mod/charmhelpers/tests/core/test_unitdata.py new file mode 100644 index 0000000..c6a85ec --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/core/test_unitdata.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2015 Canonical Ltd. +# +# Authors: +# Kapil Thangavelu +# +try: + from StringIO import StringIO +except Exception: + from io import StringIO + +import os +import shutil +import tempfile +import unittest + +from mock import patch + +from charmhelpers.core.unitdata import Storage, HookData, kv + + +class HookDataTest(unittest.TestCase): + + def setUp(self): + self.charm_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(self.charm_dir)) + self.change_environment(CHARM_DIR=self.charm_dir) + + def change_environment(self, **kw): + original_env = dict(os.environ) + + @self.addCleanup + def cleanup_env(): + os.environ.clear() + os.environ.update(original_env) + + os.environ.update(kw) + + @patch('charmhelpers.core.hookenv.hook_name') + @patch('charmhelpers.core.hookenv.execution_environment') + @patch('charmhelpers.core.hookenv.charm_dir') + def test_hook_data_records(self, cdir, ctx, name): + name.return_value = 'config-changed' + ctx.return_value = { + 'rels': {}, 'conf': {'a': 1}, 'env': {}, 'unit': 'someunit'} + cdir.return_value = self.charm_dir + with open(os.path.join(self.charm_dir, 'revision'), 'w') as fh: + fh.write('1') + hook_data = HookData() + + with hook_data(): + self.assertEqual(kv(), hook_data.kv) + self.assertEqual(kv().get('charm_revisions'), ['1']) + self.assertEqual(kv().get('unit'), 'someunit') + self.assertEqual(list(hook_data.conf), ['a']) + self.assertEqual(tuple(hook_data.conf.a), (None, 1)) + + +class StorageTest(unittest.TestCase): + + def test_init_kv_multiple(self): + with tempfile.NamedTemporaryFile() as fh: + kv = Storage(fh.name) + with kv.hook_scope('xyz'): + kv.set('x', 1) + kv.close() + self.assertEqual(os.stat(fh.name).st_mode & 0o777, 0o600) + + kv = Storage(fh.name) + with kv.hook_scope('abc'): + self.assertEqual(kv.get('x'), 1) + kv.close() + + def test_hook_scope(self): + kv = Storage(':memory:') + try: + with kv.hook_scope('install') as rev: + self.assertEqual(rev, 1) + kv.set('a', 1) + raise RuntimeError('x') + except RuntimeError: + self.assertEqual(kv.get('a'), None) + + with kv.hook_scope('config-changed') as rev: + self.assertEqual(rev, 1) + kv.set('a', 1) + self.assertEqual(kv.get('a'), 1) + + kv.revision = None + + with kv.hook_scope('start') as rev: + self.assertEqual(rev, 2) + kv.set('a', False) + kv.set('a', True) + self.assertEqual(kv.get('a'), True) + + # History doesn't decode values by default + history = [h[:-1] for h in kv.gethistory('a')] + self.assertEqual( + history, + [(1, 'a', '1', 'config-changed'), + (2, 'a', 'true', 'start')]) + + history = [h[:-1] for h in kv.gethistory('a', deserialize=True)] + self.assertEqual( + history, + [(1, 'a', 1, 'config-changed'), + (2, 'a', True, 'start')]) + + def test_delta_no_previous_and_history(self): + kv = Storage(':memory:') + with kv.hook_scope('install'): + data = {'a': 0, 'c': False} + delta = kv.delta(data, 'settings.') + self.assertEqual(delta, { + 'a': (None, False), 'c': (None, False)}) + kv.update(data, 'settings.') + + with kv.hook_scope('config'): + data = {'a': 1, 'c': True} + delta = kv.delta(data, 'settings.') + self.assertEqual(delta, { + 'a': (0, 1), 'c': (False, True)}) + kv.update(data, 'settings.') + # strip the time + history = [h[:-1] for h in kv.gethistory('settings.a')] + self.assertEqual( + history, + [(1, 'settings.a', '0', 'install'), + (2, 'settings.a', '1', 'config')]) + + def test_unset(self): + kv = Storage(':memory:') + with kv.hook_scope('install'): + kv.set('a', True) + with kv.hook_scope('start'): + kv.set('a', False) + with kv.hook_scope('config-changed'): + kv.unset('a') + history = [h[:-1] for h in kv.gethistory('a')] + + self.assertEqual(history, [ + (1, 'a', 'true', 'install'), + (2, 'a', 'false', 'start'), + (3, 'a', '"DELETED"', "config-changed")]) + + def test_flush_and_close_on_closed(self): + kv = Storage(':memory:') + kv.close() + kv.flush(False) + kv.close() + + def test_multi_value_set_skips(self): + # pure coverage test + kv = Storage(':memory:') + kv.set('x', 1) + self.assertEqual(kv.set('x', 1), 1) + + def test_debug(self): + # pure coverage test... + io = StringIO() + kv = Storage(':memory:') + kv.debug(io) + + def test_record(self): + kv = Storage(':memory:') + kv.set('config', {'x': 1, 'b': False}) + config = kv.get('config', record=True) + self.assertEqual(config.b, False) + self.assertEqual(config.x, 1) + self.assertEqual(kv.set('config.x', 1), 1) + try: + config.z + except AttributeError: + pass + else: + self.fail('attribute error should fire on nonexistant') + + def test_delta(self): + kv = Storage(':memory:') + kv.update({'a': 1, 'b': 2.2}, prefix="x") + delta = kv.delta({'a': 0, 'c': False}, prefix='x') + self.assertEqual( + delta, + {'a': (1, 0), 'b': (2.2, None), 'c': (None, False)}) + self.assertEqual(delta.a.previous, 1) + self.assertEqual(delta.a.current, 0) + self.assertEqual(delta.c.previous, None) + self.assertEqual(delta.a.current, False) + + def test_update(self): + kv = Storage(':memory:') + kv.update({'v_a': 1, 'v_b': 2.2}) + self.assertEqual(kv.getrange('v_'), {'v_a': 1, 'v_b': 2.2}) + + kv.update({'a': False, 'b': True}, prefix='x_') + self.assertEqual( + kv.getrange('x_', True), {'a': False, 'b': True}) + + def test_keyrange(self): + kv = Storage(':memory:') + kv.set('docker.net_mtu', 1) + kv.set('docker.net_nack', True) + kv.set('docker.net_type', 'vxlan') + self.assertEqual( + kv.getrange('docker'), + {'docker.net_mtu': 1, 'docker.net_type': 'vxlan', + 'docker.net_nack': True}) + self.assertEqual( + kv.getrange('docker.', True), + {'net_mtu': 1, 'net_type': 'vxlan', 'net_nack': True}) + + def test_get_set_unset(self): + kv = Storage(':memory:') + kv.hook_scope('test') + kv.set('hello', 'saucy') + kv.set('hello', 'world') + self.assertEqual(kv.get('hello'), 'world') + kv.flush() + kv.unset('hello') + self.assertEqual(kv.get('hello'), None) + kv.flush(False) + self.assertEqual(kv.get('hello'), 'world') + + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/mod/charmhelpers/tests/fetch/__init__.py b/nrpe/mod/charmhelpers/tests/fetch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/fetch/python/__init__.py b/nrpe/mod/charmhelpers/tests/fetch/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/fetch/python/test_debug.py b/nrpe/mod/charmhelpers/tests/fetch/python/test_debug.py new file mode 100755 index 0000000..76ee450 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/python/test_debug.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# coding: utf-8 + +from unittest import TestCase + +from charmhelpers.fetch.python import debug + +import mock + +__author__ = "Jorge Niedbalski " + +TO_PATCH = [ + "log", + "open_port", + "close_port", + "Rpdb", + "_error", +] + + +class DebugTestCase(TestCase): + """Test cases for charmhelpers.contrib.python.debug""" + + def setUp(self): + TestCase.setUp(self) + self.patch_all() + self.log.return_value = True + self.close_port.return_value = True + + self.wrapped_function = mock.Mock(return_value=True) + self.set_trace = debug.set_trace + + def patch(self, method): + _m = mock.patch.object(debug, method) + _mock = _m.start() + self.addCleanup(_m.stop) + return _mock + + def patch_all(self): + for method in TO_PATCH: + setattr(self, method, self.patch(method)) + + def test_debug_set_trace(self): + """Check if set_trace works + """ + self.set_trace() + self.open_port.assert_called_with(4444) + + def test_debug_set_trace_ex(self): + """Check if set_trace raises exception + """ + self.set_trace() + self.Rpdb.set_trace.side_effect = Exception() + self.assertTrue(self._error.called) diff --git a/nrpe/mod/charmhelpers/tests/fetch/python/test_packages.py b/nrpe/mod/charmhelpers/tests/fetch/python/test_packages.py new file mode 100644 index 0000000..443417d --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/python/test_packages.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# coding: utf-8 + +import mock +import six + +from unittest import TestCase +from charmhelpers.fetch.python import packages + +__author__ = "Jorge Niedbalski " + +TO_PATCH = [ + "apt_install", + "charm_dir", + "log", + "pip_execute", +] + + +class PipTestCase(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.patch_all() + + self.log.return_value = True + self.apt_install.return_value = True + + def patch(self, method): + _m = mock.patch.object(packages, method) + _mock = _m.start() + self.addCleanup(_m.stop) + return _mock + + def patch_all(self): + for method in TO_PATCH: + setattr(self, method, self.patch(method)) + + def test_pip_install_requirements(self): + """ + Check if pip_install_requirements works correctly + """ + packages.pip_install_requirements("test_requirements.txt") + self.pip_execute.assert_called_with(["install", + "-r test_requirements.txt"]) + + packages.pip_install_requirements("test_requirements.txt", + "test_constraints.txt") + self.pip_execute.assert_called_with(["install", + "-r test_requirements.txt", + "-c test_constraints.txt"]) + + packages.pip_install_requirements("test_requirements.txt", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["install", + "--proxy=proxy_addr:8080", + "-r test_requirements.txt"]) + + packages.pip_install_requirements("test_requirements.txt", + log="output.log", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["install", + "--log=output.log", + "--proxy=proxy_addr:8080", + "-r test_requirements.txt"]) + + def test_pip_install(self): + """ + Check if pip_install works correctly with a single package + """ + packages.pip_install("mock") + self.pip_execute.assert_called_with(["install", + "mock"]) + packages.pip_install("mock", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["install", + "--proxy=proxy_addr:8080", + "mock"]) + packages.pip_install("mock", + log="output.log", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["install", + "--log=output.log", + "--proxy=proxy_addr:8080", + "mock"]) + + def test_pip_install_upgrade(self): + """ + Check if pip_install works correctly with a single package + """ + packages.pip_install("mock", upgrade=True) + self.pip_execute.assert_called_with(["install", + "--upgrade", + "mock"]) + + def test_pip_install_multiple(self): + """ + Check if pip_install works correctly with multiple packages + """ + packages.pip_install(["mock", "nose"]) + self.pip_execute.assert_called_with(["install", + "mock", "nose"]) + + @mock.patch('subprocess.check_call') + @mock.patch('os.path.join') + def test_pip_install_venv(self, join, check_call): + """ + Check if pip_install works correctly with multiple packages + """ + join.return_value = 'joined-path' + packages.pip_install(["mock", "nose"], venv=True) + check_call.assert_called_with(["joined-path", "install", + "mock", "nose"]) + + def test_pip_uninstall(self): + """ + Check if pip_uninstall works correctly with a single package + """ + packages.pip_uninstall("mock") + self.pip_execute.assert_called_with(["uninstall", + "-q", + "-y", + "mock"]) + packages.pip_uninstall("mock", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["uninstall", + "-q", + "-y", + "--proxy=proxy_addr:8080", + "mock"]) + packages.pip_uninstall("mock", + log="output.log", + proxy="proxy_addr:8080") + + self.pip_execute.assert_called_with(["uninstall", + "-q", + "-y", + "--log=output.log", + "--proxy=proxy_addr:8080", + "mock"]) + + def test_pip_uninstall_multiple(self): + """ + Check if pip_uninstall works correctly with multiple packages + """ + packages.pip_uninstall(["mock", "nose"]) + self.pip_execute.assert_called_with(["uninstall", + "-q", + "-y", + "mock", "nose"]) + + def test_pip_list(self): + """ + Checks if pip_list works correctly + """ + packages.pip_list() + self.pip_execute.assert_called_with(["list"]) + + @mock.patch('os.path.join') + @mock.patch('subprocess.check_call') + @mock.patch.object(packages, 'pip_install') + def test_pip_create_virtualenv(self, pip_install, check_call, join): + """ + Checks if pip_create_virtualenv works correctly + """ + join.return_value = 'joined-path' + packages.pip_create_virtualenv() + if six.PY2: + self.apt_install.assert_called_with('python-virtualenv') + expect_flags = [] + else: + self.apt_install.assert_called_with(['python3-virtualenv', 'virtualenv']) + expect_flags = ['--python=python3'] + check_call.assert_called_with(['virtualenv', 'joined-path'] + expect_flags) diff --git a/nrpe/mod/charmhelpers/tests/fetch/python/test_version.py b/nrpe/mod/charmhelpers/tests/fetch/python/test_version.py new file mode 100644 index 0000000..9e32af3 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/python/test_version.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# coding: utf-8 + +from unittest import TestCase +from charmhelpers.fetch.python import version + +import sys + +__author__ = "Jorge Niedbalski " + + +class VersionTestCase(TestCase): + + def setUp(self): + TestCase.setUp(self) + + def test_current_version(self): + """ + Check if version.current_version and version.current_version_string + works correctly + """ + self.assertEquals(version.current_version(), + sys.version_info) + self.assertEquals(version.current_version_string(), + "{0}.{1}.{2}".format(sys.version_info.major, + sys.version_info.minor, + sys.version_info.micro)) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_archiveurl.py b/nrpe/mod/charmhelpers/tests/fetch/test_archiveurl.py new file mode 100644 index 0000000..b3733b5 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_archiveurl.py @@ -0,0 +1,131 @@ +import os + +from unittest import TestCase +from mock import ( + MagicMock, + patch, + mock_open, + Mock, + ANY +) +from charmhelpers.fetch import ( + archiveurl, + UnhandledSource, +) + +import six +if six.PY3: + from urllib.parse import urlparse + from urllib.error import URLError +else: + from urllib2 import URLError + from urlparse import urlparse + + +class ArchiveUrlFetchHandlerTest(TestCase): + + def setUp(self): + super(ArchiveUrlFetchHandlerTest, self).setUp() + self.valid_urls = ( + "http://example.com/foo.tar.gz", + "http://example.com/foo.tgz", + "http://example.com/foo.tar.bz2", + "http://example.com/foo.tbz2", + "http://example.com/foo.zip", + "http://example.com/foo.zip?bar=baz&x=y#whee", + "ftp://example.com/foo.tar.gz", + "https://example.com/foo.tgz", + "file://example.com/foo.tar.bz2", + ) + self.invalid_urls = ( + "git://example.com/foo.tar.gz", + "http://example.com/foo", + "http://example.com/foobar=baz&x=y#tar.gz", + "http://example.com/foobar?h=baz.zip", + "bzr+ssh://example.com/foo.tar.gz", + "lp:example/foo.tgz", + "file//example.com/foo.tar.bz2", + "garbage", + ) + self.fh = archiveurl.ArchiveUrlFetchHandler() + + def test_handles_archive_urls(self): + for url in self.valid_urls: + result = self.fh.can_handle(url) + self.assertEqual(result, True, url) + for url in self.invalid_urls: + result = self.fh.can_handle(url) + self.assertNotEqual(result, True, url) + + @patch('charmhelpers.fetch.archiveurl.urlopen') + def test_downloads(self, _urlopen): + for url in self.valid_urls: + response = MagicMock() + response.read.return_value = "bar" + _urlopen.return_value = response + + _open = mock_open() + with patch('charmhelpers.fetch.archiveurl.open', + _open, create=True): + self.fh.download(url, "foo") + + response.read.assert_called_with() + _open.assert_called_once_with("foo", 'wb') + _open().write.assert_called_with("bar") + + @patch('charmhelpers.fetch.archiveurl.check_hash') + @patch('charmhelpers.fetch.archiveurl.mkdir') + @patch('charmhelpers.fetch.archiveurl.extract') + def test_installs(self, _extract, _mkdir, _check_hash): + self.fh.download = MagicMock() + + for url in self.valid_urls: + filename = urlparse(url).path + dest = os.path.join('foo', 'fetched', os.path.basename(filename)) + _extract.return_value = dest + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + where = self.fh.install(url, checksum='deadbeef') + self.fh.download.assert_called_with(url, dest) + _extract.assert_called_with(dest, None) + _check_hash.assert_called_with(dest, 'deadbeef', 'sha1') + self.assertEqual(where, dest) + _check_hash.reset_mock() + + url = "http://www.example.com/archive.tar.gz" + + self.fh.download.side_effect = URLError('fail') + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.assertRaises(UnhandledSource, self.fh.install, url) + + self.fh.download.side_effect = OSError('fail') + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.assertRaises(UnhandledSource, self.fh.install, url) + + @patch('charmhelpers.fetch.archiveurl.check_hash') + @patch('charmhelpers.fetch.archiveurl.mkdir') + @patch('charmhelpers.fetch.archiveurl.extract') + def test_install_with_hash_in_url(self, _extract, _mkdir, _check_hash): + self.fh.download = MagicMock() + url = "file://example.com/foo.tar.bz2#sha512=beefdead" + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.fh.install(url) + _check_hash.assert_called_with(ANY, 'beefdead', 'sha512') + + @patch('charmhelpers.fetch.archiveurl.mkdir') + @patch('charmhelpers.fetch.archiveurl.extract') + def test_install_with_duplicate_hash_in_url(self, _extract, _mkdir): + self.fh.download = MagicMock() + url = "file://example.com/foo.tar.bz2#sha512=a&sha512=b" + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + with self.assertRaisesRegexp( + TypeError, "Expected 1 hash value, not 2"): + self.fh.install(url) + + @patch('charmhelpers.fetch.archiveurl.urlretrieve') + @patch('charmhelpers.fetch.archiveurl.check_hash') + def test_download_and_validate(self, vfmock, urlmock): + urlmock.return_value = ('/tmp/tmpebM9Hv', Mock()) + dlurl = 'http://example.com/foo.tgz' + dlhash = '988881adc9fc3655077dc2d4d757d480b5ea0e11' + self.fh.download_and_validate(dlurl, dlhash) + vfmock.assert_called_with('/tmp/tmpebM9Hv', dlhash, 'sha1') diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_bzrurl.py b/nrpe/mod/charmhelpers/tests/fetch/test_bzrurl.py new file mode 100644 index 0000000..4b71209 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_bzrurl.py @@ -0,0 +1,136 @@ +import os +import shutil +import subprocess +import tempfile +from testtools import TestCase +from mock import ( + MagicMock, + patch, +) + +import six +if six.PY3: + from urllib.parse import urlparse +else: + from urlparse import urlparse + +try: + from charmhelpers.fetch import ( + bzrurl, + UnhandledSource, + ) +except ImportError: + bzrurl = None + UnhandledSource = None + + +class BzrUrlFetchHandlerTest(TestCase): + + def setUp(self): + super(BzrUrlFetchHandlerTest, self).setUp() + self.valid_urls = ( + "bzr+ssh://example.com/branch-name", + "bzr+ssh://example.com/branch-name/", + "lp:lp-branch-name", + "lp:example/lp-branch-name", + ) + self.invalid_urls = ( + "http://example.com/foo.tar.gz", + "http://example.com/foo.tgz", + "http://example.com/foo.tar.bz2", + "http://example.com/foo.tbz2", + "http://example.com/foo.zip", + "http://example.com/foo.zip?bar=baz&x=y#whee", + "ftp://example.com/foo.tar.gz", + "https://example.com/foo.tgz", + "file://example.com/foo.tar.bz2", + "git://example.com/foo.tar.gz", + "http://example.com/foo", + "http://example.com/foobar=baz&x=y#tar.gz", + "http://example.com/foobar?h=baz.zip", + "abc:example", + "file//example.com/foo.tar.bz2", + "garbage", + ) + self.fh = bzrurl.BzrUrlFetchHandler() + + def test_handles_bzr_urls(self): + for url in self.valid_urls: + result = self.fh.can_handle(url) + self.assertEqual(result, True, url) + for url in self.invalid_urls: + result = self.fh.can_handle(url) + self.assertNotEqual(result, True, url) + + @patch('charmhelpers.fetch.bzrurl.check_output') + def test_branch(self, check_output): + dest_path = "/destination/path" + for url in self.valid_urls: + self.fh.remote_branch = MagicMock() + self.fh.load_plugins = MagicMock() + self.fh.branch(url, dest_path) + + check_output.assert_called_with(['bzr', 'branch', url, dest_path], stderr=-2) + + for url in self.invalid_urls: + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.assertRaises(UnhandledSource, self.fh.branch, + url, dest_path) + + @patch('charmhelpers.fetch.bzrurl.check_output') + def test_branch_revno(self, check_output): + dest_path = "/destination/path" + for url in self.valid_urls: + self.fh.remote_branch = MagicMock() + self.fh.load_plugins = MagicMock() + self.fh.branch(url, dest_path, revno=42) + + check_output.assert_called_with(['bzr', 'branch', '-r', '42', + url, dest_path], stderr=-2) + + for url in self.invalid_urls: + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.assertRaises(UnhandledSource, self.fh.branch, url, + dest_path) + + def test_branch_functional(self): + src = None + dst = None + try: + src = tempfile.mkdtemp() + subprocess.check_output(['bzr', 'init', src], stderr=subprocess.STDOUT) + dst = tempfile.mkdtemp() + os.rmdir(dst) + self.fh.branch(src, dst) + assert os.path.exists(os.path.join(dst, '.bzr')) + self.fh.branch(src, dst) # idempotent + assert os.path.exists(os.path.join(dst, '.bzr')) + finally: + if src: + shutil.rmtree(src, ignore_errors=True) + if dst: + shutil.rmtree(dst, ignore_errors=True) + + def test_installs(self): + self.fh.branch = MagicMock() + + for url in self.valid_urls: + branch_name = urlparse(url).path.strip("/").split("/")[-1] + dest = os.path.join('foo', 'fetched') + dest_dir = os.path.join(dest, os.path.basename(branch_name)) + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + where = self.fh.install(url) + self.assertEqual(where, dest_dir) + + @patch('charmhelpers.fetch.bzrurl.mkdir') + def test_installs_dir(self, _mkdir): + self.fh.branch = MagicMock() + + for url in self.valid_urls: + branch_name = urlparse(url).path.strip("/").split("/")[-1] + dest = os.path.join('opt', 'f') + dest_dir = os.path.join(dest, os.path.basename(branch_name)) + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + where = self.fh.install(url, dest) + self.assertEqual(where, dest_dir) + _mkdir.assert_called_with(dest, perms=0o755) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_fetch.py b/nrpe/mod/charmhelpers/tests/fetch/test_fetch.py new file mode 100644 index 0000000..202467f --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_fetch.py @@ -0,0 +1,269 @@ +import six +import os +import yaml + +from testtools import TestCase +from mock import ( + patch, + MagicMock, + call, +) + +from charmhelpers import fetch + +if six.PY3: + from urllib.parse import urlparse + builtin_open = 'builtins.open' +else: + from urlparse import urlparse + builtin_open = '__builtin__.open' + + +FAKE_APT_CACHE = { + # an installed package + 'vim': { + 'current_ver': '2:7.3.547-6ubuntu5' + }, + # a uninstalled installation candidate + 'emacs': { + } +} + + +def fake_apt_cache(in_memory=True, progress=None): + def _get(package): + pkg = MagicMock() + if package not in FAKE_APT_CACHE: + raise KeyError + pkg.name = package + if 'current_ver' in FAKE_APT_CACHE[package]: + pkg.current_ver.ver_str = FAKE_APT_CACHE[package]['current_ver'] + else: + pkg.current_ver = None + return pkg + cache = MagicMock() + cache.__getitem__.side_effect = _get + return cache + + +def getenv(update=None): + # return a copy of os.environ with update applied. + # this was necessary because some modules modify os.environment directly + copy = os.environ.copy() + if update is not None: + copy.update(update) + return copy + + +class FetchTest(TestCase): + + @patch('charmhelpers.fetch.log') + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_single_source(self, add_source, config, log): + config.side_effect = ['source', 'key'] + fetch.configure_sources() + add_source.assert_called_with('source', 'key') + + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_null_source(self, add_source, config): + config.side_effect = [None, None] + fetch.configure_sources() + self.assertEqual(add_source.call_count, 0) + + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_empty_source(self, add_source, config): + config.side_effect = ['', ''] + fetch.configure_sources() + self.assertEqual(add_source.call_count, 0) + + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_single_source_no_key(self, add_source, config): + config.side_effect = ['source', None] + fetch.configure_sources() + add_source.assert_called_with('source', None) + + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_multiple_sources(self, add_source, config): + sources = ["sourcea", "sourceb"] + keys = ["keya", None] + config.side_effect = [ + yaml.dump(sources), + yaml.dump(keys) + ] + fetch.configure_sources() + add_source.assert_has_calls([ + call('sourcea', 'keya'), + call('sourceb', None) + ]) + + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_missing_keys(self, add_source, config): + sources = ["sourcea", "sourceb"] + keys = ["keya"] # Second key is missing + config.side_effect = [ + yaml.dump(sources), + yaml.dump(keys) + ] + self.assertRaises(fetch.SourceConfigError, fetch.configure_sources) + + @patch.object(fetch, '_fetch_update') + @patch.object(fetch, 'config') + @patch.object(fetch, 'add_source') + def test_configure_sources_update_called_ubuntu(self, add_source, config, + update): + config.side_effect = ['source', 'key'] + fetch.configure_sources(update=True) + add_source.assert_called_with('source', 'key') + self.assertTrue(update.called) + + +class InstallTest(TestCase): + + def setUp(self): + super(InstallTest, self).setUp() + self.valid_urls = ( + "http://example.com/foo.tar.gz", + "http://example.com/foo.tgz", + "http://example.com/foo.tar.bz2", + "http://example.com/foo.tbz2", + "http://example.com/foo.zip", + "http://example.com/foo.zip?bar=baz&x=y#whee", + "ftp://example.com/foo.tar.gz", + "https://example.com/foo.tgz", + "file://example.com/foo.tar.bz2", + "bzr+ssh://example.com/branch-name", + "bzr+ssh://example.com/branch-name/", + "lp:branch-name", + "lp:example/branch-name", + ) + self.invalid_urls = ( + "git://example.com/foo.tar.gz", + "http://example.com/foo", + "http://example.com/foobar=baz&x=y#tar.gz", + "http://example.com/foobar?h=baz.zip", + "abc:example", + "file//example.com/foo.tar.bz2", + "garbage", + ) + + @patch('charmhelpers.fetch.log') + @patch('charmhelpers.fetch.plugins') + def test_installs_remote(self, _plugins, _log): + h1 = MagicMock(name="h1") + h1.can_handle.return_value = "Nope" + + h2 = MagicMock(name="h2") + h2.can_handle.return_value = True + h2.install.side_effect = fetch.UnhandledSource() + + h3 = MagicMock(name="h3") + h3.can_handle.return_value = True + h3.install.return_value = "foo" + + _plugins.return_value = [h1, h2, h3] + for url in self.valid_urls: + result = fetch.install_remote(url) + + h1.can_handle.assert_called_with(url) + h2.can_handle.assert_called_with(url) + h3.can_handle.assert_called_with(url) + + h1.install.assert_not_called() + h2.install.assert_called_with(url) + h3.install.assert_called_with(url) + + self.assertEqual(result, "foo") + + fetch.install_remote('url', extra_arg=True) + h2.install.assert_called_with('url', extra_arg=True) + + @patch('charmhelpers.fetch.install_remote') + @patch('charmhelpers.fetch.config') + def test_installs_from_config(self, _config, _instrem): + for url in self.valid_urls: + _config.return_value = {"foo": url} + fetch.install_from_config("foo") + _instrem.assert_called_with(url) + + +class PluginTest(TestCase): + + @patch('charmhelpers.fetch.importlib.import_module') + def test_imports_plugins(self, import_): + fetch_handlers = ['a.foo', 'b.foo', 'c.foo'] + module = MagicMock() + import_.return_value = module + plugins = fetch.plugins(fetch_handlers) + + self.assertEqual(len(fetch_handlers), len(plugins)) + module.foo.assert_has_calls(([call()] * len(fetch_handlers))) + + @patch('charmhelpers.fetch.importlib.import_module') + def test_imports_plugins_default(self, import_): + module = MagicMock() + import_.return_value = module + plugins = fetch.plugins() + + self.assertEqual(len(fetch.FETCH_HANDLERS), len(plugins)) + for handler in fetch.FETCH_HANDLERS: + classname = handler.rsplit('.', 1)[-1] + getattr(module, classname).assert_called_with() + + @patch('charmhelpers.fetch.log') + @patch('charmhelpers.fetch.importlib.import_module') + def test_skips_and_logs_missing_plugins(self, import_, log_): + fetch_handlers = ['a.foo', 'b.foo', 'c.foo'] + import_.side_effect = (NotImplementedError, NotImplementedError, + MagicMock()) + plugins = fetch.plugins(fetch_handlers) + + self.assertEqual(1, len(plugins)) + self.assertEqual(2, log_.call_count) + + @patch('charmhelpers.fetch.log') + @patch.object(fetch.importlib, 'import_module') + def test_plugins_are_valid(self, import_module, log_): + plugins = fetch.plugins() + self.assertEqual(len(fetch.FETCH_HANDLERS), len(plugins)) + + +class BaseFetchHandlerTest(TestCase): + + def setUp(self): + super(BaseFetchHandlerTest, self).setUp() + self.test_urls = ( + "http://example.com/foo?bar=baz&x=y#blarg", + "https://example.com/foo", + "ftp://example.com/foo", + "file://example.com/foo", + "git://github.com/foo/bar", + "bzr+ssh://bazaar.launchpad.net/foo/bar", + "bzr+http://bazaar.launchpad.net/foo/bar", + "garbage", + ) + self.fh = fetch.BaseFetchHandler() + + def test_handles_nothing(self): + for url in self.test_urls: + self.assertNotEqual(self.fh.can_handle(url), True) + + def test_install_throws_unhandled(self): + for url in self.test_urls: + self.assertRaises(fetch.UnhandledSource, self.fh.install, url) + + def test_parses_urls(self): + sample_url = "http://example.com/foo?bar=baz&x=y#blarg" + p = self.fh.parse_url(sample_url) + self.assertEqual(p, urlparse(sample_url)) + + def test_returns_baseurl(self): + sample_url = "http://example.com/foo?bar=baz&x=y#blarg" + expected_url = "http://example.com/foo" + u = self.fh.base_url(sample_url) + self.assertEqual(u, expected_url) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_fetch_centos.py b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_centos.py new file mode 100644 index 0000000..43ca852 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_centos.py @@ -0,0 +1,315 @@ +import subprocess +import os + +from tests.helpers import patch_open +from testtools import TestCase +from mock import ( + patch, + MagicMock, + call, +) +from charmhelpers.fetch import centos as fetch + + +def getenv(update=None): + # return a copy of os.environ with update applied. + # this was necessary because some modules modify os.environment directly + copy = os.environ.copy() + if update is not None: + copy.update(update) + return copy + + +class FetchTest(TestCase): + + @patch("charmhelpers.fetch.log") + @patch('yum.YumBase.doPackageLists') + def test_filter_packages_missing_centos(self, yumBase, log): + + class MockPackage: + def __init__(self, name): + self.base_package_name = name + + yum_dict = { + 'installed': { + MockPackage('vim') + }, + 'available': { + MockPackage('vim') + } + } + import yum + yum.YumBase.return_value.doPackageLists.return_value = yum_dict + result = fetch.filter_installed_packages(['vim', 'emacs']) + self.assertEquals(result, ['emacs']) + + @patch("charmhelpers.fetch.log") + def test_filter_packages_none_missing_centos(self, log): + + class MockPackage: + def __init__(self, name): + self.base_package_name = name + + yum_dict = { + 'installed': { + MockPackage('vim') + }, + 'available': { + MockPackage('vim') + } + } + import yum + yum.yumBase.return_value.doPackageLists.return_value = yum_dict + result = fetch.filter_installed_packages(['vim']) + self.assertEquals(result, []) + + @patch('charmhelpers.fetch.centos.log') + @patch('yum.YumBase.doPackageLists') + def test_filter_packages_not_available_centos(self, yumBase, log): + + class MockPackage: + def __init__(self, name): + self.base_package_name = name + + yum_dict = { + 'installed': { + MockPackage('vim') + } + } + import yum + yum.YumBase.return_value.doPackageLists.return_value = yum_dict + + result = fetch.filter_installed_packages(['vim', 'joe']) + self.assertEquals(result, ['joe']) + + @patch('charmhelpers.fetch.centos.log') + def test_add_source_none_centos(self, log): + fetch.add_source(source=None) + self.assertTrue(log.called) + + @patch('charmhelpers.fetch.centos.log') + @patch('os.listdir') + def test_add_source_http_centos(self, listdir, log): + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + listdir.assert_called_with('/etc/yum.repos.d/') + mock_file.write.assert_has_calls([ + call("[archive.ubuntu.com_ubuntu raring-backports main]\n"), + call("name=archive.ubuntu.com/ubuntu raring-backports main\n"), + call("baseurl=http://archive.ubuntu.com/ubuntu raring" + "-backports main\n\n")]) + + @patch('charmhelpers.fetch.centos.log') + @patch('os.listdir') + @patch('subprocess.check_call') + def test_add_source_http_and_key_id_centos(self, check_call, + listdir, log): + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + key_id = "akey" + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source, key=key_id) + listdir.assert_called_with('/etc/yum.repos.d/') + mock_file.write.assert_has_calls([ + call("[archive.ubuntu.com_ubuntu raring-backports main]\n"), + call("name=archive.ubuntu.com/ubuntu raring-backports main\n"), + call("baseurl=http://archive.ubuntu.com/ubuntu raring" + "-backports main\n\n")]) + check_call.assert_called_with(['rpm', '--import', key_id]) + + @patch('charmhelpers.fetch.centos.log') + @patch('os.listdir') + @patch('subprocess.check_call') + def test_add_source_https_and_key_id_centos(self, check_call, + listdir, log): + source = "https://USER:PASS@private-ppa.launchpad.net/project/awesome" + key_id = "GPGPGP" + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source, key=key_id) + listdir.assert_called_with('/etc/yum.repos.d/') + mock_file.write.assert_has_calls([ + call("[_USER:PASS@private-ppa.launchpad" + ".net_project_awesome]\n"), + call("name=/USER:PASS@private-ppa.launchpad.net" + "/project/awesome\n"), + call("baseurl=https://USER:PASS@private-ppa.launchpad.net" + "/project/awesome\n\n")]) + check_call.assert_called_with(['rpm', '--import', key_id]) + + @patch('charmhelpers.fetch.centos.log') + @patch.object(fetch, 'NamedTemporaryFile') + @patch('os.listdir') + @patch('subprocess.check_call') + def test_add_source_http_and_key_centos(self, check_call, + listdir, temp_file, log): + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + key = ''' + -----BEGIN PGP PUBLIC KEY BLOCK----- + [...] + -----END PGP PUBLIC KEY BLOCK----- + ''' + file_mock = MagicMock() + file_mock.name = 'temporary_file' + temp_file.return_value.__enter__.return_value = file_mock + listdir.return_value = [] + + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source, key=key) + listdir.assert_called_with('/etc/yum.repos.d/') + self.assertTrue(log.called) + check_call.assert_called_with(['rpm', '--import', file_mock.name]) + file_mock.write.assert_called_once_with(key) + file_mock.flush.assert_called_once_with() + file_mock.seek.assert_called_once_with(0) + + +class YumTests(TestCase): + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_yum_upgrade_non_fatal(self, log, mock_call): + options = ['--foo', '--bar'] + fetch.upgrade(options) + + mock_call.assert_called_with(['yum', '--assumeyes', + '--foo', '--bar', 'upgrade'], + env=getenv()) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.centos.log') + def test_yum_upgrade_fatal(self, log, mock_call): + options = ['--foo', '--bar'] + fetch.upgrade(options, fatal=True) + + mock_call.assert_called_with(['yum', '--assumeyes', + '--foo', '--bar', 'upgrade'], + env=getenv()) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_installs_yum_packages(self, log, mock_call): + packages = ['foo', 'bar'] + options = ['--foo', '--bar'] + + fetch.install(packages, options) + + mock_call.assert_called_with(['yum', '--assumeyes', + '--foo', '--bar', 'install', + 'foo', 'bar'], + env=getenv()) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_installs_yum_packages_without_options(self, log, mock_call): + packages = ['foo', 'bar'] + fetch.install(packages) + + mock_call.assert_called_with(['yum', '--assumeyes', + 'install', 'foo', 'bar'], + env=getenv()) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_installs_yum_packages_as_string(self, log, mock_call): + packages = 'foo bar' + fetch.install(packages) + + mock_call.assert_called_with(['yum', '--assumeyes', + 'install', 'foo bar'], + env=getenv()) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.centos.log') + def test_installs_yum_packages_with_possible_errors(self, log, mock_call): + packages = ['foo', 'bar'] + options = ['--foo', '--bar'] + + fetch.install(packages, options, fatal=True) + + mock_call.assert_called_with(['yum', '--assumeyes', + '--foo', '--bar', + 'install', 'foo', 'bar'], + env=getenv()) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.centos.log') + def test_purges_yum_packages_as_string_fatal(self, log, mock_call): + packages = 'irrelevant names' + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.purge, packages, fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.centos.log') + def test_purges_yum_packages_fatal(self, log, mock_call): + packages = ['irrelevant', 'names'] + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.purge, packages, fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_purges_yum_packages_as_string_nofatal(self, log, mock_call): + packages = 'foo bar' + fetch.purge(packages) + + self.assertTrue(log.called) + mock_call.assert_called_with(['yum', '--assumeyes', + 'remove', 'foo bar'], + env=getenv()) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.centos.log') + def test_purges_yum_packages_nofatal(self, log, mock_call): + packages = ['foo', 'bar'] + fetch.purge(packages) + + self.assertTrue(log.called) + mock_call.assert_called_with(['yum', '--assumeyes', + 'remove', 'foo', 'bar'], + env=getenv()) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.centos.log') + def test_yum_update_fatal(self, log, check_call): + fetch.update(fatal=True) + check_call.assert_called_with(['yum', '--assumeyes', 'update'], + env=getenv()) + self.assertTrue(log.called) + + @patch('subprocess.check_output') + @patch('charmhelpers.fetch.centos.log') + def test_yum_search(self, log, check_output): + package = ['irrelevant'] + + from charmhelpers.fetch.centos import yum_search + yum_search(package) + check_output.assert_called_with(['yum', 'search', 'irrelevant']) + self.assertTrue(log.called) + + @patch('subprocess.check_call') + @patch('time.sleep') + def test_run_yum_command_retries_if_fatal(self, check_call, sleep): + """The _run_yum_command function retries the command if it can't get + the YUM lock.""" + self.called = False + + def side_effect(*args, **kwargs): + """ + First, raise an exception (can't acquire lock), then return 0 + (the lock is grabbed). + """ + if not self.called: + self.called = True + raise subprocess.CalledProcessError( + returncode=1, cmd="some command") + else: + return 0 + + check_call.side_effect = side_effect + check_call.return_value = 0 + from charmhelpers.fetch.centos import _run_yum_command + _run_yum_command(["some", "command"], fatal=True) + self.assertTrue(sleep.called) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu.py b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu.py new file mode 100644 index 0000000..1eda712 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu.py @@ -0,0 +1,1111 @@ +import six +import subprocess +import io +import os + +from tests.helpers import patch_open +from testtools import TestCase +from mock import ( + patch, + MagicMock, + call, + sentinel, +) +from charmhelpers.fetch import ubuntu as fetch + +if six.PY3: + builtin_open = 'builtins.open' +else: + builtin_open = '__builtin__.open' + +# mocked return of openstack.get_distrib_codename() +FAKE_CODENAME = 'precise' + +url = 'deb ' + fetch.CLOUD_ARCHIVE_URL +UCA_SOURCES = [ + ('cloud:precise-folsom/proposed', url + ' precise-proposed/folsom main'), + ('cloud:precise-folsom', url + ' precise-updates/folsom main'), + ('cloud:precise-folsom/updates', url + ' precise-updates/folsom main'), + ('cloud:precise-grizzly/proposed', url + ' precise-proposed/grizzly main'), + ('cloud:precise-grizzly', url + ' precise-updates/grizzly main'), + ('cloud:precise-grizzly/updates', url + ' precise-updates/grizzly main'), + ('cloud:precise-havana/proposed', url + ' precise-proposed/havana main'), + ('cloud:precise-havana', url + ' precise-updates/havana main'), + ('cloud:precise-havana/updates', url + ' precise-updates/havana main'), + ('cloud:precise-icehouse/proposed', + url + ' precise-proposed/icehouse main'), + ('cloud:precise-icehouse', url + ' precise-updates/icehouse main'), + ('cloud:precise-icehouse/updates', url + ' precise-updates/icehouse main'), +] + +PGP_KEY_ASCII_ARMOR = """-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: SKS 1.1.5 +Comment: Hostname: keyserver.ubuntu.com + +mI0EUCEyTAEEAMuUxyfiegCCwn4J/c0nw5PUTSJdn5FqiUTq6iMfij65xf1vl0g/Mxqw0gfg +AJIsCDvO9N9dloLAwF6FUBMg5My7WyhRPTAKF505TKJboyX3Pp4J1fU1LV8QFVOp87vUh1Rz +B6GU7cSglhnbL85gmbJTllkzkb3h4Yw7W+edjcQ/ABEBAAG0K0xhdW5jaHBhZCBQUEEgZm9y +IFVidW50dSBDbG91ZCBBcmNoaXZlIFRlYW2IuAQTAQIAIgUCUCEyTAIbAwYLCQgHAwIGFQgC +CQoLBBYCAwECHgECF4AACgkQimhEop9oEE7kJAP/eTBgq3Mhbvo0d8elMOuqZx3nmU7gSyPh +ep0zYIRZ5TJWl/7PRtvp0CJA6N6ZywYTQ/4ANHhpibcHZkh8K0AzUvsGXnJRSFoJeqyDbD91 +EhoO+4ZfHs2HvRBQEDZILMa2OyuB497E5Mmyua3HDEOrG2cVLllsUZzpTFCx8NgeMHk= +=jLBm +-----END PGP PUBLIC KEY BLOCK-----""" + +PGP_KEY_BIN_PGP = b'\x98\x8d\x04P!2L\x01\x04\x00\xcb\x94\xc7\'\xe2z\x00\x82\xc2~\t\xfd\xcd\'\xc3\x93\xd4M"]\x9f\x91j\x89D\xea\xea#\x1f\x8a>\xb9\xc5\xfdo\x97H?3\x1a\xb0\xd2\x07\xe0\x00\x92,\x08;\xce\xf4\xdf]\x96\x82\xc0\xc0^\x85P\x13 \xe4\xcc\xbb[(Q=0\n\x17\x9d9L\xa2[\xa3%\xf7>\x9e\t\xd5\xf55-_\x10\x15S\xa9\xf3\xbb\xd4\x87Ts\x07\xa1\x94\xed\xc4\xa0\x96\x19\xdb/\xce`\x99\xb2S\x96Y3\x91\xbd\xe1\xe1\x8c;[\xe7\x9d\x8d\xc4?\x00\x11\x01\x00\x01\xb4+Launchpad PPA for Ubuntu Cloud Archive Team\x88\xb8\x04\x13\x01\x02\x00"\x05\x02P!2L\x02\x1b\x03\x06\x0b\t\x08\x07\x03\x02\x06\x15\x08\x02\t\n\x0b\x04\x16\x02\x03\x01\x02\x1e\x01\x02\x17\x80\x00\n\t\x10\x8ahD\xa2\x9fh\x10N\xe4$\x03\xffy0`\xabs!n\xfa4w\xc7\xa50\xeb\xaag\x1d\xe7\x99N\xe0K#\xe1z\x9d3`\x84Y\xe52V\x97\xfe\xcfF\xdb\xe9\xd0"@\xe8\xde\x99\xcb\x06\x13C\xfe\x004xi\x89\xb7\x07fH|+@3R\xfb\x06^rQHZ\tz\xac\x83l?u\x12\x1a\x0e\xfb\x86_\x1e\xcd\x87\xbd\x10P\x106H,\xc6\xb6;+\x81\xe3\xde\xc4\xe4\xc9\xb2\xb9\xad\xc7\x0cC\xab\x1bg\x15.YlQ\x9c\xe9LP\xb1\xf0\xd8\x1e0y' # noqa + +# a keyid can be retrieved by the ASCII armor-encoded key using this: +# cat testkey.asc | gpg --with-colons --import-options import-show --dry-run +# --import +PGP_KEY_ID = '8a6844a29f68104e' + +FAKE_APT_CACHE = { + # an installed package + 'vim': { + 'current_ver': '2:7.3.547-6ubuntu5' + }, + # a uninstalled installation candidate + 'emacs': { + } +} + + +def fake_apt_cache(in_memory=True, progress=None): + def _get(package): + pkg = MagicMock() + if package not in FAKE_APT_CACHE: + raise KeyError + pkg.name = package + if 'current_ver' in FAKE_APT_CACHE[package]: + pkg.current_ver.ver_str = FAKE_APT_CACHE[package]['current_ver'] + else: + pkg.current_ver = None + return pkg + cache = MagicMock() + cache.__getitem__.side_effect = _get + return cache + + +class FetchTest(TestCase): + + def setUp(self): + super(FetchTest, self).setUp() + self.patch(fetch, 'get_apt_dpkg_env', lambda: {}) + + @patch("charmhelpers.fetch.ubuntu.log") + @patch.object(fetch, 'apt_cache') + def test_filter_packages_missing_ubuntu(self, cache, log): + cache.side_effect = fake_apt_cache + result = fetch.filter_installed_packages(['vim', 'emacs']) + self.assertEquals(result, ['emacs']) + + @patch("charmhelpers.fetch.ubuntu.log") + @patch.object(fetch, 'apt_cache') + def test_filter_packages_none_missing_ubuntu(self, cache, log): + cache.side_effect = fake_apt_cache + result = fetch.filter_installed_packages(['vim']) + self.assertEquals(result, []) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'apt_cache') + def test_filter_packages_not_available_ubuntu(self, cache, log): + cache.side_effect = fake_apt_cache + result = fetch.filter_installed_packages(['vim', 'joe']) + self.assertEquals(result, ['joe']) + log.assert_called_with('Package joe has no installation candidate.', + level='WARNING') + + @patch('charmhelpers.fetch.ubuntu.filter_installed_packages') + def test_filter_missing_packages(self, filter_installed_packages): + filter_installed_packages.return_value = ['pkga'] + self.assertEqual(['pkgb'], + fetch.filter_missing_packages(['pkga', 'pkgb'])) + + @patch.object(fetch, 'log', lambda *args, **kwargs: None) + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + def test_import_apt_key_radix(self, dearmor_gpg_key, + w_keyfile): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + with patch('subprocess.check_output') as _subp_check_output: + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + _subp_check_output.side_effect = check_output_side_effect + + fetch.import_key(PGP_KEY_ID) + _subp_check_output.assert_called_with(curl_cmd, env=None) + w_keyfile.assert_called_once_with(key_name=PGP_KEY_ID, + key_material=PGP_KEY_BIN_PGP) + + @patch.object(fetch, 'log', lambda *args, **kwargs: None) + @patch.object(os, 'getenv') + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + def test_import_apt_key_radix_https_proxy(self, dearmor_gpg_key, + w_keyfile, getenv): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + def get_env_side_effect(var): + return { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'JUJU_CHARM_HTTPS_PROXY': None, + }[var] + getenv.side_effect = get_env_side_effect + + with patch('subprocess.check_output') as _subp_check_output: + proxy_settings = { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'https_proxy': 'http://squid.internal:3128', + } + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + _subp_check_output.side_effect = check_output_side_effect + + fetch.import_key(PGP_KEY_ID) + _subp_check_output.assert_called_with(curl_cmd, env=proxy_settings) + w_keyfile.assert_called_once_with(key_name=PGP_KEY_ID, + key_material=PGP_KEY_BIN_PGP) + + @patch.object(fetch, 'log', lambda *args, **kwargs: None) + @patch.object(os, 'getenv') + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + def test_import_apt_key_radix_charm_https_proxy(self, dearmor_gpg_key, + w_keyfile, getenv): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + def get_env_side_effect(var): + return { + 'HTTPS_PROXY': None, + 'JUJU_CHARM_HTTPS_PROXY': 'http://squid.internal:3128', + }[var] + getenv.side_effect = get_env_side_effect + + with patch('subprocess.check_output') as _subp_check_output: + proxy_settings = { + 'HTTPS_PROXY': 'http://squid.internal:3128', + 'https_proxy': 'http://squid.internal:3128', + } + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + _subp_check_output.side_effect = check_output_side_effect + + fetch.import_key(PGP_KEY_ID) + _subp_check_output.assert_called_with(curl_cmd, env=proxy_settings) + w_keyfile.assert_called_once_with(key_name=PGP_KEY_ID, + key_material=PGP_KEY_BIN_PGP) + + @patch.object(fetch, 'log', lambda *args, **kwargs: None) + @patch.object(fetch, '_dearmor_gpg_key') + @patch('subprocess.check_output') + def test_import_bad_apt_key(self, check_output, dearmor_gpg_key): + """Ensure error when importing apt key fails""" + errmsg = ('Invalid GPG key material. Check your network setup' + ' (MTU, routing, DNS) and/or proxy server settings' + ' as well as destination keyserver status.') + bad_keyid = 'foo' + + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(bad_keyid)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): 'foobar', + }[' '.join(command)] + check_output.side_effect = check_output_side_effect + + def dearmor_side_effect(key_asc): + raise fetch.GPGKeyError(errmsg) + dearmor_gpg_key.side_effect = dearmor_side_effect + try: + fetch.import_key(bad_keyid) + assert False + except fetch.GPGKeyError as e: + self.assertEqual(str(e), errmsg) + + @patch('charmhelpers.fetch.ubuntu.log') + def test_add_source_none_ubuntu(self, log): + fetch.add_source(source=None) + self.assertTrue(log.called) + + @patch('subprocess.check_call') + def test_add_source_ppa(self, check_call): + source = "ppa:test-ppa" + fetch.add_source(source=source) + check_call.assert_called_with( + ['add-apt-repository', '--yes', source], env={}) + + @patch("charmhelpers.fetch.ubuntu.log") + @patch('subprocess.check_call') + @patch('time.sleep') + def test_add_source_ppa_retries_30_times(self, sleep, check_call, log): + self.call_count = 0 + + def side_effect(*args, **kwargs): + """Raise an 3 times, then return 0 """ + self.call_count += 1 + if self.call_count <= fetch.CMD_RETRY_COUNT: + raise subprocess.CalledProcessError( + returncode=1, cmd="some add-apt-repository command") + else: + return 0 + check_call.side_effect = side_effect + + source = "ppa:test-ppa" + fetch.add_source(source=source) + check_call.assert_called_with( + ['add-apt-repository', '--yes', source], env={}) + sleep.assert_called_with(10) + self.assertTrue(fetch.CMD_RETRY_COUNT, sleep.call_count) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_call') + def test_add_source_http_ubuntu(self, check_call, log): + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + fetch.add_source(source=source) + check_call.assert_called_with( + ['add-apt-repository', '--yes', source], env={}) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_call') + def test_add_source_https(self, check_call, log): + source = "https://example.com" + fetch.add_source(source=source) + check_call.assert_called_with( + ['add-apt-repository', '--yes', source], env={}) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_call') + def test_add_source_deb(self, check_call, log): + """add-apt-repository behaves differently when using the deb prefix. + + $ add-apt-repository --yes \ + "http://special.example.com/ubuntu precise-special main" + $ grep special /etc/apt/sources.list + deb http://special.example.com/ubuntu precise precise-special main + deb-src http://special.example.com/ubuntu precise precise-special main + + $ add-apt-repository --yes \ + "deb http://special.example.com/ubuntu precise-special main" + $ grep special /etc/apt/sources.list + deb http://special.example.com/ubuntu precise precise-special main + deb-src http://special.example.com/ubuntu precise precise-special main + deb http://special.example.com/ubuntu precise-special main + deb-src http://special.example.com/ubuntu precise-special main + """ + source = "deb http://archive.ubuntu.com/ubuntu raring-backports main" + fetch.add_source(source=source) + check_call.assert_called_with( + ['add-apt-repository', '--yes', source], env={}) + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_add_source_http_and_key_id(self, check_call, check_output, log, + dearmor_gpg_key, + w_keyfile): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + check_output.side_effect = check_output_side_effect + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + check_call.return_value = 0 # Successful exit code + fetch.add_source(source=source, key=PGP_KEY_ID) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + check_output.assert_has_calls([ + call(['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)], + env=None), + ]) + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_add_source_https_and_key_id(self, check_call, check_output, log, + dearmor_gpg_key, + w_keyfile): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + check_output.side_effect = check_output_side_effect + + check_call.return_value = 0 + + source = "https://USER:PASS@private-ppa.launchpad.net/project/awesome" + fetch.add_source(source=source, key=PGP_KEY_ID) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + check_output.assert_has_calls([ + call(['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)], + env=None), + ]) + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'get_distrib_codename') + @patch('subprocess.check_call') + @patch('subprocess.Popen') + def test_add_source_http_and_key_gpg1(self, popen, check_call, + get_distrib_codename, log, + dearmor_gpg_key, + w_keyfile): + + def check_call_side_effect(*args, **kwargs): + # Make sure the gpg key has already been added before the + # add-apt-repository call, as the update could fail otherwise. + popen.assert_called_with( + ['gpg', '--with-colons', '--with-fingerprint'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + return 0 + + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + key = PGP_KEY_ASCII_ARMOR + key_bytes = PGP_KEY_ASCII_ARMOR.encode('utf-8') + get_distrib_codename.return_value = 'trusty' + check_call.side_effect = check_call_side_effect + + expected_key = '35F77D63B5CEC106C577ED856E85A86E4652B4E6' + if six.PY3: + popen.return_value.communicate.return_value = [b""" +pub:-:1024:1:6E85A86E4652B4E6:2009-01-18:::-:Launchpad PPA for Landscape: +fpr:::::::::35F77D63B5CEC106C577ED856E85A86E4652B4E6: + """, b''] + else: + popen.return_value.communicate.return_value = [""" +pub:-:1024:1:6E85A86E4652B4E6:2009-01-18:::-:Launchpad PPA for Landscape: +fpr:::::::::35F77D63B5CEC106C577ED856E85A86E4652B4E6: + """, ''] + + dearmor_gpg_key.return_value = PGP_KEY_BIN_PGP + + fetch.add_source(source=source, key=key) + popen.assert_called_with( + ['gpg', '--with-colons', '--with-fingerprint'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + dearmor_gpg_key.assert_called_with(key_bytes) + w_keyfile.assert_called_with(key_name=expected_key, + key_material=PGP_KEY_BIN_PGP) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'get_distrib_codename') + @patch('subprocess.check_call') + @patch('subprocess.Popen') + def test_add_source_http_and_key_gpg2(self, popen, check_call, + get_distrib_codename, log, + dearmor_gpg_key, + w_keyfile): + + def check_call_side_effect(*args, **kwargs): + # Make sure the gpg key has already been added before the + # add-apt-repository call, as the update could fail otherwise. + popen.assert_called_with( + ['gpg', '--with-colons', '--with-fingerprint'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + return 0 + + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + key = PGP_KEY_ASCII_ARMOR + key_bytes = PGP_KEY_ASCII_ARMOR.encode('utf-8') + get_distrib_codename.return_value = 'bionic' + check_call.side_effect = check_call_side_effect + + expected_key = '35F77D63B5CEC106C577ED856E85A86E4652B4E6' + + if six.PY3: + popen.return_value.communicate.return_value = [b""" +fpr:::::::::35F77D63B5CEC106C577ED856E85A86E4652B4E6: +uid:-::::1232306042::52FE92E6867B4C099AA1A1877A804A965F41A98C::ppa::::::::::0: + """, b''] + else: + # python2 on a distro with gpg2 (unlikely, but possible) + popen.return_value.communicate.return_value = [""" +fpr:::::::::35F77D63B5CEC106C577ED856E85A86E4652B4E6: +uid:-::::1232306042::52FE92E6867B4C099AA1A1877A804A965F41A98C::ppa::::::::::0: + """, ''] + + dearmor_gpg_key.return_value = PGP_KEY_BIN_PGP + + fetch.add_source(source=source, key=key) + popen.assert_called_with( + ['gpg', '--with-colons', '--with-fingerprint'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + dearmor_gpg_key.assert_called_with(key_bytes) + w_keyfile.assert_called_with(key_name=expected_key, + key_material=PGP_KEY_BIN_PGP) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + + def test_add_source_cloud_invalid_pocket(self): + source = "cloud:havana-updates" + self.assertRaises(fetch.SourceConfigError, + fetch.add_source, source) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + @patch.object(fetch, 'get_distrib_codename') + def test_add_source_cloud_pocket_style(self, get_distrib_codename, + apt_install, + filter_pkg, log): + source = "cloud:precise-updates/havana" + get_distrib_codename.return_value = 'precise' + result = ('# Ubuntu Cloud Archive\n' + 'deb http://ubuntu-cloud.archive.canonical.com/ubuntu' + ' precise-updates/havana main\n') + + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + @patch.object(fetch, 'get_distrib_codename') + def test_add_source_cloud_os_style(self, get_distrib_codename, apt_install, + filter_pkg, log): + source = "cloud:precise-havana" + get_distrib_codename.return_value = 'precise' + result = ('# Ubuntu Cloud Archive\n' + 'deb http://ubuntu-cloud.archive.canonical.com/ubuntu' + ' precise-updates/havana main\n') + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'filter_installed_packages') + @patch.object(fetch, 'apt_install') + def test_add_source_cloud_distroless_style(self, apt_install, + filter_pkg, log): + source = "cloud:havana" + result = ('# Ubuntu Cloud Archive\n' + 'deb http://ubuntu-cloud.archive.canonical.com/ubuntu' + ' precise-updates/havana main\n') + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + mock_file.write.assert_called_with(result) + filter_pkg.assert_called_with(['ubuntu-cloud-keyring']) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'get_distrib_codename') + @patch('platform.machine') + def test_add_source_proposed_x86_64(self, _machine, + get_distrib_codename, log): + source = "proposed" + result = ('# Proposed\n' + 'deb http://archive.ubuntu.com/ubuntu precise-proposed' + ' main universe multiverse restricted\n') + get_distrib_codename.return_value = 'precise' + _machine.return_value = 'x86_64' + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + mock_file.write.assert_called_with(result) + + @patch('charmhelpers.fetch.ubuntu.log') + @patch.object(fetch, 'get_distrib_codename') + @patch('platform.machine') + def test_add_source_proposed_ppc64le(self, _machine, + get_distrib_codename, log): + source = "proposed" + result = ( + "# Proposed\n" + "deb http://ports.ubuntu.com/ubuntu-ports precise-proposed main " + "universe multiverse restricted\n") + get_distrib_codename.return_value = 'precise' + _machine.return_value = 'ppc64le' + with patch_open() as (mock_open, mock_file): + fetch.add_source(source=source) + mock_file.write.assert_called_with(result) + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_add_source_http_and_key_id_ubuntu(self, check_call, check_output, + log, dearmor_gpg_key, + w_keyfile): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + check_output.side_effect = check_output_side_effect + check_call.return_value = 0 + source = "http://archive.ubuntu.com/ubuntu raring-backports main" + key_id = PGP_KEY_ID + fetch.add_source(source=source, key=key_id) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + check_output.assert_has_calls([ + call(['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)], + env=None), + ]) + + @patch.object(fetch, '_write_apt_gpg_keyfile') + @patch.object(fetch, '_dearmor_gpg_key') + @patch('charmhelpers.fetch.ubuntu.log') + @patch('subprocess.check_output') + @patch('subprocess.check_call') + def test_add_source_https_and_key_id_ubuntu(self, check_call, check_output, + log, dearmor_gpg_key, + w_keyfile): + def dearmor_side_effect(key_asc): + return { + PGP_KEY_ASCII_ARMOR: PGP_KEY_BIN_PGP, + }[key_asc] + dearmor_gpg_key.side_effect = dearmor_side_effect + + curl_cmd = ['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)] + + def check_output_side_effect(command, env): + return { + ' '.join(curl_cmd): PGP_KEY_ASCII_ARMOR, + }[' '.join(command)] + check_output.side_effect = check_output_side_effect + check_call.return_value = 0 + + source = "https://USER:PASS@private-ppa.launchpad.net/project/awesome" + fetch.add_source(source=source, key=PGP_KEY_ID) + check_call.assert_any_call( + ['add-apt-repository', '--yes', source], env={}), + check_output.assert_has_calls([ + call(['curl', ('https://keyserver.ubuntu.com' + '/pks/lookup?op=get&options=mr' + '&exact=on&search=0x{}').format(PGP_KEY_ID)], + env=None), + ]) + + @patch('charmhelpers.fetch.ubuntu.log') + def test_configure_bad_install_source(self, log): + try: + fetch.add_source('foo', fail_invalid=True) + self.fail("Calling add_source('foo') should fail") + except fetch.SourceConfigError as e: + self.assertEqual(str(e), "Unknown source: 'foo'") + + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_configure_install_source_uca_staging(self, _lsb): + """Test configuring installation source from UCA staging sources""" + _lsb.return_value = FAKE_CODENAME + # staging pockets are configured as PPAs + with patch('subprocess.check_call') as _subp: + src = 'cloud:precise-folsom/staging' + fetch.add_source(src) + cmd = ['add-apt-repository', '-y', + 'ppa:ubuntu-cloud-archive/folsom-staging'] + _subp.assert_called_with(cmd, env={}) + + @patch(builtin_open) + @patch('charmhelpers.fetch.ubuntu.apt_install') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + @patch('charmhelpers.fetch.ubuntu.filter_installed_packages') + def test_configure_install_source_uca_repos( + self, _fip, _lsb, _install, _open): + """Test configuring installation source from UCA sources""" + _lsb.return_value = FAKE_CODENAME + _file = MagicMock(spec=io.FileIO) + _open.return_value = _file + _fip.side_effect = lambda x: x + for src, url in UCA_SOURCES: + actual_url = "# Ubuntu Cloud Archive\n{}\n".format(url) + fetch.add_source(src) + _install.assert_called_with(['ubuntu-cloud-keyring'], + fatal=True) + _open.assert_called_with( + '/etc/apt/sources.list.d/cloud-archive.list', + 'w' + ) + _file.__enter__().write.assert_called_with(actual_url) + + def test_configure_install_source_bad_uca(self): + """Test configuring installation source from bad UCA source""" + try: + fetch.add_source('cloud:foo-bar', fail_invalid=True) + self.fail("add_source('cloud:foo-bar') should fail") + except fetch.SourceConfigError as e: + _e = ('Invalid Cloud Archive release specified: foo-bar' + ' on this Ubuntuversion') + self.assertTrue(str(e).startswith(_e)) + + @patch('charmhelpers.fetch.ubuntu.log') + def test_add_unparsable_source(self, log_): + source = "propsed" # Minor typo + fetch.add_source(source=source) + self.assertEqual(1, log_.call_count) + + @patch('charmhelpers.fetch.ubuntu.log') + def test_add_distro_source(self, log): + source = "distro" + # distro is a noop but test validate no exception is thrown + fetch.add_source(source=source) + + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_is_distro( + self, mock_get_distrib_codename, mock_add_cloud_pocket): + mock_get_distrib_codename.return_value = 'focal' + fetch.add_source('ussuri') + mock_add_cloud_pocket.assert_not_called() + + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_is_cloud_pocket( + self, mock_get_distrib_codename, mock_add_cloud_pocket): + mock_get_distrib_codename.return_value = 'bionic' + fetch.add_source('ussuri') + mock_add_cloud_pocket.assert_called_once_with("bionic-ussuri") + + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_impossible_version( + self, mock_get_distrib_codename, mock_add_cloud_pocket): + mock_get_distrib_codename.return_value = 'xenial' + try: + fetch.add_source('ussuri') + self.fail("add_source('ussuri') on xenial should fail") + except fetch.SourceConfigError: + pass + mock_add_cloud_pocket.assert_not_called() + + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_impossible_ubuntu( + self, mock_get_distrib_codename, mock_add_cloud_pocket): + mock_get_distrib_codename.return_value = 'bambam' + try: + fetch.add_source('ussuri') + self.fail("add_source('ussuri') on bambam should fail") + except fetch.SourceConfigError: + pass + mock_add_cloud_pocket.assert_not_called() + + @patch('charmhelpers.fetch.ubuntu._add_proposed') + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_proposed_is_distro_proposed( + self, mock_get_distrib_codename, mock_add_cloud_pocket, + mock_add_proposed): + mock_get_distrib_codename.return_value = 'focal' + fetch.add_source('ussuri/proposed') + mock_add_cloud_pocket.assert_not_called() + mock_add_proposed.assert_called_once_with() + + @patch('charmhelpers.fetch.ubuntu._add_proposed') + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_proposed_is_cloud_pocket( + self, mock_get_distrib_codename, mock_add_cloud_pocket, + mock_add_proposed): + mock_get_distrib_codename.return_value = 'bionic' + fetch.add_source('ussuri/proposed') + mock_add_cloud_pocket.assert_called_once_with("bionic-ussuri/proposed") + mock_add_proposed.assert_not_called() + + @patch('charmhelpers.fetch.ubuntu._add_proposed') + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_proposed_impossible_version( + self, mock_get_distrib_codename, mock_add_cloud_pocket, + mock_add_proposed): + mock_get_distrib_codename.return_value = 'xenial' + try: + fetch.add_source('ussuri/proposed') + self.fail("add_source('ussuri/proposed') on xenial should fail") + except fetch.SourceConfigError: + pass + mock_add_cloud_pocket.assert_not_called() + mock_add_proposed.assert_not_called() + + @patch('charmhelpers.fetch.ubuntu._add_proposed') + @patch('charmhelpers.fetch.ubuntu._add_cloud_pocket') + @patch('charmhelpers.fetch.ubuntu.get_distrib_codename') + def test_add_bare_openstack_proposed_impossible_ubuntu( + self, mock_get_distrib_codename, mock_add_cloud_pocket, + mock_add_proposed): + mock_get_distrib_codename.return_value = 'bambam' + try: + fetch.add_source('ussuri/proposed') + self.fail("add_source('ussuri/proposed') on bambam should fail") + except fetch.SourceConfigError: + pass + mock_add_cloud_pocket.assert_not_called() + mock_add_proposed.assert_not_called() + + +class AptTests(TestCase): + + def setUp(self): + super(AptTests, self).setUp() + self.patch(fetch, 'get_apt_dpkg_env', lambda: {}) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_apt_upgrade_non_fatal(self, log, mock_call): + options = ['--foo', '--bar'] + fetch.apt_upgrade(options) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'upgrade'], + env={}) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_apt_upgrade_fatal(self, log, mock_call): + options = ['--foo', '--bar'] + fetch.apt_upgrade(options, fatal=True) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'upgrade'], + env={}) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_apt_dist_upgrade_fatal(self, log, mock_call): + options = ['--foo', '--bar'] + fetch.apt_upgrade(options, fatal=True, dist=True) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'dist-upgrade'], + env={}) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_installs_apt_packages(self, log, mock_call): + packages = ['foo', 'bar'] + options = ['--foo', '--bar'] + + fetch.apt_install(packages, options) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'install', 'foo', 'bar'], + env={}) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_installs_apt_packages_without_options(self, log, mock_call): + packages = ['foo', 'bar'] + + fetch.apt_install(packages) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--option=Dpkg::Options::=--force-confold', + 'install', 'foo', 'bar'], + env={}) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_installs_apt_packages_as_string(self, log, mock_call): + packages = 'foo bar' + options = ['--foo', '--bar'] + + fetch.apt_install(packages, options) + + mock_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'install', 'foo bar'], + env={}) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_installs_apt_packages_with_possible_errors(self, log, + check_call): + packages = ['foo', 'bar'] + options = ['--foo', '--bar'] + + fetch.apt_install(packages, options, fatal=True) + + check_call.assert_called_with( + ['apt-get', '--assume-yes', + '--foo', '--bar', 'install', 'foo', 'bar'], + env={}) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_purges_apt_packages_as_string_fatal(self, log, mock_call): + packages = 'irrelevant names' + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.apt_purge, packages, fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_purges_apt_packages_fatal(self, log, mock_call): + packages = ['irrelevant', 'names'] + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.apt_purge, packages, fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_purges_apt_packages_as_string_nofatal(self, log, mock_call): + packages = 'foo bar' + + fetch.apt_purge(packages) + + self.assertTrue(log.called) + mock_call.assert_called_with( + ['apt-get', '--assume-yes', 'purge', 'foo bar'], + env={}) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_purges_apt_packages_nofatal(self, log, mock_call): + packages = ['foo', 'bar'] + + fetch.apt_purge(packages) + + self.assertTrue(log.called) + mock_call.assert_called_with( + ['apt-get', '--assume-yes', 'purge', 'foo', 'bar'], + env={}) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_mark_apt_packages_as_string_fatal(self, log, mock_call): + packages = 'irrelevant names' + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.apt_mark, packages, sentinel.mark, + fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_mark_apt_packages_fatal(self, log, mock_call): + packages = ['irrelevant', 'names'] + mock_call.side_effect = OSError('fail') + + self.assertRaises(OSError, fetch.apt_mark, packages, sentinel.mark, + fatal=True) + self.assertTrue(log.called) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_mark_apt_packages_as_string_nofatal(self, log, mock_call): + packages = 'foo bar' + + fetch.apt_mark(packages, sentinel.mark) + + self.assertTrue(log.called) + mock_call.assert_called_with( + ['apt-mark', sentinel.mark, 'foo bar'], + universal_newlines=True) + + @patch('subprocess.call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_mark_apt_packages_nofatal(self, log, mock_call): + packages = ['foo', 'bar'] + + fetch.apt_mark(packages, sentinel.mark) + + self.assertTrue(log.called) + mock_call.assert_called_with( + ['apt-mark', sentinel.mark, 'foo', 'bar'], + universal_newlines=True) + + @patch('subprocess.check_call') + @patch('charmhelpers.fetch.ubuntu.log') + def test_mark_apt_packages_nofatal_abortonfatal(self, log, mock_call): + packages = ['foo', 'bar'] + + fetch.apt_mark(packages, sentinel.mark, fatal=True) + + self.assertTrue(log.called) + mock_call.assert_called_with( + ['apt-mark', sentinel.mark, 'foo', 'bar'], + universal_newlines=True) + + @patch('charmhelpers.fetch.ubuntu.apt_mark') + def test_apt_hold(self, apt_mark): + fetch.apt_hold(sentinel.packages) + apt_mark.assert_called_once_with(sentinel.packages, 'hold', + fatal=False) + + @patch('charmhelpers.fetch.ubuntu.apt_mark') + def test_apt_hold_fatal(self, apt_mark): + fetch.apt_hold(sentinel.packages, fatal=sentinel.fatal) + apt_mark.assert_called_once_with(sentinel.packages, 'hold', + fatal=sentinel.fatal) + + @patch('charmhelpers.fetch.ubuntu.apt_mark') + def test_apt_unhold(self, apt_mark): + fetch.apt_unhold(sentinel.packages) + apt_mark.assert_called_once_with(sentinel.packages, 'unhold', + fatal=False) + + @patch('charmhelpers.fetch.ubuntu.apt_mark') + def test_apt_unhold_fatal(self, apt_mark): + fetch.apt_unhold(sentinel.packages, fatal=sentinel.fatal) + apt_mark.assert_called_once_with(sentinel.packages, 'unhold', + fatal=sentinel.fatal) + + @patch('subprocess.check_call') + def test_apt_update_fatal(self, check_call): + fetch.apt_update(fatal=True) + check_call.assert_called_with( + ['apt-get', 'update'], + env={}) + + @patch('subprocess.call') + def test_apt_update_nonfatal(self, call): + fetch.apt_update() + call.assert_called_with( + ['apt-get', 'update'], + env={}) + + @patch('subprocess.check_call') + @patch('time.sleep') + def test_run_apt_command_retries_if_fatal(self, check_call, sleep): + """The _run_apt_command function retries the command if it can't get + the APT lock.""" + self.called = False + + def side_effect(*args, **kwargs): + """ + First, raise an exception (can't acquire lock), then return 0 + (the lock is grabbed). + """ + if not self.called: + self.called = True + raise subprocess.CalledProcessError( + returncode=100, cmd="some command") + else: + return 0 + + check_call.side_effect = side_effect + check_call.return_value = 0 + + from charmhelpers.fetch.ubuntu import _run_apt_command + _run_apt_command(["some", "command"], fatal=True) + self.assertTrue(sleep.called) + + @patch.object(fetch, 'apt_cache') + def test_get_upstream_version(self, cache): + cache.side_effect = fake_apt_cache + self.assertEqual(fetch.get_upstream_version('vim'), '7.3.547') + self.assertEqual(fetch.get_upstream_version('emacs'), None) + self.assertEqual(fetch.get_upstream_version('unknown'), None) + + @patch('charmhelpers.fetch.ubuntu._run_apt_command') + def test_apt_autoremove_fatal(self, run_apt_command): + fetch.apt_autoremove(purge=True, fatal=True) + run_apt_command.assert_called_with( + ['apt-get', '--assume-yes', 'autoremove', '--purge'], + True + ) + + @patch('charmhelpers.fetch.ubuntu._run_apt_command') + def test_apt_autoremove_nonfatal(self, run_apt_command): + fetch.apt_autoremove(purge=False, fatal=False) + run_apt_command.assert_called_with( + ['apt-get', '--assume-yes', 'autoremove'], + False + ) + + +class TestAptDpkgEnv(TestCase): + + @patch.object(fetch, 'get_system_env') + def test_get_apt_dpkg_env(self, mock_get_system_env): + mock_get_system_env.return_value = '/a/path' + self.assertEquals( + fetch.get_apt_dpkg_env(), + {'DEBIAN_FRONTEND': 'noninteractive', 'PATH': '/a/path'}) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu_apt_pkg.py b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu_apt_pkg.py new file mode 100644 index 0000000..23856c9 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_fetch_ubuntu_apt_pkg.py @@ -0,0 +1,278 @@ +import mock +import subprocess +import unittest + +from charmhelpers.fetch import ubuntu_apt_pkg as apt_pkg + + +class Test_apt_pkg_Cache(unittest.TestCase): + """Borrow PatchHelper methods from ``charms.openstack``.""" + def setUp(self): + self._patches = {} + self._patches_start = {} + + def tearDown(self): + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def patch(self, patchee, name=None, **kwargs): + """Patch a patchable thing. Uses mock.patch() to do the work. + Automatically unpatches at the end of the test. + + The mock gets added to the test object (self) using 'name' or the last + part of the patchee string, after the final dot. + + :param patchee: representing module.object that is to be + patched. + :param name: optional name to call the mock. + :param **kwargs: any other args to pass to mock.patch() + """ + mocked = mock.patch(patchee, **kwargs) + if name is None: + name = patchee.split('.')[-1] + started = mocked.start() + self._patches[name] = mocked + self._patches_start[name] = started + setattr(self, name, started) + + def patch_object(self, obj, attr, name=None, **kwargs): + """Patch a patchable thing. Uses mock.patch.object() to do the work. + Automatically unpatches at the end of the test. + + The mock gets added to the test object (self) using 'name' or the attr + passed in the arguments. + + :param obj: an object that needs to have an attribute patched. + :param attr: that represents the attribute being patched. + :param name: optional name to call the mock. + :param **kwargs: any other args to pass to mock.patch() + """ + mocked = mock.patch.object(obj, attr, **kwargs) + if name is None: + name = attr + started = mocked.start() + self._patches[name] = mocked + self._patches_start[name] = started + setattr(self, name, started) + + def test_apt_cache_show(self): + self.patch_object(apt_pkg.subprocess, 'check_output') + apt_cache = apt_pkg.Cache() + self.check_output.return_value = ( + 'Package: dpkg\n' + 'Version: 1.19.0.5ubuntu2.1\n' + 'Bugs: https://bugs.launchpad.net/ubuntu/+filebug\n' + 'Description-en: Debian package management system\n' + ' Multiline description\n' + '\n' + 'Package: lsof\n' + 'Architecture: amd64\n' + 'Version: 4.91+dfsg-1ubuntu1\n' + '\n' + 'N: There is 1 additional record.\n') + self.assertEquals( + apt_cache._apt_cache_show(['package']), + {'dpkg': { + 'package': 'dpkg', 'version': '1.19.0.5ubuntu2.1', + 'bugs': 'https://bugs.launchpad.net/ubuntu/+filebug', + 'description-en': 'Debian package management system\n' + 'Multiline description'}, + 'lsof': { + 'package': 'lsof', 'architecture': 'amd64', + 'version': '4.91+dfsg-1ubuntu1'}, + }) + self.check_output.assert_called_once_with( + ['apt-cache', 'show', '--no-all-versions', 'package'], + stderr=subprocess.STDOUT, + universal_newlines=True) + + def test_dpkg_list(self): + self.patch_object(apt_pkg.subprocess, 'check_output') + apt_cache = apt_pkg.Cache() + self.check_output.return_value = ( + 'Desired=Unknown/Install/Remove/Purge/Hold\n' + '| Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/' + 'trig-aWait/Trig-pend\n' + '|/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad)\n' + '||/ Name Version Architecture Description\n' + '+++-=============================-==================-===========-' + '=================================\n' + 'ii dpkg 1.19.0.5ubuntu2.1 amd64 ' + 'Debian package management system\n' + 'rc linux-image-4.15.0-42-generic 4.15.0-42.45 amd64 ' + 'Signed kernel image generic\n' + 'ii lsof 4.91+dfsg-1ubuntu1 amd64 ' + 'utility to list open files\n') + expect = { + 'dpkg': { + 'name': 'dpkg', + 'version': '1.19.0.5ubuntu2.1', + 'architecture': 'amd64', + 'description': 'Debian package management system' + }, + 'lsof': { + 'name': 'lsof', + 'version': '4.91+dfsg-1ubuntu1', + 'architecture': 'amd64', + 'description': 'utility to list open files' + }, + } + self.assertEquals( + apt_cache._dpkg_list(['package']), expect) + self.check_output.side_effect = subprocess.CalledProcessError( + 1, '', output=self.check_output.return_value) + self.assertEquals(apt_cache._dpkg_list(['package']), expect) + self.check_output.side_effect = subprocess.CalledProcessError(2, '') + with self.assertRaises(subprocess.CalledProcessError): + _ = apt_cache._dpkg_list(['package']) + + def test_version_compare(self): + self.patch_object(apt_pkg.subprocess, 'check_call') + self.assertEquals(apt_pkg.version_compare('2', '1'), 1) + self.check_call.assert_called_once_with( + ['dpkg', '--compare-versions', '2', 'gt', '1'], + stderr=subprocess.STDOUT, + universal_newlines=True) + self.check_call.side_effect = [ + subprocess.CalledProcessError(1, '', ''), + None, + None, + ] + self.assertEquals(apt_pkg.version_compare('2', '2'), 0) + self.check_call.side_effect = [ + subprocess.CalledProcessError(1, '', ''), + subprocess.CalledProcessError(1, '', ''), + None, + ] + self.assertEquals(apt_pkg.version_compare('1', '2'), -1) + self.check_call.side_effect = subprocess.CalledProcessError(2, '', '') + self.assertRaises(subprocess.CalledProcessError, + apt_pkg.version_compare, '2', '2') + + def test_apt_cache(self): + self.patch_object(apt_pkg.subprocess, 'check_output') + apt_cache = apt_pkg.Cache() + self.check_output.side_effect = [ + ('Package: dpkg\n' + 'Version: 1.19.0.6ubuntu0\n' + 'Bugs: https://bugs.launchpad.net/ubuntu/+filebug\n' + 'Description-en: Debian package management system\n' + ' Multiline description\n' + '\n' + 'Package: lsof\n' + 'Architecture: amd64\n' + 'Version: 4.91+dfsg-1ubuntu1\n' + '\n' + 'N: There is 1 additional record.\n'), + ('Desired=Unknown/Install/Remove/Purge/Hold\n' + '| Status=Not/Inst/Conf-files/Unpacked/halF-conf/Half-inst/' + 'trig-aWait/Trig-pend\n' + '|/ Err?=(none)/Reinst-required (Status,Err: uppercase=bad)\n' + '||/ Name Version Architecture Description\n' + '+++-=============================-==================-===========-' + '=================================\n' + 'ii dpkg 1.19.0.5ubuntu2.1 amd64 ' + 'Debian package management system\n' + 'rc linux-image-4.15.0-42-generic 4.15.0-42.45 amd64 ' + 'Signed kernel image generic\n' + 'ii lsof 4.91+dfsg-1ubuntu1 amd64 ' + 'utility to list open files\n'), + ] + pkg = apt_cache['dpkg'] + self.assertEquals(pkg.name, 'dpkg') + self.assertEquals(pkg.current_ver.ver_str, '1.19.0.5ubuntu2.1') + self.assertEquals(pkg.architecture, 'amd64') + self.check_output.side_effect = [ + subprocess.CalledProcessError(100, ''), + subprocess.CalledProcessError(1, ''), + ] + with self.assertRaises(KeyError): + pkg = apt_cache['nonexistent'] + self.check_output.side_effect = [ + ('Package: dpkg\n' + 'Version: 1.19.0.6ubuntu0\n' + 'Bugs: https://bugs.launchpad.net/ubuntu/+filebug\n' + 'Description-en: Debian package management system\n' + ' Multiline description\n' + '\n' + 'Package: lsof\n' + 'Architecture: amd64\n' + 'Version: 4.91+dfsg-1ubuntu1\n' + '\n' + 'N: There is 1 additional record.\n'), + subprocess.CalledProcessError(42, ''), + ] + with self.assertRaises(subprocess.CalledProcessError): + # System error occurs while making dpkg inquiry + pkg = apt_cache['dpkg'] + self.check_output.side_effect = [ + subprocess.CalledProcessError(42, ''), + subprocess.CalledProcessError(1, ''), + ] + with self.assertRaises(subprocess.CalledProcessError): + pkg = apt_cache['system-error-occurs-while-making-apt-inquiry'] + + +class Test_apt_pkg_PkgVersion(unittest.TestCase): + + def test_PkgVersion(self): + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') < + apt_pkg.PkgVersion('2:20.4.1')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.1') < + apt_pkg.PkgVersion('2:20.4.0')) + + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') <= + apt_pkg.PkgVersion('2:20.4.1')) + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') <= + apt_pkg.PkgVersion('2:20.4.0')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.1') <= + apt_pkg.PkgVersion('2:20.4.0')) + + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.1') > + apt_pkg.PkgVersion('2:20.4.0')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.0') > + apt_pkg.PkgVersion('2:20.4.1')) + + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.1') >= + apt_pkg.PkgVersion('2:20.4.0')) + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') >= + apt_pkg.PkgVersion('2:20.4.0')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.0') >= + apt_pkg.PkgVersion('2:20.4.1')) + + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') == + apt_pkg.PkgVersion('2:20.4.0')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.0') == + apt_pkg.PkgVersion('2:20.4.1')) + + self.assertTrue( + apt_pkg.PkgVersion('2:20.4.0') != + apt_pkg.PkgVersion('2:20.4.1')) + self.assertFalse( + apt_pkg.PkgVersion('2:20.4.0') != + apt_pkg.PkgVersion('2:20.4.0')) + + pkgs = [apt_pkg.PkgVersion('2:20.4.0'), + apt_pkg.PkgVersion('2:21.4.0'), + apt_pkg.PkgVersion('2:17.4.0')] + pkgs.sort() + self.assertEqual( + pkgs, + [apt_pkg.PkgVersion('2:17.4.0'), + apt_pkg.PkgVersion('2:20.4.0'), + apt_pkg.PkgVersion('2:21.4.0')]) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_giturl.py b/nrpe/mod/charmhelpers/tests/fetch/test_giturl.py new file mode 100644 index 0000000..e657027 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_giturl.py @@ -0,0 +1,115 @@ +import os +import shutil +import subprocess +import tempfile +from testtools import TestCase +from mock import ( + MagicMock, + patch, +) +from charmhelpers.core.host import chdir + +import six +if six.PY3: + from urllib.parse import urlparse +else: + from urlparse import urlparse + + +try: + from charmhelpers.fetch import ( + giturl, + UnhandledSource, + ) +except ImportError: + giturl = None + UnhandledSource = None + + +class GitUrlFetchHandlerTest(TestCase): + def setUp(self): + super(GitUrlFetchHandlerTest, self).setUp() + self.valid_urls = ( + "http://example.com/git-branch", + "https://example.com/git-branch", + "git://example.com/git-branch", + ) + self.invalid_urls = ( + "file://example.com/foo.tar.bz2", + "abc:example", + "garbage", + ) + self.fh = giturl.GitUrlFetchHandler() + + def test_handles_git_urls(self): + for url in self.valid_urls: + result = self.fh.can_handle(url) + self.assertEqual(result, True, url) + for url in self.invalid_urls: + result = self.fh.can_handle(url) + self.assertNotEqual(result, True, url) + + @patch.object(giturl, 'check_output') + def test_clone(self, check_output): + dest_path = "/destination/path" + branch = "master" + for url in self.valid_urls: + self.fh.remote_branch = MagicMock() + self.fh.load_plugins = MagicMock() + self.fh.clone(url, dest_path, branch, None) + + check_output.assert_called_with( + ['git', 'clone', url, dest_path, '--branch', branch], stderr=-2) + + for url in self.invalid_urls: + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + self.assertRaises(UnhandledSource, self.fh.clone, url, + dest_path, None, + branch) + + def test_clone_functional(self): + src = None + dst = None + try: + src = tempfile.mkdtemp() + with chdir(src): + subprocess.check_output(['git', 'init']) + subprocess.check_output(['git', 'config', 'user.name', 'Joe']) + subprocess.check_output( + ['git', 'config', 'user.email', 'joe@test.com']) + subprocess.check_output(['touch', 'foo']) + subprocess.check_output(['git', 'add', 'foo']) + subprocess.check_output(['git', 'commit', '-m', 'test']) + dst = tempfile.mkdtemp() + os.rmdir(dst) + self.fh.clone(src, dst) + assert os.path.exists(os.path.join(dst, '.git')) + self.fh.clone(src, dst) # idempotent + assert os.path.exists(os.path.join(dst, '.git')) + finally: + if src: + shutil.rmtree(src, ignore_errors=True) + if dst: + shutil.rmtree(dst, ignore_errors=True) + + def test_installs(self): + self.fh.clone = MagicMock() + + for url in self.valid_urls: + branch_name = urlparse(url).path.strip("/").split("/")[-1] + dest = os.path.join('foo', 'fetched', + os.path.basename(branch_name)) + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + where = self.fh.install(url) + self.assertEqual(where, dest) + + def test_installs_specified_dest(self): + self.fh.clone = MagicMock() + + for url in self.valid_urls: + branch_name = urlparse(url).path.strip("/").split("/")[-1] + dest_repo = os.path.join('/tmp/git/', + os.path.basename(branch_name)) + with patch.dict('os.environ', {'CHARM_DIR': 'foo'}): + where = self.fh.install(url, dest="/tmp/git") + self.assertEqual(where, dest_repo) diff --git a/nrpe/mod/charmhelpers/tests/fetch/test_snap.py b/nrpe/mod/charmhelpers/tests/fetch/test_snap.py new file mode 100644 index 0000000..02e6a70 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/fetch/test_snap.py @@ -0,0 +1,79 @@ +# Copyright 2014-2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from mock import patch +from unittest import TestCase +import charmhelpers.fetch.snap as fetch_snap + +__author__ = 'Joseph Borg ' + +TEST_ENV = {'foo': 'bar'} + + +class SnapTest(TestCase): + """ + Test and install and removal of a snap. + """ + @patch.object(fetch_snap, 'log', lambda *args, **kwargs: None) + @patch('subprocess.check_call') + @patch('os.environ', TEST_ENV) + def testSnapInstall(self, check_call): + """ + Test snap install. + + :param check_call: Mock object + :return: None + """ + check_call.return_value = 0 + fetch_snap.snap_install(['hello-world', 'htop'], '--classic', '--stable') + check_call.assert_called_with(['snap', 'install', '--classic', '--stable', 'hello-world', 'htop'], env=TEST_ENV) + + @patch.object(fetch_snap, 'log', lambda *args, **kwargs: None) + @patch('subprocess.check_call') + @patch('os.environ', TEST_ENV) + def testSnapRefresh(self, check_call): + """ + Test snap refresh. + + :param check_call: Mock object + :return: None + """ + check_call.return_value = 0 + fetch_snap.snap_refresh(['hello-world', 'htop'], '--classic', '--stable') + check_call.assert_called_with(['snap', 'refresh', '--classic', '--stable', 'hello-world', 'htop'], env=TEST_ENV) + + @patch.object(fetch_snap, 'log', lambda *args, **kwargs: None) + @patch('subprocess.check_call') + @patch('os.environ', TEST_ENV) + def testSnapRemove(self, check_call): + """ + Test snap remove. + + :param check_call: Mock object + :return: None + """ + check_call.return_value = 0 + fetch_snap.snap_remove(['hello-world', 'htop']) + check_call.assert_called_with(['snap', 'remove', 'hello-world', 'htop'], env=TEST_ENV) + + def test_valid_snap_channel(self): + """ Test valid snap channel + + :return: None + """ + # Valid + self.assertTrue(fetch_snap.valid_snap_channel('edge')) + + # Invalid + with self.assertRaises(fetch_snap.InvalidSnapChannel): + fetch_snap.valid_snap_channel('DOESNOTEXIST') diff --git a/nrpe/mod/charmhelpers/tests/helpers.py b/nrpe/mod/charmhelpers/tests/helpers.py new file mode 100644 index 0000000..1058126 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/helpers.py @@ -0,0 +1,144 @@ +''' General helper functions for tests ''' +from contextlib import contextmanager +from mock import patch, MagicMock +import io + +import six +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + + +@contextmanager +def patch_open(): + '''Patch open() to allow mocking both open() itself and the file that is + yielded. + + Yields the mock for "open" and "file", respectively.''' + mock_open = MagicMock(spec=open) + mock_file = MagicMock(spec=io.FileIO) + + @contextmanager + def stub_open(*args, **kwargs): + mock_open(*args, **kwargs) + yield mock_file + + with patch(builtin_open, stub_open): + yield mock_open, mock_file + + +@contextmanager +def mock_open(filename, contents=None): + ''' Slightly simpler mock of open to return contents for filename ''' + def mock_file(name, mode='r', buffering=-1): # Python 2 signature. + if name == filename: + if (not six.PY3) or 'b' in mode: + return io.BytesIO(contents) + return io.StringIO(contents) + else: + return open(name, mode, buffering) + + with patch(builtin_open, mock_file): + yield + + +class FakeRelation(object): + ''' + A fake relation class. Lets tests specify simple relation data + for a default relation + unit (foo:0, foo/0, set in setUp()), eg: + + rel = { + 'private-address': 'foo', + 'password': 'passwd', + } + relation = FakeRelation(rel) + self.relation_get.side_effect = relation.get + passwd = self.relation_get('password') + + or more complex relations meant to be addressed by explicit relation id + + unit id combos: + + rel = { + 'mysql:0': { + 'mysql/0': { + 'private-address': 'foo', + 'password': 'passwd', + } + } + } + relation = FakeRelation(rel) + self.relation_get.side_affect = relation.get + passwd = self.relation_get('password', rid='mysql:0', unit='mysql/0') + + set_relation_context can be used to simulate being in a relation hook + context. This allows omitting a relation id or unit when calling relation + helpers as the related unit is present. + + To set the context: + + relation = FakeRelation(rel) + relation.set_relation_context('mysql-svc2/0', 'shared-db:12') + + To clear it: + + relation.clear_relation_context() + ''' + def __init__(self, relation_data): + self.relation_data = relation_data + self.remote_unit = None + self.current_relation_id = None + + def set_relation_context(self, remote_unit, relation_id): + self.remote_unit = remote_unit + self.current_relation_id = relation_id + + def clear_relation_context(self): + self.remote_unit = None + self.current_relation_id = None + + def get(self, attribute=None, unit=None, rid=None): + if not rid or rid == 'foo:0': + if self.current_relation_id: + if not unit: + unit = self.remote_unit + udata = self.relation_data[self.current_relation_id][unit] + if attribute: + return udata[attribute] + else: + return udata[unit] + if attribute is None: + return self.relation_data + elif attribute in self.relation_data: + return self.relation_data[attribute] + return None + else: + if rid not in self.relation_data: + return None + try: + relation = self.relation_data[rid][unit] + except KeyError: + return None + if attribute and attribute in relation: + return relation[attribute] + return relation + + def relation_id(self): + return self.current_relation_id + + def relation_ids(self, reltype=None): + rids = self.relation_data.keys() + if reltype: + return [r for r in rids if r.split(':')[0] == reltype] + return rids + + def related_units(self, relid=None): + try: + return self.relation_data[relid].keys() + except KeyError: + return [] + + def relation_units(self, relation_id): + if relation_id not in self.relation_data: + return None + return self.relation_data[relation_id].keys() diff --git a/nrpe/mod/charmhelpers/tests/payload/__init__.py b/nrpe/mod/charmhelpers/tests/payload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/payload/test_archive.py b/nrpe/mod/charmhelpers/tests/payload/test_archive.py new file mode 100644 index 0000000..f636917 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/payload/test_archive.py @@ -0,0 +1,137 @@ +import os +from testtools import TestCase +from mock import ( + patch, + MagicMock, +) +from charmhelpers.payload import archive +from tempfile import mkdtemp +from shutil import rmtree +import subprocess + + +class ArchiveTestCase(TestCase): + + def create_archive(self, format): + workdir = mkdtemp() + if format == "tar": + workfile = "{}/foo.tar.gz".format(workdir) + cmd = "tar czf {} hosts".format(workfile) + elif format == "zip": + workfile = "{}/foo.zip".format(workdir) + cmd = "zip {} hosts".format(workfile) + curdir = os.getcwd() + os.chdir("/etc") + subprocess.check_output(cmd, shell=True) + os.chdir(curdir) + self.addCleanup(rmtree, workdir) + return (workfile, ["hosts"]) + + @patch('os.path.isfile') + def test_gets_archive_handler_by_ext(self, _isfile): + tar_archive_handler = archive.extract_tarfile + zip_archive_handler = archive.extract_zipfile + _isfile.return_value = False + + for ext in ('tar', 'tar.gz', 'tgz', 'tar.bz2', 'tbz2', 'tbz'): + handler = archive.get_archive_handler("somefile.{}".format(ext)) + msg = "handler for extension: {}".format(ext) + self.assertEqual(handler, tar_archive_handler, msg) + + for ext in ('zip', 'jar'): + handler = archive.get_archive_handler("somefile.{}".format(ext)) + msg = "handler for extension {}".format(ext) + self.assertEqual(handler, zip_archive_handler, msg) + + @patch('zipfile.is_zipfile') + @patch('tarfile.is_tarfile') + @patch('os.path.isfile') + def test_gets_archive_hander_by_filetype(self, _isfile, _istarfile, + _iszipfile): + tar_archive_handler = archive.extract_tarfile + zip_archive_handler = archive.extract_zipfile + _isfile.return_value = True + + _istarfile.return_value = True + _iszipfile.return_value = False + handler = archive.get_archive_handler("foo") + self.assertEqual(handler, tar_archive_handler) + + _istarfile.return_value = False + _iszipfile.return_value = True + handler = archive.get_archive_handler("foo") + self.assertEqual(handler, zip_archive_handler) + + @patch('charmhelpers.core.hookenv.charm_dir') + def test_gets_archive_dest_default(self, _charmdir): + _charmdir.return_value = "foo" + thedir = archive.archive_dest_default("baz") + self.assertEqual(thedir, os.path.join("foo", "archives", "baz")) + + thedir = archive.archive_dest_default("baz/qux") + self.assertEqual(thedir, os.path.join("foo", "archives", "qux")) + + def test_extracts_tarfile(self): + destdir = mkdtemp() + self.addCleanup(rmtree, destdir) + tar_file, contents = self.create_archive("tar") + archive.extract_tarfile(tar_file, destdir) + for path in [os.path.join(destdir, item) for item in contents]: + self.assertTrue(os.path.exists(path)) + + def test_extracts_zipfile(self): + destdir = mkdtemp() + self.addCleanup(rmtree, destdir) + try: + zip_file, contents = self.create_archive("zip") + except subprocess.CalledProcessError as e: + if e.returncode == 127: + self.skip("Skipping - zip is not installed") + else: + raise + archive.extract_zipfile(zip_file, destdir) + for path in [os.path.join(destdir, item) for item in contents]: + self.assertTrue(os.path.exists(path)) + + @patch('charmhelpers.core.host.mkdir') + @patch('charmhelpers.payload.archive.get_archive_handler') + @patch('charmhelpers.payload.archive.archive_dest_default') + def test_extracts(self, _defdest, _gethandler, _mkdir): + archive_name = "foo" + archive_handler = MagicMock() + _gethandler.return_value = archive_handler + + dest = archive.extract(archive_name, "bar") + + _gethandler.assert_called_with(archive_name) + archive_handler.assert_called_with(archive_name, "bar") + _defdest.assert_not_called() + _mkdir.assert_called_with("bar") + self.assertEqual(dest, "bar") + + @patch('charmhelpers.core.host.mkdir') + @patch('charmhelpers.payload.archive.get_archive_handler') + def test_unhandled_extract_raises_exc(self, _gethandler, _mkdir): + archive_name = "foo" + _gethandler.return_value = None + + self.assertRaises(archive.ArchiveError, archive.extract, + archive_name) + + _gethandler.assert_called_with(archive_name) + _mkdir.assert_not_called() + + @patch('charmhelpers.core.host.mkdir') + @patch('charmhelpers.payload.archive.get_archive_handler') + @patch('charmhelpers.payload.archive.archive_dest_default') + def test_extracts_default_dest(self, _defdest, _gethandler, _mkdir): + expected_dest = "bar" + archive_name = "foo" + _defdest.return_value = expected_dest + handler = MagicMock() + handler.return_value = expected_dest + _gethandler.return_value = handler + + dest = archive.extract(archive_name) + self.assertEqual(expected_dest, dest) + handler.assert_called_with(archive_name, expected_dest) diff --git a/nrpe/mod/charmhelpers/tests/payload/test_execd.py b/nrpe/mod/charmhelpers/tests/payload/test_execd.py new file mode 100644 index 0000000..20d83c5 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/payload/test_execd.py @@ -0,0 +1,153 @@ +from testtools import TestCase +from mock import patch +import os +import shutil +import stat + +from tempfile import mkdtemp + +from charmhelpers.payload import execd + + +class ExecDTestCase(TestCase): + + def setUp(self): + super(ExecDTestCase, self).setUp() + charm_dir = mkdtemp() + self.addCleanup(shutil.rmtree, charm_dir) + self.test_charm_dir = charm_dir + + env_patcher = patch.dict('os.environ', + {'CHARM_DIR': self.test_charm_dir}) + env_patcher.start() + self.addCleanup(env_patcher.stop) + + def test_default_execd_dir(self): + expected = os.path.join(self.test_charm_dir, 'exec.d') + default_dir = execd.default_execd_dir() + + self.assertEqual(expected, default_dir) + + def make_preinstall_executable(self, module_dir, execd_dir='exec.d', + error_on_preinstall=False): + """Add a charm-pre-install to module dir. + + When executed, the charm-pre-install will create a second + file in the same directory, charm-pre-install-success. + """ + module_path = os.path.join(self.test_charm_dir, execd_dir, module_dir) + os.makedirs(module_path) + + charm_pre_install_path = os.path.join(module_path, + 'charm-pre-install') + pre_install_success_path = os.path.join(module_path, + 'charm-pre-install-success') + with open(charm_pre_install_path, 'w+') as f: + if not error_on_preinstall: + f.write("#!/bin/bash\n" + "/usr/bin/touch {}".format(pre_install_success_path)) + else: + f.write("#!/bin/bash\n" + "echo stdout_from_pre_install\n" + "echo stderr_from_pre_install >&2\n" + "exit 1") + + # ensure it is executable. + perms = stat.S_IRUSR + stat.S_IXUSR + os.chmod(charm_pre_install_path, perms) + + def assert_preinstall_called_for_mod(self, module_dir, + execd_dir='exec.d'): + """Asserts that the charm-pre-install-success file exists.""" + expected_file = os.path.join(self.test_charm_dir, execd_dir, + module_dir, 'charm-pre-install-success') + files = os.listdir(os.path.dirname(expected_file)) + self.assertTrue(os.path.exists(expected_file), "files were: %s. charmdir is: %s" % (files, self.test_charm_dir)) + + def test_execd_preinstall(self): + """All charm-pre-install hooks are executed.""" + self.make_preinstall_executable(module_dir='basenode') + self.make_preinstall_executable(module_dir='mod2') + + execd.execd_preinstall() + + self.assert_preinstall_called_for_mod('basenode') + self.assert_preinstall_called_for_mod('mod2') + + def test_execd_module_list_from_env(self): + modules = ['basenode', 'mod2', 'c'] + for module in modules: + self.make_preinstall_executable(module_dir=module) + + actual_mod_paths = list(execd.execd_module_paths()) + + expected_mod_paths = [ + os.path.join(self.test_charm_dir, 'exec.d', module) + for module in modules] + self.assertSetEqual(set(actual_mod_paths), set(expected_mod_paths)) + + def test_execd_module_list_with_dir(self): + modules = ['basenode', 'mod2', 'c'] + for module in modules: + self.make_preinstall_executable(module_dir=module, + execd_dir='foo') + + actual_mod_paths = list(execd.execd_module_paths( + execd_dir=os.path.join(self.test_charm_dir, 'foo'))) + + expected_mod_paths = [ + os.path.join(self.test_charm_dir, 'foo', module) + for module in modules] + self.assertSetEqual(set(actual_mod_paths), set(expected_mod_paths)) + + def test_execd_module_paths_no_execd_dir(self): + """Empty list is returned when the exec.d doesn't exist.""" + actual_mod_paths = list(execd.execd_module_paths()) + + self.assertEqual(actual_mod_paths, []) + + def test_execd_submodule_list(self): + modules = ['basenode', 'mod2', 'c'] + for module in modules: + self.make_preinstall_executable(module_dir=module) + + submodules = list(execd.execd_submodule_paths('charm-pre-install')) + + expected = [os.path.join(self.test_charm_dir, 'exec.d', mod, + 'charm-pre-install') for mod in modules] + self.assertEqual(sorted(submodules), sorted(expected)) + + def test_execd_run(self): + modules = ['basenode', 'mod2', 'c'] + for module in modules: + self.make_preinstall_executable(module_dir=module) + + execd.execd_run('charm-pre-install') + + self.assert_preinstall_called_for_mod('basenode') + self.assert_preinstall_called_for_mod('mod2') + self.assert_preinstall_called_for_mod('c') + + @patch('charmhelpers.core.hookenv.log') + def test_execd_run_logs_exception(self, log_): + self.make_preinstall_executable(module_dir='basenode', + error_on_preinstall=True) + + execd.execd_run('charm-pre-install', die_on_error=False) + + expected_log = ('Error (1) running {}/exec.d/basenode/' + 'charm-pre-install. Output: ' + 'stdout_from_pre_install\n' + 'stderr_from_pre_install\n'.format(self.test_charm_dir)) + log_.assert_called_with(expected_log) + + @patch('charmhelpers.core.hookenv.log') + @patch('sys.exit') + def test_execd_run_dies_with_return_code(self, exit_, log): + self.make_preinstall_executable(module_dir='basenode', + error_on_preinstall=True) + + with open(os.devnull, 'wb') as devnull: + execd.execd_run('charm-pre-install', stderr=devnull) + + exit_.assert_called_with(1) diff --git a/nrpe/mod/charmhelpers/tests/tools/__init__.py b/nrpe/mod/charmhelpers/tests/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nrpe/mod/charmhelpers/tests/tools/test_charm_helper_sync.py b/nrpe/mod/charmhelpers/tests/tools/test_charm_helper_sync.py new file mode 100644 index 0000000..7c66b9b --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/tools/test_charm_helper_sync.py @@ -0,0 +1,304 @@ +import unittest +from mock import call, patch +import yaml + +import tools.charm_helpers_sync.charm_helpers_sync as sync + +import six +if not six.PY3: + builtin_open = '__builtin__.open' +else: + builtin_open = 'builtins.open' + + +INCLUDE = """ +include: + - core + - contrib.openstack + - contrib.storage + - contrib.hahelpers: + - utils + - ceph_utils + - cluster_utils + - haproxy_utils +""" + + +class HelperSyncTests(unittest.TestCase): + def test_clone_helpers(self): + '''It properly branches the correct helpers branch''' + with patch('subprocess.check_call') as check_call: + sync.clone_helpers(work_dir='/tmp/foo', repo='git:charm-helpers') + check_call.assert_called_with(['git', + 'clone', '--depth=1', + 'git:charm-helpers', + '/tmp/foo/charm-helpers']) + + def test_module_path(self): + '''It converts a python module path to a filesystem path''' + self.assertEquals(sync._module_path('some.test.module'), + 'some/test/module') + + def test_src_path(self): + '''It renders the correct path to module within charm-helpers tree''' + path = sync._src_path(src='/tmp/charm-helpers', + module='contrib.openstack') + self.assertEquals('/tmp/charm-helpers/charmhelpers/contrib/openstack', + path) + + def test_dest_path(self): + '''It correctly finds the correct install path within a charm''' + path = sync._dest_path(dest='/tmp/mycharm/hooks/charmhelpers', + module='contrib.openstack') + self.assertEquals('/tmp/mycharm/hooks/charmhelpers/contrib/openstack', + path) + + @patch(builtin_open) + @patch('os.path.exists') + @patch('os.walk') + def test_ensure_init(self, walk, exists, _open): + '''It ensures all subdirectories of a parent are python importable''' + # os walk + # os.path.join + # os.path.exists + # open + def _walk(path): + yield ('/tmp/hooks/', ['helpers'], []) + yield ('/tmp/hooks/helpers', ['foo'], []) + yield ('/tmp/hooks/helpers/foo', [], []) + walk.side_effect = _walk + exists.return_value = False + sync.ensure_init('hooks/helpers/foo/') + ex = [call('/tmp/hooks/__init__.py', 'wb'), + call('/tmp/hooks/helpers/__init__.py', 'wb'), + call('/tmp/hooks/helpers/foo/__init__.py', 'wb')] + for c in ex: + self.assertIn(c, _open.call_args_list) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.ensure_init') + @patch('os.path.isfile') + @patch('shutil.copy') + @patch('os.makedirs') + @patch('os.path.exists') + def test_sync_pyfile(self, exists, mkdirs, copy, isfile, ensure_init): + '''It correctly syncs a py src file from src to dest''' + exists.return_value = False + isfile.return_value = True + sync.sync_pyfile('/tmp/charm-helpers/core/host', + 'hooks/charmhelpers/core') + mkdirs.assert_called_with('hooks/charmhelpers/core') + copy_f = call('/tmp/charm-helpers/core/host.py', + 'hooks/charmhelpers/core') + copy_i = call('/tmp/charm-helpers/core/__init__.py', + 'hooks/charmhelpers/core') + self.assertIn(copy_f, copy.call_args_list) + self.assertIn(copy_i, copy.call_args_list) + ensure_init.assert_called_with('hooks/charmhelpers/core') + + def _test_filter_dir(self, opts, isfile, isdir): + '''It filters non-python files and non-module dirs from source''' + files = { + 'bad_file.bin': 'f', + 'some_dir': 'd', + 'good_helper.py': 'f', + 'good_helper2.py': 'f', + 'good_helper3.py': 'f', + 'bad_file.img': 'f', + } + + def _isfile(f): + try: + return files[f.split('/').pop()] == 'f' + except KeyError: + return False + + def _isdir(f): + try: + return files[f.split('/').pop()] == 'd' + except KeyError: + return False + + isfile.side_effect = _isfile + isdir.side_effect = _isdir + result = sync.get_filter(opts)(dir='/tmp/charm-helpers/core', + ls=six.iterkeys(files)) + return result + + @patch('os.path.isdir') + @patch('os.path.isfile') + def test_filter_dir_no_opts(self, isfile, isdir): + '''It filters out all non-py files by default''' + result = self._test_filter_dir(opts=None, isfile=isfile, isdir=isdir) + ex = ['bad_file.bin', 'bad_file.img', 'some_dir'] + self.assertEquals(sorted(ex), sorted(result)) + + @patch('os.path.isdir') + @patch('os.path.isfile') + def test_filter_dir_with_include(self, isfile, isdir): + '''It includes non-py files if specified as an include opt''' + result = sorted(self._test_filter_dir(opts=['inc=*.img'], + isfile=isfile, isdir=isdir)) + ex = sorted(['bad_file.bin', 'some_dir']) + self.assertEquals(ex, result) + + @patch('os.path.isdir') + @patch('os.path.isfile') + def test_filter_dir_include_all(self, isfile, isdir): + '''It does not filter anything if option specified to include all''' + self.assertEquals(sync.get_filter(opts=['inc=*']), None) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.get_filter') + @patch('tools.charm_helpers_sync.charm_helpers_sync.ensure_init') + @patch('shutil.copytree') + @patch('shutil.rmtree') + @patch('os.path.exists') + def test_sync_directory(self, exists, rmtree, copytree, ensure_init, + _filter): + '''It correctly syncs src directory to dest directory''' + _filter.return_value = None + sync.sync_directory('/tmp/charm-helpers/charmhelpers/core', + 'hooks/charmhelpers/core') + exists.return_value = True + rmtree.assert_called_with('hooks/charmhelpers/core') + copytree.assert_called_with('/tmp/charm-helpers/charmhelpers/core', + 'hooks/charmhelpers/core', ignore=None) + ensure_init.assert_called_with('hooks/charmhelpers/core') + + @patch('os.path.isfile') + def test_is_pyfile(self, isfile): + '''It correctly identifies incomplete path to a py src file as such''' + sync._is_pyfile('/tmp/charm-helpers/charmhelpers/core/host') + isfile.assert_called_with( + '/tmp/charm-helpers/charmhelpers/core/host.py' + ) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.sync_pyfile') + @patch('tools.charm_helpers_sync.charm_helpers_sync.sync_directory') + @patch('os.path.isdir') + def test_syncs_directory(self, is_dir, sync_dir, sync_pyfile): + '''It correctly syncs a module directory''' + is_dir.return_value = True + sync.sync(src='/tmp/charm-helpers', + dest='hooks/charmhelpers', + module='contrib.openstack') + + sync_dir.assert_called_with( + '/tmp/charm-helpers/charmhelpers/contrib/openstack', + 'hooks/charmhelpers/contrib/openstack', None) + + # __init__.py files leading to the directory were also synced. + sync_pyfile.assert_has_calls([ + call('/tmp/charm-helpers/charmhelpers/__init__', + 'hooks/charmhelpers'), + call('/tmp/charm-helpers/charmhelpers/contrib/__init__', + 'hooks/charmhelpers/contrib')]) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.sync_pyfile') + @patch('tools.charm_helpers_sync.charm_helpers_sync._is_pyfile') + @patch('os.path.isdir') + def test_syncs_file(self, is_dir, is_pyfile, sync_pyfile): + '''It correctly syncs a module file''' + is_dir.return_value = False + is_pyfile.return_value = True + sync.sync(src='/tmp/charm-helpers', + dest='hooks/charmhelpers', + module='contrib.openstack.utils') + sync_pyfile.assert_has_calls([ + call('/tmp/charm-helpers/charmhelpers/__init__', + 'hooks/charmhelpers'), + call('/tmp/charm-helpers/charmhelpers/contrib/__init__', + 'hooks/charmhelpers/contrib'), + call('/tmp/charm-helpers/charmhelpers/contrib/openstack/__init__', + 'hooks/charmhelpers/contrib/openstack'), + call('/tmp/charm-helpers/charmhelpers/contrib/openstack/utils', + 'hooks/charmhelpers/contrib/openstack')]) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.sync') + @patch('os.path.isdir') + @patch('os.path.exists') + def test_sync_helpers_from_config(self, exists, isdir, _sync): + '''It correctly syncs a list of included helpers''' + include = yaml.safe_load(INCLUDE)['include'] + isdir.return_value = True + exists.return_value = False + sync.sync_helpers(include=include, + src='/tmp/charm-helpers', + + dest='hooks/charmhelpers') + mods = [ + 'core', + 'contrib.openstack', + 'contrib.storage', + 'contrib.hahelpers.utils', + 'contrib.hahelpers.ceph_utils', + 'contrib.hahelpers.cluster_utils', + 'contrib.hahelpers.haproxy_utils' + ] + + ex_calls = [] + [ex_calls.append( + call('/tmp/charm-helpers', 'hooks/charmhelpers', c, []) + ) for c in mods] + self.assertEquals(ex_calls, _sync.call_args_list) + + @patch('tools.charm_helpers_sync.charm_helpers_sync.sync') + @patch('os.path.isdir') + @patch('os.path.exists') + @patch('shutil.rmtree') + def test_sync_helpers_from_config_cleanup(self, _rmtree, _exists, + isdir, _sync): + '''It correctly syncs a list of included helpers''' + include = yaml.safe_load(INCLUDE)['include'] + isdir.return_value = True + _exists.return_value = True + + sync.sync_helpers(include=include, + src='/tmp/charm-helpers', + + dest='hooks/charmhelpers') + _rmtree.assert_called_with('hooks/charmhelpers') + mods = [ + 'core', + 'contrib.openstack', + 'contrib.storage', + 'contrib.hahelpers.utils', + 'contrib.hahelpers.ceph_utils', + 'contrib.hahelpers.cluster_utils', + 'contrib.hahelpers.haproxy_utils' + ] + + ex_calls = [] + [ex_calls.append( + call('/tmp/charm-helpers', 'hooks/charmhelpers', c, []) + ) for c in mods] + self.assertEquals(ex_calls, _sync.call_args_list) + + def test_extract_option_no_globals(self): + '''It extracts option from an included item with no global options''' + inc = 'contrib.openstack.templates|inc=*.template' + result = sync.extract_options(inc) + ex = ('contrib.openstack.templates', ['inc=*.template']) + self.assertEquals(ex, result) + + def test_extract_option_with_global_as_string(self): + '''It extracts option for include with global options as str''' + inc = 'contrib.openstack.templates|inc=*.template' + result = sync.extract_options(inc, global_options='inc=foo.*') + ex = ('contrib.openstack.templates', + ['inc=*.template', 'inc=foo.*']) + self.assertEquals(ex, result) + + def test_extract_option_with_globals(self): + '''It extracts option from an included item with global options''' + inc = 'contrib.openstack.templates|inc=*.template' + result = sync.extract_options(inc, global_options=['inc=*.cfg']) + ex = ('contrib.openstack.templates', ['inc=*.template', 'inc=*.cfg']) + self.assertEquals(ex, result) + + def test_extract_multiple_options_with_globals(self): + '''It extracts multiple options from an included item''' + inc = 'contrib.openstack.templates|inc=*.template,inc=foo.*' + result = sync.extract_options(inc, global_options=['inc=*.cfg']) + ex = ('contrib.openstack.templates', + ['inc=*.template', 'inc=foo.*', 'inc=*.cfg']) + self.assertEquals(ex, result) diff --git a/nrpe/mod/charmhelpers/tests/utils.py b/nrpe/mod/charmhelpers/tests/utils.py new file mode 100644 index 0000000..7ac4699 --- /dev/null +++ b/nrpe/mod/charmhelpers/tests/utils.py @@ -0,0 +1,79 @@ +# Copyright 2016 Canonical Ltd +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit test helpers from https://github.com/openstack/charms.openstack/""" + +import contextlib +import io +import mock +import unittest + + +@contextlib.contextmanager +def patch_open(): + '''Patch open() to allow mocking both open() itself and the file that is + yielded. + + Yields the mock for "open" and "file", respectively.''' + mock_open = mock.MagicMock(spec=open) + mock_file = mock.MagicMock(spec=io.FileIO) + + @contextlib.contextmanager + def stub_open(*args, **kwargs): + mock_open(*args, **kwargs) + yield mock_file + + with mock.patch('builtins.open', stub_open): + yield mock_open, mock_file + + +class BaseTestCase(unittest.TestCase): + + def setUp(self): + self._patches = {} + self._patches_start = {} + + def tearDown(self): + for k, v in self._patches.items(): + v.stop() + setattr(self, k, None) + self._patches = None + self._patches_start = None + + def patch_object(self, obj, attr, return_value=None, name=None, new=None, + **kwargs): + if name is None: + name = attr + if new is not None: + mocked = mock.patch.object(obj, attr, new=new, **kwargs) + else: + mocked = mock.patch.object(obj, attr, **kwargs) + self._patches[name] = mocked + started = mocked.start() + if new is None: + started.return_value = return_value + self._patches_start[name] = started + setattr(self, name, started) + + def patch(self, item, return_value=None, name=None, new=None, **kwargs): + if name is None: + raise RuntimeError("Must pass 'name' to .patch()") + if new is not None: + mocked = mock.patch(item, new=new, **kwargs) + else: + mocked = mock.patch(item, **kwargs) + self._patches[name] = mocked + started = mocked.start() + if new is None: + started.return_value = return_value + self._patches_start[name] = started + setattr(self, name, started) diff --git a/nrpe/mod/charmhelpers/tools/__init__.py b/nrpe/mod/charmhelpers/tools/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tools/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/tools/charm_helpers_sync/README.md b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/README.md new file mode 100644 index 0000000..30ba07b --- /dev/null +++ b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/README.md @@ -0,0 +1,156 @@ +Script for synchronizing charm-helpers into a charm branch. + +This script is intended to be used by charm authors during the development +of their charm. It allows authors to pull in bits of a charm-helpers source +tree and embed directly into their charm, to be deployed with the rest of +their hooks and charm payload. This script is not intended to be called +by the hooks themselves, but instead by the charm author while they are +hacking on a charm offline. Consider it a method of compiling specific +revision of a charm-helpers branch into a given charm source tree. + +Some goals and benefits to using a sync tool to manage this process: + + - Reduces the burden of manually copying in upstream charm helpers code + into a charm and helps ensure we can easily keep a specific charm's + helper code up to date. + + - Allows authors to hack on their own working branch of charm-helpers, + easily sync into their WIP charm. Any changes they've made to charm + helpers can be upstreamed via a merge of their charm-helpers branch + into lp:charm-helpers, ideally at the same time they are upstreaming + the charm itself into the charm store. Separating charm helper + development from charm development can help reduce cases where charms + are shipping locally modified helpers. + + - Avoids the need to ship the *entire* charm-helpers source tree with + a charm. Authors can selectively pick and choose what subset of helpers + to include to satisfy the goals of their charm. + +Allows specifying a list of dependencies to sync in from a charm-helpers +branch. Ideally, each charm should describe its requirements in a yaml +config included in the charm, eg `charm-helpers.yaml` (NOTE: Example module +layout as of 12/18/2019): + + $ cd my-charm + $ cat >charm-helpers.yaml </charm-helpers \ + -d hooks/helpers core contrib.openstack contrib.hahelpers + +or use a specific branch using the @ notation + + $ charm-helpers-sync.py -r https://github.com//charm-helpers@branch_name \ + -d hooks/helpers core contrib.openstack contrib.hahelpers + +Script will create missing `__init__.py`'s to ensure each subdirectory is +importable, assuming the script is run from the charm's top-level directory. diff --git a/nrpe/mod/charmhelpers/tools/charm_helpers_sync/__init__.py b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nrpe/mod/charmhelpers/tools/charm_helpers_sync/charm_helpers_sync.py b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/charm_helpers_sync.py new file mode 100755 index 0000000..7c0c194 --- /dev/null +++ b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/charm_helpers_sync.py @@ -0,0 +1,261 @@ +#!/usr/bin/python + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Authors: +# Adam Gandelman + +import logging +import optparse +import os +import subprocess +import shutil +import sys +import tempfile +import yaml +from fnmatch import fnmatch + +import six + +CHARM_HELPERS_REPO = 'https://github.com/juju/charm-helpers' + + +def parse_config(conf_file): + if not os.path.isfile(conf_file): + logging.error('Invalid config file: %s.' % conf_file) + return False + return yaml.load(open(conf_file).read()) + + +def clone_helpers(work_dir, repo): + dest = os.path.join(work_dir, 'charm-helpers') + logging.info('Cloning out %s to %s.' % (repo, dest)) + branch = None + if '@' in repo: + repo, branch = repo.split('@', 1) + cmd = ['git', 'clone', '--depth=1'] + if branch is not None: + cmd += ['--branch', branch] + cmd += [repo, dest] + subprocess.check_call(cmd) + return dest + + +def _module_path(module): + return os.path.join(*module.split('.')) + + +def _src_path(src, module): + return os.path.join(src, 'charmhelpers', _module_path(module)) + + +def _dest_path(dest, module): + return os.path.join(dest, _module_path(module)) + + +def _is_pyfile(path): + return os.path.isfile(path + '.py') + + +def ensure_init(path): + ''' + ensure directories leading up to path are importable, omitting + parent directory, eg path='/hooks/helpers/foo'/: + hooks/ + hooks/helpers/__init__.py + hooks/helpers/foo/__init__.py + ''' + for d, dirs, files in os.walk(os.path.join(*path.split('/')[:2])): + _i = os.path.join(d, '__init__.py') + if not os.path.exists(_i): + logging.info('Adding missing __init__.py: %s' % _i) + open(_i, 'wb').close() + + +def sync_pyfile(src, dest): + src = src + '.py' + src_dir = os.path.dirname(src) + logging.info('Syncing pyfile: %s -> %s.' % (src, dest)) + if not os.path.exists(dest): + os.makedirs(dest) + shutil.copy(src, dest) + if os.path.isfile(os.path.join(src_dir, '__init__.py')): + shutil.copy(os.path.join(src_dir, '__init__.py'), + dest) + ensure_init(dest) + + +def get_filter(opts=None): + opts = opts or [] + if 'inc=*' in opts: + # do not filter any files, include everything + return None + + def _filter(dir, ls): + incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt] + _filter = [] + for f in ls: + _f = os.path.join(dir, f) + + if not os.path.isdir(_f) and not _f.endswith('.py') and incs: + if True not in [fnmatch(_f, inc) for inc in incs]: + logging.debug('Not syncing %s, does not match include ' + 'filters (%s)' % (_f, incs)) + _filter.append(f) + else: + logging.debug('Including file, which matches include ' + 'filters (%s): %s' % (incs, _f)) + elif (os.path.isfile(_f) and not _f.endswith('.py')): + logging.debug('Not syncing file: %s' % f) + _filter.append(f) + elif (os.path.isdir(_f) and not + os.path.isfile(os.path.join(_f, '__init__.py'))): + logging.debug('Not syncing directory: %s' % f) + _filter.append(f) + return _filter + return _filter + + +def sync_directory(src, dest, opts=None): + if os.path.exists(dest): + logging.debug('Removing existing directory: %s' % dest) + shutil.rmtree(dest) + logging.info('Syncing directory: %s -> %s.' % (src, dest)) + + shutil.copytree(src, dest, ignore=get_filter(opts)) + ensure_init(dest) + + +def sync(src, dest, module, opts=None): + + # Sync charmhelpers/__init__.py for bootstrap code. + sync_pyfile(_src_path(src, '__init__'), dest) + + # Sync other __init__.py files in the path leading to module. + m = [] + steps = module.split('.')[:-1] + while steps: + m.append(steps.pop(0)) + init = '.'.join(m + ['__init__']) + sync_pyfile(_src_path(src, init), + os.path.dirname(_dest_path(dest, init))) + + # Sync the module, or maybe a .py file. + if os.path.isdir(_src_path(src, module)): + sync_directory(_src_path(src, module), _dest_path(dest, module), opts) + elif _is_pyfile(_src_path(src, module)): + sync_pyfile(_src_path(src, module), + os.path.dirname(_dest_path(dest, module))) + else: + logging.warn('Could not sync: %s. Neither a pyfile or directory, ' + 'does it even exist?' % module) + + +def parse_sync_options(options): + if not options: + return [] + return options.split(',') + + +def extract_options(inc, global_options=None): + global_options = global_options or [] + if global_options and isinstance(global_options, six.string_types): + global_options = [global_options] + if '|' not in inc: + return (inc, global_options) + inc, opts = inc.split('|') + return (inc, parse_sync_options(opts) + global_options) + + +def sync_helpers(include, src, dest, options=None): + if os.path.exists(dest): + logging.debug('Removing existing directory: %s' % dest) + shutil.rmtree(dest) + if not os.path.isdir(dest): + os.makedirs(dest) + + global_options = parse_sync_options(options) + + for inc in include: + if isinstance(inc, str): + inc, opts = extract_options(inc, global_options) + sync(src, dest, inc, opts) + elif isinstance(inc, dict): + # could also do nested dicts here. + for k, v in six.iteritems(inc): + if isinstance(v, list): + for m in v: + inc, opts = extract_options(m, global_options) + sync(src, dest, '%s.%s' % (k, inc), opts) + + +if __name__ == '__main__': + parser = optparse.OptionParser() + parser.add_option('-c', '--config', action='store', dest='config', + default=None, help='helper config file') + parser.add_option('-D', '--debug', action='store_true', dest='debug', + default=False, help='debug') + parser.add_option('-r', '--repository', action='store', dest='repo', + help='charm-helpers git repository (overrides config)') + parser.add_option('-d', '--destination', action='store', dest='dest_dir', + help='sync destination dir (overrides config)') + (opts, args) = parser.parse_args() + + if opts.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if opts.config: + logging.info('Loading charm helper config from %s.' % opts.config) + config = parse_config(opts.config) + if not config: + logging.error('Could not parse config from %s.' % opts.config) + sys.exit(1) + else: + config = {} + + if 'repo' not in config: + config['repo'] = CHARM_HELPERS_REPO + if opts.repo: + config['repo'] = opts.repo + if opts.dest_dir: + config['destination'] = opts.dest_dir + + if 'destination' not in config: + logging.error('No destination dir. specified as option or config.') + sys.exit(1) + + if 'include' not in config: + if not args: + logging.error('No modules to sync specified as option or config.') + sys.exit(1) + config['include'] = [] + [config['include'].append(a) for a in args] + + sync_options = None + if 'options' in config: + sync_options = config['options'] + tmpd = tempfile.mkdtemp() + try: + checkout = clone_helpers(tmpd, config['repo']) + sync_helpers(config['include'], checkout, config['destination'], + options=sync_options) + except Exception as e: + logging.error("Could not sync: %s" % e) + raise e + finally: + logging.debug('Cleaning up %s' % tmpd) + shutil.rmtree(tmpd) diff --git a/nrpe/mod/charmhelpers/tools/charm_helpers_sync/example-config.yaml b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/example-config.yaml new file mode 100644 index 0000000..8563ec0 --- /dev/null +++ b/nrpe/mod/charmhelpers/tools/charm_helpers_sync/example-config.yaml @@ -0,0 +1,14 @@ +# Import from remote git repository. +repo: https://github.com/juju/charm-helpers +# install helpers to ./hooks/charmhelpers +destination: hooks/charmhelpers +include: + # include all of charmhelpers.core + - core + # all of charmhelpers.payload + - payload + # and a subset of charmhelpers.contrib.hahelpers + - contrib.hahelpers: + - openstack_common + - ceph_utils + - utils diff --git a/nrpe/mod/charmhelpers/tox.ini b/nrpe/mod/charmhelpers/tox.ini new file mode 100644 index 0000000..a115776 --- /dev/null +++ b/nrpe/mod/charmhelpers/tox.ini @@ -0,0 +1,65 @@ +[tox] +envlist = pep8,py2,py3 +skipsdist = true +sitepackages = false +# NOTE(beisner): Avoid false positives by not skipping missing interpreters. +# NOTE(beisner): Avoid pollution by not enabling sitepackages. +# NOTE(beisner): the 'py3' env is useful to "just give me whatever py3 is here." +# NOTE(beisner): the 'py3x' envs are useful to use a distinct interpreter version (will fail if not found) +ignore_basepython_conflict = true +# NOTES: +# * We avoid the new dependency resolver by pinning pip < 20.3, see +# https://github.com/pypa/pip/issues/9187 +# * Pinning dependencies requires tox >= 3.2.0, see +# https://tox.readthedocs.io/en/latest/config.html#conf-requires +# * It is also necessary to pin virtualenv as a newer virtualenv would still +# lead to fetching the latest pip in the func* tox targets, see +# https://stackoverflow.com/a/38133283 +requires = pip < 20.3 + virtualenv < 20.0 +# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci +minversion = 3.2.0 + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONHASHSEED=0 +install_command = {toxinidir}/pip.sh install {opts} {packages} +passenv = HOME TERM +commands = nosetests -s --nologcapture {posargs} --with-coverage --cover-package=charmhelpers tests/ +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py2] +basepython = python +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py3] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py34] +basepython = python3.4 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py35] +basepython = python3.5 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py36] +basepython = python3.6 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py37] +basepython = python3.7 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:py38] +basepython = python3.8 +deps = -r{toxinidir}/test-requirements.txt + +[testenv:pep8] +basepython = python3 +deps = -r{toxinidir}/test-requirements.txt +commands = flake8 -v {posargs} charmhelpers tests tools + +[flake8] +ignore = E402,E501,E741,E722,W504 diff --git a/nrpe/repo-info b/nrpe/repo-info new file mode 100644 index 0000000..6c9f8f4 --- /dev/null +++ b/nrpe/repo-info @@ -0,0 +1 @@ +candidate/21.10 diff --git a/nrpe/revision b/nrpe/revision new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/nrpe/revision @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/nrpe/templates/export_host.cfg.tmpl b/nrpe/templates/export_host.cfg.tmpl new file mode 100644 index 0000000..cac188b --- /dev/null +++ b/nrpe/templates/export_host.cfg.tmpl @@ -0,0 +1,10 @@ +#--------------------------------------------------- +# This file is Juju managed +#-------------------------------------------------- + +define host { + address {{ nagios_ipaddress }} + host_name {{ nagios_hostname }} + use {{ hostcheck_inherit }} + hostgroups machines, {{ hostgroups }} +} diff --git a/nrpe/templates/nrpe.tmpl b/nrpe/templates/nrpe.tmpl new file mode 100644 index 0000000..5cfc06b --- /dev/null +++ b/nrpe/templates/nrpe.tmpl @@ -0,0 +1,18 @@ +#-------------------------------------------------------- +# This file is managed by Juju +#-------------------------------------------------------- + +# See https://github.com/stockholmuniversity/Nagios-NRPE/blob/2.0.10/share/nrpe.cfg +server_address={{ nrpe_ipaddress or '0.0.0.0' }} +server_port={{ server_port }} +allowed_hosts={{ external_nagios_master }},{{ monitor_allowed_hosts }} +nrpe_user=nagios +nrpe_group=nagios +dont_blame_nrpe={{ dont_blame_nrpe }} +debug={{ debug }} +command_timeout=60 +pid_file=/var/run/nagios/nrpe.pid + +# All configuration snippets go into nrpe.d/ +include_dir=/etc/nagios/nrpe.d/ + diff --git a/nrpe/templates/nrpe_command.tmpl b/nrpe/templates/nrpe_command.tmpl new file mode 100644 index 0000000..f602e38 --- /dev/null +++ b/nrpe/templates/nrpe_command.tmpl @@ -0,0 +1,3 @@ +# {{ description }} +command[{{ cmd_name }}]={{ cmd_exec }} {{ cmd_params }} + diff --git a/nrpe/templates/rsync-juju.d.tmpl b/nrpe/templates/rsync-juju.d.tmpl new file mode 100644 index 0000000..a625958 --- /dev/null +++ b/nrpe/templates/rsync-juju.d.tmpl @@ -0,0 +1,11 @@ +#------------------------------------------------------------ +# This file is managed by Juju. +#------------------------------------------------------------ + +[external-nagios] + path = /var/lib/nagios/export/ + comment = External Nagios Node configs + list = false + read only = true + lock file = /var/run/rsyncd.external-nagios.lock + hosts allow = {{ external_nagios_master }} diff --git a/nrpe/test-mon.yaml b/nrpe/test-mon.yaml new file mode 100644 index 0000000..4c11bf6 --- /dev/null +++ b/nrpe/test-mon.yaml @@ -0,0 +1,24 @@ +version: '0.3' +monitors: + local: + procrunning: + jujud: + name: Juju Running + min: 1 + max: 1 + executable: jujud + rsync: + name: RSYNc Running + min: 1 + max: 1 + executable: rsync + remote: + tcp: + ssh: + name: SSH Running + port: 22 + string: SSH.* + expect: + warning: 2 + critical: 10 + timeout: 12 diff --git a/nrpe/tests/00-setup b/nrpe/tests/00-setup new file mode 100755 index 0000000..dec09eb --- /dev/null +++ b/nrpe/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python3-requests python3-distro-info -y diff --git a/nrpe/tests/10-tests b/nrpe/tests/10-tests new file mode 100755 index 0000000..6b74ebf --- /dev/null +++ b/nrpe/tests/10-tests @@ -0,0 +1,391 @@ +#!/usr/bin/python3 + +import amulet +import unittest +import time +import yaml +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils, +) +autils = AmuletUtils() +PAUSE_TIME = 30 + + +class TestDeployment(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.deployment = amulet.Deployment(series='trusty') + cls.deployment.add('mysql') + cls.deployment.add('nrpe') + cls.deployment.add('nagios') + cls.deployment.configure('mysql', {'dataset-size': '10%'}) + cls.deployment.relate('nrpe:monitors', + 'nagios:monitors') + cls.deployment.relate('nrpe:local-monitors', + 'mysql:local-monitors') + cls.deployment.expose('nagios') + try: + cls.deployment.setup(timeout=900) + cls.deployment.sentry.wait() + except amulet.helpers.TimeoutError: + msg = "Environment wasn't stood up in time" + amulet.raise_status(amulet.SKIP, msg=msg) + except Exception: + raise + + def check_nrpe_setting(self, filename, expected_settings, juju_kv, + filedelim=None): + self.nrpe_sentry = self.deployment.sentry['nrpe'][0] + if juju_kv: + self.deployment.configure('nrpe', juju_kv) + time.sleep(PAUSE_TIME) + nrpe_contents = self.nrpe_sentry.file_contents(filename) + for line in nrpe_contents.split('\n'): + if not line: + continue + line_key = line.split(filedelim)[0] + if line_key in expected_settings.keys(): + line_value = ' '.join(line.split(filedelim)[1:]) + if line_value != expected_settings[line_key]: + msg = ('Setting %s in %s did not contain expected value ' + '(%s != %s)') % (line_key, filename, line_value, + expected_settings[line_key]) + amulet.raise_status(amulet.FAIL, msg=msg) + + def test_monitors_relation_sub_monitors(self): + """ Check default monitor definitions are passed to Nagios """ + self.deployment.configure('nrpe', {'monitors': ''}) + time.sleep(PAUSE_TIME) + self.nrpe_sentry = self.deployment.sentry['nrpe'][0] + relation_data = self.nrpe_sentry.relation( + 'monitors', + 'nagios:monitors', + ) + monitors = yaml.safe_load(relation_data['monitors']) + checks = [ + 'check_mem_sub', + 'check_disk_root_sub', + 'check_swap_sub', + 'check_load_sub', + 'check_users_sub', + 'check_zombie_procs_sub', + 'check_total_procs_sub', + 'check_conntrack_sub', + 'check_swap_activity_sub', + ] + for check in checks: + if check not in monitors['monitors']['remote']['nrpe'].keys(): + amulet.raise_status( + amulet.FAIL, + msg='{} not found in monitors relation'.format(check) + ) + + def test_monitors_relation_principal_monitors(self): + """ Check monitor definitions from principal are passed to Nagios """ + self.nrpe_sentry = self.deployment.sentry['nrpe'][0] + relation_data = self.nrpe_sentry.relation( + 'monitors', + 'nagios:monitors', + ) + monitors = yaml.safe_load(relation_data['monitors']) + if 'mysql' not in monitors['monitors']['remote'].keys(): + amulet.raise_status( + amulet.FAIL, + msg='mysql remote monitor not found in monitors relation', + ) + nrpe_checks = monitors['monitors']['remote']['nrpe'].keys() + if 'check_proc_mysqld_principal' not in nrpe_checks: + amulet.raise_status( + amulet.FAIL, + msg='mysql process monitor not found in monitors relation', + ) + + def test_monitors_relation_user_monitors(self): + """ Check user configured monitor definitions are passed to Nagios """ + user_monitors = { + 'version': '0.3', + 'monitors': { + 'local': { + 'procrunning': { + 'rsync': { + 'max': 1, + 'executable': 'rsync', + 'name': 'RSYNc Running', + 'min': 1 + }, + 'jujud': { + 'max': 1, + 'executable': 'jujud', + 'name': 'Juju Running', + 'min': 1 + } + } + }, + 'remote': { + 'tcp': { + 'ssh': { + 'warning': 2, + 'critical': 10, + 'name': 'SSH Running', + 'timeout': 12, + 'port': 22, + 'string': 'SSH.*', + 'expect': None + } + } + } + } + } + self.deployment.configure( + 'nrpe', {'monitors': yaml.dump(user_monitors)} + ) + time.sleep(PAUSE_TIME) + self.nrpe_sentry = self.deployment.sentry['nrpe'][0] + relation_data = self.nrpe_sentry.relation( + 'monitors', + 'nagios:monitors', + ) + monitors = yaml.safe_load(relation_data['monitors']) + checks = ['check_proc_jujud_user', 'check_proc_jujud_user'] + for check in checks: + if check not in monitors['monitors']['remote']['nrpe'].keys(): + amulet.raise_status( + amulet.FAIL, + msg='{} not found in monitors relation'.format(check), + ) + if 'ssh' not in monitors['monitors']['remote']['tcp'].keys(): + amulet.raise_status( + amulet.FAIL, + msg='{} not found in monitors relation'.format(check), + ) + + def test_services(self): + """ Test basic services are running """ + self.nagios_sentry = self.deployment.sentry['nagios'][0] + self.nrpe_sentry = self.deployment.sentry['nrpe'][0] + commands = { + self.nrpe_sentry: ['service nagios-nrpe-server status'], + self.nagios_sentry: ['service nagios3 status'], + } + ret = autils.validate_services(commands) + if ret: + amulet.raise_status(amulet.FAIL, msg=ret) + + def test_config_nagios_master(self): + unit = self.deployment.sentry['nagios'][0] + nagios_relation = unit.relation('monitors', 'nrpe:monitors') + ipaddr = nagios_relation['private-address'] + test_config = { + 'filename': '/etc/nagios/nrpe.cfg', + 'expected_settings': { + 'allowed_hosts': '127.0.0.1,10.0.0.10,' + ipaddr + }, + 'juju_kv': {'nagios_master': '10.0.0.10'}, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + + def test_config_rsync_fragment(self): + test_config = { + 'filename': '/etc/rsync-juju.d/010-nrpe-external-master.conf', + 'expected_settings': {'hosts allow': '10.0.0.10'}, + 'juju_kv': { + 'nagios_master': '10.0.0.10', + 'export_nagios_definitions': True, + }, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + + def test_config_rsync(self): + test_config = { + 'filename': '/etc/rsyncd.conf', + 'expected_settings': {'&include': '/etc/rsync-juju.d'}, + 'juju_kv': {'export_nagios_definitions': True}, + } + self.check_nrpe_setting(**test_config) + + def test_config_server_port(self): + test_config = { + 'filename': '/etc/nagios/nrpe.cfg', + 'expected_settings': {'server_port': '5888'}, + 'juju_kv': {'server_port': '5888'}, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + self.deployment.configure('nrpe', {'server_port': '5888'}) + + def test_config_debug(self): + test_config = { + 'filename': '/etc/nagios/nrpe.cfg', + 'expected_settings': {'debug': '1'}, + 'juju_kv': {'debug': 'True'}, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + self.deployment.configure('nrpe', {'debug': 'True'}) + + def test_config_dont_blame_nrpe(self): + test_config = { + 'filename': '/etc/nagios/nrpe.cfg', + 'expected_settings': {'dont_blame_nrpe': '1'}, + 'juju_kv': {'dont_blame_nrpe': 'True'}, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + self.deployment.configure('nrpe', {'dont_blame_nrpe': 'True'}) + + def test_nagios_host_context(self): + hostname = 'bob-mysql-0' + test_config = { + 'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname), + 'expected_settings': {'host_name': hostname, + 'use': 'masterhostgroup', + 'hostgroups': 'machines, myhostgroup1'}, + 'juju_kv': { + 'nagios_host_context': 'bob', + 'hostcheck_inherit': 'masterhostgroup', + 'nagios_hostname_type': 'unit', + 'hostgroups': 'myhostgroup1', + 'export_nagios_definitions': True, + }, + } + self.check_nrpe_setting(**test_config) + + def test_nagios_hostname_type(self): + sentry = self.deployment.sentry['nrpe'][0] + hostname = sentry.run('hostname')[0] + test_config = { + 'filename': '/var/lib/nagios/export/host__%s.cfg' % (hostname), + 'expected_settings': {'host_name': hostname, + 'use': 'masterhostgroup', + 'hostgroups': 'machines, myhostgroup1'}, + 'juju_kv': { + 'nagios_host_context': 'bob', + 'hostcheck_inherit': 'masterhostgroup', + 'hostgroups': 'myhostgroup1', + 'nagios_hostname_type': 'host', + 'export_nagios_definitions': True + }, + } + self.check_nrpe_setting(**test_config) + + def test_sub_postfix(self): + check_cmd = ('/usr/lib/nagios/plugins/check_disk -u GB -w 25% -c 20% ' + '-K 5% -p /') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_disk_root_testing.cfg', + 'expected_settings': {'command[check_disk_root]': check_cmd}, + 'juju_kv': {'sub_postfix': '_testing'}, + 'filedelim': '=', + } + self.check_nrpe_setting(**test_config) + + def test_custom_disk_check_params(self): + chk_key = 'command[check_disk_root]=/usr/lib/nagios/plugins/check_disk' + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_disk_root_sub.cfg', + 'expected_settings': {chk_key: '-u GB -w 5% -c 1% -K 10% -p /'}, + 'juju_kv': {'disk_root': '-u GB -w 5% -c 1% -K 10%'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_zombie_check_params(self): + chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/' + 'check_procs') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_zombie_procs_sub.cfg', + 'expected_settings': {chk_key: '-w 6 -c 12 -s Z'}, + 'juju_kv': {'zombies': '-w 6 -c 12 -s Z'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_procs_check_params(self): + chk_key = ('command[check_zombie_procs]=/usr/lib/nagios/plugins/' + 'check_procs') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_total_procs_sub.cfg', + 'expected_settings': {chk_key: '-w 40 -c 60'}, + 'juju_kv': {'procs': '-w 40 -c 60'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_load_check_params(self): + chk_key = 'command[check_load]=/usr/lib/nagios/plugins/check_load' + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_load_sub.cfg', + 'expected_settings': {chk_key: '-w 9,9,9 -c 16,16,16'}, + 'juju_kv': {'load': '-w 9,9,9 -c 16,16,16'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_users_check_params(self): + chk_key = 'command[check_users]=/usr/lib/nagios/plugins/check_users' + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_users_sub.cfg', + 'expected_settings': {chk_key: '-w 40 -c 50'}, + 'juju_kv': {'users': '-w 40 -c 50'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_swap_check_params(self): + chk_key = 'command[check_swap]=/usr/lib/nagios/plugins/check_swap' + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_swap_sub.cfg', + 'expected_settings': {chk_key: '-w 5% -c 1%'}, + 'juju_kv': {'swap': '-w 5% -c 1%'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_swap_activity_check_params(self): + chk_key = 'command[check_swap_activity]=/usr/local/lib/nagios/plugins/check_swap_activity' + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_swap_activity_sub.cfg', + 'expected_settings': {chk_key: '-w 20 -c 700'}, + 'juju_kv': {'swap_activity': '-w 20 -c 700'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_conntrack_check_params(self): + chk_key = ('command[check_conntrack]=/usr/local/lib/nagios/plugins/' + 'check_conntrack.sh') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_conntrack_sub.cfg', + 'expected_settings': {chk_key: '-w 50 -c 70'}, + 'juju_kv': {'conntrack': '-w 50 -c 70'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_lacp_bonds(self): + chk_key = ('command[check_lacp_bond0]=/usr/local/lib/nagios/plugins/' + 'check_lacp_bond.py -i bond0') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_lacp_bond0.cfg', + 'expected_settings': {chk_key: 'bond0'}, + 'juju_kv': {'lacp_bonds': 'bond0'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_netlinks_string(self): + chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/' + 'check_netlinks.py -i eth0 -m 1500 -s 10000') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg', + 'expected_settings': {chk_key: 'eth0 mtu:1500 speed:10000'}, + 'juju_kv': {'netlinks': 'eth0 mtu:1500 speed:10000'} + } + self.check_nrpe_setting(**test_config) + + def test_custom_netlinks_yaml_list(self): + chk_key = ('command[check_netlinks_eth0]=/usr/local/lib/nagios/plugins/' + 'check_netlinks.py -i eth0 -m 1500 -s 10000') + test_config = { + 'filename': '/etc/nagios/nrpe.d/check_netlinks_eth0.cfg', + 'expected_settings': {chk_key: ['eth0 mtu:1500 speed:10000']}, + 'juju_kv': {'netlinks': "['eth0 mtu:1500 speed:10000']"} + } + self.check_nrpe_setting(**test_config) + + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/tests/11-monitors-configurations b/nrpe/tests/11-monitors-configurations new file mode 100755 index 0000000..c1d70cc --- /dev/null +++ b/nrpe/tests/11-monitors-configurations @@ -0,0 +1,60 @@ +#!/usr/bin/python3 + +import amulet +import requests + +seconds = 20000 + +d = amulet.Deployment(series='trusty') + +d.add('nagios') +d.add('mysql') +d.add('nrpe') + +d.configure('mysql', {'dataset-size': '10%'}) +d.relate('nagios:monitors', 'mysql:monitors') +d.relate('nrpe:monitors', 'nagios:monitors') +d.relate('nrpe:local-monitors', 'mysql:local-monitors') + +d.expose('nagios') + +try: + d.setup(timeout=seconds) +except amulet.helpers.TimeoutError: + amulet.raise_status(amulet.SKIP, msg="Environment wasn't stood up in time") +except: + raise + + +## +# Set relationship aliases +## +mysql_unit = d.sentry['mysql'][0] +nagios_unit = d.sentry['nagios'][0] + + +def test_nrpe_monitors_config(): + # look for procrunning in nrpe config + try: + mysql_unit.file_contents('/etc/nagios/nrpe.d/' + 'check_proc_mysqld_principal.cfg') + except IOError as e: + amulet.raise_status(amulet.ERROR, + msg="procrunning config not found. Error:" + + e.args[1]) + + +def test_nagios_monitors_response(): + # look for mysql_database requests + nagpwd = nagios_unit.file_contents('/var/lib/juju/nagios.passwd').strip() + host_url = ("http://%s/cgi-bin/nagios3/status.cgi?" + "host=mysql-0") + r = requests.get(host_url % nagios_unit.info['public-address'], + auth=('nagiosadmin', nagpwd)) + if not r.text.find('mysql-0-basic'): + amulet.raise_status(amulet.ERROR, + msg='Nagios is not monitoring the' + + ' hosts it supposed to.') + +test_nrpe_monitors_config() +test_nagios_monitors_response() diff --git a/nrpe/tests/13-monitors-config b/nrpe/tests/13-monitors-config new file mode 100755 index 0000000..2b3c7bb --- /dev/null +++ b/nrpe/tests/13-monitors-config @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import amulet +import requests + +seconds = 20000 + +d = amulet.Deployment(series='trusty') + +d.add('nagios') +d.add('mysql') +d.add('nrpe') + +d.configure('mysql', {'dataset-size': '10%'}) +d.relate('nagios:monitors', 'mysql:monitors') +d.relate('nrpe:monitors', 'nagios:monitors') +d.relate('nrpe:local-monitors', 'mysql:local-monitors') + +d.configure('nrpe', { + 'monitors': """ + version: '0.3' + monitors: + remote: {} + local: + processcount: + all: + max: 100 + """ +}) + +d.expose('nagios') + +try: + d.setup(timeout=seconds) +except amulet.helpers.TimeoutError: + amulet.raise_status(amulet.SKIP, msg="Environment wasn't stood up in time") +except: + raise + + +## +# Set relationship aliases +## +mysql_unit = d.sentry['mysql'][0] +nagios_unit = d.sentry['nagios'][0] + + +def test_nrpe_monitors_config(): + # look for procrunning in nrpe config + try: + mysql_unit.file_contents('/etc/nagios/nrpe.d/' + 'check_proc_mysqld_principal.cfg') + except IOError as e: + amulet.raise_status(amulet.ERROR, + msg="procrunning config not found. Error:" + + e.args[1]) + + +def test_nagios_monitors_response(): + # look for mysql_database requests + nagpwd = nagios_unit.file_contents('/var/lib/juju/nagios.passwd').strip() + host_url = ("http://%s/cgi-bin/nagios3/status.cgi?" + "host=mysql-0") + r = requests.get(host_url % nagios_unit.info['public-address'], + auth=('nagiosadmin', nagpwd)) + if not r.text.find('processcount'): + amulet.raise_status(amulet.ERROR, + msg='Nagios is not monitoring the' + + ' hosts it supposed to.') + +test_nrpe_monitors_config() +test_nagios_monitors_response() diff --git a/nrpe/tests/14-basic-nrpe-external-master b/nrpe/tests/14-basic-nrpe-external-master new file mode 100755 index 0000000..c4bbfaf --- /dev/null +++ b/nrpe/tests/14-basic-nrpe-external-master @@ -0,0 +1,75 @@ +#!/usr/bin/python3 + +import amulet +import re +import unittest +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils, +) +autils = AmuletUtils() + + +class TestBasicNRPEExternalMasterDeployment(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.deployment = amulet.Deployment(series='trusty') + cls.deployment.add('mysql') + cls.deployment.add('nrpe') + cls.deployment.configure('nrpe', {'nagios_hostname_type': 'unit', + 'nagios_host_context': 'mygroup'}) + cls.deployment.configure('mysql', {'dataset-size': '10%'}) + cls.deployment.relate('nrpe:nrpe-external-master', + 'mysql:nrpe-external-master') + try: + cls.deployment.setup(timeout=900) + cls.deployment.sentry.wait() + except amulet.helpers.TimeoutError: + msg = "Environment wasn't stood up in time" + amulet.raise_status(amulet.SKIP, msg=msg) + except: + raise + + def test_nrpe_external_master_relation(self): + """ + Check nagios_hostname and nagions_host_context are passed to principals + """ + nrpe_sentry = self.deployment.sentry['nrpe'][0] + mysql = self.deployment.sentry['mysql'][0] + relation = [ + 'nrpe-external-master', + 'mysql:nrpe-external-master'] + nagios_hostname = "mygroup-{}".format( + mysql.info["unit_name"].replace('/', '-')) + expected = { + 'ingress-address': autils.valid_ip, + 'private-address': autils.valid_ip, + 'nagios_hostname': nagios_hostname, + 'nagios_host_context': 'mygroup'} + ret = autils.validate_relation_data(nrpe_sentry, relation, expected) + if ret: + message = autils.relation_error("nrpe to mysql principal", ret) + amulet.raise_status(amulet.FAIL, msg=message) + + def test_exported_nagiosconfig_nrpe_external_master_principal(self): + """ + The hostname defined in exported nagios service files matches + nagios_hostname from the nrpe-external-master relation. + """ + mysql = self.deployment.sentry['mysql'][0] + nagios_hostname = "mygroup-{}".format( + mysql.info["unit_name"].replace('/', '-')) + mysql_service_file = "service__mygroup-{}_check_mysql.cfg".format( + mysql.info["unit_name"].replace('/', '-')) + content = mysql.file_contents("/var/lib/nagios/export/{}".format( + mysql_service_file)) + for line in content.split('\n'): + host_match = re.match('.*host_name\s+([-\w]+)',line) + if host_match: + service_hostname = host_match.groups()[0] + if service_hostname != nagios_hostname: + message = 'Invalid host_name {} in {}. Expected {}'.format( + service_hostname, mysql_service_file, nagios_hostname) + amulet.raise_status(amulet.FAIL, msg=message) + +if __name__ == '__main__': + unittest.main() diff --git a/nrpe/tests/charmhelpers b/nrpe/tests/charmhelpers new file mode 120000 index 0000000..488c89e --- /dev/null +++ b/nrpe/tests/charmhelpers @@ -0,0 +1 @@ +../mod/charmhelpers/charmhelpers \ No newline at end of file diff --git a/nrpe/tests/functional/requirements.txt b/nrpe/tests/functional/requirements.txt new file mode 100644 index 0000000..bbe8435 --- /dev/null +++ b/nrpe/tests/functional/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/openstack-charmers/zaza.git#egg=zaza +python-openstackclient diff --git a/nrpe/tests/functional/tests/bundles/bionic.yaml b/nrpe/tests/functional/tests/bundles/bionic.yaml new file mode 100644 index 0000000..f83462a --- /dev/null +++ b/nrpe/tests/functional/tests/bundles/bionic.yaml @@ -0,0 +1,19 @@ +series: bionic +applications: + rabbitmq-server: + charm: cs:rabbitmq-server + num_units: 1 + container: + charm: cs:ubuntu + num_units: 1 + to: ["lxd:rabbitmq-server/0"] + nagios: + charm: cs:nagios + num_units: 1 +relations: + - - rabbitmq-server:nrpe-external-master + - nrpe:nrpe-external-master + - - container:juju-info + - nrpe:general-info + - - nrpe:monitors + - nagios:monitors diff --git a/nrpe/tests/functional/tests/bundles/focal.yaml b/nrpe/tests/functional/tests/bundles/focal.yaml new file mode 100644 index 0000000..33bea11 --- /dev/null +++ b/nrpe/tests/functional/tests/bundles/focal.yaml @@ -0,0 +1,21 @@ +series: focal +applications: + rabbitmq-server: + charm: cs:rabbitmq-server + num_units: 1 + constraints: root-disk=8G + container: + charm: cs:ubuntu + num_units: 1 + to: ["lxd:rabbitmq-server/0"] + nagios: + charm: cs:nagios + num_units: 1 + series: bionic +relations: + - - rabbitmq-server:nrpe-external-master + - nrpe:nrpe-external-master + - - container:juju-info + - nrpe:general-info + - - nrpe:monitors + - nagios:monitors diff --git a/nrpe/tests/functional/tests/bundles/overlays/local-charm-overlay.yaml.j2 b/nrpe/tests/functional/tests/bundles/overlays/local-charm-overlay.yaml.j2 new file mode 100644 index 0000000..005f302 --- /dev/null +++ b/nrpe/tests/functional/tests/bundles/overlays/local-charm-overlay.yaml.j2 @@ -0,0 +1,3 @@ +applications: + nrpe: + charm: "{{ CHARM_BUILD_DIR }}/{{ charm_name }}" diff --git a/nrpe/tests/functional/tests/bundles/xenial.yaml b/nrpe/tests/functional/tests/bundles/xenial.yaml new file mode 100644 index 0000000..89ef991 --- /dev/null +++ b/nrpe/tests/functional/tests/bundles/xenial.yaml @@ -0,0 +1,19 @@ +series: xenial +applications: + rabbitmq-server: + charm: cs:rabbitmq-server + num_units: 1 + container: + charm: cs:ubuntu + num_units: 1 + to: ["lxd:rabbitmq-server/0"] + nagios: + charm: cs:nagios + num_units: 1 +relations: + - - rabbitmq-server:nrpe-external-master + - nrpe:nrpe-external-master + - - container:juju-info + - nrpe:general-info + - - nrpe:monitors + - nagios:monitors diff --git a/nrpe/tests/functional/tests/nrpe_tests.py b/nrpe/tests/functional/tests/nrpe_tests.py new file mode 100644 index 0000000..9f288b2 --- /dev/null +++ b/nrpe/tests/functional/tests/nrpe_tests.py @@ -0,0 +1,280 @@ +"""Zaza functional tests.""" +import logging +import pprint +import unittest + +import yaml + +import zaza.model as model + + +class TestBase(unittest.TestCase): + """Base Class for charm functional tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for tests.""" + cls.model_name = model.get_juju_model() + cls.application_name = "nrpe" + cls.lead_unit_name = model.get_lead_unit_name( + cls.application_name, model_name=cls.model_name + ) + cls.units = model.get_units(cls.application_name, model_name=cls.model_name) + cls.nrpe_ip = model.get_app_ips(cls.application_name)[0] + + +class TestNrpe(TestBase): + """Class for charm functional tests.""" + + def test_01_nrpe_check(self): + """Verify nrpe check exists.""" + logging.debug( + "Verify the nrpe checks are created and have the required content..." + ) + + nrpe_checks = { + "check_conntrack.cfg": + "command[check_conntrack]=/usr/local/lib/nagios/plugins/" + "check_conntrack.sh", + "check_disk_root.cfg": + "command[check_disk_root]=/usr/lib/nagios/plugins/check_disk", + "check_load.cfg": "command[check_load]=/usr/lib/nagios/plugins/check_load", + "check_mem.cfg": + "command[check_mem]=/usr/local/lib/nagios/plugins/check_mem.pl", + "check_rabbitmq.cfg": + "command[check_rabbitmq]=" + "/usr/local/lib/nagios/plugins/check_rabbitmq.py", + "check_swap_activity.cfg": + "command[check_swap_activity]=" + "/usr/local/lib/nagios/plugins/check_swap_activity", + } + + for nrpe_check in nrpe_checks: + logging.info("Checking content of '{}' nrpe check".format(nrpe_check)) + cmd = "cat /etc/nagios/nrpe.d/" + nrpe_check + result = model.run_on_unit(self.lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning( + "Unable to find nrpe check {} at /etc/nagios/nrpe.d/".format( + nrpe_check + ) + ) + + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(nrpe_checks[nrpe_check] in content) + + def test_02_enable_swap(self): + """Check swap checks are applied.""" + swap = "-w 40% -c 25%" + model.set_application_config(self.application_name, {"swap": swap}) + model.block_until_all_units_idle() + cmd = "cat /etc/nagios/nrpe.d/check_swap.cfg" + result = model.run_on_unit(self.lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning( + "Unable to find nrpe check check_swap.cfg at /etc/nagios/nrpe.d/" + ) + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(swap in content) + + def test_02_remove_check(self): + """Verify swap check is removed.""" + model.set_application_config(self.application_name, {"swap": ""}) + model.block_until_all_units_idle() + cmd = "cat /etc/nagios/nrpe.d/check_swap.cfg" + result = model.run_on_unit(self.lead_unit_name, cmd) + self.assertTrue(result.get("Code") != 0) + + def test_03_user_monitor(self): + """Verify user monitors are applied.""" + user_monitors = { + "version": "0.3", + "monitors": { + "local": { + "procrunning": { + "rsync": { + "max": 1, + "executable": "rsync", + "name": "RSYNc Running", + "min": 1, + }, + "jujud": { + "max": 1, + "executable": "jujud", + "name": "Juju Running", + "min": 1, + }, + } + }, + "remote": { + "tcp": { + "ssh": { + "warning": 2, + "critical": 10, + "name": "SSH Running", + "timeout": 12, + "port": 22, + "string": "SSH.*", + "expect": None, + } + } + }, + }, + } + model.set_application_config( + self.application_name, {"monitors": yaml.dump(user_monitors)} + ) + model.block_until_all_units_idle() + + local_nrpe_checks = { + "check_proc_jujud_user.cfg": + "command[check_proc_jujud_user]=/usr/lib/nagios/plugins/" + "check_procs -w 1 -c 1 -C jujud", + "check_proc_rsync_user.cfg": + "command[check_proc_rsync_user]=/usr/lib/nagios/plugins/" + "check_procs -w 1 -c 1 -C rsync", + } + + for nrpe_check in local_nrpe_checks: + logging.info("Checking content of '{}' nrpe check".format(nrpe_check)) + cmd = "cat /etc/nagios/nrpe.d/" + nrpe_check + result = model.run_on_unit(self.lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning( + "Unable to find nrpe check {} at /etc/nagios/nrpe.d/".format( + nrpe_check + ) + ) + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(local_nrpe_checks[nrpe_check] in content) + + remote_nrpe_checks = { + "check_tcp_H_HOSTADDRESS__E_p22_s_SSH____eNone_w2_c10_t12_t10.cfg": + "/usr/lib/nagios/plugins/check_tcp -H $HOSTADDRESS$ " + "-E -p 22 -s 'SSH.*' -e None -w 2 -c 10 -t 12 -t 10" + } + for nrpe_check in remote_nrpe_checks: + logging.info( + "Checking content of '{}' nrpe command in nagios unit".format( + nrpe_check + ) + ) + cmd = "cat /etc/nagios3/conf.d/commands/" + nrpe_check + nagios_lead_unit_name = model.get_lead_unit_name( + "nagios", model_name=self.model_name + ) + result = model.run_on_unit(nagios_lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning( + "Unable to find nrpe command {} at " + "/etc/nagios3/conf.d/commands/ in nagios unit".format(nrpe_check) + ) + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(remote_nrpe_checks[nrpe_check] in content) + + def test_04_check_nagios_ip_is_allowed(self): + """Verify nagios ip is allowed in nrpe.cfg.""" + nagios_ip = model.get_app_ips("nagios")[0] + line = "allowed_hosts=127.0.0.1,{}/32".format(nagios_ip) + cmd = "cat /etc/nagios/nrpe.cfg" + result = model.run_on_unit(self.lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning("Unable to find nrpe config file at /etc/nagios/nrpe.cfg") + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(line in content) + + def test_05_netlinks(self): + """Check netlinks checks are applied.""" + netlinks = "- ens3 mtu:9000 speed:10000" + model.set_application_config(self.application_name, {"netlinks": netlinks}) + model.block_until_all_units_idle() + cmd = "cat /etc/nagios/nrpe.d/check_netlinks_ens3.cfg" + line = ( + "command[check_netlinks_ens3]=/usr/local/lib/nagios/plugins/" + "check_netlinks.py -i ens3 -m 9000 -s 1000" + ) + result = model.run_on_unit(self.lead_unit_name, cmd) + code = result.get("Code") + if code != "0": + logging.warning( + "Unable to find nrpe check at " + "/etc/nagios/nrpe.d/check_netlinks_ens3.cfg" + ) + raise model.CommandRunFailed(cmd, result) + content = result.get("Stdout") + self.assertTrue(line in content) + + def test_06_container_checks(self): + """Check that certain checks are enabled on hosts but disabled on containers.""" + # Enable appropriate config to enable various checks for testing whether they + # get created on containers versus hosts. + model.set_application_config(self.application_name, { + "disk_root": "-u GB -w 25% -c 20% -K 5%", + "zombies": "-w 3 -c 6 -s Z", + "procs": "-k -w 250 -c 300", + "load": "auto", + "conntrack": "-w 80 -c 90", + "users": "-w 20 -c 25", + "swap": "-w 40% -c 25%", + "swap_activity": "-i 5 -w 10240 -c 40960", + "mem": "-C -h -u -w 85 -c 90", + "lacp_bonds": "lo", # Enable a bogus lacp check on the loopback interface + "netlinks": "- ens3 mtu:9000 speed:10000", # Copied from test_05_netlinks + "xfs_errors": "5", + }) + model.block_until_all_units_idle() + + host_checks = self._get_unit_check_files("rabbitmq-server/0") + container_checks = self._get_unit_check_files("container/0") + expected_shared_checks = set([ + "check_conntrack.cfg", # I think this should be host-only, but am not sure. + "check_total_procs.cfg", + "check_users.cfg", + "check_zombie_procs.cfg", # This also feels host-only to me; thoughts? + ]) + expected_host_only_checks = set([ + "check_arp_cache.cfg", + "check_disk_root.cfg", + "check_lacp_lo.cfg", + "check_load.cfg", + "check_mem.cfg", + "check_netlinks_ens3.cfg", + "check_ro_filesystem.cfg", + "check_swap.cfg", + "check_swap_activity.cfg", + "check_xfs_errors.cfg", + ]) + self.assertTrue(expected_shared_checks.issubset(host_checks), + self._get_set_comparison(expected_shared_checks, + host_checks)) + self.assertTrue(expected_shared_checks.issubset(container_checks), + self._get_set_comparison(expected_shared_checks, + container_checks)) + self.assertTrue(expected_host_only_checks.issubset(host_checks), + self._get_set_comparison(expected_host_only_checks, + host_checks)) + self.assertTrue(expected_host_only_checks.isdisjoint(container_checks), + self._get_set_comparison(expected_host_only_checks, + container_checks)) + + def _get_unit_check_files(self, unit): + cmdline = "ls /etc/nagios/nrpe.d/" + result = model.run_on_unit(unit, cmdline) + self.assertEqual(result["Code"], "0") + return set(result["Stdout"].splitlines()) + + def _get_set_comparison(self, expected_checks, actual_checks): + return pprint.pformat({ + 'Expected:': expected_checks, + 'Actual:': actual_checks, + }) diff --git a/nrpe/tests/functional/tests/tests.yaml b/nrpe/tests/functional/tests/tests.yaml new file mode 100644 index 0000000..a661597 --- /dev/null +++ b/nrpe/tests/functional/tests/tests.yaml @@ -0,0 +1,13 @@ +charm_name: nrpe +gate_bundles: + - focal + - bionic + - xenial +smoke_bundles: + - focal +tests: + - tests.nrpe_tests.TestNrpe +target_deploy_status: + container: + workload-status: active + workload-status-message-prefix: "" diff --git a/nrpe/tests/tests.yaml b/nrpe/tests/tests.yaml new file mode 100644 index 0000000..757bcc2 --- /dev/null +++ b/nrpe/tests/tests.yaml @@ -0,0 +1,2 @@ +makefile: + - lint diff --git a/nrpe/tests/unit/__init__.py b/nrpe/tests/unit/__init__.py new file mode 100644 index 0000000..e0310a0 --- /dev/null +++ b/nrpe/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Unit tests.""" diff --git a/nrpe/tests/unit/requirements.txt b/nrpe/tests/unit/requirements.txt new file mode 100644 index 0000000..9d85006 --- /dev/null +++ b/nrpe/tests/unit/requirements.txt @@ -0,0 +1,4 @@ +coverage +six +PyYAML +netifaces diff --git a/nrpe/tests/unit/test_nrpe_helpers.py b/nrpe/tests/unit/test_nrpe_helpers.py new file mode 100644 index 0000000..949eb92 --- /dev/null +++ b/nrpe/tests/unit/test_nrpe_helpers.py @@ -0,0 +1,118 @@ +"""Unit tests for hooks/nrpe_helpers.py module.""" +import unittest +from unittest import mock + +import netifaces + +import nrpe_helpers +from nrpe_helpers import match_cidr_to_ifaces + + +class TestMatchCidrToIfaces(unittest.TestCase): + """Test match_cidr_to_ifaces helper function.""" + + mock_iface_ip_data = { + "lo": "127.0.0.1", + "eno1": "10.0.0.0", + "eno2": "10.1.0.0", + "fan-252": "252.0.0.1", + } + + def test_single_dev_match(self): + """Test single interface match.""" + self._run_mocked_test("10.0.0.0/16", ["eno1"]) + + def test_multi_dev_match(self): + """Test multiple interface match.""" + self._run_mocked_test("10.0.0.0/8", ["eno1", "eno2"]) + + def test_no_dev_match(self): + """Test no interface match.""" + self._run_mocked_test("192.168.0.0/16", []) + + def test_cidr_with_host_bits_set(self): + """Test invalid CIDR input (e.g. "eno1").""" + with self.assertRaises(Exception): + match_cidr_to_ifaces("10.1.2.3/8") # Should be 10.0.0.0/8 + + def test_iface_passed_in_as_cidr(self): + """Test invalid CIDR input (e.g. "eno1").""" + with self.assertRaises(Exception): + match_cidr_to_ifaces("eno1") + + @mock.patch("netifaces.ifaddresses") + @mock.patch("netifaces.interfaces") + def _run_mocked_test(self, cidr, matches, ifaces_mock, addrs_mock): + iface_ip_tuples = list(self.mock_iface_ip_data.items()) + ifaces_mock.return_value = [t[0] for t in iface_ip_tuples] + addrs_mock.side_effect = [ + {netifaces.AF_INET: [{"addr": t[1]}]} for t in iface_ip_tuples + ] + self.assertEqual(match_cidr_to_ifaces(cidr), matches) + + +class TestIngressAddress(unittest.TestCase): + """Test functions to provide a suitable ingress address.""" + + @mock.patch("nrpe_helpers.hookenv.config") + @mock.patch("nrpe_helpers.hookenv.network_get") + def test_get_bind_address(self, mock_network_get, mock_config): + """Prove we get a local IP address for interface binding.""" + mock_config.return_value = "private" + mock_network_get.return_value = { + "bind-addresses": [ + { + "mac-address": "06:f1:3a:74:ad:fe", + "interface-name": "ens5", + "addresses": [ + { + "hostname": "", + "address": "172.31.29.247", + "cidr": "172.31.16.0/20", + } + ], + "macaddress": "06:f1:3a:74:ad:fe", + "interfacename": "ens5", + } + ], + "egress-subnets": ["3.8.134.119/32"], + "ingress-addresses": ["3.8.134.119"], + } + self.assertEqual(nrpe_helpers.get_ingress_address("mockbinding"), + "172.31.29.247") + + @mock.patch("nrpe_helpers.hookenv.config") + @mock.patch("nrpe_helpers.hookenv.network_get") + def test_get_private_address(self, mock_network_get, mock_config): + """Prove we get a local IP address for Nagios relation.""" + mock_config.return_value = "private" + mock_network_get.return_value = { + "bind-addresses": [ + { + "mac-address": "06:f1:3a:74:ad:fe", + "interface-name": "ens5", + "addresses": [ + { + "hostname": "", + "address": "172.31.29.247", + "cidr": "172.31.16.0/20", + } + ], + "macaddress": "06:f1:3a:74:ad:fe", + "interfacename": "ens5", + } + ], + "egress-subnets": ["3.8.134.119/32"], + "ingress-addresses": ["3.8.134.119"], + } + self.assertEqual(nrpe_helpers.get_ingress_address("mockbinding", external=True), + "172.31.29.247") + + @mock.patch("nrpe_helpers.hookenv.config") + @mock.patch("nrpe_helpers.hookenv.unit_get") + def test_get_public_address(self, mock_unit_get, mock_config): + """Prove we get a public IP address for Nagios relation.""" + mock_config.return_value = "public" + mock_unit_get.return_value = "1.2.3.4" + self.assertEqual(nrpe_helpers.get_ingress_address("mockbinding", external=True), + "1.2.3.4") diff --git a/nrpe/tox.ini b/nrpe/tox.ini new file mode 100644 index 0000000..d179d89 --- /dev/null +++ b/nrpe/tox.ini @@ -0,0 +1,75 @@ +[tox] +skipsdist=True +skip_missing_interpreters = True +envlist = lint, unit, func + +[testenv] +basepython = python3 +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/lib/:{toxinidir}/hooks/ +passenv = + HOME + PATH + CHARM_BUILD_DIR + PYTEST_KEEP_MODEL + PYTEST_CLOUD_NAME + PYTEST_CLOUD_REGION + PYTEST_MODEL + MODEL_SETTINGS + HTTP_PROXY + HTTPS_PROXY + NO_PROXY + SNAP_HTTP_PROXY + SNAP_HTTPS_PROXY + OS_REGION_NAME + OS_AUTH_VERSION + OS_AUTH_URL + OS_PROJECT_DOMAIN_NAME + OS_USERNAME + OS_PASSWORD + OS_PROJECT_ID + OS_USER_DOMAIN_NAME + OS_PROJECT_NAME + OS_IDENTITY_API_VERSION + +[testenv:lint] +commands = + flake8 + black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod|tests)/" . +deps = + black + flake8 + flake8-docstrings + flake8-import-order + pep8-naming + flake8-colors + +[flake8] +exclude = + .git, + __pycache__, + .tox, + charmhelpers, + mod, + .build + +max-line-length = 88 +max-complexity = 14 + +[testenv:black] +commands = + black --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|dist|charmhelpers|mod|tests)/" . +deps = + black + +[testenv:unit] +commands = + coverage run -m unittest discover -s {toxinidir}/tests/unit -v + coverage report --omit tests/*,mod/*,.tox/* + coverage html --omit tests/*,mod/*,.tox/* +deps = -r{toxinidir}/tests/unit/requirements.txt + +[testenv:func] +changedir = {toxinidir}/tests/functional +commands = functest-run-suite {posargs} +deps = -r{toxinidir}/tests/functional/requirements.txt diff --git a/nrpe/version b/nrpe/version new file mode 100644 index 0000000..48a4708 --- /dev/null +++ b/nrpe/version @@ -0,0 +1 @@ +cs-nrpe-charmers-nrpe-27-190-ge3888d2-dirty \ No newline at end of file diff --git a/ubuntu/LICENSE b/ubuntu/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/ubuntu/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/ubuntu/README.md b/ubuntu/README.md new file mode 100644 index 0000000..09841bc --- /dev/null +++ b/ubuntu/README.md @@ -0,0 +1,49 @@ +# Overview + +This charm provides a blank [Ubuntu](http://ubuntu.com) image. It does not provide any applications other than a blank cloud image for you to manage manually, it is intended for testing and development. + +# Usage + +Step by step instructions on using this charm: + + juju deploy ubuntu + +You can then ssh to the instance with: + + juju ssh ubuntu/0 + +## Scale out Usage + +This charm is not designed to be used at scale since it does not have any relationships, however you can bulk add machines with `add-unit`: + + juju add-unit ubuntu # Add one more + juju add-unit -n5 ubuntu # Add 5 at a time + + +You can also alias names in order to organize a bunch of empty instances: + + juju deploy ubuntu mytestmachine1 + juju deploy ubuntu mytestmachine2 + +and so on. + +## Known Limitations and Issues + +This charm does not provide anything other than a blank server, so it does not relate to other charms. + +# Configuration + +This charm has no configuration options. + +# Contact Information + +## Upstream + +- [Ubuntu](http://ubuntu.com) +- [Bug tracker](http://bugs.launchpad.net/ubuntu) +- [Ubuntu Server Mailing list](https://lists.ubuntu.com/archives/ubuntu-server/) + +## Charm Contact Information + +- Author: Juju Charm Community +- Report bugs at: [Github](http://github.com/juju-solutions/charm-ubuntu/issues) diff --git a/ubuntu/config.yaml b/ubuntu/config.yaml new file mode 100644 index 0000000..fe0a7df --- /dev/null +++ b/ubuntu/config.yaml @@ -0,0 +1,6 @@ +options: + hostname: + type: string + default: "" + description: Override hostname of machine, when empty uses default machine hostname + diff --git a/ubuntu/dispatch b/ubuntu/dispatch new file mode 100755 index 0000000..1aa2949 --- /dev/null +++ b/ubuntu/dispatch @@ -0,0 +1,4 @@ +#!/bin/sh + +JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv \ + exec ./src/charm.py diff --git a/ubuntu/hooks/install b/ubuntu/hooks/install new file mode 100755 index 0000000..1aa2949 --- /dev/null +++ b/ubuntu/hooks/install @@ -0,0 +1,4 @@ +#!/bin/sh + +JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv \ + exec ./src/charm.py diff --git a/ubuntu/hooks/start b/ubuntu/hooks/start new file mode 100755 index 0000000..1aa2949 --- /dev/null +++ b/ubuntu/hooks/start @@ -0,0 +1,4 @@ +#!/bin/sh + +JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv \ + exec ./src/charm.py diff --git a/ubuntu/hooks/upgrade-charm b/ubuntu/hooks/upgrade-charm new file mode 100755 index 0000000..1aa2949 --- /dev/null +++ b/ubuntu/hooks/upgrade-charm @@ -0,0 +1,4 @@ +#!/bin/sh + +JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv \ + exec ./src/charm.py diff --git a/ubuntu/icon.svg b/ubuntu/icon.svg new file mode 100644 index 0000000..a5576ed --- /dev/null +++ b/ubuntu/icon.svg @@ -0,0 +1,279 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/ubuntu/manifest.yaml b/ubuntu/manifest.yaml new file mode 100644 index 0000000..d43d894 --- /dev/null +++ b/ubuntu/manifest.yaml @@ -0,0 +1,25 @@ +analysis: + attributes: + - name: language + result: python + - name: framework + result: operator +bases: +- architectures: + - amd64 + channel: '16.04' + name: ubuntu +- architectures: + - amd64 + channel: '18.04' + name: ubuntu +- architectures: + - amd64 + channel: '20.04' + name: ubuntu +- architectures: + - amd64 + channel: '22.04' + name: ubuntu +charmcraft-started-at: '2022-06-30T17:08:07.778172Z' +charmcraft-version: 1.7.1 diff --git a/ubuntu/metadata.yaml b/ubuntu/metadata.yaml new file mode 100644 index 0000000..f2eb46a --- /dev/null +++ b/ubuntu/metadata.yaml @@ -0,0 +1,12 @@ +name: ubuntu +summary: A pristine Ubuntu Server +maintainer: Charmers +description: | + This simply deploys the Ubuntu Cloud/Server image +tags: + - misc +series: + - focal + - bionic + - xenial + - jammy diff --git a/ubuntu/revision b/ubuntu/revision new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/ubuntu/revision @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/ubuntu/src/charm.py b/ubuntu/src/charm.py new file mode 100755 index 0000000..6e6448e --- /dev/null +++ b/ubuntu/src/charm.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +import logging +from pathlib import Path +from subprocess import check_call, check_output + +from ops.charm import CharmBase +from ops.main import main +from ops.model import ActiveStatus + + +log = logging.getLogger(__name__) + + +class UbuntuCharm(CharmBase): + def __init__(self, *args): + """Initialize charm. + + Setup hook event observers and any other basic initialization. + """ + super().__init__(*args) + + for event in ( + self.on.install, + self.on.leader_elected, + self.on.upgrade_charm, + self.on.post_series_upgrade, + ): + self.framework.observe(event, self._set_version) + self.framework.observe(self.on.config_changed, self._update_hostname) + + def _set_version(self, _): + """Set application version. + + Invoked for relevant hook events and, on the leader unit, determine and + set the application-level workload version to the Ubuntu version upon + which the charm is running. + """ + self.unit.status = ActiveStatus() + if not self.unit.is_leader(): + return + try: + output = check_output(["lsb_release", "-r", "-s"]) + version = output.decode("utf8").strip() + self.unit.set_workload_version(version) + except Exception: + log.exception("Error getting release") + + def _update_hostname(self, event): + """Update the machine hostname based on the config option.""" + hostname = self.config["hostname"] + if not hostname: + return + + Path("/etc/hostname").write_text(hostname) + check_call(["hostname", hostname]) + + +if __name__ == "__main__": + main(UbuntuCharm) diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/INSTALLER b/ubuntu/venv/PyYAML-6.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/LICENSE b/ubuntu/venv/PyYAML-6.0.dist-info/LICENSE new file mode 100644 index 0000000..2f1b8e1 --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2021 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/METADATA b/ubuntu/venv/PyYAML-6.0.dist-info/METADATA new file mode 100644 index 0000000..92b9f36 --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/METADATA @@ -0,0 +1,45 @@ +Metadata-Version: 2.1 +Name: PyYAML +Version: 6.0 +Summary: YAML parser and emitter for Python +Home-page: https://pyyaml.org/ +Download-URL: https://pypi.org/project/PyYAML/ +Author: Kirill Simonov +Author-email: xi@resolvent.net +License: MIT +Project-URL: Bug Tracker, https://github.com/yaml/pyyaml/issues +Project-URL: CI, https://github.com/yaml/pyyaml/actions +Project-URL: Documentation, https://pyyaml.org/wiki/PyYAMLDocumentation +Project-URL: Mailing lists, http://lists.sourceforge.net/lists/listinfo/yaml-core +Project-URL: Source Code, https://github.com/yaml/pyyaml +Platform: Any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup +Requires-Python: >=3.6 +License-File: LICENSE + +YAML is a data serialization format designed for human readability +and interaction with scripting languages. PyYAML is a YAML parser +and emitter for Python. + +PyYAML features a complete YAML 1.1 parser, Unicode support, pickle +support, capable extension API, and sensible error messages. PyYAML +supports standard YAML tags and provides Python-specific tags that +allow to represent an arbitrary Python object. + +PyYAML is applicable for a broad range of tasks from complex +configuration files to object serialization and persistence. diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/RECORD b/ubuntu/venv/PyYAML-6.0.dist-info/RECORD new file mode 100644 index 0000000..bb3ea3a --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/RECORD @@ -0,0 +1,42 @@ +PyYAML-6.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +PyYAML-6.0.dist-info/LICENSE,sha256=jTko-dxEkP1jVwfLiOsmvXZBAqcoKVQwfT5RZ6V36KQ,1101 +PyYAML-6.0.dist-info/METADATA,sha256=Q56A1jQxEnzMMCyqgYkqM5Egvr9y18VExGF3ciGLjlY,2005 +PyYAML-6.0.dist-info/RECORD,, +PyYAML-6.0.dist-info/WHEEL,sha256=HDpOoUm88OZsXwJ1fG8bFxqAdtiJNfIcxp3_RuQe18I,103 +PyYAML-6.0.dist-info/top_level.txt,sha256=rpj0IVMTisAjh_1vG3Ccf9v5jpCQwAz6cD1IVU5ZdhQ,11 +_yaml/__init__.py,sha256=04Ae_5osxahpJHa3XBZUAf4wi6XX32gR8D6X6p64GEA,1402 +_yaml/__pycache__/__init__.cpython-38.pyc,, +yaml/__init__.py,sha256=NDS7S8XgA72-hY6LRmGzUWTPvzGzjWVrWk-OGA-77AA,12309 +yaml/__pycache__/__init__.cpython-38.pyc,, +yaml/__pycache__/composer.cpython-38.pyc,, +yaml/__pycache__/constructor.cpython-38.pyc,, +yaml/__pycache__/cyaml.cpython-38.pyc,, +yaml/__pycache__/dumper.cpython-38.pyc,, +yaml/__pycache__/emitter.cpython-38.pyc,, +yaml/__pycache__/error.cpython-38.pyc,, +yaml/__pycache__/events.cpython-38.pyc,, +yaml/__pycache__/loader.cpython-38.pyc,, +yaml/__pycache__/nodes.cpython-38.pyc,, +yaml/__pycache__/parser.cpython-38.pyc,, +yaml/__pycache__/reader.cpython-38.pyc,, +yaml/__pycache__/representer.cpython-38.pyc,, +yaml/__pycache__/resolver.cpython-38.pyc,, +yaml/__pycache__/scanner.cpython-38.pyc,, +yaml/__pycache__/serializer.cpython-38.pyc,, +yaml/__pycache__/tokens.cpython-38.pyc,, +yaml/composer.py,sha256=_Ko30Wr6eDWUeUpauUGT3Lcg9QPBnOPVlTnIMRGJ9FM,4883 +yaml/constructor.py,sha256=kNgkfaeLUkwQYY_Q6Ff1Tz2XVw_pG1xVE9Ak7z-viLA,28639 +yaml/cyaml.py,sha256=6ZrAG9fAYvdVe2FK_w0hmXoG7ZYsoYUwapG8CiC72H0,3851 +yaml/dumper.py,sha256=PLctZlYwZLp7XmeUdwRuv4nYOZ2UBnDIUy8-lKfLF-o,2837 +yaml/emitter.py,sha256=jghtaU7eFwg31bG0B7RZea_29Adi9CKmXq_QjgQpCkQ,43006 +yaml/error.py,sha256=Ah9z-toHJUbE9j-M8YpxgSRM5CgLCcwVzJgLLRF2Fxo,2533 +yaml/events.py,sha256=50_TksgQiE4up-lKo_V-nBy-tAIxkIPQxY5qDhKCeHw,2445 +yaml/loader.py,sha256=UVa-zIqmkFSCIYq_PgSGm4NSJttHY2Rf_zQ4_b1fHN0,2061 +yaml/nodes.py,sha256=gPKNj8pKCdh2d4gr3gIYINnPOaOxGhJAUiYhGRnPE84,1440 +yaml/parser.py,sha256=ilWp5vvgoHFGzvOZDItFoGjD6D42nhlZrZyjAwa0oJo,25495 +yaml/reader.py,sha256=0dmzirOiDG4Xo41RnuQS7K9rkY3xjHiVasfDMNTqCNw,6794 +yaml/representer.py,sha256=IuWP-cAW9sHKEnS0gCqSa894k1Bg4cgTxaDwIcbRQ-Y,14190 +yaml/resolver.py,sha256=9L-VYfm4mWHxUD1Vg4X7rjDRK_7VZd6b92wzq7Y2IKY,9004 +yaml/scanner.py,sha256=YEM3iLZSaQwXcQRg2l2R4MdT0zGP2F9eHkKGKnHyWQY,51279 +yaml/serializer.py,sha256=ChuFgmhU01hj4xgI8GaKv6vfM2Bujwa9i7d2FAHj7cA,4165 +yaml/tokens.py,sha256=lTQIzSVw8Mg9wv459-TjiOQe6wVziqaRlqX2_89rp54,2573 diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/WHEEL b/ubuntu/venv/PyYAML-6.0.dist-info/WHEEL new file mode 100644 index 0000000..637e142 --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.37.1) +Root-Is-Purelib: false +Tag: cp38-cp38-linux_x86_64 + diff --git a/ubuntu/venv/PyYAML-6.0.dist-info/top_level.txt b/ubuntu/venv/PyYAML-6.0.dist-info/top_level.txt new file mode 100644 index 0000000..e6475e9 --- /dev/null +++ b/ubuntu/venv/PyYAML-6.0.dist-info/top_level.txt @@ -0,0 +1,2 @@ +_yaml +yaml diff --git a/ubuntu/venv/_yaml/__init__.py b/ubuntu/venv/_yaml/__init__.py new file mode 100644 index 0000000..7baa8c4 --- /dev/null +++ b/ubuntu/venv/_yaml/__init__.py @@ -0,0 +1,33 @@ +# This is a stub package designed to roughly emulate the _yaml +# extension module, which previously existed as a standalone module +# and has been moved into the `yaml` package namespace. +# It does not perfectly mimic its old counterpart, but should get +# close enough for anyone who's relying on it even when they shouldn't. +import yaml + +# in some circumstances, the yaml module we imoprted may be from a different version, so we need +# to tread carefully when poking at it here (it may not have the attributes we expect) +if not getattr(yaml, '__with_libyaml__', False): + from sys import version_info + + exc = ModuleNotFoundError if version_info >= (3, 6) else ImportError + raise exc("No module named '_yaml'") +else: + from yaml._yaml import * + import warnings + warnings.warn( + 'The _yaml extension module is now located at yaml._yaml' + ' and its location is subject to change. To use the' + ' LibYAML-based parser and emitter, import from `yaml`:' + ' `from yaml import CLoader as Loader, CDumper as Dumper`.', + DeprecationWarning + ) + del warnings + # Don't `del yaml` here because yaml is actually an existing + # namespace member of _yaml. + +__name__ = '_yaml' +# If the module is top-level (i.e. not a part of any specific package) +# then the attribute should be set to ''. +# https://docs.python.org/3.8/library/types.html +__package__ = '' diff --git a/ubuntu/venv/easy_install.py b/ubuntu/venv/easy_install.py new file mode 100644 index 0000000..d87e984 --- /dev/null +++ b/ubuntu/venv/easy_install.py @@ -0,0 +1,5 @@ +"""Run the EasyInstall command""" + +if __name__ == '__main__': + from setuptools.command.easy_install import main + main() diff --git a/ubuntu/venv/ops-1.5.0.dist-info/INSTALLER b/ubuntu/venv/ops-1.5.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/ubuntu/venv/ops-1.5.0.dist-info/LICENSE.txt b/ubuntu/venv/ops-1.5.0.dist-info/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ubuntu/venv/ops-1.5.0.dist-info/METADATA b/ubuntu/venv/ops-1.5.0.dist-info/METADATA new file mode 100644 index 0000000..2d81a1a --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/METADATA @@ -0,0 +1,151 @@ +Metadata-Version: 2.1 +Name: ops +Version: 1.5.0 +Summary: The Python library behind great charms +Home-page: https://github.com/canonical/operator +Author: The Charmcraft team at Canonical Ltd. +Author-email: charmcraft@lists.launchpad.net +License: Apache-2.0 +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: POSIX :: Linux +Requires-Python: >=3.5 +Description-Content-Type: text/markdown +License-File: LICENSE.txt +Requires-Dist: PyYAML + +# The Charmed Operator Framework + +This Charmed Operator Framework simplifies [operator](https://charmhub.io/about) development +for [model-driven application management](https://juju.is/model-driven-operations). + +Operators emerged from the Kubernetes community; an operator is software that drives lifecycle +management, configuration, integration and daily actions for an application. Operators simplify +software management and operations. They capture reusable app domain knowledge from experts in a +software component that can be shared. + +This project extends the operator pattern to enable +[charmed operators](https://juju.is/universal-operators), not just for Kubernetes but also +operators for traditional Linux or Windows application management. + +Operators use a [Charmed Operator Lifecycle Manager +(Charmed OLM)](https://juju.is/operator-lifecycle-manager) to coordinate their work in a cluster. +The system uses Golang for concurrent event processing under the hood, but enables the operators to +be written in Python. + +## Simple, composable operators + +Operators should 'do one thing and do it well'. Each operator drives a single microservice and can +be [composed with other operators](https://juju.is/integration) to deliver a complex application. + +It is better to have small, reusable operators that each drive a single microservice very well. +The operator handles instantiation, scaling, configuration, optimisation, networking, service mesh, +observability, and day-2 operations specific to that microservice. + +Operator composition takes place through declarative integration in the OLM. Operators declare +integration endpoints, and discover lines of integration between those endpoints dynamically at +runtime. + +## Pure Python operators + +The framework provides a standard Python library and object model that represents the application +graph, and an event distribution mechanism for distributed system coordination and communication. + +The OLM is written in Golang for efficient concurrency in event handling and distribution. +Operators can be written in any language. We recommend this Python framework for ease of design, +development and collaboration. + +## Better collaboration + +Operator developers publish Python libraries that make it easy to integrate your operator with +their operator. The framework includes standard tools to distribute these integration libraries and +keep them up to date. + +Development collaboration happens at [Charmhub.io](https://charmhub.io/) where operators are +published along with integration libraries. Design and code review discussions are hosted in the +Charmhub [discourse]. We recommend the [Open Operator Manifesto](https://charmhub.io/manifesto) +as a guideline for high quality operator engineering. + +## Event serialization and operator services + +Distributed systems can be hard! So this framework exists to make it much simpler to reason about +operator behaviour, especially in complex deployments. The Charmed OLM provides +[operator services](https://juju.is/operator-services) such as provisioning, event delivery, +leader election and model management. + +Coordination between operators is provided by a cluster-wide event distribution system. Events are +serialized to avoid race conditions in any given container or machine. This greatly simplifies the +development of operators for high availability, scale-out and integrated applications. + +## Model-driven Operator Lifecycle Manager + +A key goal of the project is to improve the user experience for admins working with multiple +different operators. + +We embrace [model-driven operations](https://juju.is/model-driven-operations) in the Charmed +Operator Lifecycle Manager. The model encompasses capacity, storage, networking, the application +graph and administrative access. + +Admins describe the application graph of integrated microservices, and the OLM then drives +instantiation. A change in the model is propagated to all affected operators, reducing the +duplication of effort and repetition normally found in operating a complex topology of services. + +Administrative actions, updates, configuration and integration are all driven through the OLM. + +# Getting started + +A package of operator code is called a charmed operator or “charm. You will use `charmcraft` to +register your operator name, and publish it when you are ready. There are more details on how to +get a complete development environment setup over in the +[documentation](https://juju.is/docs/sdk/dev-setup) + +Charmed Operators written using the Charmed Operator Framework are just Python code. The goal +is to feel natural for somebody used to coding in Python, and reasonably easy to learn for somebody +who is not a pythonista. + +The dependencies of the operator framework are kept as minimal as possible; currently that's Python +3.5 or greater, and `PyYAML` (both are included by default in Ubuntu's cloud images from 16.04 on). + +For a brief intro on how to get started, check out the +[Hello, World!](https://juju.is/docs/sdk/hello-world) section of the documentation! + +# Testing your charmed operators + +The operator framework provides a testing harness, so you can check your charmed operator does the +right thing in different scenarios, without having to create a full deployment. +`pydoc3 ops.testing` has the details, including this example: + +```python +harness = Harness(MyCharm) +# Do initial setup here +relation_id = harness.add_relation('db', 'postgresql') +# Now instantiate the charm to see events as the model changes +harness.begin() +harness.add_relation_unit(relation_id, 'postgresql/0') +harness.update_relation_data(relation_id, 'postgresql/0', {'key': 'val'}) +# Check that charm has properly handled the relation_joined event for postgresql/0 +self.assertEqual(harness.charm. ...) +``` + +## Talk to us + +If you need help, have ideas, or would just like to chat with us, reach out on +the Charmhub [Mattermost]. + +We also pay attention to the Charmhub [Discourse] + +You can deep dive into the [API docs] if that's your thing. + +[discourse]: https://discourse.charmhub.io +[api docs]: https://ops.rtfd.io/ +[sdk docs]: https://juju.is/docs/sdk +[mattermost]: https://chat.charmhub.io/charmhub/channels/charm-dev + +## Operator Framework development + +See [HACKING.md](HACKING.md) for details on dev environments, testing, etc. + diff --git a/ubuntu/venv/ops-1.5.0.dist-info/RECORD b/ubuntu/venv/ops-1.5.0.dist-info/RECORD new file mode 100644 index 0000000..a889cbf --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/RECORD @@ -0,0 +1,63 @@ +ops-1.5.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +ops-1.5.0.dist-info/LICENSE.txt,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358 +ops-1.5.0.dist-info/METADATA,sha256=zGx-dEEbiqH8b5lapdT2h6mSACG8VvdmgPUiEJorFtY,7001 +ops-1.5.0.dist-info/RECORD,, +ops-1.5.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92 +ops-1.5.0.dist-info/top_level.txt,sha256=enC05wWafSg8iDKIvj3gvtAtEP2kYCyN5Gmd689q-_I,4 +ops/__init__.py,sha256=jbUCTFsrtEpa6EBUy6wZm72pPvtMUt2lrWvxxKghtf8,2206 +ops/__pycache__/__init__.cpython-38.pyc,, +ops/__pycache__/charm.cpython-38.pyc,, +ops/__pycache__/framework.cpython-38.pyc,, +ops/__pycache__/jujuversion.cpython-38.pyc,, +ops/__pycache__/log.cpython-38.pyc,, +ops/__pycache__/main.cpython-38.pyc,, +ops/__pycache__/model.cpython-38.pyc,, +ops/__pycache__/pebble.cpython-38.pyc,, +ops/__pycache__/storage.cpython-38.pyc,, +ops/__pycache__/testing.cpython-38.pyc,, +ops/__pycache__/version.cpython-38.pyc,, +ops/_private/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +ops/_private/__pycache__/__init__.cpython-38.pyc,, +ops/_private/__pycache__/yaml.cpython-38.pyc,, +ops/_private/yaml.py,sha256=nRRAeUlTnxcJpgRqi5kSPU7iggYhE5nGagtH6xo2Qco,1104 +ops/_vendor/__init__.py,sha256=aDJt1DWgZGUkvdmpLRNaqDO2JV1aSfKgcFVqiKtU0es,574 +ops/_vendor/__pycache__/__init__.cpython-38.pyc,, +ops/_vendor/websocket/__init__.py,sha256=WRqB2mYirdZh3uiWtw7qjLYj4SC3gYXDkqzZa7BRFjg,787 +ops/_vendor/websocket/__pycache__/__init__.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_abnf.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_app.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_cookiejar.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_core.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_exceptions.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_handshake.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_http.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_logging.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_socket.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_ssl_compat.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_url.cpython-38.pyc,, +ops/_vendor/websocket/__pycache__/_utils.cpython-38.pyc,, +ops/_vendor/websocket/_abnf.py,sha256=Qso7_UIZqDQfdfoWaXTAoUMKf4lpcolyz1XKcowClXg,13142 +ops/_vendor/websocket/_app.py,sha256=3b7aWh32VU_2hc6Ff2QEkoexeJBioSESjBE7QHoQdkM,15006 +ops/_vendor/websocket/_cookiejar.py,sha256=k6MwBzOccxIu1TDmmClWqEBGHfXujfm6Wc0iY4QN7Rw,2135 +ops/_vendor/websocket/_core.py,sha256=Qbr5c0dD5UVZ_esGSGGCruvXh8OIt8zTSUeeutZq1eM,19358 +ops/_vendor/websocket/_exceptions.py,sha256=cjjLpOXOUvlZ0j7xdvCW_zTNStphzwF4kIUpfEnCzz8,2175 +ops/_vendor/websocket/_handshake.py,sha256=T4G5YbmusrXG3uerqmn7OknEu-CxsS_wcStJ-DFTE3o,5915 +ops/_vendor/websocket/_http.py,sha256=wSW-I_MK0I1z6h8kWTzopKHyYc5f1LNoLmvJaNZVFRc,11344 +ops/_vendor/websocket/_logging.py,sha256=_QF3W-nlTu2AYRA9xTrN-8cVwNNabwLgxNMSr5S7i9E,1972 +ops/_vendor/websocket/_socket.py,sha256=nmK2kZ1KgRQbCBSVTvzWqHfeO_IyCpm9hzDThWzbf44,4843 +ops/_vendor/websocket/_ssl_compat.py,sha256=sdMsgIT6Cej-f8JREXCw53WRfIF1c57uwfuAZpss1rg,1337 +ops/_vendor/websocket/_url.py,sha256=SbnhpWkmkvcUds4wls-4fvkL7RB4AgXOy-kYjS5Adpo,4807 +ops/_vendor/websocket/_utils.py,sha256=D-H7nHYGSO6ERfx-KkJa685fZfSFbARq9_2619M6XsM,3524 +ops/charm.py,sha256=HlG9AX2gpQnCoYPOSOH17DKVI92Y5GZ7ft-w4d9Z2NE,41849 +ops/framework.py,sha256=oZGoFdAnqz1qcJH6NGzwGqMiTGpiLLO7bucyrFUE4ao,48073 +ops/jujuversion.py,sha256=FYW7rsd-KS3IdMSn__63T7nrYKVj2oVtZYyluFPVBbc,4189 +ops/lib/__init__.py,sha256=9vSrYlkhIW3IK2HHwwJONxI_k8xB6FTqYD7Tgzq3Y2A,9212 +ops/lib/__pycache__/__init__.cpython-38.pyc,, +ops/log.py,sha256=qMbcCDwRsPJJdinVqCEcbZQSzEqEfGhlY7cCoRKQiYg,2525 +ops/main.py,sha256=KJEd2g6zEIOSA-6yh8_QfEcsfknFuG5N2tBQEWmImxU,16783 +ops/model.py,sha256=JJN5HNq4o87Lf4sW4S7a15QWh4Zc37Zgdv8fYsieElQ,95925 +ops/pebble.py,sha256=nJaeOjeS8R4CCyEccHbbq-OxNZiqD_5P_jV6-Y-xLMM,88948 +ops/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +ops/storage.py,sha256=ZfgWzi6ZWoHcTJqOQcQmRTThInhA2mi0oGqlfVYQpiQ,14964 +ops/testing.py,sha256=sMxpaeuGs9gohQPOMQfduLXHIsVlDjDWMh0TuxBoUq0,93183 +ops/version.py,sha256=LEt5sKSOkipQfWAOTeLpjmRnmQD5VJaLW3r9GI2pskM,46 diff --git a/ubuntu/venv/ops-1.5.0.dist-info/WHEEL b/ubuntu/venv/ops-1.5.0.dist-info/WHEEL new file mode 100644 index 0000000..becc9a6 --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.37.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/ubuntu/venv/ops-1.5.0.dist-info/top_level.txt b/ubuntu/venv/ops-1.5.0.dist-info/top_level.txt new file mode 100644 index 0000000..2d81d3b --- /dev/null +++ b/ubuntu/venv/ops-1.5.0.dist-info/top_level.txt @@ -0,0 +1 @@ +ops diff --git a/ubuntu/venv/ops/__init__.py b/ubuntu/venv/ops/__init__.py new file mode 100644 index 0000000..44cb77a --- /dev/null +++ b/ubuntu/venv/ops/__init__.py @@ -0,0 +1,44 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Charmed Operator Framework. + +The Charmed Operator Framework allows the development of operators in a simple +and straightforward way, using standard Python structures to allow for clean, +maintainable, and reusable code. + +A Kubernetes operator is a container that drives lifecycle management, +configuration, integration and daily actions for an application. Operators +simplify software management and operations. They capture reusable app domain +knowledge from experts in a software component that can be shared. + +The Charmed Operator Framework extends the "operator pattern" to enable Charmed +Operators, packaged as and often referred to as "charms". Charms are not just +for Kubernetes but also operators for traditional Linux or Windows application +management. Operators use an Operator Lifecycle Manager (OLM), like Juju, to +coordinate their work in a cluster. The system uses Golang for concurrent event +processing under the hood, but enables the operators to be written in Python. + +Operators should do one thing and do it well. Each operator drives a single +application or service and can be composed with other operators to deliver a +complex application or service. An operator handles instantiation, scaling, +configuration, optimisation, networking, service mesh, observability, +and day-2 operations specific to that application. + +Full developer documentation is available at https://juju.is/docs/sdk. +""" + +# Import here the bare minimum to break the circular import between modules +from . import charm # noqa: F401 (imported but unused) +from .version import version as __version__ # noqa: F401 (imported but unused) diff --git a/ubuntu/venv/ops/_private/__init__.py b/ubuntu/venv/ops/_private/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/ops/_private/yaml.py b/ubuntu/venv/ops/_private/yaml.py new file mode 100644 index 0000000..82802d4 --- /dev/null +++ b/ubuntu/venv/ops/_private/yaml.py @@ -0,0 +1,31 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Internal YAML helpers.""" + +import yaml + +# Use C speedups if available +_safe_loader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) +_safe_dumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) + + +def safe_load(stream): + """Same as yaml.safe_load, but use fast C loader if available.""" + return yaml.load(stream, Loader=_safe_loader) + + +def safe_dump(data, stream=None, **kwargs): + """Same as yaml.safe_dump, but use fast C dumper if available.""" + return yaml.dump(data, stream=stream, Dumper=_safe_dumper, **kwargs) diff --git a/ubuntu/venv/ops/_vendor/__init__.py b/ubuntu/venv/ops/_vendor/__init__.py new file mode 100644 index 0000000..9c53588 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ubuntu/venv/ops/_vendor/websocket/__init__.py b/ubuntu/venv/ops/_vendor/websocket/__init__.py new file mode 100644 index 0000000..a9fa463 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/__init__.py @@ -0,0 +1,26 @@ +""" +__init__.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from ._abnf import * +from ._app import WebSocketApp +from ._core import * +from ._exceptions import * +from ._logging import * +from ._socket import * + +__version__ = "1.2.1" diff --git a/ubuntu/venv/ops/_vendor/websocket/_abnf.py b/ubuntu/venv/ops/_vendor/websocket/_abnf.py new file mode 100644 index 0000000..6a4d490 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_abnf.py @@ -0,0 +1,423 @@ +""" + +""" + +""" +_abnf.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import array +import os +import struct +import sys + +from ._exceptions import * +from ._utils import validate_utf8 +from threading import Lock + +try: + # If wsaccel is available, use compiled routines to mask data. + # wsaccel only provides around a 10% speed boost compared + # to the websocket-client _mask() implementation. + # Note that wsaccel is unmaintained. + from wsaccel.xormask import XorMaskerSimple + + def _mask(_m, _d): + return XorMaskerSimple(_m).process(_d) + +except ImportError: + # wsaccel is not available, use websocket-client _mask() + native_byteorder = sys.byteorder + + def _mask(mask_value, data_value): + datalen = len(data_value) + data_value = int.from_bytes(data_value, native_byteorder) + mask_value = int.from_bytes(mask_value * (datalen // 4) + mask_value[: datalen % 4], native_byteorder) + return (data_value ^ mask_value).to_bytes(datalen, native_byteorder) + + +__all__ = [ + 'ABNF', 'continuous_frame', 'frame_buffer', + 'STATUS_NORMAL', + 'STATUS_GOING_AWAY', + 'STATUS_PROTOCOL_ERROR', + 'STATUS_UNSUPPORTED_DATA_TYPE', + 'STATUS_STATUS_NOT_AVAILABLE', + 'STATUS_ABNORMAL_CLOSED', + 'STATUS_INVALID_PAYLOAD', + 'STATUS_POLICY_VIOLATION', + 'STATUS_MESSAGE_TOO_BIG', + 'STATUS_INVALID_EXTENSION', + 'STATUS_UNEXPECTED_CONDITION', + 'STATUS_BAD_GATEWAY', + 'STATUS_TLS_HANDSHAKE_ERROR', +] + +# closing frame status codes. +STATUS_NORMAL = 1000 +STATUS_GOING_AWAY = 1001 +STATUS_PROTOCOL_ERROR = 1002 +STATUS_UNSUPPORTED_DATA_TYPE = 1003 +STATUS_STATUS_NOT_AVAILABLE = 1005 +STATUS_ABNORMAL_CLOSED = 1006 +STATUS_INVALID_PAYLOAD = 1007 +STATUS_POLICY_VIOLATION = 1008 +STATUS_MESSAGE_TOO_BIG = 1009 +STATUS_INVALID_EXTENSION = 1010 +STATUS_UNEXPECTED_CONDITION = 1011 +STATUS_BAD_GATEWAY = 1014 +STATUS_TLS_HANDSHAKE_ERROR = 1015 + +VALID_CLOSE_STATUS = ( + STATUS_NORMAL, + STATUS_GOING_AWAY, + STATUS_PROTOCOL_ERROR, + STATUS_UNSUPPORTED_DATA_TYPE, + STATUS_INVALID_PAYLOAD, + STATUS_POLICY_VIOLATION, + STATUS_MESSAGE_TOO_BIG, + STATUS_INVALID_EXTENSION, + STATUS_UNEXPECTED_CONDITION, + STATUS_BAD_GATEWAY, +) + + +class ABNF(object): + """ + ABNF frame class. + See http://tools.ietf.org/html/rfc5234 + and http://tools.ietf.org/html/rfc6455#section-5.2 + """ + + # operation code values. + OPCODE_CONT = 0x0 + OPCODE_TEXT = 0x1 + OPCODE_BINARY = 0x2 + OPCODE_CLOSE = 0x8 + OPCODE_PING = 0x9 + OPCODE_PONG = 0xa + + # available operation code value tuple + OPCODES = (OPCODE_CONT, OPCODE_TEXT, OPCODE_BINARY, OPCODE_CLOSE, + OPCODE_PING, OPCODE_PONG) + + # opcode human readable string + OPCODE_MAP = { + OPCODE_CONT: "cont", + OPCODE_TEXT: "text", + OPCODE_BINARY: "binary", + OPCODE_CLOSE: "close", + OPCODE_PING: "ping", + OPCODE_PONG: "pong" + } + + # data length threshold. + LENGTH_7 = 0x7e + LENGTH_16 = 1 << 16 + LENGTH_63 = 1 << 63 + + def __init__(self, fin=0, rsv1=0, rsv2=0, rsv3=0, + opcode=OPCODE_TEXT, mask=1, data=""): + """ + Constructor for ABNF. Please check RFC for arguments. + """ + self.fin = fin + self.rsv1 = rsv1 + self.rsv2 = rsv2 + self.rsv3 = rsv3 + self.opcode = opcode + self.mask = mask + if data is None: + data = "" + self.data = data + self.get_mask_key = os.urandom + + def validate(self, skip_utf8_validation=False): + """ + Validate the ABNF frame. + + Parameters + ---------- + skip_utf8_validation: skip utf8 validation. + """ + if self.rsv1 or self.rsv2 or self.rsv3: + raise WebSocketProtocolException("rsv is not implemented, yet") + + if self.opcode not in ABNF.OPCODES: + raise WebSocketProtocolException("Invalid opcode %r", self.opcode) + + if self.opcode == ABNF.OPCODE_PING and not self.fin: + raise WebSocketProtocolException("Invalid ping frame.") + + if self.opcode == ABNF.OPCODE_CLOSE: + l = len(self.data) + if not l: + return + if l == 1 or l >= 126: + raise WebSocketProtocolException("Invalid close frame.") + if l > 2 and not skip_utf8_validation and not validate_utf8(self.data[2:]): + raise WebSocketProtocolException("Invalid close frame.") + + code = 256 * self.data[0] + self.data[1] + if not self._is_valid_close_status(code): + raise WebSocketProtocolException("Invalid close opcode.") + + @staticmethod + def _is_valid_close_status(code): + return code in VALID_CLOSE_STATUS or (3000 <= code < 5000) + + def __str__(self): + return "fin=" + str(self.fin) \ + + " opcode=" + str(self.opcode) \ + + " data=" + str(self.data) + + @staticmethod + def create_frame(data, opcode, fin=1): + """ + Create frame to send text, binary and other data. + + Parameters + ---------- + data: + data to send. This is string value(byte array). + If opcode is OPCODE_TEXT and this value is unicode, + data value is converted into unicode string, automatically. + opcode: + operation code. please see OPCODE_XXX. + fin: + fin flag. if set to 0, create continue fragmentation. + """ + if opcode == ABNF.OPCODE_TEXT and isinstance(data, str): + data = data.encode("utf-8") + # mask must be set if send data from client + return ABNF(fin, 0, 0, 0, opcode, 1, data) + + def format(self): + """ + Format this object to string(byte array) to send data to server. + """ + if any(x not in (0, 1) for x in [self.fin, self.rsv1, self.rsv2, self.rsv3]): + raise ValueError("not 0 or 1") + if self.opcode not in ABNF.OPCODES: + raise ValueError("Invalid OPCODE") + length = len(self.data) + if length >= ABNF.LENGTH_63: + raise ValueError("data is too long") + + frame_header = chr(self.fin << 7 | + self.rsv1 << 6 | self.rsv2 << 5 | self.rsv3 << 4 | + self.opcode).encode('latin-1') + if length < ABNF.LENGTH_7: + frame_header += chr(self.mask << 7 | length).encode('latin-1') + elif length < ABNF.LENGTH_16: + frame_header += chr(self.mask << 7 | 0x7e).encode('latin-1') + frame_header += struct.pack("!H", length) + else: + frame_header += chr(self.mask << 7 | 0x7f).encode('latin-1') + frame_header += struct.pack("!Q", length) + + if not self.mask: + return frame_header + self.data + else: + mask_key = self.get_mask_key(4) + return frame_header + self._get_masked(mask_key) + + def _get_masked(self, mask_key): + s = ABNF.mask(mask_key, self.data) + + if isinstance(mask_key, str): + mask_key = mask_key.encode('utf-8') + + return mask_key + s + + @staticmethod + def mask(mask_key, data): + """ + Mask or unmask data. Just do xor for each byte + + Parameters + ---------- + mask_key: + 4 byte string. + data: + data to mask/unmask. + """ + if data is None: + data = "" + + if isinstance(mask_key, str): + mask_key = mask_key.encode('latin-1') + + if isinstance(data, str): + data = data.encode('latin-1') + + return _mask(array.array("B", mask_key), array.array("B", data)) + + +class frame_buffer(object): + _HEADER_MASK_INDEX = 5 + _HEADER_LENGTH_INDEX = 6 + + def __init__(self, recv_fn, skip_utf8_validation): + self.recv = recv_fn + self.skip_utf8_validation = skip_utf8_validation + # Buffers over the packets from the layer beneath until desired amount + # bytes of bytes are received. + self.recv_buffer = [] + self.clear() + self.lock = Lock() + + def clear(self): + self.header = None + self.length = None + self.mask = None + + def has_received_header(self): + return self.header is None + + def recv_header(self): + header = self.recv_strict(2) + b1 = header[0] + fin = b1 >> 7 & 1 + rsv1 = b1 >> 6 & 1 + rsv2 = b1 >> 5 & 1 + rsv3 = b1 >> 4 & 1 + opcode = b1 & 0xf + b2 = header[1] + has_mask = b2 >> 7 & 1 + length_bits = b2 & 0x7f + + self.header = (fin, rsv1, rsv2, rsv3, opcode, has_mask, length_bits) + + def has_mask(self): + if not self.header: + return False + return self.header[frame_buffer._HEADER_MASK_INDEX] + + def has_received_length(self): + return self.length is None + + def recv_length(self): + bits = self.header[frame_buffer._HEADER_LENGTH_INDEX] + length_bits = bits & 0x7f + if length_bits == 0x7e: + v = self.recv_strict(2) + self.length = struct.unpack("!H", v)[0] + elif length_bits == 0x7f: + v = self.recv_strict(8) + self.length = struct.unpack("!Q", v)[0] + else: + self.length = length_bits + + def has_received_mask(self): + return self.mask is None + + def recv_mask(self): + self.mask = self.recv_strict(4) if self.has_mask() else "" + + def recv_frame(self): + + with self.lock: + # Header + if self.has_received_header(): + self.recv_header() + (fin, rsv1, rsv2, rsv3, opcode, has_mask, _) = self.header + + # Frame length + if self.has_received_length(): + self.recv_length() + length = self.length + + # Mask + if self.has_received_mask(): + self.recv_mask() + mask = self.mask + + # Payload + payload = self.recv_strict(length) + if has_mask: + payload = ABNF.mask(mask, payload) + + # Reset for next frame + self.clear() + + frame = ABNF(fin, rsv1, rsv2, rsv3, opcode, has_mask, payload) + frame.validate(self.skip_utf8_validation) + + return frame + + def recv_strict(self, bufsize): + shortage = bufsize - sum(map(len, self.recv_buffer)) + while shortage > 0: + # Limit buffer size that we pass to socket.recv() to avoid + # fragmenting the heap -- the number of bytes recv() actually + # reads is limited by socket buffer and is relatively small, + # yet passing large numbers repeatedly causes lots of large + # buffers allocated and then shrunk, which results in + # fragmentation. + bytes_ = self.recv(min(16384, shortage)) + self.recv_buffer.append(bytes_) + shortage -= len(bytes_) + + unified = bytes("", 'utf-8').join(self.recv_buffer) + + if shortage == 0: + self.recv_buffer = [] + return unified + else: + self.recv_buffer = [unified[bufsize:]] + return unified[:bufsize] + + +class continuous_frame(object): + + def __init__(self, fire_cont_frame, skip_utf8_validation): + self.fire_cont_frame = fire_cont_frame + self.skip_utf8_validation = skip_utf8_validation + self.cont_data = None + self.recving_frames = None + + def validate(self, frame): + if not self.recving_frames and frame.opcode == ABNF.OPCODE_CONT: + raise WebSocketProtocolException("Illegal frame") + if self.recving_frames and \ + frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): + raise WebSocketProtocolException("Illegal frame") + + def add(self, frame): + if self.cont_data: + self.cont_data[1] += frame.data + else: + if frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY): + self.recving_frames = frame.opcode + self.cont_data = [frame.opcode, frame.data] + + if frame.fin: + self.recving_frames = None + + def is_fire(self, frame): + return frame.fin or self.fire_cont_frame + + def extract(self, frame): + data = self.cont_data + self.cont_data = None + frame.data = data[1] + if not self.fire_cont_frame and data[0] == ABNF.OPCODE_TEXT and not self.skip_utf8_validation and not validate_utf8(frame.data): + raise WebSocketPayloadException( + "cannot decode: " + repr(frame.data)) + + return [data[0], frame] diff --git a/ubuntu/venv/ops/_vendor/websocket/_app.py b/ubuntu/venv/ops/_vendor/websocket/_app.py new file mode 100644 index 0000000..61925ba --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_app.py @@ -0,0 +1,412 @@ +""" + +""" + +""" +_app.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import selectors +import sys +import threading +import time +import traceback +from ._abnf import ABNF +from ._core import WebSocket, getdefaulttimeout +from ._exceptions import * +from . import _logging + + +__all__ = ["WebSocketApp"] + + +class Dispatcher: + """ + Dispatcher + """ + def __init__(self, app, ping_timeout): + self.app = app + self.ping_timeout = ping_timeout + + def read(self, sock, read_callback, check_callback): + while self.app.keep_running: + sel = selectors.DefaultSelector() + sel.register(self.app.sock.sock, selectors.EVENT_READ) + + r = sel.select(self.ping_timeout) + if r: + if not read_callback(): + break + check_callback() + sel.close() + + +class SSLDispatcher: + """ + SSLDispatcher + """ + def __init__(self, app, ping_timeout): + self.app = app + self.ping_timeout = ping_timeout + + def read(self, sock, read_callback, check_callback): + while self.app.keep_running: + r = self.select() + if r: + if not read_callback(): + break + check_callback() + + def select(self): + sock = self.app.sock.sock + if sock.pending(): + return [sock,] + + sel = selectors.DefaultSelector() + sel.register(sock, selectors.EVENT_READ) + + r = sel.select(self.ping_timeout) + sel.close() + + if len(r) > 0: + return r[0][0] + + +class WebSocketApp(object): + """ + Higher level of APIs are provided. The interface is like JavaScript WebSocket object. + """ + + def __init__(self, url, header=None, + on_open=None, on_message=None, on_error=None, + on_close=None, on_ping=None, on_pong=None, + on_cont_message=None, + keep_running=True, get_mask_key=None, cookie=None, + subprotocols=None, + on_data=None): + """ + WebSocketApp initialization + + Parameters + ---------- + url: str + Websocket url. + header: list or dict + Custom header for websocket handshake. + on_open: function + Callback object which is called at opening websocket. + on_open has one argument. + The 1st argument is this class object. + on_message: function + Callback object which is called when received data. + on_message has 2 arguments. + The 1st argument is this class object. + The 2nd argument is utf-8 data received from the server. + on_error: function + Callback object which is called when we get error. + on_error has 2 arguments. + The 1st argument is this class object. + The 2nd argument is exception object. + on_close: function + Callback object which is called when connection is closed. + on_close has 3 arguments. + The 1st argument is this class object. + The 2nd argument is close_status_code. + The 3rd argument is close_msg. + on_cont_message: function + Callback object which is called when a continuation + frame is received. + on_cont_message has 3 arguments. + The 1st argument is this class object. + The 2nd argument is utf-8 string which we get from the server. + The 3rd argument is continue flag. if 0, the data continue + to next frame data + on_data: function + Callback object which is called when a message received. + This is called before on_message or on_cont_message, + and then on_message or on_cont_message is called. + on_data has 4 argument. + The 1st argument is this class object. + The 2nd argument is utf-8 string which we get from the server. + The 3rd argument is data type. ABNF.OPCODE_TEXT or ABNF.OPCODE_BINARY will be came. + The 4th argument is continue flag. If 0, the data continue + keep_running: bool + This parameter is obsolete and ignored. + get_mask_key: function + A callable function to get new mask keys, see the + WebSocket.set_mask_key's docstring for more information. + cookie: str + Cookie value. + subprotocols: list + List of available sub protocols. Default is None. + """ + self.url = url + self.header = header if header is not None else [] + self.cookie = cookie + + self.on_open = on_open + self.on_message = on_message + self.on_data = on_data + self.on_error = on_error + self.on_close = on_close + self.on_ping = on_ping + self.on_pong = on_pong + self.on_cont_message = on_cont_message + self.keep_running = False + self.get_mask_key = get_mask_key + self.sock = None + self.last_ping_tm = 0 + self.last_pong_tm = 0 + self.subprotocols = subprotocols + + def send(self, data, opcode=ABNF.OPCODE_TEXT): + """ + send message + + Parameters + ---------- + data: str + Message to send. If you set opcode to OPCODE_TEXT, + data must be utf-8 string or unicode. + opcode: int + Operation code of data. Default is OPCODE_TEXT. + """ + + if not self.sock or self.sock.send(data, opcode) == 0: + raise WebSocketConnectionClosedException( + "Connection is already closed.") + + def close(self, **kwargs): + """ + Close websocket connection. + """ + self.keep_running = False + if self.sock: + self.sock.close(**kwargs) + self.sock = None + + def _send_ping(self, interval, event, payload): + while not event.wait(interval): + self.last_ping_tm = time.time() + if self.sock: + try: + self.sock.ping(payload) + except Exception as ex: + _logging.warning("send_ping routine terminated: {}".format(ex)) + break + + def run_forever(self, sockopt=None, sslopt=None, + ping_interval=0, ping_timeout=None, + ping_payload="", + http_proxy_host=None, http_proxy_port=None, + http_no_proxy=None, http_proxy_auth=None, + skip_utf8_validation=False, + host=None, origin=None, dispatcher=None, + suppress_origin=False, proxy_type=None): + """ + Run event loop for WebSocket framework. + + This loop is an infinite loop and is alive while websocket is available. + + Parameters + ---------- + sockopt: tuple + Values for socket.setsockopt. + sockopt must be tuple + and each element is argument of sock.setsockopt. + sslopt: dict + Optional dict object for ssl socket option. + ping_interval: int or float + Automatically send "ping" command + every specified period (in seconds). + If set to 0, no ping is sent periodically. + ping_timeout: int or float + Timeout (in seconds) if the pong message is not received. + ping_payload: str + Payload message to send with each ping. + http_proxy_host: str + HTTP proxy host name. + http_proxy_port: int or str + HTTP proxy port. If not set, set to 80. + http_no_proxy: list + Whitelisted host names that don't use the proxy. + skip_utf8_validation: bool + skip utf8 validation. + host: str + update host header. + origin: str + update origin header. + dispatcher: Dispatcher object + customize reading data from socket. + suppress_origin: bool + suppress outputting origin header. + + Returns + ------- + teardown: bool + False if caught KeyboardInterrupt, True if other exception was raised during a loop + """ + + if ping_timeout is not None and ping_timeout <= 0: + raise WebSocketException("Ensure ping_timeout > 0") + if ping_interval is not None and ping_interval < 0: + raise WebSocketException("Ensure ping_interval >= 0") + if ping_timeout and ping_interval and ping_interval <= ping_timeout: + raise WebSocketException("Ensure ping_interval > ping_timeout") + if not sockopt: + sockopt = [] + if not sslopt: + sslopt = {} + if self.sock: + raise WebSocketException("socket is already opened") + thread = None + self.keep_running = True + self.last_ping_tm = 0 + self.last_pong_tm = 0 + + def teardown(close_frame=None): + """ + Tears down the connection. + + Parameters + ---------- + close_frame: ABNF frame + If close_frame is set, the on_close handler is invoked + with the statusCode and reason from the provided frame. + """ + + if thread and thread.is_alive(): + event.set() + thread.join() + self.keep_running = False + if self.sock: + self.sock.close() + close_status_code, close_reason = self._get_close_args( + close_frame if close_frame else None) + self.sock = None + + # Finally call the callback AFTER all teardown is complete + self._callback(self.on_close, close_status_code, close_reason) + + try: + self.sock = WebSocket( + self.get_mask_key, sockopt=sockopt, sslopt=sslopt, + fire_cont_frame=self.on_cont_message is not None, + skip_utf8_validation=skip_utf8_validation, + enable_multithread=True) + self.sock.settimeout(getdefaulttimeout()) + self.sock.connect( + self.url, header=self.header, cookie=self.cookie, + http_proxy_host=http_proxy_host, + http_proxy_port=http_proxy_port, http_no_proxy=http_no_proxy, + http_proxy_auth=http_proxy_auth, subprotocols=self.subprotocols, + host=host, origin=origin, suppress_origin=suppress_origin, + proxy_type=proxy_type) + if not dispatcher: + dispatcher = self.create_dispatcher(ping_timeout) + + self._callback(self.on_open) + + if ping_interval: + event = threading.Event() + thread = threading.Thread( + target=self._send_ping, args=(ping_interval, event, ping_payload)) + thread.daemon = True + thread.start() + + def read(): + if not self.keep_running: + return teardown() + + op_code, frame = self.sock.recv_data_frame(True) + if op_code == ABNF.OPCODE_CLOSE: + return teardown(frame) + elif op_code == ABNF.OPCODE_PING: + self._callback(self.on_ping, frame.data) + elif op_code == ABNF.OPCODE_PONG: + self.last_pong_tm = time.time() + self._callback(self.on_pong, frame.data) + elif op_code == ABNF.OPCODE_CONT and self.on_cont_message: + self._callback(self.on_data, frame.data, + frame.opcode, frame.fin) + self._callback(self.on_cont_message, + frame.data, frame.fin) + else: + data = frame.data + if op_code == ABNF.OPCODE_TEXT: + data = data.decode("utf-8") + self._callback(self.on_data, data, frame.opcode, True) + self._callback(self.on_message, data) + + return True + + def check(): + if (ping_timeout): + has_timeout_expired = time.time() - self.last_ping_tm > ping_timeout + has_pong_not_arrived_after_last_ping = self.last_pong_tm - self.last_ping_tm < 0 + has_pong_arrived_too_late = self.last_pong_tm - self.last_ping_tm > ping_timeout + + if (self.last_ping_tm and + has_timeout_expired and + (has_pong_not_arrived_after_last_ping or has_pong_arrived_too_late)): + raise WebSocketTimeoutException("ping/pong timed out") + return True + + dispatcher.read(self.sock.sock, read, check) + except (Exception, KeyboardInterrupt, SystemExit) as e: + self._callback(self.on_error, e) + if isinstance(e, SystemExit): + # propagate SystemExit further + raise + teardown() + return not isinstance(e, KeyboardInterrupt) + + def create_dispatcher(self, ping_timeout): + timeout = ping_timeout or 10 + if self.sock.is_ssl(): + return SSLDispatcher(self, timeout) + + return Dispatcher(self, timeout) + + def _get_close_args(self, close_frame): + """ + _get_close_args extracts the close code and reason from the close body + if it exists (RFC6455 says WebSocket Connection Close Code is optional) + """ + # Need to catch the case where close_frame is None + # Otherwise the following if statement causes an error + if not self.on_close or not close_frame: + return [None, None] + + # Extract close frame status code + if close_frame.data and len(close_frame.data) >= 2: + close_status_code = 256 * close_frame.data[0] + close_frame.data[1] + reason = close_frame.data[2:].decode('utf-8') + return [close_status_code, reason] + else: + # Most likely reached this because len(close_frame_data.data) < 2 + return [None, None] + + def _callback(self, callback, *args): + if callback: + try: + callback(self, *args) + + except Exception as e: + _logging.error("error from callback {}: {}".format(callback, e)) + if self.on_error: + self.on_error(self, e) diff --git a/ubuntu/venv/ops/_vendor/websocket/_cookiejar.py b/ubuntu/venv/ops/_vendor/websocket/_cookiejar.py new file mode 100644 index 0000000..dcf5031 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_cookiejar.py @@ -0,0 +1,67 @@ +""" + +""" + +""" +_cookiejar.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import http.cookies + + +class SimpleCookieJar(object): + def __init__(self): + self.jar = dict() + + def add(self, set_cookie): + if set_cookie: + simpleCookie = http.cookies.SimpleCookie(set_cookie) + + for k, v in simpleCookie.items(): + domain = v.get("domain") + if domain: + if not domain.startswith("."): + domain = "." + domain + cookie = self.jar.get(domain) if self.jar.get(domain) else http.cookies.SimpleCookie() + cookie.update(simpleCookie) + self.jar[domain.lower()] = cookie + + def set(self, set_cookie): + if set_cookie: + simpleCookie = http.cookies.SimpleCookie(set_cookie) + + for k, v in simpleCookie.items(): + domain = v.get("domain") + if domain: + if not domain.startswith("."): + domain = "." + domain + self.jar[domain.lower()] = simpleCookie + + def get(self, host): + if not host: + return "" + + cookies = [] + for domain, simpleCookie in self.jar.items(): + host = host.lower() + if host.endswith(domain) or host == domain[1:]: + cookies.append(self.jar.get(domain)) + + return "; ".join(filter( + None, sorted( + ["%s=%s" % (k, v.value) for cookie in filter(None, cookies) for k, v in cookie.items()] + ))) diff --git a/ubuntu/venv/ops/_vendor/websocket/_core.py b/ubuntu/venv/ops/_vendor/websocket/_core.py new file mode 100644 index 0000000..f92f8a6 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_core.py @@ -0,0 +1,597 @@ +""" +_core.py +==================================== +WebSocket Python client +""" + +""" +_core.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import socket +import struct +import threading +import time + +# websocket modules +from ._abnf import * +from ._exceptions import * +from ._handshake import * +from ._http import * +from ._logging import * +from ._socket import * +from ._ssl_compat import * +from ._utils import * + +__all__ = ['WebSocket', 'create_connection'] + + +class WebSocket(object): + """ + Low level WebSocket interface. + + This class is based on the WebSocket protocol `draft-hixie-thewebsocketprotocol-76 `_ + + We can connect to the websocket server and send/receive data. + The following example is an echo client. + + >>> import websocket + >>> ws = websocket.WebSocket() + >>> ws.connect("ws://echo.websocket.org") + >>> ws.send("Hello, Server") + >>> ws.recv() + 'Hello, Server' + >>> ws.close() + + Parameters + ---------- + get_mask_key: func + A callable function to get new mask keys, see the + WebSocket.set_mask_key's docstring for more information. + sockopt: tuple + Values for socket.setsockopt. + sockopt must be tuple and each element is argument of sock.setsockopt. + sslopt: dict + Optional dict object for ssl socket options. + fire_cont_frame: bool + Fire recv event for each cont frame. Default is False. + enable_multithread: bool + If set to True, lock send method. + skip_utf8_validation: bool + Skip utf8 validation. + """ + + def __init__(self, get_mask_key=None, sockopt=None, sslopt=None, + fire_cont_frame=False, enable_multithread=True, + skip_utf8_validation=False, **_): + """ + Initialize WebSocket object. + + Parameters + ---------- + sslopt: dict + Optional dict object for ssl socket options. + """ + self.sock_opt = sock_opt(sockopt, sslopt) + self.handshake_response = None + self.sock = None + + self.connected = False + self.get_mask_key = get_mask_key + # These buffer over the build-up of a single frame. + self.frame_buffer = frame_buffer(self._recv, skip_utf8_validation) + self.cont_frame = continuous_frame( + fire_cont_frame, skip_utf8_validation) + + if enable_multithread: + self.lock = threading.Lock() + self.readlock = threading.Lock() + else: + self.lock = NoLock() + self.readlock = NoLock() + + def __iter__(self): + """ + Allow iteration over websocket, implying sequential `recv` executions. + """ + while True: + yield self.recv() + + def __next__(self): + return self.recv() + + def next(self): + return self.__next__() + + def fileno(self): + return self.sock.fileno() + + def set_mask_key(self, func): + """ + Set function to create mask key. You can customize mask key generator. + Mainly, this is for testing purpose. + + Parameters + ---------- + func: func + callable object. the func takes 1 argument as integer. + The argument means length of mask key. + This func must return string(byte array), + which length is argument specified. + """ + self.get_mask_key = func + + def gettimeout(self): + """ + Get the websocket timeout (in seconds) as an int or float + + Returns + ---------- + timeout: int or float + returns timeout value (in seconds). This value could be either float/integer. + """ + return self.sock_opt.timeout + + def settimeout(self, timeout): + """ + Set the timeout to the websocket. + + Parameters + ---------- + timeout: int or float + timeout time (in seconds). This value could be either float/integer. + """ + self.sock_opt.timeout = timeout + if self.sock: + self.sock.settimeout(timeout) + + timeout = property(gettimeout, settimeout) + + def getsubprotocol(self): + """ + Get subprotocol + """ + if self.handshake_response: + return self.handshake_response.subprotocol + else: + return None + + subprotocol = property(getsubprotocol) + + def getstatus(self): + """ + Get handshake status + """ + if self.handshake_response: + return self.handshake_response.status + else: + return None + + status = property(getstatus) + + def getheaders(self): + """ + Get handshake response header + """ + if self.handshake_response: + return self.handshake_response.headers + else: + return None + + def is_ssl(self): + try: + return isinstance(self.sock, ssl.SSLSocket) + except: + return False + + headers = property(getheaders) + + def connect(self, url, **options): + """ + Connect to url. url is websocket url scheme. + ie. ws://host:port/resource + You can customize using 'options'. + If you set "header" list object, you can set your own custom header. + + >>> ws = WebSocket() + >>> ws.connect("ws://echo.websocket.org/", + ... header=["User-Agent: MyProgram", + ... "x-custom: header"]) + + Parameters + ---------- + header: list or dict + Custom http header list or dict. + cookie: str + Cookie value. + origin: str + Custom origin url. + connection: str + Custom connection header value. + Default value "Upgrade" set in _handshake.py + suppress_origin: bool + Suppress outputting origin header. + host: str + Custom host header string. + timeout: int or float + Socket timeout time. This value is an integer or float. + If you set None for this value, it means "use default_timeout value" + http_proxy_host: str + HTTP proxy host name. + http_proxy_port: str or int + HTTP proxy port. Default is 80. + http_no_proxy: list + Whitelisted host names that don't use the proxy. + http_proxy_auth: tuple + HTTP proxy auth information. Tuple of username and password. Default is None. + redirect_limit: int + Number of redirects to follow. + subprotocols: list + List of available subprotocols. Default is None. + socket: socket + Pre-initialized stream socket. + """ + self.sock_opt.timeout = options.get('timeout', self.sock_opt.timeout) + self.sock, addrs = connect(url, self.sock_opt, proxy_info(**options), + options.pop('socket', None)) + + try: + self.handshake_response = handshake(self.sock, *addrs, **options) + for attempt in range(options.pop('redirect_limit', 3)): + if self.handshake_response.status in SUPPORTED_REDIRECT_STATUSES: + url = self.handshake_response.headers['location'] + self.sock.close() + self.sock, addrs = connect(url, self.sock_opt, proxy_info(**options), + options.pop('socket', None)) + self.handshake_response = handshake(self.sock, *addrs, **options) + self.connected = True + except: + if self.sock: + self.sock.close() + self.sock = None + raise + + def send(self, payload, opcode=ABNF.OPCODE_TEXT): + """ + Send the data as string. + + Parameters + ---------- + payload: str + Payload must be utf-8 string or unicode, + If the opcode is OPCODE_TEXT. + Otherwise, it must be string(byte array). + opcode: int + Operation code (opcode) to send. + """ + + frame = ABNF.create_frame(payload, opcode) + return self.send_frame(frame) + + def send_frame(self, frame): + """ + Send the data frame. + + >>> ws = create_connection("ws://echo.websocket.org/") + >>> frame = ABNF.create_frame("Hello", ABNF.OPCODE_TEXT) + >>> ws.send_frame(frame) + >>> cont_frame = ABNF.create_frame("My name is ", ABNF.OPCODE_CONT, 0) + >>> ws.send_frame(frame) + >>> cont_frame = ABNF.create_frame("Foo Bar", ABNF.OPCODE_CONT, 1) + >>> ws.send_frame(frame) + + Parameters + ---------- + frame: ABNF frame + frame data created by ABNF.create_frame + """ + if self.get_mask_key: + frame.get_mask_key = self.get_mask_key + data = frame.format() + length = len(data) + if (isEnabledForTrace()): + trace("++Sent raw: " + repr(data)) + trace("++Sent decoded: " + frame.__str__()) + with self.lock: + while data: + l = self._send(data) + data = data[l:] + + return length + + def send_binary(self, payload): + return self.send(payload, ABNF.OPCODE_BINARY) + + def ping(self, payload=""): + """ + Send ping data. + + Parameters + ---------- + payload: str + data payload to send server. + """ + if isinstance(payload, str): + payload = payload.encode("utf-8") + self.send(payload, ABNF.OPCODE_PING) + + def pong(self, payload=""): + """ + Send pong data. + + Parameters + ---------- + payload: str + data payload to send server. + """ + if isinstance(payload, str): + payload = payload.encode("utf-8") + self.send(payload, ABNF.OPCODE_PONG) + + def recv(self): + """ + Receive string data(byte array) from the server. + + Returns + ---------- + data: string (byte array) value. + """ + with self.readlock: + opcode, data = self.recv_data() + if opcode == ABNF.OPCODE_TEXT: + return data.decode("utf-8") + elif opcode == ABNF.OPCODE_TEXT or opcode == ABNF.OPCODE_BINARY: + return data + else: + return '' + + def recv_data(self, control_frame=False): + """ + Receive data with operation code. + + Parameters + ---------- + control_frame: bool + a boolean flag indicating whether to return control frame + data, defaults to False + + Returns + ------- + opcode, frame.data: tuple + tuple of operation code and string(byte array) value. + """ + opcode, frame = self.recv_data_frame(control_frame) + return opcode, frame.data + + def recv_data_frame(self, control_frame=False): + """ + Receive data with operation code. + + Parameters + ---------- + control_frame: bool + a boolean flag indicating whether to return control frame + data, defaults to False + + Returns + ------- + frame.opcode, frame: tuple + tuple of operation code and string(byte array) value. + """ + while True: + frame = self.recv_frame() + if (isEnabledForTrace()): + trace("++Rcv raw: " + repr(frame.format())) + trace("++Rcv decoded: " + frame.__str__()) + if not frame: + # handle error: + # 'NoneType' object has no attribute 'opcode' + raise WebSocketProtocolException( + "Not a valid frame %s" % frame) + elif frame.opcode in (ABNF.OPCODE_TEXT, ABNF.OPCODE_BINARY, ABNF.OPCODE_CONT): + self.cont_frame.validate(frame) + self.cont_frame.add(frame) + + if self.cont_frame.is_fire(frame): + return self.cont_frame.extract(frame) + + elif frame.opcode == ABNF.OPCODE_CLOSE: + self.send_close() + return frame.opcode, frame + elif frame.opcode == ABNF.OPCODE_PING: + if len(frame.data) < 126: + self.pong(frame.data) + else: + raise WebSocketProtocolException( + "Ping message is too long") + if control_frame: + return frame.opcode, frame + elif frame.opcode == ABNF.OPCODE_PONG: + if control_frame: + return frame.opcode, frame + + def recv_frame(self): + """ + Receive data as frame from server. + + Returns + ------- + self.frame_buffer.recv_frame(): ABNF frame object + """ + return self.frame_buffer.recv_frame() + + def send_close(self, status=STATUS_NORMAL, reason=bytes('', encoding='utf-8')): + """ + Send close data to the server. + + Parameters + ---------- + status: int + Status code to send. See STATUS_XXX. + reason: str or bytes + The reason to close. This must be string or bytes. + """ + if status < 0 or status >= ABNF.LENGTH_16: + raise ValueError("code is invalid range") + self.connected = False + self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE) + + def close(self, status=STATUS_NORMAL, reason=bytes('', encoding='utf-8'), timeout=3): + """ + Close Websocket object + + Parameters + ---------- + status: int + Status code to send. See STATUS_XXX. + reason: bytes + The reason to close. + timeout: int or float + Timeout until receive a close frame. + If None, it will wait forever until receive a close frame. + """ + if self.connected: + if status < 0 or status >= ABNF.LENGTH_16: + raise ValueError("code is invalid range") + + try: + self.connected = False + self.send(struct.pack('!H', status) + reason, ABNF.OPCODE_CLOSE) + sock_timeout = self.sock.gettimeout() + self.sock.settimeout(timeout) + start_time = time.time() + while timeout is None or time.time() - start_time < timeout: + try: + frame = self.recv_frame() + if frame.opcode != ABNF.OPCODE_CLOSE: + continue + if isEnabledForError(): + recv_status = struct.unpack("!H", frame.data[0:2])[0] + if recv_status >= 3000 and recv_status <= 4999: + debug("close status: " + repr(recv_status)) + elif recv_status != STATUS_NORMAL: + error("close status: " + repr(recv_status)) + break + except: + break + self.sock.settimeout(sock_timeout) + self.sock.shutdown(socket.SHUT_RDWR) + except: + pass + + self.shutdown() + + def abort(self): + """ + Low-level asynchronous abort, wakes up other threads that are waiting in recv_* + """ + if self.connected: + self.sock.shutdown(socket.SHUT_RDWR) + + def shutdown(self): + """ + close socket, immediately. + """ + if self.sock: + self.sock.close() + self.sock = None + self.connected = False + + def _send(self, data): + return send(self.sock, data) + + def _recv(self, bufsize): + try: + return recv(self.sock, bufsize) + except WebSocketConnectionClosedException: + if self.sock: + self.sock.close() + self.sock = None + self.connected = False + raise + + +def create_connection(url, timeout=None, class_=WebSocket, **options): + """ + Connect to url and return websocket object. + + Connect to url and return the WebSocket object. + Passing optional timeout parameter will set the timeout on the socket. + If no timeout is supplied, + the global default timeout setting returned by getdefaulttimeout() is used. + You can customize using 'options'. + If you set "header" list object, you can set your own custom header. + + >>> conn = create_connection("ws://echo.websocket.org/", + ... header=["User-Agent: MyProgram", + ... "x-custom: header"]) + + Parameters + ---------- + class_: class + class to instantiate when creating the connection. It has to implement + settimeout and connect. It's __init__ should be compatible with + WebSocket.__init__, i.e. accept all of it's kwargs. + header: list or dict + custom http header list or dict. + cookie: str + Cookie value. + origin: str + custom origin url. + suppress_origin: bool + suppress outputting origin header. + host: str + custom host header string. + timeout: int or float + socket timeout time. This value could be either float/integer. + If set to None, it uses the default_timeout value. + http_proxy_host: str + HTTP proxy host name. + http_proxy_port: str or int + HTTP proxy port. If not set, set to 80. + http_no_proxy: list + Whitelisted host names that don't use the proxy. + http_proxy_auth: tuple + HTTP proxy auth information. tuple of username and password. Default is None. + enable_multithread: bool + Enable lock for multithread. + redirect_limit: int + Number of redirects to follow. + sockopt: tuple + Values for socket.setsockopt. + sockopt must be a tuple and each element is an argument of sock.setsockopt. + sslopt: dict + Optional dict object for ssl socket options. + subprotocols: list + List of available subprotocols. Default is None. + skip_utf8_validation: bool + Skip utf8 validation. + socket: socket + Pre-initialized stream socket. + """ + sockopt = options.pop("sockopt", []) + sslopt = options.pop("sslopt", {}) + fire_cont_frame = options.pop("fire_cont_frame", False) + enable_multithread = options.pop("enable_multithread", True) + skip_utf8_validation = options.pop("skip_utf8_validation", False) + websock = class_(sockopt=sockopt, sslopt=sslopt, + fire_cont_frame=fire_cont_frame, + enable_multithread=enable_multithread, + skip_utf8_validation=skip_utf8_validation, **options) + websock.settimeout(timeout if timeout is not None else getdefaulttimeout()) + websock.connect(url, **options) + return websock diff --git a/ubuntu/venv/ops/_vendor/websocket/_exceptions.py b/ubuntu/venv/ops/_vendor/websocket/_exceptions.py new file mode 100644 index 0000000..2d5b053 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_exceptions.py @@ -0,0 +1,84 @@ +""" +Define WebSocket exceptions +""" + +""" +_exceptions.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +class WebSocketException(Exception): + """ + WebSocket exception class. + """ + pass + + +class WebSocketProtocolException(WebSocketException): + """ + If the WebSocket protocol is invalid, this exception will be raised. + """ + pass + + +class WebSocketPayloadException(WebSocketException): + """ + If the WebSocket payload is invalid, this exception will be raised. + """ + pass + + +class WebSocketConnectionClosedException(WebSocketException): + """ + If remote host closed the connection or some network error happened, + this exception will be raised. + """ + pass + + +class WebSocketTimeoutException(WebSocketException): + """ + WebSocketTimeoutException will be raised at socket timeout during read/write data. + """ + pass + + +class WebSocketProxyException(WebSocketException): + """ + WebSocketProxyException will be raised when proxy error occurred. + """ + pass + + +class WebSocketBadStatusException(WebSocketException): + """ + WebSocketBadStatusException will be raised when we get bad handshake status code. + """ + + def __init__(self, message, status_code, status_message=None, resp_headers=None): + msg = message % (status_code, status_message) + super(WebSocketBadStatusException, self).__init__(msg) + self.status_code = status_code + self.resp_headers = resp_headers + + +class WebSocketAddressException(WebSocketException): + """ + If the websocket address info cannot be found, this exception will be raised. + """ + pass diff --git a/ubuntu/venv/ops/_vendor/websocket/_handshake.py b/ubuntu/venv/ops/_vendor/websocket/_handshake.py new file mode 100644 index 0000000..da1a8d4 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_handshake.py @@ -0,0 +1,190 @@ +""" +_handshake.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import hashlib +import hmac +import os +from base64 import encodebytes as base64encode +from http import client as HTTPStatus +from ._cookiejar import SimpleCookieJar +from ._exceptions import * +from ._http import * +from ._logging import * +from ._socket import * + +__all__ = ["handshake_response", "handshake", "SUPPORTED_REDIRECT_STATUSES"] + +# websocket supported version. +VERSION = 13 + +SUPPORTED_REDIRECT_STATUSES = (HTTPStatus.MOVED_PERMANENTLY, HTTPStatus.FOUND, HTTPStatus.SEE_OTHER,) +SUCCESS_STATUSES = SUPPORTED_REDIRECT_STATUSES + (HTTPStatus.SWITCHING_PROTOCOLS,) + +CookieJar = SimpleCookieJar() + + +class handshake_response(object): + + def __init__(self, status, headers, subprotocol): + self.status = status + self.headers = headers + self.subprotocol = subprotocol + CookieJar.add(headers.get("set-cookie")) + + +def handshake(sock, hostname, port, resource, **options): + headers, key = _get_handshake_headers(resource, hostname, port, options) + + header_str = "\r\n".join(headers) + send(sock, header_str) + dump("request header", header_str) + + status, resp = _get_resp_headers(sock) + if status in SUPPORTED_REDIRECT_STATUSES: + return handshake_response(status, resp, None) + success, subproto = _validate(resp, key, options.get("subprotocols")) + if not success: + raise WebSocketException("Invalid WebSocket Header") + + return handshake_response(status, resp, subproto) + + +def _pack_hostname(hostname): + # IPv6 address + if ':' in hostname: + return '[' + hostname + ']' + + return hostname + + +def _get_handshake_headers(resource, host, port, options): + headers = [ + "GET %s HTTP/1.1" % resource, + "Upgrade: websocket" + ] + if port == 80 or port == 443: + hostport = _pack_hostname(host) + else: + hostport = "%s:%d" % (_pack_hostname(host), port) + if "host" in options and options["host"] is not None: + headers.append("Host: %s" % options["host"]) + else: + headers.append("Host: %s" % hostport) + + if "suppress_origin" not in options or not options["suppress_origin"]: + if "origin" in options and options["origin"] is not None: + headers.append("Origin: %s" % options["origin"]) + else: + headers.append("Origin: http://%s" % hostport) + + key = _create_sec_websocket_key() + + # Append Sec-WebSocket-Key & Sec-WebSocket-Version if not manually specified + if 'header' not in options or 'Sec-WebSocket-Key' not in options['header']: + key = _create_sec_websocket_key() + headers.append("Sec-WebSocket-Key: %s" % key) + else: + key = options['header']['Sec-WebSocket-Key'] + + if 'header' not in options or 'Sec-WebSocket-Version' not in options['header']: + headers.append("Sec-WebSocket-Version: %s" % VERSION) + + if 'connection' not in options or options['connection'] is None: + headers.append('Connection: Upgrade') + else: + headers.append(options['connection']) + + subprotocols = options.get("subprotocols") + if subprotocols: + headers.append("Sec-WebSocket-Protocol: %s" % ",".join(subprotocols)) + + if "header" in options: + header = options["header"] + if isinstance(header, dict): + header = [ + ": ".join([k, v]) + for k, v in header.items() + if v is not None + ] + headers.extend(header) + + server_cookie = CookieJar.get(host) + client_cookie = options.get("cookie", None) + + cookie = "; ".join(filter(None, [server_cookie, client_cookie])) + + if cookie: + headers.append("Cookie: %s" % cookie) + + headers.append("") + headers.append("") + + return headers, key + + +def _get_resp_headers(sock, success_statuses=SUCCESS_STATUSES): + status, resp_headers, status_message = read_headers(sock) + if status not in success_statuses: + raise WebSocketBadStatusException("Handshake status %d %s", status, status_message, resp_headers) + return status, resp_headers + + +_HEADERS_TO_CHECK = { + "upgrade": "websocket", + "connection": "upgrade", +} + + +def _validate(headers, key, subprotocols): + subproto = None + for k, v in _HEADERS_TO_CHECK.items(): + r = headers.get(k, None) + if not r: + return False, None + r = [x.strip().lower() for x in r.split(',')] + if v not in r: + return False, None + + if subprotocols: + subproto = headers.get("sec-websocket-protocol", None) + if not subproto or subproto.lower() not in [s.lower() for s in subprotocols]: + error("Invalid subprotocol: " + str(subprotocols)) + return False, None + subproto = subproto.lower() + + result = headers.get("sec-websocket-accept", None) + if not result: + return False, None + result = result.lower() + + if isinstance(result, str): + result = result.encode('utf-8') + + value = (key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode('utf-8') + hashed = base64encode(hashlib.sha1(value).digest()).strip().lower() + success = hmac.compare_digest(hashed, result) + + if success: + return True, subproto + else: + return False, None + + +def _create_sec_websocket_key(): + randomness = os.urandom(16) + return base64encode(randomness).decode('utf-8').strip() diff --git a/ubuntu/venv/ops/_vendor/websocket/_http.py b/ubuntu/venv/ops/_vendor/websocket/_http.py new file mode 100644 index 0000000..9ddf01d --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_http.py @@ -0,0 +1,335 @@ +""" +_http.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import errno +import os +import socket +import sys + +from ._exceptions import * +from ._logging import * +from ._socket import* +from ._ssl_compat import * +from ._url import * + +from base64 import encodebytes as base64encode + +__all__ = ["proxy_info", "connect", "read_headers"] + +try: + from python_socks.sync import Proxy + from python_socks._errors import * + from python_socks._types import ProxyType + HAVE_PYTHON_SOCKS = True +except: + HAVE_PYTHON_SOCKS = False + + class ProxyError(Exception): + pass + + class ProxyTimeoutError(Exception): + pass + + class ProxyConnectionError(Exception): + pass + + +class proxy_info(object): + + def __init__(self, **options): + self.proxy_host = options.get("http_proxy_host", None) + if self.proxy_host: + self.proxy_port = options.get("http_proxy_port", 0) + self.auth = options.get("http_proxy_auth", None) + self.no_proxy = options.get("http_no_proxy", None) + self.proxy_protocol = options.get("proxy_type", "http") + # Note: If timeout not specified, default python-socks timeout is 60 seconds + self.proxy_timeout = options.get("timeout", None) + if self.proxy_protocol not in ['http', 'socks4', 'socks4a', 'socks5', 'socks5h']: + raise ProxyError("Only http, socks4, socks5 proxy protocols are supported") + else: + self.proxy_port = 0 + self.auth = None + self.no_proxy = None + self.proxy_protocol = "http" + + +def _start_proxied_socket(url, options, proxy): + if not HAVE_PYTHON_SOCKS: + raise WebSocketException("Python Socks is needed for SOCKS proxying but is not available") + + hostname, port, resource, is_secure = parse_url(url) + + if proxy.proxy_protocol == "socks5": + rdns = False + proxy_type = ProxyType.SOCKS5 + if proxy.proxy_protocol == "socks4": + rdns = False + proxy_type = ProxyType.SOCKS4 + # socks5h and socks4a send DNS through proxy + if proxy.proxy_protocol == "socks5h": + rdns = True + proxy_type = ProxyType.SOCKS5 + if proxy.proxy_protocol == "socks4a": + rdns = True + proxy_type = ProxyType.SOCKS4 + + ws_proxy = Proxy.create( + proxy_type=proxy_type, + host=proxy.proxy_host, + port=int(proxy.proxy_port), + username=proxy.auth[0] if proxy.auth else None, + password=proxy.auth[1] if proxy.auth else None, + rdns=rdns) + + sock = ws_proxy.connect(hostname, port, timeout=proxy.proxy_timeout) + + if is_secure and HAVE_SSL: + sock = _ssl_socket(sock, options.sslopt, hostname) + elif is_secure: + raise WebSocketException("SSL not available.") + + return sock, (hostname, port, resource) + + +def connect(url, options, proxy, socket): + # Use _start_proxied_socket() only for socks4 or socks5 proxy + # Use _tunnel() for http proxy + # TODO: Use python-socks for http protocol also, to standardize flow + if proxy.proxy_host and not socket and not (proxy.proxy_protocol == "http"): + return _start_proxied_socket(url, options, proxy) + + hostname, port, resource, is_secure = parse_url(url) + + if socket: + return socket, (hostname, port, resource) + + addrinfo_list, need_tunnel, auth = _get_addrinfo_list( + hostname, port, is_secure, proxy) + if not addrinfo_list: + raise WebSocketException( + "Host not found.: " + hostname + ":" + str(port)) + + sock = None + try: + sock = _open_socket(addrinfo_list, options.sockopt, options.timeout) + if need_tunnel: + sock = _tunnel(sock, hostname, port, auth) + + if is_secure: + if HAVE_SSL: + sock = _ssl_socket(sock, options.sslopt, hostname) + else: + raise WebSocketException("SSL not available.") + + return sock, (hostname, port, resource) + except: + if sock: + sock.close() + raise + + +def _get_addrinfo_list(hostname, port, is_secure, proxy): + phost, pport, pauth = get_proxy_info( + hostname, is_secure, proxy.proxy_host, proxy.proxy_port, proxy.auth, proxy.no_proxy) + try: + # when running on windows 10, getaddrinfo without socktype returns a socktype 0. + # This generates an error exception: `_on_error: exception Socket type must be stream or datagram, not 0` + # or `OSError: [Errno 22] Invalid argument` when creating socket. Force the socket type to SOCK_STREAM. + if not phost: + addrinfo_list = socket.getaddrinfo( + hostname, port, 0, socket.SOCK_STREAM, socket.SOL_TCP) + return addrinfo_list, False, None + else: + pport = pport and pport or 80 + # when running on windows 10, the getaddrinfo used above + # returns a socktype 0. This generates an error exception: + # _on_error: exception Socket type must be stream or datagram, not 0 + # Force the socket type to SOCK_STREAM + addrinfo_list = socket.getaddrinfo(phost, pport, 0, socket.SOCK_STREAM, socket.SOL_TCP) + return addrinfo_list, True, pauth + except socket.gaierror as e: + raise WebSocketAddressException(e) + + +def _open_socket(addrinfo_list, sockopt, timeout): + err = None + for addrinfo in addrinfo_list: + family, socktype, proto = addrinfo[:3] + sock = socket.socket(family, socktype, proto) + sock.settimeout(timeout) + for opts in DEFAULT_SOCKET_OPTION: + sock.setsockopt(*opts) + for opts in sockopt: + sock.setsockopt(*opts) + + address = addrinfo[4] + err = None + while not err: + try: + sock.connect(address) + except socket.error as error: + error.remote_ip = str(address[0]) + try: + eConnRefused = (errno.ECONNREFUSED, errno.WSAECONNREFUSED) + except: + eConnRefused = (errno.ECONNREFUSED, ) + if error.errno == errno.EINTR: + continue + elif error.errno in eConnRefused: + err = error + continue + else: + if sock: + sock.close() + raise error + else: + break + else: + continue + break + else: + if err: + raise err + + return sock + + +def _wrap_sni_socket(sock, sslopt, hostname, check_hostname): + context = ssl.SSLContext(sslopt.get('ssl_version', ssl.PROTOCOL_TLS)) + + if sslopt.get('cert_reqs', ssl.CERT_NONE) != ssl.CERT_NONE: + cafile = sslopt.get('ca_certs', None) + capath = sslopt.get('ca_cert_path', None) + if cafile or capath: + context.load_verify_locations(cafile=cafile, capath=capath) + elif hasattr(context, 'load_default_certs'): + context.load_default_certs(ssl.Purpose.SERVER_AUTH) + if sslopt.get('certfile', None): + context.load_cert_chain( + sslopt['certfile'], + sslopt.get('keyfile', None), + sslopt.get('password', None), + ) + # see + # https://github.com/liris/websocket-client/commit/b96a2e8fa765753e82eea531adb19716b52ca3ca#commitcomment-10803153 + context.verify_mode = sslopt['cert_reqs'] + if HAVE_CONTEXT_CHECK_HOSTNAME: + context.check_hostname = check_hostname + if 'ciphers' in sslopt: + context.set_ciphers(sslopt['ciphers']) + if 'cert_chain' in sslopt: + certfile, keyfile, password = sslopt['cert_chain'] + context.load_cert_chain(certfile, keyfile, password) + if 'ecdh_curve' in sslopt: + context.set_ecdh_curve(sslopt['ecdh_curve']) + + return context.wrap_socket( + sock, + do_handshake_on_connect=sslopt.get('do_handshake_on_connect', True), + suppress_ragged_eofs=sslopt.get('suppress_ragged_eofs', True), + server_hostname=hostname, + ) + + +def _ssl_socket(sock, user_sslopt, hostname): + sslopt = dict(cert_reqs=ssl.CERT_REQUIRED) + sslopt.update(user_sslopt) + + certPath = os.environ.get('WEBSOCKET_CLIENT_CA_BUNDLE') + if certPath and os.path.isfile(certPath) \ + and user_sslopt.get('ca_certs', None) is None: + sslopt['ca_certs'] = certPath + elif certPath and os.path.isdir(certPath) \ + and user_sslopt.get('ca_cert_path', None) is None: + sslopt['ca_cert_path'] = certPath + + if sslopt.get('server_hostname', None): + hostname = sslopt['server_hostname'] + + check_hostname = sslopt["cert_reqs"] != ssl.CERT_NONE and sslopt.pop( + 'check_hostname', True) + sock = _wrap_sni_socket(sock, sslopt, hostname, check_hostname) + + if not HAVE_CONTEXT_CHECK_HOSTNAME and check_hostname: + match_hostname(sock.getpeercert(), hostname) + + return sock + + +def _tunnel(sock, host, port, auth): + debug("Connecting proxy...") + connect_header = "CONNECT %s:%d HTTP/1.1\r\n" % (host, port) + connect_header += "Host: %s:%d\r\n" % (host, port) + + # TODO: support digest auth. + if auth and auth[0]: + auth_str = auth[0] + if auth[1]: + auth_str += ":" + auth[1] + encoded_str = base64encode(auth_str.encode()).strip().decode().replace('\n', '') + connect_header += "Proxy-Authorization: Basic %s\r\n" % encoded_str + connect_header += "\r\n" + dump("request header", connect_header) + + send(sock, connect_header) + + try: + status, resp_headers, status_message = read_headers(sock) + except Exception as e: + raise WebSocketProxyException(str(e)) + + if status != 200: + raise WebSocketProxyException( + "failed CONNECT via proxy status: %r" % status) + + return sock + + +def read_headers(sock): + status = None + status_message = None + headers = {} + trace("--- response header ---") + + while True: + line = recv_line(sock) + line = line.decode('utf-8').strip() + if not line: + break + trace(line) + if not status: + + status_info = line.split(" ", 2) + status = int(status_info[1]) + if len(status_info) > 2: + status_message = status_info[2] + else: + kv = line.split(":", 1) + if len(kv) == 2: + key, value = kv + if key.lower() == "set-cookie" and headers.get("set-cookie"): + headers["set-cookie"] = headers.get("set-cookie") + "; " + value.strip() + else: + headers[key.lower()] = value.strip() + else: + raise WebSocketException("Invalid header") + + trace("-----------------------") + + return status, headers, status_message diff --git a/ubuntu/venv/ops/_vendor/websocket/_logging.py b/ubuntu/venv/ops/_vendor/websocket/_logging.py new file mode 100644 index 0000000..480d43b --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_logging.py @@ -0,0 +1,90 @@ +""" + +""" + +""" +_logging.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import logging + +_logger = logging.getLogger('websocket') +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +_logger.addHandler(NullHandler()) + +_traceEnabled = False + +__all__ = ["enableTrace", "dump", "error", "warning", "debug", "trace", + "isEnabledForError", "isEnabledForDebug", "isEnabledForTrace"] + + +def enableTrace(traceable, handler=logging.StreamHandler()): + """ + Turn on/off the traceability. + + Parameters + ---------- + traceable: bool + If set to True, traceability is enabled. + """ + global _traceEnabled + _traceEnabled = traceable + if traceable: + _logger.addHandler(handler) + _logger.setLevel(logging.DEBUG) + + +def dump(title, message): + if _traceEnabled: + _logger.debug("--- " + title + " ---") + _logger.debug(message) + _logger.debug("-----------------------") + + +def error(msg): + _logger.error(msg) + + +def warning(msg): + _logger.warning(msg) + + +def debug(msg): + _logger.debug(msg) + + +def trace(msg): + if _traceEnabled: + _logger.debug(msg) + + +def isEnabledForError(): + return _logger.isEnabledFor(logging.ERROR) + + +def isEnabledForDebug(): + return _logger.isEnabledFor(logging.DEBUG) + + +def isEnabledForTrace(): + return _traceEnabled diff --git a/ubuntu/venv/ops/_vendor/websocket/_socket.py b/ubuntu/venv/ops/_vendor/websocket/_socket.py new file mode 100644 index 0000000..eb573d4 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_socket.py @@ -0,0 +1,182 @@ +""" + +""" + +""" +_socket.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import errno +import selectors +import socket + +from ._exceptions import * +from ._ssl_compat import * +from ._utils import * + +DEFAULT_SOCKET_OPTION = [(socket.SOL_TCP, socket.TCP_NODELAY, 1)] +if hasattr(socket, "SO_KEEPALIVE"): + DEFAULT_SOCKET_OPTION.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)) +if hasattr(socket, "TCP_KEEPIDLE"): + DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPIDLE, 30)) +if hasattr(socket, "TCP_KEEPINTVL"): + DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPINTVL, 10)) +if hasattr(socket, "TCP_KEEPCNT"): + DEFAULT_SOCKET_OPTION.append((socket.SOL_TCP, socket.TCP_KEEPCNT, 3)) + +_default_timeout = None + +__all__ = ["DEFAULT_SOCKET_OPTION", "sock_opt", "setdefaulttimeout", "getdefaulttimeout", + "recv", "recv_line", "send"] + + +class sock_opt(object): + + def __init__(self, sockopt, sslopt): + if sockopt is None: + sockopt = [] + if sslopt is None: + sslopt = {} + self.sockopt = sockopt + self.sslopt = sslopt + self.timeout = None + + +def setdefaulttimeout(timeout): + """ + Set the global timeout setting to connect. + + Parameters + ---------- + timeout: int or float + default socket timeout time (in seconds) + """ + global _default_timeout + _default_timeout = timeout + + +def getdefaulttimeout(): + """ + Get default timeout + + Returns + ---------- + _default_timeout: int or float + Return the global timeout setting (in seconds) to connect. + """ + return _default_timeout + + +def recv(sock, bufsize): + if not sock: + raise WebSocketConnectionClosedException("socket is already closed.") + + def _recv(): + try: + return sock.recv(bufsize) + except SSLWantReadError: + pass + except socket.error as exc: + error_code = extract_error_code(exc) + if error_code is None: + raise + if error_code != errno.EAGAIN or error_code != errno.EWOULDBLOCK: + raise + + sel = selectors.DefaultSelector() + sel.register(sock, selectors.EVENT_READ) + + r = sel.select(sock.gettimeout()) + sel.close() + + if r: + return sock.recv(bufsize) + + try: + if sock.gettimeout() == 0: + bytes_ = sock.recv(bufsize) + else: + bytes_ = _recv() + except socket.timeout as e: + message = extract_err_message(e) + raise WebSocketTimeoutException(message) + except SSLError as e: + message = extract_err_message(e) + if isinstance(message, str) and 'timed out' in message: + raise WebSocketTimeoutException(message) + else: + raise + + if not bytes_: + raise WebSocketConnectionClosedException( + "Connection to remote host was lost.") + + return bytes_ + + +def recv_line(sock): + line = [] + while True: + c = recv(sock, 1) + line.append(c) + if c == b'\n': + break + return b''.join(line) + + +def send(sock, data): + if isinstance(data, str): + data = data.encode('utf-8') + + if not sock: + raise WebSocketConnectionClosedException("socket is already closed.") + + def _send(): + try: + return sock.send(data) + except SSLWantWriteError: + pass + except socket.error as exc: + error_code = extract_error_code(exc) + if error_code is None: + raise + if error_code != errno.EAGAIN or error_code != errno.EWOULDBLOCK: + raise + + sel = selectors.DefaultSelector() + sel.register(sock, selectors.EVENT_WRITE) + + w = sel.select(sock.gettimeout()) + sel.close() + + if w: + return sock.send(data) + + try: + if sock.gettimeout() == 0: + return sock.send(data) + else: + return _send() + except socket.timeout as e: + message = extract_err_message(e) + raise WebSocketTimeoutException(message) + except Exception as e: + message = extract_err_message(e) + if isinstance(message, str) and "timed out" in message: + raise WebSocketTimeoutException(message) + else: + raise diff --git a/ubuntu/venv/ops/_vendor/websocket/_ssl_compat.py b/ubuntu/venv/ops/_vendor/websocket/_ssl_compat.py new file mode 100644 index 0000000..9e5460c --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_ssl_compat.py @@ -0,0 +1,44 @@ +""" +_ssl_compat.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +__all__ = ["HAVE_SSL", "ssl", "SSLError", "SSLWantReadError", "SSLWantWriteError"] + +try: + import ssl + from ssl import SSLError + from ssl import SSLWantReadError + from ssl import SSLWantWriteError + HAVE_CONTEXT_CHECK_HOSTNAME = False + if hasattr(ssl, 'SSLContext') and hasattr(ssl.SSLContext, 'check_hostname'): + HAVE_CONTEXT_CHECK_HOSTNAME = True + + __all__.append("HAVE_CONTEXT_CHECK_HOSTNAME") + HAVE_SSL = True +except ImportError: + # dummy class of SSLError for environment without ssl support + class SSLError(Exception): + pass + + class SSLWantReadError(Exception): + pass + + class SSLWantWriteError(Exception): + pass + + ssl = None + HAVE_SSL = False diff --git a/ubuntu/venv/ops/_vendor/websocket/_url.py b/ubuntu/venv/ops/_vendor/websocket/_url.py new file mode 100644 index 0000000..f2a5501 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_url.py @@ -0,0 +1,176 @@ +""" + +""" +""" +_url.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import socket +import struct + +from urllib.parse import unquote, urlparse + + +__all__ = ["parse_url", "get_proxy_info"] + + +def parse_url(url): + """ + parse url and the result is tuple of + (hostname, port, resource path and the flag of secure mode) + + Parameters + ---------- + url: str + url string. + """ + if ":" not in url: + raise ValueError("url is invalid") + + scheme, url = url.split(":", 1) + + parsed = urlparse(url, scheme="http") + if parsed.hostname: + hostname = parsed.hostname + else: + raise ValueError("hostname is invalid") + port = 0 + if parsed.port: + port = parsed.port + + is_secure = False + if scheme == "ws": + if not port: + port = 80 + elif scheme == "wss": + is_secure = True + if not port: + port = 443 + else: + raise ValueError("scheme %s is invalid" % scheme) + + if parsed.path: + resource = parsed.path + else: + resource = "/" + + if parsed.query: + resource += "?" + parsed.query + + return hostname, port, resource, is_secure + + +DEFAULT_NO_PROXY_HOST = ["localhost", "127.0.0.1"] + + +def _is_ip_address(addr): + try: + socket.inet_aton(addr) + except socket.error: + return False + else: + return True + + +def _is_subnet_address(hostname): + try: + addr, netmask = hostname.split("/") + return _is_ip_address(addr) and 0 <= int(netmask) < 32 + except ValueError: + return False + + +def _is_address_in_network(ip, net): + ipaddr = struct.unpack('!I', socket.inet_aton(ip))[0] + netaddr, netmask = net.split('/') + netaddr = struct.unpack('!I', socket.inet_aton(netaddr))[0] + + netmask = (0xFFFFFFFF << (32 - int(netmask))) & 0xFFFFFFFF + return ipaddr & netmask == netaddr + + +def _is_no_proxy_host(hostname, no_proxy): + if not no_proxy: + v = os.environ.get("no_proxy", os.environ.get("NO_PROXY", "")).replace(" ", "") + if v: + no_proxy = v.split(",") + if not no_proxy: + no_proxy = DEFAULT_NO_PROXY_HOST + + if '*' in no_proxy: + return True + if hostname in no_proxy: + return True + if _is_ip_address(hostname): + return any([_is_address_in_network(hostname, subnet) for subnet in no_proxy if _is_subnet_address(subnet)]) + for domain in [domain for domain in no_proxy if domain.startswith('.')]: + if hostname.endswith(domain): + return True + return False + + +def get_proxy_info( + hostname, is_secure, proxy_host=None, proxy_port=0, proxy_auth=None, + no_proxy=None, proxy_type='http'): + """ + Try to retrieve proxy host and port from environment + if not provided in options. + Result is (proxy_host, proxy_port, proxy_auth). + proxy_auth is tuple of username and password + of proxy authentication information. + + Parameters + ---------- + hostname: str + Websocket server name. + is_secure: bool + Is the connection secure? (wss) looks for "https_proxy" in env + before falling back to "http_proxy" + proxy_host: str + http proxy host name. + http_proxy_port: str or int + http proxy port. + http_no_proxy: list + Whitelisted host names that don't use the proxy. + http_proxy_auth: tuple + HTTP proxy auth information. Tuple of username and password. Default is None. + proxy_type: str + Specify the proxy protocol (http, socks4, socks4a, socks5, socks5h). Default is "http". + Use socks4a or socks5h if you want to send DNS requests through the proxy. + """ + if _is_no_proxy_host(hostname, no_proxy): + return None, 0, None + + if proxy_host: + port = proxy_port + auth = proxy_auth + return proxy_host, port, auth + + env_keys = ["http_proxy"] + if is_secure: + env_keys.insert(0, "https_proxy") + + for key in env_keys: + value = os.environ.get(key, os.environ.get(key.upper(), "")).replace(" ", "") + if value: + proxy = urlparse(value) + auth = (unquote(proxy.username), unquote(proxy.password)) if proxy.username else None + return proxy.hostname, proxy.port, auth + + return None, 0, None diff --git a/ubuntu/venv/ops/_vendor/websocket/_utils.py b/ubuntu/venv/ops/_vendor/websocket/_utils.py new file mode 100644 index 0000000..feed027 --- /dev/null +++ b/ubuntu/venv/ops/_vendor/websocket/_utils.py @@ -0,0 +1,104 @@ +""" +_url.py +websocket - WebSocket client library for Python + +Copyright 2021 engn33r + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +__all__ = ["NoLock", "validate_utf8", "extract_err_message", "extract_error_code"] + + +class NoLock(object): + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass + + +try: + # If wsaccel is available we use compiled routines to validate UTF-8 + # strings. + from wsaccel.utf8validator import Utf8Validator + + def _validate_utf8(utfbytes): + return Utf8Validator().validate(utfbytes)[0] + +except ImportError: + # UTF-8 validator + # python implementation of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + + _UTF8_ACCEPT = 0 + _UTF8_REJECT = 12 + + _UTF8D = [ + # The first part of the table maps bytes to character classes that + # to reduce the size of the transition table and create bitmasks. + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, + + # The second part is a transition table that maps a combination + # of a state of the automaton and a character class to a state. + 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, + 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, + 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, + 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, + 12,36,12,12,12,12,12,12,12,12,12,12, ] + + def _decode(state, codep, ch): + tp = _UTF8D[ch] + + codep = (ch & 0x3f) | (codep << 6) if ( + state != _UTF8_ACCEPT) else (0xff >> tp) & ch + state = _UTF8D[256 + state + tp] + + return state, codep + + def _validate_utf8(utfbytes): + state = _UTF8_ACCEPT + codep = 0 + for i in utfbytes: + state, codep = _decode(state, codep, i) + if state == _UTF8_REJECT: + return False + + return True + + +def validate_utf8(utfbytes): + """ + validate utf8 byte string. + utfbytes: utf byte string to check. + return value: if valid utf8 string, return true. Otherwise, return false. + """ + return _validate_utf8(utfbytes) + + +def extract_err_message(exception): + if exception.args: + return exception.args[0] + else: + return None + + +def extract_error_code(exception): + if exception.args and len(exception.args) > 1: + return exception.args[0] if isinstance(exception.args[0], int) else None diff --git a/ubuntu/venv/ops/charm.py b/ubuntu/venv/ops/charm.py new file mode 100644 index 0000000..eb1a5b7 --- /dev/null +++ b/ubuntu/venv/ops/charm.py @@ -0,0 +1,1066 @@ +# Copyright 2019-2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base objects for the Charm, events and metadata.""" + +import enum +import os +import pathlib +import typing + +from ops import model +from ops._private import yaml +from ops.framework import EventBase, EventSource, Framework, Object, ObjectEvents + + +class HookEvent(EventBase): + """Events raised by Juju to progress a charm's lifecycle. + + Hooks are callback methods of a charm class (a subclass of + :class:`CharmBase`) that are invoked in response to events raised + by Juju. These callback methods are the means by which a charm + governs the lifecycle of its application. + + The :class:`HookEvent` class is the base of a type hierarchy of events + related to the charm's lifecycle. + + :class:`HookEvent` subtypes are grouped into the following categories + + - Core lifecycle events + - Relation events + - Storage events + - Metric events + """ + + +class ActionEvent(EventBase): + """Events raised by Juju when an administrator invokes a Juju Action. + + This class is the data type of events triggered when an administrator + invokes a Juju Action. Callbacks bound to these events may be used + for responding to the administrator's Juju Action request. + + To read the parameters for the action, see the instance variable :attr:`params`. + To respond with the result of the action, call :meth:`set_results`. To add + progress messages that are visible as the action is progressing use + :meth:`log`. + + Attributes: + params: The parameters passed to the action. + """ + + def defer(self): + """Action events are not deferable like other events. + + This is because an action runs synchronously and the administrator + is waiting for the result. + """ + raise RuntimeError('cannot defer action events') + + def restore(self, snapshot: dict) -> None: + """Used by the operator framework to record the action. + + Not meant to be called directly by charm code. + """ + env_action_name = os.environ.get('JUJU_ACTION_NAME') + event_action_name = self.handle.kind[:-len('_action')].replace('_', '-') + if event_action_name != env_action_name: + # This could only happen if the dev manually emits the action, or from a bug. + raise RuntimeError('action event kind does not match current action') + # Params are loaded at restore rather than __init__ because + # the model is not available in __init__. + self.params = self.framework.model._backend.action_get() + + def set_results(self, results: typing.Mapping) -> None: + """Report the result of the action. + + Args: + results: The result of the action as a Dict + """ + self.framework.model._backend.action_set(results) + + def log(self, message: str) -> None: + """Send a message that a user will see while the action is running. + + Args: + message: The message for the user. + """ + self.framework.model._backend.action_log(message) + + def fail(self, message: str = '') -> None: + """Report that this action has failed. + + Args: + message: Optional message to record why it has failed. + """ + self.framework.model._backend.action_fail(message) + + +class InstallEvent(HookEvent): + """Event triggered when a charm is installed. + + This event is triggered at the beginning of a charm's + lifecycle. Any associated callback method should be used to + perform one-time setup operations, such as installing prerequisite + software. + """ + + +class StartEvent(HookEvent): + """Event triggered immediately after first configuration change. + + This event is triggered immediately after the first + :class:`ConfigChangedEvent`. Callback methods bound to the event should be + used to ensure that the charm’s software is in a running state. Note that + the charm’s software should be configured so as to persist in this state + through reboots without further intervention on Juju’s part. + """ + + +class StopEvent(HookEvent): + """Event triggered when a charm is shut down. + + This event is triggered when an application's removal is requested + by the client. The event fires immediately before the end of the + unit’s destruction sequence. Callback methods bound to this event + should be used to ensure that the charm’s software is not running, + and that it will not start again on reboot. + """ + + +class RemoveEvent(HookEvent): + """Event triggered when a unit is about to be terminated. + + This event fires prior to Juju removing the charm and terminating its unit. + """ + + +class ConfigChangedEvent(HookEvent): + """Event triggered when a configuration change is requested. + + This event fires in several different situations. + + - immediately after the :class:`install ` event. + - after a :class:`relation is created `. + - after a :class:`leader is elected `. + - after changing charm configuration using the GUI or command line + interface + - when the charm :class:`starts `. + - when a new unit :class:`joins a relation `. + - when there is a :class:`change to an existing relation `. + + Any callback method bound to this event cannot assume that the + software has already been started; it should not start stopped + software, but should (if appropriate) restart running software to + take configuration changes into account. + """ + + +class UpdateStatusEvent(HookEvent): + """Event triggered by a status update request from Juju. + + This event is periodically triggered by Juju so that it can + provide constant feedback to the administrator about the status of + the application the charm is modeling. Any callback method bound + to this event should determine the "health" of the application and + set the status appropriately. + + The interval between :class:`update-status ` events can + be configured model-wide, e.g. ``juju model-config + update-status-hook-interval=1m``. + """ + + +class UpgradeCharmEvent(HookEvent): + """Event triggered by request to upgrade the charm. + + This event will be triggered when an administrator executes ``juju + upgrade-charm``. The event fires after Juju has unpacked the upgraded charm + code, and so this event will be handled by the callback method bound to the + event in the new codebase. The associated callback method is invoked + provided there is no existing error state. The callback method should be + used to reconcile current state written by an older version of the charm + into whatever form that is needed by the current charm version. + """ + + +class PreSeriesUpgradeEvent(HookEvent): + """Event triggered to prepare a unit for series upgrade. + + This event triggers when an administrator executes ``juju upgrade-series + MACHINE prepare``. The event will fire for each unit that is running on the + specified machine. Any callback method bound to this event must prepare the + charm for an upgrade to the series. This may include things like exporting + database content to a version neutral format, or evacuating running + instances to other machines. + + It can be assumed that only after all units on a machine have executed the + callback method associated with this event, the administrator will initiate + steps to actually upgrade the series. After the upgrade has been completed, + the :class:`PostSeriesUpgradeEvent` will fire. + """ + + +class PostSeriesUpgradeEvent(HookEvent): + """Event triggered after a series upgrade. + + This event is triggered after the administrator has done a distribution + upgrade (or rolled back and kept the same series). It is called in response + to ``juju upgrade-series MACHINE complete``. Associated charm callback + methods are expected to do whatever steps are necessary to reconfigure their + applications for the new series. This may include things like populating the + upgraded version of a database. Note however charms are expected to check if + the series has actually changed or whether it was rolled back to the + original series. + """ + + +class LeaderElectedEvent(HookEvent): + """Event triggered when a new leader has been elected. + + Juju will trigger this event when a new leader unit is chosen for + a given application. + + This event fires at least once after Juju selects a leader + unit. Callback methods bound to this event may take any action + required for the elected unit to assert leadership. Note that only + the elected leader unit will receive this event. + """ + + +class LeaderSettingsChangedEvent(HookEvent): + """Event triggered when leader changes any settings. + + DEPRECATED NOTICE + + This event has been deprecated in favor of using a Peer relation, + and having the leader set a value in the Application data bag for + that peer relation. (see :class:`RelationChangedEvent`). + """ + + +class CollectMetricsEvent(HookEvent): + """Event triggered by Juju to collect metrics. + + Juju fires this event every five minutes for the lifetime of the + unit. Callback methods bound to this event may use the :meth:`add_metrics` + method of this class to send measurements to Juju. + + Note that associated callback methods are currently sandboxed in + how they can interact with Juju. + """ + + def add_metrics(self, metrics: typing.Mapping, labels: typing.Mapping = None) -> None: + """Record metrics that have been gathered by the charm for this unit. + + Args: + metrics: A collection of {key: float} pairs that contains the + metrics that have been gathered + labels: {key:value} strings that can be applied to the + metrics that are being gathered + """ + self.framework.model._backend.add_metrics(metrics, labels) + + +class RelationEvent(HookEvent): + """A base class representing the various relation lifecycle events. + + Relation lifecycle events are generated when application units + participate in relations. Units can only participate in relations + after they have been "started", and before they have been + "stopped". Within that time window, the unit may participate in + several different relations at a time, including multiple + relations with the same name. + + Attributes: + relation: The :class:`~ops.model.Relation` involved in this event + app: The remote :class:`~ops.model.Application` that has triggered this + event + unit: The remote :class:`~ops.model.Unit` that has triggered this event. This may be + ``None`` if the relation event was triggered as an + :class:`~ops.model.Application` level event + + """ + + def __init__(self, handle, relation, app=None, unit=None): + super().__init__(handle) + + if unit is not None and unit.app != app: + raise RuntimeError( + 'cannot create RelationEvent with application {} and unit {}'.format(app, unit)) + + self.relation = relation + self.app = app + self.unit = unit + + def snapshot(self) -> dict: + """Used by the framework to serialize the event to disk. + + Not meant to be called by charm code. + """ + snapshot = { + 'relation_name': self.relation.name, + 'relation_id': self.relation.id, + } + if self.app: + snapshot['app_name'] = self.app.name + if self.unit: + snapshot['unit_name'] = self.unit.name + return snapshot + + def restore(self, snapshot: dict) -> None: + """Used by the framework to deserialize the event from disk. + + Not meant to be called by charm code. + """ + self.relation = self.framework.model.get_relation( + snapshot['relation_name'], snapshot['relation_id']) + + app_name = snapshot.get('app_name') + if app_name: + self.app = self.framework.model.get_app(app_name) + else: + self.app = None + + unit_name = snapshot.get('unit_name') + if unit_name: + self.unit = self.framework.model.get_unit(unit_name) + else: + self.unit = None + + +class RelationCreatedEvent(RelationEvent): + """Event triggered when a new relation is created. + + This is triggered when a new relation to another app is added in Juju. This + can occur before units for those applications have started. All existing + relations should be established before start. + """ + + +class RelationJoinedEvent(RelationEvent): + """Event triggered when a new unit joins a relation. + + This event is triggered whenever a new unit of a related + application joins the relation. The event fires only when that + remote unit is first observed by the unit. Callback methods bound + to this event may set any local unit settings that can be + determined using no more than the name of the joining unit and the + remote ``private-address`` setting, which is always available when + the relation is created and is by convention not deleted. + """ + + +class RelationChangedEvent(RelationEvent): + """Event triggered when relation data changes. + + This event is triggered whenever there is a change to the data bucket for a + related application or unit. Look at ``event.relation.data[event.unit/app]`` + to see the new information, where ``event`` is the event object passed to + the callback method bound to this event. + + This event always fires once, after :class:`RelationJoinedEvent`, and + will subsequently fire whenever that remote unit changes its settings for + the relation. Callback methods bound to this event should be the only ones + that rely on remote relation settings. They should not error if the settings + are incomplete, since it can be guaranteed that when the remote unit or + application changes its settings, the event will fire again. + + The settings that may be queried, or set, are determined by the relation’s + interface. + """ + + +class RelationDepartedEvent(RelationEvent): + """Event triggered when a unit leaves a relation. + + This is the inverse of the :class:`RelationJoinedEvent`, representing when a + unit is leaving the relation (the unit is being removed, the app is being + removed, the relation is being removed). For remaining units, this event is + emitted once for each departing unit. For departing units, this event is + emitted once for each remaining unit. + + Callback methods bound to this event may be used to remove all + references to the departing remote unit, because there’s no + guarantee that it’s still part of the system; it’s perfectly + probable (although not guaranteed) that the system running that + unit has already shut down. + + Once all callback methods bound to this event have been run for such a + relation, the unit agent will fire the :class:`RelationBrokenEvent`. + + Attributes: + departing_unit: The :class:`~ops.model.Unit` that is departing. This + can facilitate determining e.g. whether *you* are the departing + unit. + """ + + def __init__(self, handle, relation, app=None, unit=None, + departing_unit_name=None): + super().__init__(handle, relation, app=app, unit=unit) + + self._departing_unit_name = departing_unit_name + + @property + def departing_unit(self) -> typing.Optional[model.Unit]: + """The `ops.model.Unit` that is departing, if any.""" + # doing this on init would fail because `framework` gets patched in + # post-init + if not self._departing_unit_name: + return None + return self.framework.model.get_unit(self._departing_unit_name) + + def snapshot(self) -> dict: + """Used by the framework to serialize the event to disk. + + Not meant to be called by charm code. + """ + snapshot = super().snapshot() + if self._departing_unit_name: + snapshot['departing_unit'] = self.departing_unit.name + return snapshot + + def restore(self, snapshot: dict) -> None: + """Used by the framework to deserialize the event from disk. + + Not meant to be called by charm code. + """ + super().restore(snapshot) + + self._departing_unit_name = snapshot.get('departing_unit') + + +class RelationBrokenEvent(RelationEvent): + """Event triggered when a relation is removed. + + If a relation is being removed (``juju remove-relation`` or ``juju + remove-application``), once all the units have been removed, this event will + fire to signal that the relationship has been fully terminated. + + The event indicates that the current relation is no longer valid, and that + the charm’s software must be configured as though the relation had never + existed. It will only be called after every callback method bound to + :class:`RelationDepartedEvent` has been run. If a callback method + bound to this event is being executed, it is guaranteed that no remote units + are currently known locally. + """ + + +class StorageEvent(HookEvent): + """Base class representing storage-related events. + + Juju can provide a variety of storage types to a charms. The + charms can define several different types of storage that are + allocated from Juju. Changes in state of storage trigger sub-types + of :class:`StorageEvent`. + """ + + def __init__(self, handle, storage): + super().__init__(handle) + self.storage = storage + + def snapshot(self) -> dict: + """Used by the framework to serialize the event to disk. + + Not meant to be called by charm code. + """ + snapshot = {} + + if isinstance(self.storage, model.Storage): + snapshot["storage_name"] = self.storage.name + snapshot["storage_index"] = self.storage.index + snapshot["storage_location"] = str(self.storage.location) + return snapshot + + def restore(self, snapshot: dict) -> None: + """Used by the framework to deserialize the event from disk. + + Not meant to be called by charm code. + """ + storage_name = snapshot.get("storage_name") + storage_index = snapshot.get("storage_index") + storage_location = snapshot.get("storage_location") + + if storage_name and storage_index is not None: + storages = self.framework.model.storages[storage_name] + self.storage = next((s for s in storages if s.index == storage_index), None,) + if self.storage is None: + msg = 'failed loading storage (name={!r}, index={!r}) from snapshot' \ + .format(storage_name, storage_index) + raise RuntimeError(msg) + self.storage.location = storage_location + + +class StorageAttachedEvent(StorageEvent): + """Event triggered when new storage becomes available. + + This event is triggered when new storage is available for the + charm to use. + + Callback methods bound to this event allow the charm to run code + when storage has been added. Such methods will be run before the + :class:`InstallEvent` fires, so that the installation routine may + use the storage. The name prefix of this hook will depend on the + storage key defined in the ``metadata.yaml`` file. + """ + + +class StorageDetachingEvent(StorageEvent): + """Event triggered prior to removal of storage. + + This event is triggered when storage a charm has been using is + going away. + + Callback methods bound to this event allow the charm to run code + before storage is removed. Such methods will be run before storage + is detached, and always before the :class:`StopEvent` fires, thereby + allowing the charm to gracefully release resources before they are + removed and before the unit terminates. The name prefix of the + hook will depend on the storage key defined in the ``metadata.yaml`` + file. + """ + + +class WorkloadEvent(HookEvent): + """Base class representing workload-related events. + + Workload events are generated for all containers that the charm + expects in metadata. Workload containers currently only trigger + a PebbleReadyEvent. + + Attributes: + workload: The :class:`~ops.model.Container` involved in this event. + Workload currently only can be a Container but in future may + be other types that represent the specific workload type e.g. + a Machine. + """ + + def __init__(self, handle, workload): + super().__init__(handle) + + self.workload = workload + + def snapshot(self) -> dict: + """Used by the framework to serialize the event to disk. + + Not meant to be called by charm code. + """ + snapshot = {} + if isinstance(self.workload, model.Container): + snapshot['container_name'] = self.workload.name + return snapshot + + def restore(self, snapshot: dict) -> None: + """Used by the framework to deserialize the event from disk. + + Not meant to be called by charm code. + """ + container_name = snapshot.get('container_name') + if container_name: + self.workload = self.framework.model.unit.get_container(container_name) + else: + self.workload = None + + +class PebbleReadyEvent(WorkloadEvent): + """Event triggered when pebble is ready for a workload. + + This event is triggered when the Pebble process for a workload/container + starts up, allowing the charm to configure how services should be launched. + + Callback methods bound to this event allow the charm to run code after + a workload has started its Pebble instance and is ready to receive instructions + regarding what services should be started. The name prefix of the hook + will depend on the container key defined in the ``metadata.yaml`` file. + """ + + +class CharmEvents(ObjectEvents): + """Events generated by Juju pertaining to application lifecycle. + + This class is used to create an event descriptor (``self.on``) attribute for + a charm class that inherits from :class:`CharmBase`. The event descriptor + may be used to set up event handlers for corresponding events. + + By default the following events will be provided through + :class:`CharmBase`:: + + self.on.install + self.on.start + self.on.remove + self.on.update_status + self.on.config_changed + self.on.upgrade_charm + self.on.pre_series_upgrade + self.on.post_series_upgrade + self.on.leader_elected + self.on.collect_metrics + + + In addition to these, depending on the charm's metadata (``metadata.yaml``), + named relation and storage events may also be defined. These named events + are created by :class:`CharmBase` using charm metadata. The named events may be + accessed as ``self.on[].`` + """ + + install = EventSource(InstallEvent) + start = EventSource(StartEvent) + stop = EventSource(StopEvent) + remove = EventSource(RemoveEvent) + update_status = EventSource(UpdateStatusEvent) + config_changed = EventSource(ConfigChangedEvent) + upgrade_charm = EventSource(UpgradeCharmEvent) + pre_series_upgrade = EventSource(PreSeriesUpgradeEvent) + post_series_upgrade = EventSource(PostSeriesUpgradeEvent) + leader_elected = EventSource(LeaderElectedEvent) + leader_settings_changed = EventSource(LeaderSettingsChangedEvent) + collect_metrics = EventSource(CollectMetricsEvent) + + +class CharmBase(Object): + """Base class that represents the charm overall. + + :class:`CharmBase` is used to create a charm. This is done by inheriting + from :class:`CharmBase` and customising the sub class as required. So to + create your own charm, say ``MyCharm``, define a charm class and set up the + required event handlers (“hooks”) in its constructor:: + + import logging + + from ops.charm import CharmBase + from ops.main import main + + logger = logging.getLogger(__name__) + + def MyCharm(CharmBase): + def __init__(self, *args): + logger.debug('Initializing Charm') + + super().__init__(*args) + + self.framework.observe(self.on.config_changed, self._on_config_changed) + self.framework.observe(self.on.stop, self._on_stop) + # ... + + if __name__ == "__main__": + main(MyCharm) + + As shown in the example above, a charm class is instantiated by + :func:`~ops.main.main` rather than charm authors directly instantiating a + charm. + + Args: + framework: The framework responsible for managing the Model and events for this + charm. + key: Ignored; will remove after deprecation period of the signature change. + + """ + + # note that without the #: below, sphinx will copy the whole of CharmEvents + # docstring inline which is less than ideal. + #: Used to set up event handlers; see :class:`CharmEvents`. + on = CharmEvents() + + def __init__(self, framework: Framework, key: typing.Optional = None): + super().__init__(framework, None) + + for relation_name in self.framework.meta.relations: + relation_name = relation_name.replace('-', '_') + self.on.define_event(relation_name + '_relation_created', RelationCreatedEvent) + self.on.define_event(relation_name + '_relation_joined', RelationJoinedEvent) + self.on.define_event(relation_name + '_relation_changed', RelationChangedEvent) + self.on.define_event(relation_name + '_relation_departed', RelationDepartedEvent) + self.on.define_event(relation_name + '_relation_broken', RelationBrokenEvent) + + for storage_name in self.framework.meta.storages: + storage_name = storage_name.replace('-', '_') + self.on.define_event(storage_name + '_storage_attached', StorageAttachedEvent) + self.on.define_event(storage_name + '_storage_detaching', StorageDetachingEvent) + + for action_name in self.framework.meta.actions: + action_name = action_name.replace('-', '_') + self.on.define_event(action_name + '_action', ActionEvent) + + for container_name in self.framework.meta.containers: + container_name = container_name.replace('-', '_') + self.on.define_event(container_name + '_pebble_ready', PebbleReadyEvent) + + @property + def app(self) -> model.Application: + """Application that this unit is part of.""" + return self.framework.model.app + + @property + def unit(self) -> model.Unit: + """Unit that this execution is responsible for.""" + return self.framework.model.unit + + @property + def meta(self) -> 'CharmMeta': + """Metadata of this charm.""" + return self.framework.meta + + @property + def charm_dir(self) -> pathlib.Path: + """Root directory of the charm as it is running.""" + return self.framework.charm_dir + + @property + def config(self) -> model.ConfigData: + """A mapping containing the charm's config and current values.""" + return self.model.config + + +class CharmMeta: + """Object containing the metadata for the charm. + + This is read from ``metadata.yaml`` and/or ``actions.yaml``. Generally + charms will define this information, rather than reading it at runtime. This + class is mostly for the framework to understand what the charm has defined. + + The :attr:`maintainers`, :attr:`tags`, :attr:`terms`, :attr:`series`, and + :attr:`extra_bindings` attributes are all lists of strings. The :attr:`containers`, + :attr:`requires`, :attr:`provides`, :attr:`peers`, :attr:`relations`, + :attr:`storages`, :attr:`resources`, and :attr:`payloads` attributes are all + mappings of names to instances of the respective :class:`RelationMeta`, + :class:`StorageMeta`, :class:`ResourceMeta`, or :class:`PayloadMeta`. + + The :attr:`relations` attribute is a convenience accessor which includes all + of the ``requires``, ``provides``, and ``peers`` :class:`RelationMeta` + items. If needed, the role of the relation definition can be obtained from + its :attr:`role ` attribute. + + Attributes: + name: The name of this charm + summary: Short description of what this charm does + description: Long description for this charm + maintainers: A list of strings of the email addresses of the maintainers + of this charm. + tags: Charm store tag metadata for categories associated with this charm. + terms: Charm store terms that should be agreed to before this charm can + be deployed. (Used for things like licensing issues.) + series: The list of supported OS series that this charm can support. + The first entry in the list is the default series that will be + used by deploy if no other series is requested by the user. + subordinate: True/False whether this charm is intended to be used as a + subordinate charm. + min_juju_version: If supplied, indicates this charm needs features that + are not available in older versions of Juju. + containers: A dict of {name: :class:`ContainerMeta` } for each of the 'containers' + declared by this charm in the `matadata.yaml` file. + requires: A dict of {name: :class:`RelationMeta` } for each 'requires' relation. + provides: A dict of {name: :class:`RelationMeta` } for each 'provides' relation. + peers: A dict of {name: :class:`RelationMeta` } for each 'peer' relation. + relations: A dict containing all :class:`RelationMeta` attributes (merged from other + sections) + storages: A dict of {name: :class:`StorageMeta`} for each defined storage. + resources: A dict of {name: :class:`ResourceMeta`} for each defined resource. + payloads: A dict of {name: :class:`PayloadMeta`} for each defined payload. + extra_bindings: A dict of additional named bindings that a charm can use + for network configuration. + actions: A dict of {name: :class:`ActionMeta`} for actions that the charm has defined. + Args: + raw: a mapping containing the contents of metadata.yaml + actions_raw: a mapping containing the contents of actions.yaml + + """ + + def __init__(self, raw: dict = None, actions_raw: dict = None): + raw = raw or {} + actions_raw = actions_raw or {} + + self.name = raw.get('name', '') + self.summary = raw.get('summary', '') + self.description = raw.get('description', '') + self.maintainers = [] + if 'maintainer' in raw: + self.maintainers.append(raw['maintainer']) + if 'maintainers' in raw: + self.maintainers.extend(raw['maintainers']) + self.tags = raw.get('tags', []) + self.terms = raw.get('terms', []) + self.series = raw.get('series', []) + self.subordinate = raw.get('subordinate', False) + self.min_juju_version = raw.get('min-juju-version') + self.requires = {name: RelationMeta(RelationRole.requires, name, rel) + for name, rel in raw.get('requires', {}).items()} + self.provides = {name: RelationMeta(RelationRole.provides, name, rel) + for name, rel in raw.get('provides', {}).items()} + self.peers = {name: RelationMeta(RelationRole.peer, name, rel) + for name, rel in raw.get('peers', {}).items()} + self.relations = {} + self.relations.update(self.requires) + self.relations.update(self.provides) + self.relations.update(self.peers) + self.storages = {name: StorageMeta(name, storage) + for name, storage in raw.get('storage', {}).items()} + self.resources = {name: ResourceMeta(name, res) + for name, res in raw.get('resources', {}).items()} + self.payloads = {name: PayloadMeta(name, payload) + for name, payload in raw.get('payloads', {}).items()} + self.extra_bindings = raw.get('extra-bindings', {}) + self.actions = {name: ActionMeta(name, action) for name, action in actions_raw.items()} + # This is taken from Charm Metadata v2, but only the "containers" and + # "containers.name" fields that we need right now for Pebble. See: + # https://discourse.charmhub.io/t/charm-metadata-v2/3674 + self.containers = {name: ContainerMeta(name, container) + for name, container in raw.get('containers', {}).items()} + + @classmethod + def from_yaml( + cls, metadata: typing.Union[str, typing.TextIO], + actions: typing.Optional[typing.Union[str, typing.TextIO]] = None): + """Instantiate a CharmMeta from a YAML description of metadata.yaml. + + Args: + metadata: A YAML description of charm metadata (name, relations, etc.) + This can be a simple string, or a file-like object. (passed to `yaml.safe_load`). + actions: YAML description of Actions for this charm (eg actions.yaml) + """ + meta = yaml.safe_load(metadata) + raw_actions = {} + if actions is not None: + raw_actions = yaml.safe_load(actions) + if raw_actions is None: + raw_actions = {} + return cls(meta, raw_actions) + + +class RelationRole(enum.Enum): + """An annotation for a charm's role in a relation. + + For each relation a charm's role may be + + - A Peer + - A service consumer in the relation ('requires') + - A service provider in the relation ('provides') + """ + peer = 'peer' + requires = 'requires' + provides = 'provides' + + def is_peer(self) -> bool: + """Return whether the current role is peer. + + A convenience to avoid having to import charm. + """ + return self is RelationRole.peer + + +class RelationMeta: + """Object containing metadata about a relation definition. + + Should not be constructed directly by charm code. Is gotten from one of + :attr:`CharmMeta.peers`, :attr:`CharmMeta.requires`, :attr:`CharmMeta.provides`, + or :attr:`CharmMeta.relations`. + + Attributes: + role: This is :class:`RelationRole`; one of peer/requires/provides + relation_name: Name of this relation from metadata.yaml + interface_name: Optional definition of the interface protocol. + limit: Optional definition of maximum number of connections to this relation endpoint. + scope: "global" (default) or "container" scope based on how the relation should be used. + """ + + VALID_SCOPES = ['global', 'container'] + + def __init__(self, role: RelationRole, relation_name: str, raw: dict): + if not isinstance(role, RelationRole): + raise TypeError("role should be a Role, not {!r}".format(role)) + self._default_scope = self.VALID_SCOPES[0] + self.role = role + self.relation_name = relation_name + self.interface_name = raw['interface'] + + self.limit = raw.get('limit') + if self.limit and not isinstance(self.limit, int): + raise TypeError("limit should be an int, not {}".format(type(self.limit))) + + self.scope = raw.get('scope') or self._default_scope + if self.scope not in self.VALID_SCOPES: + raise TypeError("scope should be one of {}; not '{}'".format( + ', '.join("'{}'".format(s) for s in self.VALID_SCOPES), self.scope)) + + +class StorageMeta: + """Object containing metadata about a storage definition. + + Attributes: + storage_name: Name of storage + type: Storage type + description: A text description of the storage + read_only: Whether or not the storage is read only + minimum_size: Minimum size of storage + location: Mount point of storage + multiple_range: Range of numeric qualifiers when multiple storage units are used + """ + + def __init__(self, name, raw): + self.storage_name = name + self.type = raw['type'] + self.description = raw.get('description', '') + self.shared = raw.get('shared', False) + self.read_only = raw.get('read-only', False) + self.minimum_size = raw.get('minimum-size') + self.location = raw.get('location') + self.multiple_range = None + if 'multiple' in raw: + range = raw['multiple']['range'] + if '-' not in range: + self.multiple_range = (int(range), int(range)) + else: + range = range.split('-') + self.multiple_range = (int(range[0]), int(range[1]) if range[1] else None) + + +class ResourceMeta: + """Object containing metadata about a resource definition. + + Attributes: + resource_name: Name of resource + filename: Name of file + description: A text description of resource + """ + + def __init__(self, name, raw): + self.resource_name = name + self.type = raw['type'] + self.filename = raw.get('filename', None) + self.description = raw.get('description', '') + + +class PayloadMeta: + """Object containing metadata about a payload definition. + + Attributes: + payload_name: Name of payload + type: Payload type + """ + + def __init__(self, name, raw): + self.payload_name = name + self.type = raw['type'] + + +class ActionMeta: + """Object containing metadata about an action's definition.""" + + def __init__(self, name, raw=None): + raw = raw or {} + self.name = name + self.title = raw.get('title', '') + self.description = raw.get('description', '') + self.parameters = raw.get('params', {}) # {: } + self.required = raw.get('required', []) # [, ...] + + +class ContainerMeta: + """Metadata about an individual container. + + NOTE: this is extremely lightweight right now, and just includes the fields we need for + Pebble interaction. + + Attributes: + name: Name of container (key in the YAML) + mounts: :class:`ContainerStorageMeta` mounts available to the container + """ + + def __init__(self, name, raw): + self.name = name + self._mounts = {} + + # This is not guaranteed to be populated/is not enforced yet + if raw: + self._populate_mounts(raw.get('mounts', [])) + + @property + def mounts(self) -> typing.Dict: + """An accessor for the mounts in a container. + + Dict keys match key name in :class:`StorageMeta` + + Example:: + + storage: + foo: + type: filesystem + location: /test + containers: + bar: + mounts: + - storage: foo + - location: /test/mount + """ + return self._mounts + + def _populate_mounts(self, mounts: typing.List): + """Populate a list of container mountpoints. + + Since Charm Metadata v2 specifies the mounts as a List, do a little data manipulation + to convert the values to "friendly" names which contain a list of mountpoints + under each key. + """ + for mount in mounts: + storage = mount.get("storage", "") + mount = mount.get("location", "") + + if not mount: + continue + + if storage in self._mounts: + self._mounts[storage].add_location(mount) + else: + self._mounts[storage] = ContainerStorageMeta(storage, mount) + + +class ContainerStorageMeta: + """Metadata about storage for an individual container. + + Attributes: + storage: a name for the mountpoint, which should exist the keys for :class:`StorageMeta` + for the charm + location: the location `storage` is mounted at + locations: a list of mountpoints for the key + + If multiple locations are specified for the same storage, such as Kubernetes subPath mounts, + `location` will not be an accessible attribute, as it would not be possible to determine + which mount point was desired, and `locations` should be iterated over. + """ + + def __init__(self, storage, location): + self.storage = storage + self._locations = [location] + + def add_location(self, location): + """Add an additional mountpoint to a known storage.""" + self._locations.append(location) + + @property + def locations(self) -> typing.List: + """An accessor for the list of locations for a mount.""" + return self._locations + + def __getattr__(self, name): + if name == "location": + if len(self._locations) == 1: + return self._locations[0] + else: + raise RuntimeError( + "container has more than one mountpoint with the same backing storage. " + "Request .locations to see a list" + ) + else: + raise AttributeError( + "{.__class__.__name__} has no such attribute: {}!".format(self, name) + ) diff --git a/ubuntu/venv/ops/framework.py b/ubuntu/venv/ops/framework.py new file mode 100644 index 0000000..5cb9ad0 --- /dev/null +++ b/ubuntu/venv/ops/framework.py @@ -0,0 +1,1289 @@ +# Copyright 2020-2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Operator Framework infrastructure.""" + +import collections +import collections.abc +import inspect +import keyword +import logging +import marshal +import os +import pathlib +import pdb +import re +import sys +import types +import typing +import weakref + +from ops import charm +from ops.storage import NoSnapshotError, SQLiteStorage + +if typing.TYPE_CHECKING: + from pathlib import Path + + from typing_extensions import Protocol, Type + + from ops.charm import CharmMeta + from ops.model import Model + + class _Serializable(Protocol): + handle = None # type: Handle + def snapshot(self) -> dict: ... # noqa: E704 + def restore(self, snapshot: dict) -> "Object": ... # noqa: E704 + + _ObjectType = typing.TypeVar("_ObjectType", bound="Object") + _EventType = typing.TypeVar("_EventType", bound=Type["EventBase"]) + _ObserverCallback = typing.Callable[[typing.Any], None] + _Path = _Kind = str + + # This type is used to denote either a Handle instance or an instance of + # an Object (or subclass). This is used by methods and classes which can be + # called with either of those (they need a Handle, but will accept an Object + # from which they will then extract the Handle). + _ParentHandle = typing.Union["Handle", _ObjectType] + + # used to type Framework Attributes + _ObserverPath = typing.List[typing.Tuple['_Path', str, '_Path', str]] + _ObjectPath = typing.Tuple[typing.Optional['_Path'], '_Kind'] + _PathToObserverMapping = typing.Dict[str, '_ObserverCallback'] + _PathToObjectMapping = typing.Dict[str, 'Object'] + + +logger = logging.getLogger(__name__) + + +class Handle: + """Handle defines a name for an object in the form of a hierarchical path. + + The provided parent is the object (or that object's handle) that this handle + sits under, or None if the object identified by this handle stands by itself + as the root of its own hierarchy. + + The handle kind is a string that defines a namespace so objects with the + same parent and kind will have unique keys. + + The handle key is a string uniquely identifying the object. No other objects + under the same parent and kind may have the same key. + """ + + def __init__(self, parent: typing.Optional["_ParentHandle"], kind: str, key: str): + if isinstance(parent, Object): + # if it's not an Object, it will be either a Handle (good) or None (no parent) + parent = parent.handle + self._parent = parent + self._kind = kind + self._key = key + if parent: + if key: + self._path = "{}/{}[{}]".format(parent, kind, key) + else: + self._path = "{}/{}".format(parent, kind) + else: + if key: + self._path = "{}[{}]".format(kind, key) + else: + self._path = "{}".format(kind) + + def nest(self, kind: str, key: str): + """Create a new handle as child of the current one.""" + return Handle(self, kind, key) + + def __hash__(self): + return hash((self.parent, self.kind, self.key)) + + def __eq__(self, other): + return (self.parent, self.kind, self.key) == (other.parent, other.kind, other.key) + + def __str__(self): + return self.path + + @property + def parent(self): + """Return own parent handle.""" + return self._parent + + @property + def kind(self): + """Return the handle's kind.""" + return self._kind + + @property + def key(self): + """Return the handle's key.""" + return self._key + + @property + def path(self): + """Return the handle's path.""" + return self._path + + @classmethod + def from_path(cls, path): + """Build a handle from the indicated path.""" + handle = None + for pair in path.split("/"): + pair = pair.split("[") + good = False + if len(pair) == 1: + kind, key = pair[0], None + good = True + elif len(pair) == 2: + kind, key = pair + if key and key[-1] == ']': + key = key[:-1] + good = True + if not good: + raise RuntimeError("attempted to restore invalid handle path {}".format(path)) + handle = Handle(handle, kind, key) # noqa + return handle + + +class EventBase: + """The base for all the different Events. + + Inherit this and override 'snapshot' and 'restore' methods to build a custom event. + """ + + # gets patched in by `Framework.restore()`, if this event is being re-emitted + # after being loaded from snapshot, or by `BoundEvent.emit()` if this + # event is being fired for the first time. + # TODO this is hard to debug, this should be refactored + framework = None # type: Framework + + def __init__(self, handle: Handle): + self.handle = handle + self.deferred = False # type: bool + + def __repr__(self): + return "<%s via %s>" % (self.__class__.__name__, self.handle) + + def defer(self): + """Defer the event to the future. + + Deferring an event from a handler puts that handler into a queue, to be + called again the next time the charm is invoked. This invocation may be + the result of an action, or any event other than metric events. The + queue of events will be dispatched before the new event is processed. + + From the above you may deduce, but it's important to point out: + + * ``defer()`` does not interrupt the execution of the current event + handler. In almost all cases, a call to ``defer()`` should be followed + by an explicit ``return`` from the handler; + + * the re-execution of the deferred event handler starts from the top of + the handler method (not where defer was called); + + * only the handlers that actually called ``defer()`` are called again + (that is: despite talking about “deferring an event” it is actually + the handler/event combination that is deferred); and + + * any deferred events get processed before the event (or action) that + caused the current invocation of the charm. + + The general desire to call ``defer()`` happens when some precondition + isn't yet met. However, care should be exercised as to whether it is + better to defer this event so that you see it again, or whether it is + better to just wait for the event that indicates the precondition has + been met. + + For example, if ``config-changed`` is fired, and you are waiting for + different config, there is no reason to defer the event because there + will be a *different* ``config-changed`` event when the config actually + changes, rather than checking to see if maybe config has changed prior + to every other event that occurs. + + Similarly, if you need 2 events to occur before you are ready to + proceed (say event A and B). When you see event A, you could chose to + ``defer()`` it because you haven't seen B yet. However, that leads to: + + 1. event A fires, calls defer() + + 2. event B fires, event A handler is called first, still hasn't seen B + happen, so is deferred again. Then B happens, which progresses since + it has seen A. + + 3. At some future time, event C happens, which also checks if A can + proceed. + + """ + logger.debug("Deferring %s.", self) + self.deferred = True + + def snapshot(self) -> dict: + """Return the snapshot data that should be persisted. + + Subclasses must override to save any custom state. + """ + return None + + def restore(self, snapshot): + """Restore the value state from the given snapshot. + + Subclasses must override to restore their custom state. + """ + self.deferred = False + + +class EventSource: + """EventSource wraps an event type with a descriptor to facilitate observing and emitting. + + It is generally used as: + + class SomethingHappened(EventBase): + pass + + class SomeObject(Object): + something_happened = EventSource(SomethingHappened) + + With that, instances of that type will offer the someobj.something_happened + attribute which is a BoundEvent and may be used to emit and observe the event. + """ + + def __init__(self, event_type): + if not isinstance(event_type, type) or not issubclass(event_type, EventBase): + raise RuntimeError( + 'Event requires a subclass of EventBase as an argument, got {}'.format(event_type)) + self.event_type = event_type + self.event_kind = None + self.emitter_type = None + + def _set_name(self, emitter_type, event_kind): + if self.event_kind is not None: + raise RuntimeError( + 'EventSource({}) reused as {}.{} and {}.{}'.format( + self.event_type.__name__, + self.emitter_type.__name__, + self.event_kind, + emitter_type.__name__, + event_kind, + )) + self.event_kind = event_kind + self.emitter_type = emitter_type + + def __get__(self, emitter, emitter_type=None): + if emitter is None: + return self + # Framework might not be available if accessed as CharmClass.on.event + # rather than charm_instance.on.event, but in that case it couldn't be + # emitted anyway, so there's no point to registering it. + framework = getattr(emitter, 'framework', None) + if framework is not None: + framework.register_type(self.event_type, emitter, self.event_kind) + return BoundEvent(emitter, self.event_type, self.event_kind) + + +class BoundEvent: + """Event bound to an Object.""" + + def __repr__(self): + return ''.format( + self.event_type.__name__, + type(self.emitter).__name__, + self.event_kind, + hex(id(self)), + ) + + def __init__(self, emitter: "_ObjectType", + event_type: "_EventType", event_kind: str): + self.emitter = emitter + self.event_type = event_type + self.event_kind = event_kind + + def emit(self, *args, **kwargs): + """Emit event to all registered observers. + + The current storage state is committed before and after each observer is notified. + """ + framework = self.emitter.framework + key = framework._next_event_key() + event = self.event_type(Handle(self.emitter, self.event_kind, key), *args, **kwargs) + event.framework = framework + framework._emit(event) + + +class HandleKind: + """Helper descriptor to define the Object.handle_kind field. + + The handle_kind for an object defaults to its type name, but it may + be explicitly overridden if desired. + """ + + def __get__(self, obj, obj_type): + kind = obj_type.__dict__.get("handle_kind") + if kind: + return kind + return obj_type.__name__ + + +class _Metaclass(type): + """Helper class to ensure proper instantiation of Object-derived classes. + + This class currently has a single purpose: events derived from EventSource + that are class attributes of Object-derived classes need to be told what + their name is in that class. For example, in + + class SomeObject(Object): + something_happened = EventSource(SomethingHappened) + + the instance of EventSource needs to know it's called 'something_happened'. + + Starting from python 3.6 we could use __set_name__ on EventSource for this, + but until then this (meta)class does the equivalent work. + + TODO: when we drop support for 3.5 drop this class, and rename _set_name in + EventSource to __set_name__; everything should continue to work. + + """ + + def __new__(cls, *a, **kw): + k = super().__new__(cls, *a, **kw) + # k is now the Object-derived class; loop over its class attributes + for n, v in vars(k).items(): + # we could do duck typing here if we want to support + # non-EventSource-derived shenanigans. We don't. + if isinstance(v, EventSource): + # this is what 3.6+ does automatically for us: + v._set_name(k, n) + return k + + +class Object(metaclass=_Metaclass): + """Initialize an Object as a new leaf in :class:`Framework`, identified by `key`. + + Args: + parent: parent node in the tree. + key: unique identifier for this object. + + Every object belongs to exactly one framework. + + Every object has a parent, which might be a framework. + + We track a "path to object," which is the path to the parent, plus the object's unique + identifier. Event handlers use this identity to track the destination of their events, and the + Framework uses this id to track persisted state between event executions. + + The Framework should raise an error if it ever detects that two objects with the same id have + been created. + + """ + framework = None # type: Framework + handle = None # type: Handle + handle_kind = HandleKind() # type: str + + def __init__(self, parent, key): + kind = self.handle_kind + if isinstance(parent, Framework): + self.framework = parent + # Avoid Framework instances having a circular reference to themselves. + if self.framework is self: + self.framework = weakref.proxy(self.framework) + self.handle = Handle(None, kind, key) + else: + self.framework = parent.framework + self.handle = Handle(parent, kind, key) + self.framework._track(self) + + # TODO Detect conflicting handles here. + + @property + def model(self) -> "Model": + """Shortcut for more simple access the model.""" + return self.framework.model + + +class ObjectEvents(Object): + """Convenience type to allow defining .on attributes at class level.""" + + handle_kind = "on" + + def __init__(self, parent=None, key=None): + if parent is not None: + super().__init__(parent, key) + else: + self._cache = weakref.WeakKeyDictionary() + + def __get__(self, emitter, emitter_type): + if emitter is None: + return self + instance = self._cache.get(emitter) + if instance is None: + # Same type, different instance, more data. Doing this unusual construct + # means people can subclass just this one class to have their own 'on'. + instance = self._cache[emitter] = type(self)(emitter) + return instance + + @classmethod + def define_event(cls, event_kind, event_type): + """Define an event on this type at runtime. + + cls: a type to define an event on. + + event_kind: an attribute name that will be used to access the + event. Must be a valid python identifier, not be a keyword + or an existing attribute. + + event_type: a type of the event to define. + + """ + prefix = 'unable to define an event with event_kind that ' + if not event_kind.isidentifier(): + raise RuntimeError(prefix + 'is not a valid python identifier: ' + event_kind) + elif keyword.iskeyword(event_kind): + raise RuntimeError(prefix + 'is a python keyword: ' + event_kind) + try: + getattr(cls, event_kind) + raise RuntimeError( + prefix + 'overlaps with an existing type {} attribute: {}'.format(cls, event_kind)) + except AttributeError: + pass + + event_descriptor = EventSource(event_type) + event_descriptor._set_name(cls, event_kind) + setattr(cls, event_kind, event_descriptor) + + def _event_kinds(self): + event_kinds = [] + # We have to iterate over the class rather than instance to allow for properties which + # might call this method (e.g., event views), leading to infinite recursion. + for attr_name, attr_value in inspect.getmembers(type(self)): + if isinstance(attr_value, EventSource): + # We actually care about the bound_event, however, since it + # provides the most info for users of this method. + event_kinds.append(attr_name) + return event_kinds + + def events(self): + """Return a mapping of event_kinds to bound_events for all available events.""" + return {event_kind: getattr(self, event_kind) for event_kind in self._event_kinds()} + + def __getitem__(self, key): + return PrefixedEvents(self, key) + + def __repr__(self): + k = type(self) + event_kinds = ', '.join(sorted(self._event_kinds())) + return '<{}.{}: {}>'.format(k.__module__, k.__qualname__, event_kinds) + + +class PrefixedEvents: + """Events to be found in all events using a specific prefix.""" + + def __init__(self, emitter, key): + self._emitter = emitter + self._prefix = key.replace("-", "_") + '_' + + def __getattr__(self, name): + return getattr(self._emitter, self._prefix + name) + + +class PreCommitEvent(EventBase): + """Events that will be emitted first on commit.""" + + +class CommitEvent(EventBase): + """Events that will be emitted second on commit.""" + + +class FrameworkEvents(ObjectEvents): + """Manager of all framework events.""" + pre_commit = EventSource(PreCommitEvent) + commit = EventSource(CommitEvent) + + +class NoTypeError(Exception): + """No class to hold it was found when restoring an event.""" + + def __init__(self, handle_path): + self.handle_path = handle_path + + def __str__(self): + return "cannot restore {} since no class was registered for it".format(self.handle_path) + + +# the message to show to the user when a pdb breakpoint goes active +_BREAKPOINT_WELCOME_MESSAGE = """ +Starting pdb to debug charm operator. +Run `h` for help, `c` to continue, or `exit`/CTRL-d to abort. +Future breakpoints may interrupt execution again. +More details at https://juju.is/docs/sdk/debugging + +""" + +_event_regex = r'^(|.*/)on/[a-zA-Z_]+\[\d+\]$' + + +class Framework(Object): + """Main interface to from the Charm to the Operator Framework internals.""" + + on = FrameworkEvents() + + # Override properties from Object so that we can set them in __init__. + model = None # type: 'Model' + meta = None # type: 'CharmMeta' + charm_dir = None # type: 'Path' + + if typing.TYPE_CHECKING: + # to help the type checker and IDEs: + _stored = None # type: 'StoredStateData' + + def __init__(self, storage: SQLiteStorage, charm_dir: "Path", + meta: "CharmMeta", model: "Model"): + super().__init__(self, None) + + self.charm_dir = charm_dir + self.meta = meta + self.model = model + # [(observer_path, method_name, parent_path, event_key)] + self._observers = [] # type: _ObserverPath + # {observer_path: observer} + self._observer = weakref.WeakValueDictionary() # type: _PathToObserverMapping + # {object_path: object} + self._objects = weakref.WeakValueDictionary() # type: _PathToObjectMapping + # {(parent_path, kind): cls} + # (parent_path, kind) is the address of _this_ object: the parent path + # plus a 'kind' string that is the name of this object. + self._type_registry = {} # type: typing.Dict[_ObjectPath, 'Type'] + self._type_known = set() # type: typing.Set['Type'] + + if isinstance(storage, (str, pathlib.Path)): + logger.warning( + "deprecated: Framework now takes a Storage not a path") + storage = SQLiteStorage(storage) + self._storage = storage # type: 'SQLiteStorage' + + # We can't use the higher-level StoredState because it relies on events. + self.register_type(StoredStateData, None, StoredStateData.handle_kind) + stored_handle = Handle(None, StoredStateData.handle_kind, '_stored') + try: + self._stored = self.load_snapshot(stored_handle) + except NoSnapshotError: + self._stored = StoredStateData(self, '_stored') + self._stored['event_count'] = 0 + + # Flag to indicate that we already presented the welcome message in a debugger breakpoint + self._breakpoint_welcomed = False # type: bool + + # Parse the env var once, which may be used multiple times later + debug_at = os.environ.get('JUJU_DEBUG_AT') + self._juju_debug_at = (set(x.strip() for x in debug_at.split(',')) + if debug_at else set()) # type: typing.Set[str] + + def set_breakpointhook(self): + """Hook into sys.breakpointhook so the builtin breakpoint() works as expected. + + This method is called by ``main``, and is not intended to be + called by users of the framework itself outside of perhaps + some testing scenarios. + + It returns the old value of sys.excepthook. + + The breakpoint function is a Python >= 3.7 feature. + + This method was added in ops 1.0; before that, it was done as + part of the Framework's __init__. + """ + old_breakpointhook = getattr(sys, 'breakpointhook', None) + if old_breakpointhook is not None: + # Hook into builtin breakpoint, so if Python >= 3.7, devs will be able to just do + # breakpoint() + sys.breakpointhook = self.breakpoint + return old_breakpointhook + + def close(self): + """Close the underlying backends.""" + self._storage.close() + + def _track(self, obj): + """Track object and ensure it is the only object created using its handle path.""" + if obj is self: + # Framework objects don't track themselves + return + if obj.handle.path in self.framework._objects: + raise RuntimeError( + 'two objects claiming to be {} have been created'.format(obj.handle.path)) + self._objects[obj.handle.path] = obj + + def _forget(self, obj): + """Stop tracking the given object. See also _track.""" + self._objects.pop(obj.handle.path, None) + + def commit(self): + """Save changes to the underlying backends.""" + # Give a chance for objects to persist data they want to before a commit is made. + self.on.pre_commit.emit() + # Make sure snapshots are saved by instances of StoredStateData. Any possible state + # modifications in on_commit handlers of instances of other classes will not be persisted. + self.on.commit.emit() + # Save our event count after all events have been emitted. + self.save_snapshot(self._stored) + self._storage.commit() + + def register_type(self, cls, parent: typing.Optional["_ParentHandle"], kind=None): + """Register a type to a handle.""" + if parent is not None and not isinstance(parent, Handle): + parent = parent.handle + if parent: + parent_path = parent.path + else: + parent_path = None + if not kind: + kind = cls.handle_kind + self._type_registry[(parent_path, kind)] = cls + self._type_known.add(cls) + + def save_snapshot(self, value: "_Serializable"): + """Save a persistent snapshot of the provided value.""" + if type(value) not in self._type_known: + raise RuntimeError( + 'cannot save {} values before registering that type'.format(type(value).__name__)) + data = value.snapshot() + + # Use marshal as a validator, enforcing the use of simple types, as we later the + # information is really pickled, which is too error-prone for future evolution of the + # stored data (e.g. if the developer stores a custom object and later changes its + # class name; when unpickling the original class will not be there and event + # data loading will fail). + try: + marshal.dumps(data) + except ValueError: + msg = "unable to save the data for {}, it must contain only simple types: {!r}" + raise ValueError(msg.format(value.__class__.__name__, data)) + + self._storage.save_snapshot(value.handle.path, data) + + def load_snapshot(self, handle: Handle) -> '_ObjectType': + """Load a persistent snapshot.""" + parent_path = None + if handle.parent: + parent_path = handle.parent.path + cls = self._type_registry.get((parent_path, handle.kind)) + if not cls: + raise NoTypeError(handle.path) + data = self._storage.load_snapshot(handle.path) + obj = cls.__new__(cls) + obj.framework = self + obj.handle = handle + obj.restore(data) + self._track(obj) + return obj + + def drop_snapshot(self, handle: Handle): + """Discard a persistent snapshot.""" + self._storage.drop_snapshot(handle.path) + + def observe(self, bound_event: BoundEvent, + observer: "_ObserverCallback"): + """Register observer to be called when bound_event is emitted. + + The bound_event is generally provided as an attribute of the object that emits + the event, and is created in this style:: + + class SomeObject: + something_happened = Event(SomethingHappened) + + That event may be observed as:: + + framework.observe(someobj.something_happened, self._on_something_happened) + + Raises: + RuntimeError: if bound_event or observer are the wrong type. + """ + if not isinstance(bound_event, BoundEvent): + raise RuntimeError( + 'Framework.observe requires a BoundEvent as second parameter, got {}'.format( + bound_event)) + if not isinstance(observer, types.MethodType): + # help users of older versions of the framework + if isinstance(observer, charm.CharmBase): + raise TypeError( + 'observer methods must now be explicitly provided;' + ' please replace observe(self.on.{0}, self)' + ' with e.g. observe(self.on.{0}, self._on_{0})'.format( + bound_event.event_kind)) + raise RuntimeError( + 'Framework.observe requires a method as third parameter, got {}'.format( + observer)) + + event_type = bound_event.event_type + event_kind = bound_event.event_kind + emitter = bound_event.emitter + + self.register_type(event_type, emitter, event_kind) + + if hasattr(emitter, "handle"): + emitter_path = emitter.handle.path + else: + raise RuntimeError( + 'event emitter {} must have a "handle" attribute'.format( + type(emitter).__name__)) + + # Validate that the method has an acceptable call signature. + sig = inspect.signature(observer) + # Self isn't included in the params list, so the first arg will be the event. + extra_params = list(sig.parameters.values())[1:] + + method_name = observer.__name__ + observer = observer.__self__ + if not sig.parameters: + raise TypeError( + '{}.{} must accept event parameter'.format(type(observer).__name__, method_name)) + elif any(param.default is inspect.Parameter.empty for param in extra_params): + # Allow for additional optional params, since there's no reason to exclude them, but + # required params will break. + raise TypeError( + '{}.{} has extra required parameter'.format(type(observer).__name__, method_name)) + + # TODO Prevent the exact same parameters from being registered more than once. + + self._observer[observer.handle.path] = observer + self._observers.append((observer.handle.path, method_name, emitter_path, event_kind)) + + def _next_event_key(self): + """Return the next event key that should be used, incrementing the internal counter.""" + # Increment the count first; this means the keys will start at 1, and 0 + # means no events have been emitted. + self._stored['event_count'] += 1 + return str(self._stored['event_count']) + + def _emit(self, event: EventBase): + """See BoundEvent.emit for the public way to call this.""" + saved = False + event_path = event.handle.path + event_kind = event.handle.kind + parent_path = event.handle.parent.path + # TODO Track observers by (parent_path, event_kind) rather than as a list of + # all observers. Avoiding linear search through all observers for every event + for observer_path, method_name, _parent_path, _event_kind in self._observers: + if _parent_path != parent_path: + continue + if _event_kind and _event_kind != event_kind: + continue + if not saved: + # Save the event for all known observers before the first notification + # takes place, so that either everyone interested sees it, or nobody does. + self.save_snapshot(event) + saved = True + # Again, only commit this after all notices are saved. + self._storage.save_notice(event_path, observer_path, method_name) + if saved: + self._reemit(event_path) + + def reemit(self): + """Reemit previously deferred events to the observers that deferred them. + + Only the specific observers that have previously deferred the event will be + notified again. Observers that asked to be notified about events after it's + been first emitted won't be notified, as that would mean potentially observing + events out of order. + """ + self._reemit() + + def _reemit(self, single_event_path=None): + + class EventContext: + """Handles toggling the hook-is-running state in backends. + + This allows e.g. harness logic to know if it is executing within a running hook context + or not. It sets backend._hook_is_running equal to the name of the currently running + hook (e.g. "set-leader") and reverts back to the empty string when the hook execution + is completed. + """ + + def __init__(self, framework, event_name): + self._event = event_name + self._backend = None + if framework.model is not None: + self._backend = framework.model._backend + + def __enter__(self): + if self._backend: + self._backend._hook_is_running = self._event + return self + + def __exit__(self, exception_type, exception, traceback): + if self._backend: + self._backend._hook_is_running = '' + + last_event_path = None + deferred = True + for event_path, observer_path, method_name in self._storage.notices(single_event_path): + event_handle = Handle.from_path(event_path) + + if last_event_path != event_path: + if not deferred and last_event_path is not None: + self._storage.drop_snapshot(last_event_path) + last_event_path = event_path + deferred = False + + try: + event = self.load_snapshot(event_handle) + except NoTypeError: + self._storage.drop_notice(event_path, observer_path, method_name) + continue + + event.deferred = False + observer = self._observer.get(observer_path) + if observer: + if single_event_path is None: + logger.debug("Re-emitting %s.", event) + custom_handler = getattr(observer, method_name, None) + if custom_handler: + event_is_from_juju = isinstance(event, charm.HookEvent) + event_is_action = isinstance(event, charm.ActionEvent) + with EventContext(self, event_handle.kind): + if ( + event_is_from_juju or event_is_action + ) and self._juju_debug_at.intersection({'all', 'hook'}): + # Present the welcome message and run under PDB. + self._show_debug_code_message() + pdb.runcall(custom_handler, event) + else: + # Regular call to the registered method. + custom_handler(event) + + if event.deferred: + deferred = True + else: + self._storage.drop_notice(event_path, observer_path, method_name) + # We intentionally consider this event to be dead and reload it from + # scratch in the next path. + self.framework._forget(event) + + if not deferred and last_event_path is not None: + self._storage.drop_snapshot(last_event_path) + + def _show_debug_code_message(self): + """Present the welcome message (only once!) when using debugger functionality.""" + if not self._breakpoint_welcomed: + self._breakpoint_welcomed = True + print(_BREAKPOINT_WELCOME_MESSAGE, file=sys.stderr, end='') + + def breakpoint(self, name=None): + """Add breakpoint, optionally named, at the place where this method is called. + + For the breakpoint to be activated the JUJU_DEBUG_AT environment variable + must be set to "all" or to the specific name parameter provided, if any. In every + other situation calling this method does nothing. + + The framework also provides a standard breakpoint named "hook", that will + stop execution when a hook event is about to be handled. + + For those reasons, the "all" and "hook" breakpoint names are reserved. + """ + # If given, validate the name comply with all the rules + if name is not None: + if not isinstance(name, str): + raise TypeError('breakpoint names must be strings') + if name in ('hook', 'all'): + raise ValueError('breakpoint names "all" and "hook" are reserved') + if not re.match(r'^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$', name): + raise ValueError('breakpoint names must look like "foo" or "foo-bar"') + + indicated_breakpoints = self._juju_debug_at + if not indicated_breakpoints: + return + + if 'all' in indicated_breakpoints or name in indicated_breakpoints: + self._show_debug_code_message() + + # If we call set_trace() directly it will open the debugger *here*, so indicating + # it to use our caller's frame + code_frame = inspect.currentframe().f_back + pdb.Pdb().set_trace(code_frame) + else: + logger.warning( + "Breakpoint %r skipped (not found in the requested breakpoints: %s)", + name, indicated_breakpoints) + + def remove_unreferenced_events(self): + """Remove events from storage that are not referenced. + + In older versions of the framework, events that had no observers would get recorded but + never deleted. This makes a best effort to find these events and remove them from the + database. + """ + event_regex = re.compile(_event_regex) + to_remove = [] + for handle_path in self._storage.list_snapshots(): + if event_regex.match(handle_path): + notices = self._storage.notices(handle_path) + if next(notices, None) is None: + # There are no notices for this handle_path, it is valid to remove it + to_remove.append(handle_path) + for handle_path in to_remove: + self._storage.drop_snapshot(handle_path) + + +class StoredStateData(Object): + """Manager of the stored data.""" + + def __init__(self, parent, attr_name): + super().__init__(parent, attr_name) + self._cache = {} + self.dirty = False + + def __getitem__(self, key): + return self._cache.get(key) + + def __setitem__(self, key, value): + self._cache[key] = value + self.dirty = True + + def __contains__(self, key): + return key in self._cache + + def snapshot(self): + """Return the current state.""" + return self._cache + + def restore(self, snapshot): + """Restore current state to the given snapshot.""" + self._cache = snapshot + self.dirty = False + + def on_commit(self, event): + """Save changes to the storage backend.""" + if self.dirty: + self.framework.save_snapshot(self) + self.dirty = False + + +class BoundStoredState: + """Stored state data bound to a specific Object.""" + + def __init__(self, parent, attr_name): + parent.framework.register_type(StoredStateData, parent) + + handle = Handle(parent, StoredStateData.handle_kind, attr_name) + try: + data = parent.framework.load_snapshot(handle) + except NoSnapshotError: + data = StoredStateData(parent, attr_name) + + # __dict__ is used to avoid infinite recursion. + self.__dict__["_data"] = data + self.__dict__["_attr_name"] = attr_name + + parent.framework.observe(parent.framework.on.commit, self._data.on_commit) + + def __getattr__(self, key): + # "on" is the only reserved key that can't be used in the data map. + if key == "on": + return self._data.on + if key not in self._data: + raise AttributeError("attribute '{}' is not stored".format(key)) + return _wrap_stored(self._data, self._data[key]) + + def __setattr__(self, key, value): + if key == "on": + raise AttributeError("attribute 'on' is reserved and cannot be set") + + value = _unwrap_stored(self._data, value) + + if not isinstance(value, (type(None), int, float, str, bytes, list, dict, set)): + raise AttributeError( + 'attribute {!r} cannot be a {}: must be int/float/dict/list/etc'.format( + key, type(value).__name__)) + + self._data[key] = _unwrap_stored(self._data, value) + + def set_default(self, **kwargs): + """Set the value of any given key if it has not already been set.""" + for k, v in kwargs.items(): + if k not in self._data: + self._data[k] = v + + +class StoredState: + """A class used to store data the charm needs persisted across invocations. + + Example:: + + class MyClass(Object): + _stored = StoredState() + + Instances of `MyClass` can transparently save state between invocations by + setting attributes on `_stored`. Initial state should be set with + `set_default` on the bound object, that is:: + + class MyClass(Object): + _stored = StoredState() + + def __init__(self, parent, key): + super().__init__(parent, key) + self._stored.set_default(seen=set()) + self.framework.observe(self.on.seen, self._on_seen) + + def _on_seen(self, event): + self._stored.seen.add(event.uuid) + + """ + + def __init__(self): + self.parent_type = None + self.attr_name = None + + def __get__(self, parent, parent_type=None): + if self.parent_type is not None and self.parent_type not in parent_type.mro(): + # the StoredState instance is being shared between two unrelated classes + # -> unclear what is exepcted of us -> bail out + raise RuntimeError( + 'StoredState shared by {} and {}'.format( + self.parent_type.__name__, parent_type.__name__)) + + if parent is None: + # accessing via the class directly (e.g. MyClass.stored) + return self + + bound = None + if self.attr_name is not None: + bound = parent.__dict__.get(self.attr_name) + if bound is not None: + # we already have the thing from a previous pass, huzzah + return bound + + # need to find ourselves amongst the parent's bases + for cls in parent_type.mro(): + for attr_name, attr_value in cls.__dict__.items(): + if attr_value is not self: + continue + # we've found ourselves! is it the first time? + if bound is not None: + # the StoredState instance is being stored in two different + # attributes -> unclear what is expected of us -> bail out + raise RuntimeError("StoredState shared by {0}.{1} and {0}.{2}".format( + cls.__name__, self.attr_name, attr_name)) + # we've found ourselves for the first time; save where, and bind the object + self.attr_name = attr_name + self.parent_type = cls + bound = BoundStoredState(parent, attr_name) + + if bound is not None: + # cache the bound object to avoid the expensive lookup the next time + # (don't use setattr, to keep things symmetric with the fast-path lookup above) + parent.__dict__[self.attr_name] = bound + return bound + + raise AttributeError( + 'cannot find {} attribute in type {}'.format( + self.__class__.__name__, parent_type.__name__)) + + +def _wrap_stored(parent_data, value): + t = type(value) + if t is dict: + return StoredDict(parent_data, value) + if t is list: + return StoredList(parent_data, value) + if t is set: + return StoredSet(parent_data, value) + return value + + +def _unwrap_stored(parent_data, value): + t = type(value) + if t is StoredDict or t is StoredList or t is StoredSet: + return value._under + return value + + +def _wrapped_repr(obj): + t = type(obj) + if obj._under: + return "{}.{}({!r})".format(t.__module__, t.__name__, obj._under) + else: + return "{}.{}()".format(t.__module__, t.__name__) + + +class StoredDict(collections.abc.MutableMapping): + """A dict-like object that uses the StoredState as backend.""" + + def __init__(self, stored_data, under): + self._stored_data = stored_data + self._under = under + + def __getitem__(self, key): + return _wrap_stored(self._stored_data, self._under[key]) + + def __setitem__(self, key, value): + self._under[key] = _unwrap_stored(self._stored_data, value) + self._stored_data.dirty = True + + def __delitem__(self, key): + del self._under[key] + self._stored_data.dirty = True + + def __iter__(self): + return self._under.__iter__() + + def __len__(self): + return len(self._under) + + def __eq__(self, other): + if isinstance(other, StoredDict): + return self._under == other._under + elif isinstance(other, collections.abc.Mapping): + return self._under == other + else: + return NotImplemented + + __repr__ = _wrapped_repr + + +class StoredList(collections.abc.MutableSequence): + """A list-like object that uses the StoredState as backend.""" + + def __init__(self, stored_data, under): + self._stored_data = stored_data + self._under = under + + def __getitem__(self, index): + return _wrap_stored(self._stored_data, self._under[index]) + + def __setitem__(self, index, value): + self._under[index] = _unwrap_stored(self._stored_data, value) + self._stored_data.dirty = True + + def __delitem__(self, index): + del self._under[index] + self._stored_data.dirty = True + + def __len__(self): + return len(self._under) + + def insert(self, index, value): + """Insert value before index.""" + self._under.insert(index, value) + self._stored_data.dirty = True + + def append(self, value): + """Append value to the end of the list.""" + self._under.append(value) + self._stored_data.dirty = True + + def __eq__(self, other): + if isinstance(other, StoredList): + return self._under == other._under + elif isinstance(other, collections.abc.Sequence): + return self._under == other + else: + return NotImplemented + + def __lt__(self, other): + if isinstance(other, StoredList): + return self._under < other._under + elif isinstance(other, collections.abc.Sequence): + return self._under < other + else: + return NotImplemented + + def __le__(self, other): + if isinstance(other, StoredList): + return self._under <= other._under + elif isinstance(other, collections.abc.Sequence): + return self._under <= other + else: + return NotImplemented + + def __gt__(self, other): + if isinstance(other, StoredList): + return self._under > other._under + elif isinstance(other, collections.abc.Sequence): + return self._under > other + else: + return NotImplemented + + def __ge__(self, other): + if isinstance(other, StoredList): + return self._under >= other._under + elif isinstance(other, collections.abc.Sequence): + return self._under >= other + else: + return NotImplemented + + __repr__ = _wrapped_repr + + +class StoredSet(collections.abc.MutableSet): + """A set-like object that uses the StoredState as backend.""" + + def __init__(self, stored_data, under): + self._stored_data = stored_data + self._under = under + + def add(self, key): + """Add a key to a set. + + This has no effect if the key is already present. + """ + self._under.add(key) + self._stored_data.dirty = True + + def discard(self, key): + """Remove a key from a set if it is a member. + + If the key is not a member, do nothing. + """ + self._under.discard(key) + self._stored_data.dirty = True + + def __contains__(self, key): + return key in self._under + + def __iter__(self): + return self._under.__iter__() + + def __len__(self): + return len(self._under) + + @classmethod + def _from_iterable(cls, it): + """Construct an instance of the class from any iterable input. + + Per https://docs.python.org/3/library/collections.abc.html + if the Set mixin is being used in a class with a different constructor signature, + you will need to override _from_iterable() with a classmethod that can construct + new instances from an iterable argument. + """ + return set(it) + + def __le__(self, other): + if isinstance(other, StoredSet): + return self._under <= other._under + elif isinstance(other, collections.abc.Set): + return self._under <= other + else: + return NotImplemented + + def __ge__(self, other): + if isinstance(other, StoredSet): + return self._under >= other._under + elif isinstance(other, collections.abc.Set): + return self._under >= other + else: + return NotImplemented + + def __eq__(self, other): + if isinstance(other, StoredSet): + return self._under == other._under + elif isinstance(other, collections.abc.Set): + return self._under == other + else: + return NotImplemented + + __repr__ = _wrapped_repr diff --git a/ubuntu/venv/ops/jujuversion.py b/ubuntu/venv/ops/jujuversion.py new file mode 100644 index 0000000..d95cba0 --- /dev/null +++ b/ubuntu/venv/ops/jujuversion.py @@ -0,0 +1,113 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A helper to work with the Juju version.""" + +import os +import re +from functools import total_ordering + + +@total_ordering +class JujuVersion: + """Helper to work with the Juju version. + + It knows how to parse the ``JUJU_VERSION`` environment variable, and exposes different + capabilities according to the specific version, allowing also to compare with other + versions. + """ + + PATTERN = r'''^ + (?P\d{1,9})\.(?P\d{1,9}) # and numbers are always there + ((?:\.|-(?P[a-z]+))(?P\d{1,9}))? # sometimes with . or - + (\.(?P\d{1,9}))?$ # and sometimes with a number. + ''' + + def __init__(self, version: str): + m = re.match(self.PATTERN, version, re.VERBOSE) + if not m: + raise RuntimeError('"{}" is not a valid Juju version string'.format(version)) + + d = m.groupdict() + self.major = int(m.group('major')) + self.minor = int(m.group('minor')) + self.tag = d['tag'] or '' + self.patch = int(d['patch'] or 0) + self.build = int(d['build'] or 0) + + def __repr__(self): + if self.tag: + s = '{}.{}-{}{}'.format(self.major, self.minor, self.tag, self.patch) + else: + s = '{}.{}.{}'.format(self.major, self.minor, self.patch) + if self.build > 0: + s += '.{}'.format(self.build) + return s + + def __eq__(self, other: 'JujuVersion') -> bool: + if self is other: + return True + if isinstance(other, str): + other = type(self)(other) + elif not isinstance(other, JujuVersion): + raise RuntimeError('cannot compare Juju version "{}" with "{}"'.format(self, other)) + return ( + self.major == other.major + and self.minor == other.minor + and self.tag == other.tag + and self.build == other.build + and self.patch == other.patch) + + def __lt__(self, other: 'JujuVersion') -> bool: + if self is other: + return False + if isinstance(other, str): + other = type(self)(other) + elif not isinstance(other, JujuVersion): + raise RuntimeError('cannot compare Juju version "{}" with "{}"'.format(self, other)) + if self.major != other.major: + return self.major < other.major + elif self.minor != other.minor: + return self.minor < other.minor + elif self.tag != other.tag: + if not self.tag: + return False + elif not other.tag: + return True + return self.tag < other.tag + elif self.patch != other.patch: + return self.patch < other.patch + elif self.build != other.build: + return self.build < other.build + return False + + @classmethod + def from_environ(cls) -> 'JujuVersion': + """Build a JujuVersion from JUJU_VERSION.""" + v = os.environ.get('JUJU_VERSION') + if v is None: + v = '0.0.0' + return cls(v) + + def has_app_data(self) -> bool: + """Determine whether this juju version knows about app data.""" + return (self.major, self.minor, self.patch) >= (2, 7, 0) + + def is_dispatch_aware(self) -> bool: + """Determine whether this juju version knows about dispatch.""" + return (self.major, self.minor, self.patch) >= (2, 8, 0) + + def has_controller_storage(self) -> bool: + """Determine whether this juju version supports controller-side storage.""" + return (self.major, self.minor, self.patch) >= (2, 8, 0) diff --git a/ubuntu/venv/ops/lib/__init__.py b/ubuntu/venv/ops/lib/__init__.py new file mode 100644 index 0000000..1f4c85c --- /dev/null +++ b/ubuntu/venv/ops/lib/__init__.py @@ -0,0 +1,263 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Infrastructure for the opslib functionality.""" + +import logging +import os +import re +import sys +from ast import literal_eval +from importlib.machinery import ModuleSpec +from importlib.util import module_from_spec +from pkgutil import get_importer +from types import ModuleType +from typing import List + +__all__ = ('use', 'autoimport') + +logger = logging.getLogger(__name__) + +_libraries = None + +_libline_re = re.compile(r'''^LIB([A-Z]+)\s*=\s*([0-9]+|['"][a-zA-Z0-9_.\-@]+['"])''') +_libname_re = re.compile(r'''^[a-z][a-z0-9]+$''') + +# Not perfect, but should do for now. +_libauthor_re = re.compile(r'''^[A-Za-z0-9_+.-]+@[a-z0-9_-]+(?:\.[a-z0-9_-]+)*\.[a-z]{2,3}$''') + + +def use(name: str, api: int, author: str) -> ModuleType: + """Use a library from the ops libraries. + + Args: + name: the name of the library requested. + api: the API version of the library. + author: the author of the library. If not given, requests the + one in the standard library. + + Raises: + ImportError: if the library cannot be found. + TypeError: if the name, api, or author are the wrong type. + ValueError: if the name, api, or author are invalid. + """ + if not isinstance(name, str): + raise TypeError("invalid library name: {!r} (must be a str)".format(name)) + if not isinstance(author, str): + raise TypeError("invalid library author: {!r} (must be a str)".format(author)) + if not isinstance(api, int): + raise TypeError("invalid library API: {!r} (must be an int)".format(api)) + if api < 0: + raise ValueError('invalid library api: {} (must be ≥0)'.format(api)) + if not _libname_re.match(name): + raise ValueError("invalid library name: {!r} (chars and digits only)".format(name)) + if not _libauthor_re.match(author): + raise ValueError("invalid library author email: {!r}".format(author)) + + if _libraries is None: + autoimport() + + versions = _libraries.get((name, author), ()) + for lib in versions: + if lib.api == api: + return lib.import_module() + + others = ', '.join(str(lib.api) for lib in versions) + if others: + msg = 'cannot find "{}" from "{}" with API version {} (have {})'.format( + name, author, api, others) + else: + msg = 'cannot find library "{}" from "{}"'.format(name, author) + + raise ImportError(msg, name=name) + + +def autoimport(): + """Find all libs in the path and enable use of them. + + You only need to call this if you've installed a package or + otherwise changed sys.path in the current run, and need to see the + changes. Otherwise libraries are found on first call of `use`. + """ + global _libraries + _libraries = {} + for spec in _find_all_specs(sys.path): + lib = _parse_lib(spec) + if lib is None: + continue + + versions = _libraries.setdefault((lib.name, lib.author), []) + versions.append(lib) + versions.sort(reverse=True) + + +def _find_all_specs(path): + for sys_dir in path: + if sys_dir == "": + sys_dir = "." + try: + top_dirs = os.listdir(sys_dir) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as e: + logger.debug("Tried to look for ops.lib packages under '%s': %s", sys_dir, e) + continue + logger.debug("Looking for ops.lib packages under '%s'", sys_dir) + for top_dir in top_dirs: + opslib = os.path.join(sys_dir, top_dir, 'opslib') + try: + lib_dirs = os.listdir(opslib) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as e: + logger.debug(" Tried '%s': %s", opslib, e) # *lots* of things checked here + continue + else: + logger.debug(" Trying '%s'", opslib) + finder = get_importer(opslib) + if finder is None: + logger.debug(" Finder for '%s' is None", opslib) + continue + if not hasattr(finder, 'find_spec'): + logger.debug(" Finder for '%s' has no find_spec", opslib) + continue + for lib_dir in lib_dirs: + spec_name = "{}.opslib.{}".format(top_dir, lib_dir) + spec = finder.find_spec(spec_name) + if spec is None: + logger.debug(" No spec for %r", spec_name) + continue + if spec.loader is None: + # a namespace package; not supported + logger.debug(" No loader for %r (probably a namespace package)", spec_name) + continue + + logger.debug(" Found %r", spec_name) + yield spec + + +# only the first this many lines of a file are looked at for the LIB* constants +_MAX_LIB_LINES = 99 +# these keys, with these types, are needed to have an opslib +_NEEDED_KEYS = {'NAME': str, 'AUTHOR': str, 'API': int, 'PATCH': int} + + +def _join_and(keys: List[str]) -> str: + if len(keys) == 0: + return "" + if len(keys) == 1: + return keys[0] + return ", ".join(keys[:-1]) + ", and " + keys[-1] + + +class _Missing: + """Helper to get the difference between what was found and what was needed when logging.""" + + def __init__(self, found): + self._found = found + + def __str__(self): + exp = set(_NEEDED_KEYS) + got = set(self._found) + if len(got) == 0: + return "missing {}".format(_join_and(sorted(exp))) + return "got {}, but missing {}".format( + _join_and(sorted(got)), + _join_and(sorted(exp - got))) + + +def _parse_lib(spec): + if spec.origin is None: + # "can't happen" + logger.warning("No origin for %r (no idea why; please report)", spec.name) + return None + + logger.debug(" Parsing %r", spec.name) + + try: + with open(spec.origin, 'rt', encoding='utf-8') as f: + libinfo = {} + for n, line in enumerate(f): + if len(libinfo) == len(_NEEDED_KEYS): + break + if n > _MAX_LIB_LINES: + logger.debug( + " Missing opslib metadata after reading to line %d: %s", + _MAX_LIB_LINES, _Missing(libinfo)) + return None + m = _libline_re.match(line) + if m is None: + continue + key, value = m.groups() + if key in _NEEDED_KEYS: + value = literal_eval(value) + if not isinstance(value, _NEEDED_KEYS[key]): + logger.debug( + " Bad type for %s: expected %s, got %s", + key, _NEEDED_KEYS[key].__name__, type(value).__name__) + return None + libinfo[key] = value + else: + if len(libinfo) != len(_NEEDED_KEYS): + logger.debug( + " Missing opslib metadata after reading to end of file: %s", + _Missing(libinfo)) + return None + except Exception as e: + logger.debug(" Failed: %s", e) + return None + + lib = _Lib(spec, libinfo['NAME'], libinfo['AUTHOR'], libinfo['API'], libinfo['PATCH']) + logger.debug(" Success: found library %s", lib) + + return lib + + +class _Lib: + + def __init__(self, spec: ModuleSpec, name: str, author: str, api: int, patch: int): + self.spec = spec + self.name = name + self.author = author + self.api = api + self.patch = patch + + self._module = None + + def __repr__(self): + return "<_Lib {}>".format(self) + + def __str__(self): + return "{0.name} by {0.author}, API {0.api}, patch {0.patch}".format(self) + + def import_module(self) -> ModuleType: + if self._module is None: + module = module_from_spec(self.spec) + self.spec.loader.exec_module(module) + self._module = module + return self._module + + def __eq__(self, other): + if not isinstance(other, _Lib): + return NotImplemented + a = (self.name, self.author, self.api, self.patch) + b = (other.name, other.author, other.api, other.patch) + return a == b + + def __lt__(self, other): + if not isinstance(other, _Lib): + return NotImplemented + a = (self.name, self.author, self.api, self.patch) + b = (other.name, other.author, other.api, other.patch) + return a < b diff --git a/ubuntu/venv/ops/log.py b/ubuntu/venv/ops/log.py new file mode 100644 index 0000000..b816a1e --- /dev/null +++ b/ubuntu/venv/ops/log.py @@ -0,0 +1,70 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interface to emit messages to the Juju logging system.""" + +import logging +import sys +import typing + +if typing.TYPE_CHECKING: + from types import TracebackType + from typing import Type + + from ops.model import _ModelBackend # pyright: reportPrivateUsage=false + + +class JujuLogHandler(logging.Handler): + """A handler for sending logs to Juju via juju-log.""" + + def __init__(self, model_backend: "_ModelBackend", level: int = logging.DEBUG): + super().__init__(level) + self.model_backend = model_backend + + def emit(self, record: logging.LogRecord): + """Send the specified logging record to the Juju backend. + + This method is not used directly by the Operator Framework code, but by + :class:`logging.Handler` itself as part of the logging machinery. + """ + self.model_backend.juju_log(record.levelname, self.format(record)) + + +def setup_root_logging(model_backend: "_ModelBackend", debug: bool = False): + """Setup python logging to forward messages to juju-log. + + By default, logging is set to DEBUG level, and messages will be filtered by Juju. + Charmers can also set their own default log level with:: + + logging.getLogger().setLevel(logging.INFO) + + Args: + model_backend: a ModelBackend to use for juju-log + debug: if True, write logs to stderr as well as to juju-log. + """ + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + logger.addHandler(JujuLogHandler(model_backend)) + if debug: + handler = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + def except_hook(etype: "Type[BaseException]", value: BaseException, tb: "TracebackType"): + logger.error( + "Uncaught exception while in charm code:", + exc_info=(etype, value, tb)) + + sys.excepthook = except_hook diff --git a/ubuntu/venv/ops/main.py b/ubuntu/venv/ops/main.py new file mode 100644 index 0000000..691f7b3 --- /dev/null +++ b/ubuntu/venv/ops/main.py @@ -0,0 +1,435 @@ +# Copyright 2019-2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Main entry point to the Operator Framework.""" + +import inspect +import logging +import os +import shutil +import subprocess +import sys +import typing +import warnings +from pathlib import Path + +import yaml + +import ops.charm +import ops.framework +import ops.model +import ops.storage +from ops.jujuversion import JujuVersion +from ops.log import setup_root_logging + +CHARM_STATE_FILE = '.unit-state.db' + + +logger = logging.getLogger() + + +def _exe_path(path: Path) -> typing.Optional[Path]: + """Find and return the full path to the given binary. + + Here path is the absolute path to a binary, but might be missing an extension. + """ + p = shutil.which(path.name, mode=os.F_OK, path=str(path.parent)) + if p is None: + return None + return Path(p) + + +def _get_charm_dir(): + charm_dir = os.environ.get("JUJU_CHARM_DIR") + if charm_dir is None: + # Assume $JUJU_CHARM_DIR/lib/op/main.py structure. + charm_dir = Path('{}/../../..'.format(__file__)).resolve() + else: + charm_dir = Path(charm_dir).resolve() + return charm_dir + + +def _create_event_link(charm, bound_event, link_to): + """Create a symlink for a particular event. + + Args: + charm: A charm object. + bound_event: An event for which to create a symlink. + link_to: What the event link should point to + """ + if issubclass(bound_event.event_type, ops.charm.HookEvent): + event_dir = charm.framework.charm_dir / 'hooks' + event_path = event_dir / bound_event.event_kind.replace('_', '-') + elif issubclass(bound_event.event_type, ops.charm.ActionEvent): + if not bound_event.event_kind.endswith("_action"): + raise RuntimeError( + 'action event name {} needs _action suffix'.format(bound_event.event_kind)) + event_dir = charm.framework.charm_dir / 'actions' + # The event_kind is suffixed with "_action" while the executable is not. + event_path = event_dir / bound_event.event_kind[:-len('_action')].replace('_', '-') + else: + raise RuntimeError( + 'cannot create a symlink: unsupported event type {}'.format(bound_event.event_type)) + + event_dir.mkdir(exist_ok=True) + if not event_path.exists(): + target_path = os.path.relpath(link_to, str(event_dir)) + + # Ignore the non-symlink files or directories + # assuming the charm author knows what they are doing. + logger.debug( + 'Creating a new relative symlink at %s pointing to %s', + event_path, target_path) + event_path.symlink_to(target_path) + + +def _setup_event_links(charm_dir, charm): + """Set up links for supported events that originate from Juju. + + Whether a charm can handle an event or not can be determined by + introspecting which events are defined on it. + + Hooks or actions are created as symlinks to the charm code file + which is determined by inspecting symlinks provided by the charm + author at hooks/install or hooks/start. + + Args: + charm_dir: A root directory of the charm. + charm: An instance of the Charm class. + + """ + # XXX: on windows this function does not accomplish what it wants to: + # it creates symlinks with no extension pointing to a .py + # and juju only knows how to handle .exe, .bat, .cmd, and .ps1 + # so it does its job, but does not accomplish anything as the + # hooks aren't 'callable'. + link_to = os.path.realpath(os.environ.get("JUJU_DISPATCH_PATH", sys.argv[0])) + for bound_event in charm.on.events().values(): + # Only events that originate from Juju need symlinks. + if issubclass(bound_event.event_type, (ops.charm.HookEvent, ops.charm.ActionEvent)): + _create_event_link(charm, bound_event, link_to) + + +def _emit_charm_event(charm, event_name): + """Emits a charm event based on a Juju event name. + + Args: + charm: A charm instance to emit an event from. + event_name: A Juju event name to emit on a charm. + """ + event_to_emit = None + try: + event_to_emit = getattr(charm.on, event_name) + except AttributeError: + logger.debug("Event %s not defined for %s.", event_name, charm) + + # If the event is not supported by the charm implementation, do + # not error out or try to emit it. This is to support rollbacks. + if event_to_emit is not None: + args, kwargs = _get_event_args(charm, event_to_emit) + logger.debug('Emitting Juju event %s.', event_name) + event_to_emit.emit(*args, **kwargs) + + +def _get_event_args(charm, bound_event): + event_type = bound_event.event_type + model = charm.framework.model + + relation = None + if issubclass(event_type, ops.charm.WorkloadEvent): + workload_name = os.environ['JUJU_WORKLOAD_NAME'] + container = model.unit.get_container(workload_name) + return [container], {} + elif issubclass(event_type, ops.charm.StorageEvent): + storage_id = os.environ.get("JUJU_STORAGE_ID", "") + if storage_id: + storage_name = storage_id.split("/")[0] + else: + # Before JUJU_STORAGE_ID exists, take the event name as + # _storage_ and replace it with + storage_name = "-".join(bound_event.event_kind.split("_")[:-2]) + + storages = model.storages[storage_name] + id, storage_location = model._backend._storage_event_details() + if len(storages) == 1: + storage = storages[0] + else: + # If there's more than one value, pick the right one. We'll realize the key on lookup + storage = next((s for s in storages if s.id == id), None) + storage.location = storage_location + return [storage], {} + elif issubclass(event_type, ops.charm.RelationEvent): + relation_name = os.environ['JUJU_RELATION'] + relation_id = int(os.environ['JUJU_RELATION_ID'].split(':')[-1]) + relation = model.get_relation(relation_name, relation_id) + + remote_app_name = os.environ.get('JUJU_REMOTE_APP', '') + remote_unit_name = os.environ.get('JUJU_REMOTE_UNIT', '') + departing_unit_name = os.environ.get('JUJU_DEPARTING_UNIT', '') + + if not remote_app_name and remote_unit_name: + if '/' not in remote_unit_name: + raise RuntimeError('invalid remote unit name: {}'.format(remote_unit_name)) + remote_app_name = remote_unit_name.split('/')[0] + + kwargs = {} + if remote_app_name: + kwargs['app'] = model.get_app(remote_app_name) + if remote_unit_name: + kwargs['unit'] = model.get_unit(remote_unit_name) + if departing_unit_name: + kwargs['departing_unit_name'] = departing_unit_name + + if relation: + return [relation], kwargs + return [], {} + + +class _Dispatcher: + """Encapsulate how to figure out what event Juju wants us to run. + + Also knows how to run “legacy” hooks when Juju called us via a top-level + ``dispatch`` binary. + + Args: + charm_dir: the toplevel directory of the charm + + Attributes: + event_name: the name of the event to run + is_dispatch_aware: are we running under a Juju that knows about the + dispatch binary, and is that binary present? + + """ + + def __init__(self, charm_dir: Path): + self._charm_dir = charm_dir + self._exec_path = Path(os.environ.get('JUJU_DISPATCH_PATH', sys.argv[0])) + + dispatch = charm_dir / 'dispatch' + if JujuVersion.from_environ().is_dispatch_aware() and _exe_path(dispatch) is not None: + self._init_dispatch() + else: + self._init_legacy() + + def ensure_event_links(self, charm): + """Make sure necessary symlinks are present on disk.""" + if self.is_dispatch_aware: + # links aren't needed + return + + # When a charm is force-upgraded and a unit is in an error state Juju + # does not run upgrade-charm and instead runs the failed hook followed + # by config-changed. Given the nature of force-upgrading the hook setup + # code is not triggered on config-changed. + # + # 'start' event is included as Juju does not fire the install event for + # K8s charms (see LP: #1854635). + if (self.event_name in ('install', 'start', 'upgrade_charm') + or self.event_name.endswith('_storage_attached')): + _setup_event_links(self._charm_dir, charm) + + def run_any_legacy_hook(self): + """Run any extant legacy hook. + + If there is both a dispatch file and a legacy hook for the + current event, run the wanted legacy hook. + """ + if not self.is_dispatch_aware: + # we *are* the legacy hook + return + + dispatch_path = _exe_path(self._charm_dir / self._dispatch_path) + if dispatch_path is None: + logger.debug("Legacy %s does not exist.", self._dispatch_path) + return + + # super strange that there isn't an is_executable + if not os.access(str(dispatch_path), os.X_OK): + logger.warning("Legacy %s exists but is not executable.", self._dispatch_path) + return + + if dispatch_path.resolve() == Path(sys.argv[0]).resolve(): + logger.debug("Legacy %s is just a link to ourselves.", self._dispatch_path) + return + + argv = sys.argv.copy() + argv[0] = str(dispatch_path) + logger.info("Running legacy %s.", self._dispatch_path) + try: + subprocess.run(argv, check=True) + except subprocess.CalledProcessError as e: + logger.warning("Legacy %s exited with status %d.", self._dispatch_path, e.returncode) + sys.exit(e.returncode) + except OSError as e: + logger.warning("Unable to run legacy %s: %s", self._dispatch_path, e) + sys.exit(1) + else: + logger.debug("Legacy %s exited with status 0.", self._dispatch_path) + + def _set_name_from_path(self, path: Path): + """Sets the name attribute to that which can be inferred from the given path.""" + name = path.name.replace('-', '_') + if path.parent.name == 'actions': + name = '{}_action'.format(name) + self.event_name = name + + def _init_legacy(self): + """Set up the 'legacy' dispatcher. + + The current Juju doesn't know about 'dispatch' and calls hooks + explicitly. + """ + self.is_dispatch_aware = False + self._set_name_from_path(self._exec_path) + + def _init_dispatch(self): + """Set up the new 'dispatch' dispatcher. + + The current Juju will run 'dispatch' if it exists, and otherwise fall + back to the old behaviour. + + JUJU_DISPATCH_PATH will be set to the wanted hook, e.g. hooks/install, + in both cases. + """ + self._dispatch_path = Path(os.environ['JUJU_DISPATCH_PATH']) + + if 'OPERATOR_DISPATCH' in os.environ: + logger.debug("Charm called itself via %s.", self._dispatch_path) + sys.exit(0) + os.environ['OPERATOR_DISPATCH'] = '1' + + self.is_dispatch_aware = True + self._set_name_from_path(self._dispatch_path) + + def is_restricted_context(self): + """Return True if we are running in a restricted Juju context. + + When in a restricted context, most commands (relation-get, config-get, + state-get) are not available. As such, we change how we interact with + Juju. + """ + return self.event_name in ('collect_metrics',) + + +def _should_use_controller_storage(db_path: Path, meta: ops.charm.CharmMeta) -> bool: + """Figure out whether we want to use controller storage or not.""" + # if you've previously used local state, carry on using that + if db_path.exists(): + logger.debug("Using local storage: %s already exists", db_path) + return False + + # if you're not in k8s you don't need controller storage + if 'kubernetes' not in meta.series: + logger.debug("Using local storage: not a kubernetes charm") + return False + + # are we in a new enough Juju? + cur_version = JujuVersion.from_environ() + + if cur_version.has_controller_storage(): + logger.debug("Using controller storage: JUJU_VERSION=%s", cur_version) + return True + else: + logger.debug("Using local storage: JUJU_VERSION=%s", cur_version) + return False + + +def main(charm_class: typing.Type[ops.charm.CharmBase], use_juju_for_storage: bool = None): + """Setup the charm and dispatch the observed event. + + The event name is based on the way this executable was called (argv[0]). + + Args: + charm_class: your charm class. + use_juju_for_storage: whether to use controller-side storage. If not specified + then kubernetes charms that haven't previously used local storage and that + are running on a new enough Juju default to controller-side storage, + otherwise local storage is used. + """ + charm_dir = _get_charm_dir() + + model_backend = ops.model._ModelBackend() + debug = ('JUJU_DEBUG' in os.environ) + setup_root_logging(model_backend, debug=debug) + logger.debug("Operator Framework %s up and running.", ops.__version__) + + dispatcher = _Dispatcher(charm_dir) + dispatcher.run_any_legacy_hook() + + metadata = (charm_dir / 'metadata.yaml').read_text() + actions_meta = charm_dir / 'actions.yaml' + if actions_meta.exists(): + actions_metadata = actions_meta.read_text() + else: + actions_metadata = None + + if not yaml.__with_libyaml__: + logger.debug('yaml does not have libyaml extensions, using slower pure Python yaml loader') + meta = ops.charm.CharmMeta.from_yaml(metadata, actions_metadata) + model = ops.model.Model(meta, model_backend) + + charm_state_path = charm_dir / CHARM_STATE_FILE + + if use_juju_for_storage and not ops.storage.juju_backend_available(): + # raise an exception; the charm is broken and needs fixing. + msg = 'charm set use_juju_for_storage=True, but Juju version {} does not support it' + raise RuntimeError(msg.format(JujuVersion.from_environ())) + + if use_juju_for_storage is None: + use_juju_for_storage = _should_use_controller_storage(charm_state_path, meta) + + if use_juju_for_storage: + if dispatcher.is_restricted_context(): + # TODO: jam 2020-06-30 This unconditionally avoids running a collect metrics event + # Though we eventually expect that juju will run collect-metrics in a + # non-restricted context. Once we can determine that we are running collect-metrics + # in a non-restricted context, we should fire the event as normal. + logger.debug('"%s" is not supported when using Juju for storage\n' + 'see: https://github.com/canonical/operator/issues/348', + dispatcher.event_name) + # Note that we don't exit nonzero, because that would cause Juju to rerun the hook + return + store = ops.storage.JujuStorage() + else: + store = ops.storage.SQLiteStorage(charm_state_path) + framework = ops.framework.Framework(store, charm_dir, meta, model) + framework.set_breakpointhook() + try: + sig = inspect.signature(charm_class) + try: + sig.bind(framework) + except TypeError: + msg = ( + "the second argument, 'key', has been deprecated and will be " + "removed after the 0.7 release") + warnings.warn(msg, DeprecationWarning) + charm = charm_class(framework, None) + else: + charm = charm_class(framework) + dispatcher.ensure_event_links(charm) + + # TODO: Remove the collect_metrics check below as soon as the relevant + # Juju changes are made. Also adjust the docstring on + # EventBase.defer(). + # + # Skip reemission of deferred events for collect-metrics events because + # they do not have the full access to all hook tools. + if not dispatcher.is_restricted_context(): + framework.reemit() + + _emit_charm_event(charm, dispatcher.event_name) + + framework.commit() + finally: + framework.close() diff --git a/ubuntu/venv/ops/model.py b/ubuntu/venv/ops/model.py new file mode 100644 index 0000000..083311c --- /dev/null +++ b/ubuntu/venv/ops/model.py @@ -0,0 +1,2397 @@ +# Copyright 2019-2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Representations of Juju's model, application, unit, and other entities.""" +import datetime +import ipaddress +import json +import logging +import math +import os +import re +import shutil +import stat +import tempfile +import time +import typing +import weakref +from abc import ABC, abstractmethod +from pathlib import Path +from subprocess import PIPE, CalledProcessError, run +from typing import ( + Any, + BinaryIO, + Callable, + Dict, + Generator, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Sequence, + Set, + TextIO, + Tuple, + Type, + TypeVar, + Union, +) + +import ops +import ops.pebble as pebble +from ops._private import yaml +from ops.jujuversion import JujuVersion + +if typing.TYPE_CHECKING: + from typing_extensions import TypedDict + + _StorageDictType = Dict[str, Optional[List['Storage']]] + _BindingDictType = Dict[Union[str, 'Relation'], 'Binding'] + Numerical = Union[int, float] + + # all types that can be (de) serialized to json(/yaml) fom Python builtins + JsonObject = Union[Numerical, bool, str, + Dict[str, 'JsonObject'], + List['JsonObject'], + Tuple['JsonObject', ...]] + + # a k8s spec is a mapping from names/"types" to json/yaml spec objects + _K8sSpec = Mapping[str, JsonObject] + + _StatusDict = TypedDict('_StatusDict', {'status': str, 'message': str}) + + # the data structure we can use to initialize pebble layers with. + # todo: replace with pebble._LayerDict (a TypedDict) when pebble.py is typed + _LayerDict = Dict[str, '_LayerDict'] + _Layer = Union[str, _LayerDict, pebble.Layer] + + # mapping from relation name to a list of relation objects + _RelationMapping_Raw = Dict[str, Optional[List['Relation']]] + # mapping from relation name to relation metadata + _RelationsMeta_Raw = Dict[str, ops.charm.RelationMeta] + # mapping from container name to container metadata + _ContainerMeta_Raw = Dict[str, ops.charm.ContainerMeta] + _IPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] + _Network = Union[ipaddress.IPv4Network, ipaddress.IPv6Network] + + _ServiceInfoMapping = Mapping[str, pebble.ServiceInfo] + + # relation data is a string key: string value mapping so far as the + # controller is concerned + _RelationDataContent_Raw = Dict[str, str] + UnitOrApplication = Union['Unit', 'Application'] + UnitOrApplicationType = Union[Type['Unit'], Type['Application']] + + _AddressDict = TypedDict('_AddressDict', { + 'address': str, # Juju < 2.9 + 'value': str, # Juju >= 2.9 + 'cidr': str + }) + _BindAddressDict = TypedDict('_BindAddressDict', { + 'interface-name': str, + 'addresses': List[_AddressDict] + }) + _NetworkDict = TypedDict('_NetworkDict', { + 'bind-addresses': List[_BindAddressDict], + 'ingress-addresses': List[str], + 'egress-subnets': List[str] + }) + + +StrOrPath = typing.Union[str, Path] + +logger = logging.getLogger(__name__) + +MAX_LOG_LINE_LEN = 131071 # Max length of strings to pass to subshell. + + +class Model: + """Represents the Juju Model as seen from this unit. + + This should not be instantiated directly by Charmers, but can be accessed as `self.model` + from any class that derives from Object. + """ + + def __init__(self, meta: 'ops.charm.CharmMeta', backend: '_ModelBackend'): + self._cache = _ModelCache(meta, backend) + self._backend = backend + self._unit = self.get_unit(self._backend.unit_name) + # fixme: remove cast after typing charm.py + relations = typing.cast('_RelationsMeta_Raw', meta.relations) # type: ignore + self._relations = RelationMapping(relations, self.unit, self._backend, self._cache) + self._config = ConfigData(self._backend) + # fixme: remove cast after typing charm.py + resources = typing.cast(Iterable[str], meta.resources) # type: ignore + self._resources = Resources(list(resources), self._backend) + self._pod = Pod(self._backend) + # fixme: remove cast after typing charm.py + storages = typing.cast(Iterable[str], meta.storages) # type: ignore + self._storages = StorageMapping(list(storages), self._backend) + self._bindings = BindingMapping(self._backend) + + @property + def unit(self) -> 'Unit': + """A :class:`Unit` that represents the unit that is running this code (eg yourself).""" + return self._unit + + @property + def app(self) -> 'Application': + """A :class:`Application` that represents the application this unit is a part of.""" + return self._unit.app + + @property + def relations(self) -> 'RelationMapping': + """Mapping of endpoint to list of :class:`Relation`. + + Answers the question "what am I currently related to". + See also :meth:`.get_relation`. + """ + return self._relations + + @property + def config(self) -> 'ConfigData': + """Return a mapping of config for the current application.""" + return self._config + + @property + def resources(self) -> 'Resources': + """Access to resources for this charm. + + Use ``model.resources.fetch(resource_name)`` to get the path on disk + where the resource can be found. + """ + return self._resources + + @property + def storages(self) -> 'StorageMapping': + """Mapping of storage_name to :class:`Storage` as defined in metadata.yaml.""" + return self._storages + + @property + def pod(self) -> 'Pod': + """Use ``model.pod.set_spec`` to set the container specification for Kubernetes charms.""" + return self._pod + + @property + def name(self) -> str: + """Return the name of the Model that this unit is running in. + + This is read from the environment variable ``JUJU_MODEL_NAME``. + """ + return self._backend.model_name + + @property + def uuid(self) -> str: + """Return the identifier of the Model that this unit is running in. + + This is read from the environment variable ``JUJU_MODEL_UUID``. + """ + return self._backend.model_uuid + + def get_unit(self, unit_name: str) -> 'Unit': + """Get an arbitrary unit by name. + + Internally this uses a cache, so asking for the same unit two times will + return the same object. + """ + return self._cache.get(Unit, unit_name) + + def get_app(self, app_name: str) -> 'Application': + """Get an application by name. + + Internally this uses a cache, so asking for the same application two times will + return the same object. + """ + return self._cache.get(Application, app_name) + + def get_relation( + self, relation_name: str, + relation_id: Optional[int] = None) -> Optional['Relation']: + """Get a specific Relation instance. + + If relation_id is not given, this will return the Relation instance if the + relation is established only once or None if it is not established. If this + same relation is established multiple times the error TooManyRelatedAppsError is raised. + + Args: + relation_name: The name of the endpoint for this charm + relation_id: An identifier for a specific relation. Used to disambiguate when a + given application has more than one relation on a given endpoint. + + Raises: + TooManyRelatedAppsError: is raised if there is more than one relation to the + supplied relation_name and no relation_id was supplied + """ + return self.relations._get_unique(relation_name, relation_id) + + def get_binding(self, binding_key: Union[str, 'Relation']) -> Optional['Binding']: + """Get a network space binding. + + Args: + binding_key: The relation name or instance to obtain bindings for. + + Returns: + If ``binding_key`` is a relation name, the method returns the default binding + for that relation. If a relation instance is provided, the method first looks + up a more specific binding for that specific relation ID, and if none is found + falls back to the default binding for the relation name. + """ + return self._bindings.get(binding_key) + + +_T = TypeVar('_T', bound='UnitOrApplication') + + +class _ModelCache: + def __init__(self, meta: 'ops.charm.CharmMeta', backend: '_ModelBackend'): + if typing.TYPE_CHECKING: + # (entity type, name): instance. + _weakcachetype = weakref.WeakValueDictionary[ + Tuple['UnitOrApplicationType', str], + Optional['UnitOrApplication']] + + self._meta = meta + self._backend = backend + self._weakrefs = weakref.WeakValueDictionary() # type: _weakcachetype + + @typing.overload + def get(self, entity_type: Type['Unit'], name: str) -> 'Unit': ... # noqa + @typing.overload + def get(self, entity_type: Type['Application'], name: str) -> 'Application': ... # noqa + + def get(self, entity_type: 'UnitOrApplicationType', name: str): + """Fetch the cached entity of type `entity_type` with name `name`.""" + key = (entity_type, name) + entity = self._weakrefs.get(key) + if entity is not None: + return entity + + new_entity = entity_type(name, meta=self._meta, backend=self._backend, cache=self) + self._weakrefs[key] = new_entity + return new_entity + + +class Application: + """Represents a named application in the model. + + This might be your application, or might be an application that you are related to. + Charmers should not instantiate Application objects directly, but should use + :meth:`Model.get_app` if they need a reference to a given application. + + Attributes: + name: The name of this application (eg, 'mysql'). This name may differ from the name of + the charm, if the user has deployed it to a different name. + """ + + def __init__(self, name: str, meta: 'ops.charm.CharmMeta', + backend: '_ModelBackend', cache: _ModelCache): + self.name = name + self._backend = backend + self._cache = cache + self._is_our_app = self.name == self._backend.app_name + self._status = None + + def _invalidate(self): + self._status = None + + @property + def status(self) -> 'StatusBase': + """Used to report or read the status of the overall application. + + Can only be read and set by the lead unit of the application. + + The status of remote units is always Unknown. + + Raises: + RuntimeError: if you try to set the status of another application, or if you try to + set the status of this application as a unit that is not the leader. + InvalidStatusError: if you try to set the status to something that is not a + :class:`StatusBase` + + Example:: + + self.model.app.status = BlockedStatus('I need a human to come help me') + """ + if not self._is_our_app: + return UnknownStatus() + + if not self._backend.is_leader(): + raise RuntimeError('cannot get application status as a non-leader unit') + + if self._status: + return self._status + + s = self._backend.status_get(is_app=True) + self._status = StatusBase.from_name(s['status'], s['message']) + return self._status + + @status.setter + def status(self, value: 'StatusBase'): + if not isinstance(value, StatusBase): + raise InvalidStatusError( + 'invalid value provided for application {} status: {}'.format(self, value) + ) + + if not self._is_our_app: + raise RuntimeError('cannot to set status for a remote application {}'.format(self)) + + if not self._backend.is_leader(): + raise RuntimeError('cannot set application status as a non-leader unit') + + for _key in {'name', 'message'}: + assert isinstance(getattr(value, _key), str), 'status.%s must be a string' % _key + self._backend.status_set(value.name, value.message, is_app=True) + self._status = value + + def planned_units(self) -> int: + """Get the number of units that Juju has "planned" for this application. + + E.g., if an operator runs "juju deploy foo", then "juju add-unit -n 2 foo", the + planned unit count for foo will be 3. + + The data comes from the Juju agent, based on data it fetches from the + controller. Pending units are included in the count, and scale down events may + modify the count before some units have been fully torn down. The information in + planned_units is up-to-date as of the start of the current hook invocation. + + This method only returns data for this charm's application -- the Juju agent isn't + able to see planned unit counts for other applications in the model. + + """ + if not self._is_our_app: + raise RuntimeError( + 'cannot get planned units for a remote application {}.'.format(self)) + + return self._backend.planned_units() + + def __repr__(self): + return '<{}.{} {}>'.format(type(self).__module__, type(self).__name__, self.name) + + +class Unit: + """Represents a named unit in the model. + + This might be your unit, another unit of your application, or a unit of another application + that you are related to. + + Attributes: + name: The name of the unit (eg, 'mysql/0') + app: The Application the unit is a part of. + """ + + def __init__(self, name: str, meta: 'ops.charm.CharmMeta', + backend: '_ModelBackend', cache: '_ModelCache'): + self.name = name + + app_name = name.split('/')[0] + self.app = cache.get(Application, app_name) + + self._backend = backend + self._cache = cache + self._is_our_unit = self.name == self._backend.unit_name + self._status = None + + if self._is_our_unit and hasattr(meta, "containers"): + # fixme: remove cast when charm.py is typed + containers = typing.cast('_ContainerMeta_Raw', meta.containers) # type: ignore + self._containers = ContainerMapping(iter(containers), backend) + + def _invalidate(self): + self._status = None + + @property + def status(self) -> 'StatusBase': + """Used to report or read the status of a specific unit. + + The status of any unit other than yourself is always Unknown. + + Raises: + RuntimeError: if you try to set the status of a unit other than yourself. + InvalidStatusError: if you try to set the status to something other than + a :class:`StatusBase` + Example:: + + self.model.unit.status = MaintenanceStatus('reconfiguring the frobnicators') + """ + if not self._is_our_unit: + return UnknownStatus() + + if self._status: + return self._status + + s = self._backend.status_get(is_app=False) + self._status = StatusBase.from_name(s['status'], s['message']) + return self._status + + @status.setter + def status(self, value: 'StatusBase'): + if not isinstance(value, StatusBase): + raise InvalidStatusError( + 'invalid value provided for unit {} status: {}'.format(self, value) + ) + + if not self._is_our_unit: + raise RuntimeError('cannot set status for a remote unit {}'.format(self)) + + # fixme: if value.messages + self._backend.status_set(value.name, value.message, is_app=False) + self._status = value + + def __repr__(self): + return '<{}.{} {}>'.format(type(self).__module__, type(self).__name__, self.name) + + def is_leader(self) -> bool: + """Return whether this unit is the leader of its application. + + This can only be called for your own unit. + + Returns: + True if you are the leader, False otherwise + Raises: + RuntimeError: if called for a unit that is not yourself + """ + if self._is_our_unit: + # This value is not cached as it is not guaranteed to persist for the whole duration + # of a hook execution. + return self._backend.is_leader() + else: + raise RuntimeError( + 'leadership status of remote units ({}) is not visible to other' + ' applications'.format(self) + ) + + def set_workload_version(self, version: str) -> None: + """Record the version of the software running as the workload. + + This shouldn't be confused with the revision of the charm. This is informative only; + shown in the output of 'juju status'. + """ + if not isinstance(version, str): + raise TypeError("workload version must be a str, not {}: {!r}".format( + type(version).__name__, version)) + self._backend.application_version_set(version) + + @property + def containers(self) -> Mapping[str, 'Container']: + """Return a mapping of containers indexed by name.""" + if not self._is_our_unit: + raise RuntimeError('cannot get container for a remote unit {}'.format(self)) + return self._containers + + def get_container(self, container_name: str) -> 'Container': + """Get a single container by name. + + Raises: + ModelError: if the named container doesn't exist + """ + try: + return self.containers[container_name] + except KeyError: + raise ModelError('container {!r} not found'.format(container_name)) + + +class LazyMapping(Mapping[str, str], ABC): + """Represents a dict that isn't populated until it is accessed. + + Charm authors should generally never need to use this directly, but it forms + the basis for many of the dicts that the framework tracks. + """ + + # key-value mapping + _lazy_data = None # type: Optional[Dict[str, str]] + + @abstractmethod + def _load(self) -> Dict[str, str]: + raise NotImplementedError() + + @property + def _data(self) -> Dict[str, str]: + data = self._lazy_data + if data is None: + data = self._lazy_data = self._load() + return data + + def _invalidate(self): + self._lazy_data = None + + def __contains__(self, key: str) -> bool: + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key: str) -> str: + return self._data[key] + + def __repr__(self): + return repr(self._data) + + +class RelationMapping(Mapping[str, List['Relation']]): + """Map of relation names to lists of :class:`Relation` instances.""" + + def __init__(self, relations_meta: '_RelationsMeta_Raw', our_unit: 'Unit', + backend: '_ModelBackend', cache: '_ModelCache'): + self._peers = set() # type: Set[str] + for name, relation_meta in relations_meta.items(): + if relation_meta.role.is_peer(): + self._peers.add(name) + self._our_unit = our_unit + self._backend = backend + self._cache = cache + self._data = {r: None for r in relations_meta} # type: _RelationMapping_Raw + + def __contains__(self, key: str): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self) -> Iterable[str]: + return iter(self._data) + + def __getitem__(self, relation_name: str) -> List['Relation']: + is_peer = relation_name in self._peers + relation_list = self._data[relation_name] # type: Optional[List[Relation]] + if not isinstance(relation_list, list): + relation_list = self._data[relation_name] = [] # type: ignore + for rid in self._backend.relation_ids(relation_name): + relation = Relation(relation_name, rid, is_peer, + self._our_unit, self._backend, self._cache) + relation_list.append(relation) + return relation_list + + def _invalidate(self, relation_name: str): + """Used to wipe the cache of a given relation_name. + + Not meant to be used by Charm authors. The content of relation data is + static for the lifetime of a hook, so it is safe to cache in memory once + accessed. + """ + self._data[relation_name] = None + + def _get_unique(self, relation_name: str, relation_id: Optional[int] = None): + if relation_id is not None: + if not isinstance(relation_id, int): + raise ModelError('relation id {} must be int or None not {}'.format( + relation_id, + type(relation_id).__name__)) + for relation in self[relation_name]: + if relation.id == relation_id: + return relation + else: + # The relation may be dead, but it is not forgotten. + is_peer = relation_name in self._peers + return Relation(relation_name, relation_id, is_peer, + self._our_unit, self._backend, self._cache) + relations = self[relation_name] + num_related = len(relations) + self._backend._validate_relation_access( # pyright: reportPrivateUsage=false + relation_name, relations) + if num_related == 0: + return None + elif num_related == 1: + return self[relation_name][0] + else: + # TODO: We need something in the framework to catch and gracefully handle + # errors, ideally integrating the error catching with Juju's mechanisms. + raise TooManyRelatedAppsError(relation_name, num_related, 1) + + +class BindingMapping(Mapping[str, 'Binding']): + """Mapping of endpoints to network bindings. + + Charm authors should not instantiate this directly, but access it via + :meth:`Model.get_binding` + """ + + def __init__(self, backend: '_ModelBackend'): + self._backend = backend + self._data = {} # type: _BindingDictType + + def get(self, binding_key: Union[str, 'Relation']) -> 'Binding': + """Get a specific Binding for an endpoint/relation. + + Not used directly by Charm authors. See :meth:`Model.get_binding` + """ + if isinstance(binding_key, Relation): + binding_name = binding_key.name + relation_id = binding_key.id + elif isinstance(binding_key, str): + binding_name = binding_key + relation_id = None + else: + raise ModelError('binding key must be str or relation instance, not {}' + ''.format(type(binding_key).__name__)) + binding = self._data.get(binding_key) + if binding is None: + binding = Binding(binding_name, relation_id, self._backend) + self._data[binding_key] = binding + return binding + + # implemented to satisfy the Mapping ABC, but not meant to be used. + def __getitem__(self, item: Union[str, 'Relation']) -> 'Binding': + raise NotImplementedError() + + def __iter__(self) -> Iterable['Binding']: + raise NotImplementedError() + + def __len__(self) -> int: + raise NotImplementedError() + + +class Binding: + """Binding to a network space. + + Attributes: + name: The name of the endpoint this binding represents (eg, 'db') + """ + + def __init__(self, name: str, relation_id: Optional[int], backend: '_ModelBackend'): + self.name = name + self._relation_id = relation_id + self._backend = backend + self._network = None + + def _network_get(self, name: str, relation_id: Optional[int] = None) -> 'Network': + return Network(self._backend.network_get(name, relation_id)) + + @property + def network(self) -> 'Network': + """The network information for this binding.""" + if self._network is None: + try: + self._network = self._network_get(self.name, self._relation_id) + except RelationNotFoundError: + if self._relation_id is None: + raise + # If a relation is dead, we can still get network info associated with an + # endpoint itself + self._network = self._network_get(self.name) + return self._network + + +class Network: + """Network space details. + + Charm authors should not instantiate this directly, but should get access to the Network + definition from :meth:`Model.get_binding` and its ``network`` attribute. + + Attributes: + interfaces: A list of :class:`NetworkInterface` details. This includes the + information about how your application should be configured (eg, what + IP addresses should you bind to.) + Note that multiple addresses for a single interface are represented as multiple + interfaces. (eg, ``[NetworkInfo('ens1', '10.1.1.1/32'), + NetworkInfo('ens1', '10.1.2.1/32'])``) + ingress_addresses: A list of :class:`ipaddress.ip_address` objects representing the IP + addresses that other units should use to get in touch with you. + egress_subnets: A list of :class:`ipaddress.ip_network` representing the subnets that + other units will see you connecting from. Due to things like NAT it isn't always + possible to narrow it down to a single address, but when it is clear, the CIDRs + will be constrained to a single address. (eg, 10.0.0.1/32) + Args: + network_info: A dict of network information as returned by ``network-get``. + """ + + def __init__(self, network_info: '_NetworkDict'): + self.interfaces = [] # type: List[NetworkInterface] + # Treat multiple addresses on an interface as multiple logical + # interfaces with the same name. + for interface_info in network_info.get('bind-addresses', []): + interface_name = interface_info.get('interface-name') # type: str + addrs = interface_info.get('addresses') # type: Optional[List[_AddressDict]] + if addrs is not None: + for address_info in addrs: + self.interfaces.append(NetworkInterface(interface_name, address_info)) + self.ingress_addresses = [] # type: List[_IPAddress] + for address in network_info.get('ingress-addresses', []): + self.ingress_addresses.append(ipaddress.ip_address(address)) + self.egress_subnets = [] # type: List[_Network] + for subnet in network_info.get('egress-subnets', []): + self.egress_subnets.append(ipaddress.ip_network(subnet)) + + @property + def bind_address(self) -> Optional['_IPAddress']: + """A single address that your application should bind() to. + + For the common case where there is a single answer. This represents a single + address from :attr:`.interfaces` that can be used to configure where your + application should bind() and listen(). + """ + if self.interfaces: + return self.interfaces[0].address + else: + return None + + @property + def ingress_address(self): + """The address other applications should use to connect to your unit. + + Due to things like public/private addresses, NAT and tunneling, the address you bind() + to is not always the address other people can use to connect() to you. + This is just the first address from :attr:`.ingress_addresses`. + """ + if self.ingress_addresses: + return self.ingress_addresses[0] + else: + return None + + +class NetworkInterface: + """Represents a single network interface that the charm needs to know about. + + Charmers should not instantiate this type directly. Instead use :meth:`Model.get_binding` + to get the network information for a given endpoint. + + Attributes: + name: The name of the interface (eg. 'eth0', or 'ens1') + subnet: An :class:`ipaddress.ip_network` representation of the IP for the network + interface. This may be a single address (eg '10.0.1.2/32') + """ + + def __init__(self, name: str, address_info: '_AddressDict'): + self.name = name + # TODO: expose a hardware address here, see LP: #1864070. + + address = address_info.get('value') + if address is None: + # Compatibility with Juju <2.9: legacy address_info only had + # an 'address' field instead of 'value'. + address = address_info.get('address') + + # The value field may be empty. + address_ = ipaddress.ip_address(address) if address else None + self.address = address_ # type: Optional[_IPAddress] + cidr = address_info.get('cidr') # type: str + # The cidr field may be empty, see LP: #1864102. + if cidr: + subnet = ipaddress.ip_network(cidr) + elif address: + # If we have an address, convert it to a /32 or /128 IP network. + subnet = ipaddress.ip_network(address) + else: + subnet = None + self.subnet = subnet # type: Optional[_Network] + # TODO: expose a hostname/canonical name for the address here, see LP: #1864086. + + +class Relation: + """Represents an established relation between this application and another application. + + This class should not be instantiated directly, instead use :meth:`Model.get_relation` + or :attr:`ops.charm.RelationEvent.relation`. This is principally used by + :class:`ops.charm.RelationMeta` to represent the relationships between charms. + + Attributes: + name: The name of the local endpoint of the relation (eg 'db') + id: The identifier for a particular relation (integer) + app: An :class:`Application` representing the remote application of this relation. + For peer relations this will be the local application. + units: A set of :class:`Unit` for units that have started and joined this relation. + data: A :class:`RelationData` holding the data buckets for each entity + of a relation. Accessed via eg Relation.data[unit]['foo'] + """ + + def __init__( + self, relation_name: str, relation_id: int, is_peer: bool, our_unit: Unit, + backend: '_ModelBackend', cache: '_ModelCache'): + self.name = relation_name + self.id = relation_id + self.app = None # type: Optional[Application] + self.units = set() # type: Set[Unit] + + if is_peer: + # For peer relations, both the remote and the local app are the same. + self.app = our_unit.app + + try: + for unit_name in backend.relation_list(self.id): + unit = cache.get(Unit, unit_name) + self.units.add(unit) + if self.app is None: + # Use the app of one of the units if available. + self.app = unit.app + except RelationNotFoundError: + # If the relation is dead, just treat it as if it has no remote units. + pass + + # If we didn't get the remote app via our_unit.app or the units list, + # look it up via JUJU_REMOTE_APP or "relation-list --app". + if self.app is None: + app_name = backend.relation_remote_app_name(relation_id) + if app_name is not None: + self.app = cache.get(Application, app_name) + + self.data = RelationData(self, our_unit, backend) + + def __repr__(self): + return '<{}.{} {}:{}>'.format(type(self).__module__, + type(self).__name__, + self.name, + self.id) + + +class RelationData(Mapping['UnitOrApplication', 'RelationDataContent']): + """Represents the various data buckets of a given relation. + + Each unit and application involved in a relation has their own data bucket. + Eg: ``{entity: RelationDataContent}`` + where entity can be either a :class:`Unit` or a :class:`Application`. + + Units can read and write their own data, and if they are the leader, + they can read and write their application data. They are allowed to read + remote unit and application data. + + This class should not be created directly. It should be accessed via + :attr:`Relation.data` + """ + + def __init__(self, relation: Relation, our_unit: Unit, backend: '_ModelBackend'): + self.relation = weakref.proxy(relation) + self._data = { + our_unit: RelationDataContent(self.relation, our_unit, backend), + our_unit.app: RelationDataContent(self.relation, our_unit.app, backend), + } # type: Dict[UnitOrApplication, RelationDataContent] + self._data.update({ + unit: RelationDataContent(self.relation, unit, backend) + for unit in self.relation.units}) + # The relation might be dead so avoid a None key here. + if self.relation.app is not None: + self._data.update({ + self.relation.app: RelationDataContent(self.relation, self.relation.app, backend), + }) + + def __contains__(self, key: 'UnitOrApplication'): + return key in self._data + + def __len__(self): + return len(self._data) + + def __iter__(self): + return iter(self._data) + + def __getitem__(self, key: 'UnitOrApplication'): + return self._data[key] + + def __repr__(self): + return repr(self._data) + + +# We mix in MutableMapping here to get some convenience implementations, but whether it's actually +# mutable or not is controlled by the flag. +class RelationDataContent(LazyMapping, MutableMapping[str, str]): + """Data content of a unit or application in a relation.""" + + def __init__(self, relation: 'Relation', entity: 'UnitOrApplication', + backend: '_ModelBackend'): + self.relation = relation + self._entity = entity + self._backend = backend + self._is_app = isinstance(entity, Application) # type: bool + + def _load(self) -> '_RelationDataContent_Raw': + """Load the data from the current entity / relation.""" + try: + return self._backend.relation_get(self.relation.id, self._entity.name, self._is_app) + except RelationNotFoundError: + # Dead relations tell no tales (and have no data). + return {} + + def _is_mutable(self): + """Return if the data content can be modified.""" + if self._is_app: + is_our_app = self._backend.app_name == self._entity.name # type: bool + if not is_our_app: + return False + # Whether the application data bag is mutable or not depends on + # whether this unit is a leader or not, but this is not guaranteed + # to be always true during the same hook execution. + return self._backend.is_leader() + else: + is_our_unit = self._backend.unit_name == self._entity.name + if is_our_unit: + return True + return False + + def __setitem__(self, key: str, value: str): + if not self._is_mutable(): + raise RelationDataError('cannot set relation data for {}'.format(self._entity.name)) + if not isinstance(value, str): + raise RelationDataError('relation data values must be strings') + + self._backend.relation_set(self.relation.id, key, value, self._is_app) + + # Don't load data unnecessarily if we're only updating. + if self._lazy_data is not None: + if value == '': + # Match the behavior of Juju, which is that setting the value to an + # empty string will remove the key entirely from the relation data. + self._data.pop(key, None) + else: + self._data[key] = value + + def __delitem__(self, key: str): + # Match the behavior of Juju, which is that setting the value to an empty + # string will remove the key entirely from the relation data. + self.__setitem__(key, '') + + +class ConfigData(LazyMapping): + """Configuration data. + + This class should not be created directly. It should be accessed via :attr:`Model.config`. + """ + + def __init__(self, backend: '_ModelBackend'): + self._backend = backend + + def _load(self): + return self._backend.config_get() + + +class StatusBase: + """Status values specific to applications and units. + + To access a status by name, see :meth:`StatusBase.from_name`, most use cases will just + directly use the child class to indicate their status. + """ + + _statuses = {} # type: Dict[str, Type[StatusBase]] + + # Subclasses should override this attribute and make it a string. + name = NotImplemented + + def __init__(self, message: str = ''): + self.message = message + + def __new__(cls, *args: Any, **kwargs: Dict[Any, Any]): + """Forbid the usage of StatusBase directly.""" + if cls is StatusBase: + raise TypeError("cannot instantiate a base class") + return super().__new__(cls) + + def __eq__(self, other: 'StatusBase') -> bool: + if not isinstance(self, type(other)): + return False + return self.message == other.message + + def __repr__(self): + return "{.__class__.__name__}({!r})".format(self, self.message) + + @classmethod + def from_name(cls, name: str, message: str): + """Get the specific Status for the name (or UnknownStatus if not registered).""" + if name == 'unknown': + # unknown is special + return UnknownStatus() + else: + return cls._statuses[name](message) + + @classmethod + def register(cls, child: Type['StatusBase']): + """Register a Status for the child's name.""" + if not isinstance(getattr(child, 'name'), str): + raise TypeError("Can't register StatusBase subclass %s: " % child, + "missing required `name: str` class attribute") + cls._statuses[child.name] = child + return child + + +@StatusBase.register +class UnknownStatus(StatusBase): + """The unit status is unknown. + + A unit-agent has finished calling install, config-changed and start, but the + charm has not called status-set yet. + + """ + name = 'unknown' + + def __init__(self): + # Unknown status cannot be set and does not have a message associated with it. + super().__init__('') + + def __repr__(self): + return "UnknownStatus()" + + +@StatusBase.register +class ActiveStatus(StatusBase): + """The unit is ready. + + The unit believes it is correctly offering all the services it has been asked to offer. + """ + name = 'active' + + def __init__(self, message: str = ''): + super().__init__(message) + + +@StatusBase.register +class BlockedStatus(StatusBase): + """The unit requires manual intervention. + + An operator has to manually intervene to unblock the unit and let it proceed. + """ + name = 'blocked' + + +@StatusBase.register +class MaintenanceStatus(StatusBase): + """The unit is performing maintenance tasks. + + The unit is not yet providing services, but is actively doing work in preparation + for providing those services. This is a "spinning" state, not an error state. It + reflects activity on the unit itself, not on peers or related units. + + """ + name = 'maintenance' + + +@StatusBase.register +class WaitingStatus(StatusBase): + """A unit is unable to progress. + + The unit is unable to progress to an active state because an application to which + it is related is not running. + + """ + name = 'waiting' + + +class Resources: + """Object representing resources for the charm.""" + + def __init__(self, names: Iterable[str], backend: '_ModelBackend'): + self._backend = backend + self._paths = {name: None for name in names} # type: Dict[str, Optional[Path]] + + def fetch(self, name: str) -> Path: + """Fetch the resource from the controller or store. + + If successfully fetched, this returns a Path object to where the resource is stored + on disk, otherwise it raises a NameError. + """ + if name not in self._paths: + raise NameError('invalid resource name: {}'.format(name)) + if self._paths[name] is None: + self._paths[name] = Path(self._backend.resource_get(name)) + return typing.cast(Path, self._paths[name]) + + +class Pod: + """Represents the definition of a pod spec in Kubernetes models. + + Currently only supports simple access to setting the Juju pod spec via :attr:`.set_spec`. + """ + + def __init__(self, backend: '_ModelBackend'): + self._backend = backend + + def set_spec(self, spec: '_K8sSpec', k8s_resources: Optional['_K8sSpec'] = None): + """Set the specification for pods that Juju should start in kubernetes. + + See `juju help-tool pod-spec-set` for details of what should be passed. + + Args: + spec: The mapping defining the pod specification + k8s_resources: Additional kubernetes specific specification. + + Returns: + None + """ + if not self._backend.is_leader(): + raise ModelError('cannot set a pod spec as this unit is not a leader') + self._backend.pod_spec_set(spec, k8s_resources) + + +class StorageMapping(Mapping[str, List['Storage']]): + """Map of storage names to lists of Storage instances.""" + + def __init__(self, storage_names: Iterable[str], backend: '_ModelBackend'): + self._backend = backend + self._storage_map = {storage_name: None for storage_name in storage_names + } # type: _StorageDictType + + def __contains__(self, key: str): # pyright: reportIncompatibleMethodOverride=false + return key in self._storage_map + + def __len__(self): + return len(self._storage_map) + + def __iter__(self): + return iter(self._storage_map) + + def __getitem__(self, storage_name: str) -> List['Storage']: + storage_list = self._storage_map[storage_name] + if storage_list is None: + storage_list = self._storage_map[storage_name] = [] + for storage_index in self._backend.storage_list(storage_name): + storage = Storage(storage_name, storage_index, self._backend) + storage_list.append(storage) # type: ignore + return storage_list + + def request(self, storage_name: str, count: int = 1): + """Requests new storage instances of a given name. + + Uses storage-add tool to request additional storage. Juju will notify the unit + via -storage-attached events when it becomes available. + """ + if storage_name not in self._storage_map: + raise ModelError(('cannot add storage {!r}:' + ' it is not present in the charm metadata').format(storage_name)) + self._backend.storage_add(storage_name, count) + + def _invalidate(self, storage_name: str): + """Remove an entry from the storage map. + + Not meant to be used by charm authors -- this exists mainly for testing purposes. + """ + self._storage_map[storage_name] = None + + +class Storage: + """Represents a storage as defined in metadata.yaml. + + Attributes: + name: Simple string name of the storage + id: The index number for storage + """ + + def __init__(self, storage_name: str, storage_index: int, backend: '_ModelBackend'): + self.name = storage_name + self._index = storage_index + self._backend = backend + self._location = None + + @property + def index(self) -> int: + """The index associated with the storage (usually 0 for singular storage).""" + return self._index + + @property + def id(self) -> int: + """DEPRECATED (use ".index"): The index associated with the storage.""" + logger.warning("model.Storage.id is being replaced - please use model.Storage.index") + return self.index + + @property + def full_id(self) -> str: + """Returns the canonical storage name and id/index based identifier.""" + return '{}/{}'.format(self.name, self._index) + + @property + def location(self) -> Path: + """Return the location of the storage.""" + if self._location is None: + raw = self._backend.storage_get(self.full_id, "location") + self._location = Path(raw) + return self._location + + @location.setter + def location(self, location: str) -> None: + """Sets the location for use in events. + + For :class:`StorageAttachedEvent` and :class:`StorageDetachingEvent` in case + the actual details are gone from Juju by the time of a dynamic lookup. + """ + self._location = Path(location) + + +class MultiPushPullError(Exception): + """Aggregates multiple push/pull related exceptions into one.""" + + def __init__(self, message: str, errors: List[Tuple[str, Exception]]): + """Create an aggregation of several push/pull errors. + + Args: + message: error message + errors: list of errors with each represented by a tuple (,) + where source_path is the path being pushed/pulled from. + """ + self.errors = errors + self.message = message + + def __str__(self): + return '{} ({} errors): {}, ...'.format( + self.message, len(self.errors), self.errors[0][1]) + + def __repr__(self): + return 'MultiError({!r}, {} errors)'.format(self.message, len(self.errors)) + + +class Container: + """Represents a named container in a unit. + + This class should not be instantiated directly, instead use :meth:`Unit.get_container` + or :attr:`Unit.containers`. + + Attributes: + name: The name of the container from metadata.yaml (eg, 'postgres'). + """ + + def __init__(self, name: str, backend: '_ModelBackend', + pebble_client: Optional['pebble.Client'] = None): + self.name = name + + if pebble_client is None: + socket_path = '/charm/containers/{}/pebble.socket'.format(name) + pebble_client = backend.get_pebble(socket_path) + self._pebble = pebble_client # type: 'pebble.Client' + + @property + def pebble(self) -> 'pebble.Client': + """The low-level :class:`ops.pebble.Client` instance for this container.""" + return self._pebble + + def can_connect(self) -> bool: + """Report whether the Pebble API is reachable in the container. + + :meth:`can_connect` returns a bool that indicates whether the Pebble API is available at + the time the method is called. It does not guard against the Pebble API becoming + unavailable, and should be treated as a 'point in time' status only. + + If the Pebble API later fails, serious consideration should be given as to the reason for + this. + + Example:: + + container = self.unit.get_container("example") + if container.can_connect(): + try: + c.pull('/does/not/exist') + except ProtocolError, PathError: + # handle it + else: + event.defer() + """ + try: + # TODO: This call to `get_system_info` should be replaced with a call to a more + # appropriate endpoint that has stronger connotations of what constitutes a Pebble + # instance that is in fact 'ready'. + self._pebble.get_system_info() + except pebble.ConnectionError as e: + logger.debug("Pebble API is not ready; ConnectionError: %s", e.message()) + return False + except FileNotFoundError as e: + # In some cases, charm authors can attempt to hit the Pebble API before it has had the + # chance to create the UNIX socket in the shared volume. + logger.debug("Pebble API is not ready; UNIX socket not found:", str(e)) + return False + except pebble.APIError as e: + # An API error is only raised when the Pebble API returns invalid JSON, or the response + # cannot be read. Both of these are a likely indicator that something is wrong. + logger.warning("Pebble API is not ready; APIError: %s", str(e)) + return False + return True + + def autostart(self): + """Autostart all services marked as startup: enabled.""" + self._pebble.autostart_services() + + def replan(self): + """Replan all services: restart changed services and start startup-enabled services.""" + self._pebble.replan_services() + + def start(self, *service_names: str): + """Start given service(s) by name.""" + if not service_names: + raise TypeError('start expected at least 1 argument, got 0') + + # fixme: remove on pebble.exec signature fix + self._pebble.start_services(service_names) # type: ignore + + def restart(self, *service_names: str): + """Restart the given service(s) by name.""" + if not service_names: + raise TypeError('restart expected at least 1 argument, got 0') + + try: + # fixme: remove on pebble.exec signature fix + self._pebble.restart_services(service_names) # type: ignore + except pebble.APIError as e: + if e.code != 400: + raise e + # support old Pebble instances that don't support the "restart" action + stop = tuple(s.name for s in self.get_services(*service_names).values( + ) if s.is_running()) # type: Tuple[str, ...] + if stop: + # fixme: remove on pebble.exec signature fix + self._pebble.stop_services(stop) # type: ignore + # fixme: remove on pebble.exec signature fix + self._pebble.start_services(service_names) # type: ignore + + def stop(self, *service_names: str): + """Stop given service(s) by name.""" + if not service_names: + raise TypeError('stop expected at least 1 argument, got 0') + + # fixme: remove on pebble.exec signature fix + self._pebble.stop_services(service_names) # type: ignore + + def add_layer(self, label: str, layer: '_Layer', *, combine: bool = False): + """Dynamically add a new layer onto the Pebble configuration layers. + + Args: + label: Label for new layer (and label of layer to merge with if + combining). + layer: A YAML string, configuration layer dict, or pebble.Layer + object containing the Pebble layer to add. + combine: If combine is False (the default), append the new layer + as the top layer with the given label (must be unique). If + combine is True and the label already exists, the two layers + are combined into a single one considering the layer override + rules; if the layer doesn't exist, it is added as usual. + """ + # fixme: remove ignore once pebble.py is typed + self._pebble.add_layer(label, layer, combine=combine) # type: ignore + + def get_plan(self) -> 'pebble.Plan': + """Get the current effective pebble configuration.""" + return self._pebble.get_plan() + + def get_services(self, *service_names: str) -> '_ServiceInfoMapping': + """Fetch and return a mapping of status information indexed by service name. + + If no service names are specified, return status information for all + services, otherwise return information for only the given services. + """ + names = service_names or None + # fixme: remove on pebble.exec signature fix + services = self._pebble.get_services(names) # type: ignore + return ServiceInfoMapping(services) + + def get_service(self, service_name: str) -> 'pebble.ServiceInfo': + """Get status information for a single named service. + + Raises :class:`ModelError` if service_name is not found. + """ + services = self.get_services(service_name) + if not services: + raise ModelError('service {!r} not found'.format(service_name)) + if len(services) > 1: + raise RuntimeError('expected 1 service, got {}'.format(len(services))) + return services[service_name] + + def get_checks( + self, + *check_names: str, + level: Optional['pebble.CheckLevel'] = None) -> 'CheckInfoMapping': + """Fetch and return a mapping of check information indexed by check name. + + Args: + check_names: Optional check names to query for. If no check names + are specified, return checks with any name. + level: Optional check level to query for. If not specified, fetch + checks with any level. + """ + # fixme: remove on pebble.exec signature fix + checks = self._pebble.get_checks(names=check_names or None, level=level) # type: ignore + return CheckInfoMapping(checks) + + def get_check(self, check_name: str) -> 'pebble.CheckInfo': + """Get check information for a single named check. + + Raises :class:`ModelError` if check_name is not found. + """ + checks = self.get_checks(check_name) + if not checks: + raise ModelError('check {!r} not found'.format(check_name)) + if len(checks) > 1: + raise RuntimeError('expected 1 check, got {}'.format(len(checks))) + return checks[check_name] + + def pull(self, path: StrOrPath, *, + encoding: Optional[str] = 'utf-8') -> Union[BinaryIO, TextIO]: + """Read a file's content from the remote system. + + Args: + path: Path of the file to read from the remote system. + encoding: Encoding to use for decoding the file's bytes to str, + or None to specify no decoding. + + Returns: + A readable file-like object, whose read() method will return str + objects decoded according to the specified encoding, or bytes if + encoding is None. + """ + return self._pebble.pull(str(path), encoding=encoding) + + def push(self, + path: StrOrPath, + source: Union[bytes, str, BinaryIO, TextIO], + *, + encoding: str = 'utf-8', + make_dirs: Optional[bool] = False, + permissions: Optional[int] = None, + user_id: Optional[int] = None, + user: Optional[str] = None, + group_id: Optional[int] = None, + group: Optional[str] = None): + """Write content to a given file path on the remote system. + + Args: + path: Path of the file to write to on the remote system. + source: Source of data to write. This is either a concrete str or + bytes instance, or a readable file-like object. + encoding: Encoding to use for encoding source str to bytes, or + strings read from source if it is a TextIO type. Ignored if + source is bytes or BinaryIO. + make_dirs: If True, create parent directories if they don't exist. + permissions: Permissions (mode) to create file with (Pebble default + is 0o644). + user_id: User ID (UID) for file. + user: Username for file. User's UID must match user_id if both are + specified. + group_id: Group ID (GID) for file. + group: Group name for file. Group's GID must match group_id if + both are specified. + """ + self._pebble.push(str(path), source, encoding=encoding, + # fixme: remove these ignores on pebble.exec signature fix + make_dirs=make_dirs, # type: ignore + permissions=permissions, # type: ignore + user_id=user_id, user=user, # type: ignore + group_id=group_id, group=group) # type: ignore + + def list_files(self, path: StrOrPath, *, pattern: Optional[str] = None, + itself: bool = False) -> List['pebble.FileInfo']: + """Return list of directory entries from given path on remote system. + + Despite the name, this method returns a list of files *and* + directories, similar to :func:`os.listdir` or :func:`os.scandir`. + + Args: + path: Path of the directory to list, or path of the file to return + information about. + pattern: If specified, filter the list to just the files that match, + for example ``*.txt``. + itself: If path refers to a directory, return information about the + directory itself, rather than its contents. + """ + return self._pebble.list_files(str(path), + # fixme: remove on pebble.exec signature fix + pattern=pattern, itself=itself) # type: ignore + + def push_path(self, + source_path: Union[StrOrPath, Iterable[StrOrPath]], + dest_dir: StrOrPath): + """Recursively push a local path or files to the remote system. + + Only regular files and directories are copied; symbolic links, device files, etc. are + skipped. Pushing is attempted to completion even if errors occur during the process. All + errors are collected incrementally. After copying has completed, if any errors occurred, a + single MultiPushPullError is raised containing details for each error. + + Assuming the following files exist locally: + + * /foo/bar/baz.txt + * /foo/foobar.txt + * /quux.txt + + You could push the following ways:: + + # copy one file + container.push_path('/foo/foobar.txt', '/dst') + # Destination results: /dst/foobar.txt + + # copy a directory + container.push_path('/foo', '/dst') + # Destination results: /dst/foo/bar/baz.txt, /dst/foo/foobar.txt + + # copy a directory's contents + container.push_path('/foo/*', '/dst') + # Destination results: /dst/bar/baz.txt, /dst/foobar.txt + + # copy multiple files + container.push_path(['/foo/bar/baz.txt', 'quux.txt'], '/dst') + # Destination results: /dst/baz.txt, /dst/quux.txt + + # copy a file and a directory + container.push_path(['/foo/bar', '/quux.txt'], '/dst') + # Destination results: /dst/bar/baz.txt, /dst/quux.txt + + Args: + source_path: A single path or list of paths to push to the remote system. The + paths can be either a file or a directory. If source_path is a directory, the + directory base name is attached to the destination directory - i.e. the source path + directory is placed inside the destination directory. If a source path ends with a + trailing "/*" it will have its *contents* placed inside the destination directory. + dest_dir: Remote destination directory inside which the source dir/files will be + placed. This must be an absolute path. + """ + if os.name == 'nt': + raise RuntimeError('Container.push_path is not supported on Windows-based systems') + + if hasattr(source_path, '__iter__') and not isinstance(source_path, str): + source_paths = typing.cast(Iterable[StrOrPath], source_path) + else: + source_paths = typing.cast(Iterable[StrOrPath], [source_path]) + source_paths = [Path(p) for p in source_paths] + dest_dir = Path(dest_dir) + + def local_list(source_path: Path) -> List[pebble.FileInfo]: + paths = source_path.iterdir() if source_path.is_dir() else [source_path] + files = [self._build_fileinfo(source_path / f) for f in paths] + return files + + errors = [] # type: List[Tuple[str, Exception]] + for source_path in source_paths: + try: + for info in Container._list_recursive(local_list, source_path): + dstpath = self._build_destpath(info.path, source_path, dest_dir) + with open(info.path) as src: + self.push( + dstpath, + src, + make_dirs=True, + permissions=info.permissions, + user_id=info.user_id, + user=info.user, + group_id=info.group_id, + group=info.group) + except (OSError, pebble.Error) as err: + errors.append((str(source_path), err)) + if errors: + raise MultiPushPullError('failed to push one or more files', errors) + + def pull_path(self, + source_path: Union[StrOrPath, Iterable[StrOrPath]], + dest_dir: StrOrPath): + """Recursively pull a remote path or files to the local system. + + Only regular files and directories are copied; symbolic links, device files, etc. are + skipped. Pulling is attempted to completion even if errors occur during the process. All + errors are collected incrementally. After copying has completed, if any errors occurred, a + single MultiPushPullError is raised containing details for each error. + + Assuming the following files exist remotely: + + * /foo/bar/baz.txt + * /foo/foobar.txt + * /quux.txt + + You could pull the following ways:: + + # copy one file + container.pull_path('/foo/foobar.txt', '/dst') + # Destination results: /dst/foobar.txt + + # copy a directory + container.pull_path('/foo', '/dst') + # Destination results: /dst/foo/bar/baz.txt, /dst/foo/foobar.txt + + # copy a directory's contents + container.pull_path('/foo/*', '/dst') + # Destination results: /dst/bar/baz.txt, /dst/foobar.txt + + # copy multiple files + container.pull_path(['/foo/bar/baz.txt', 'quux.txt'], '/dst') + # Destination results: /dst/baz.txt, /dst/quux.txt + + # copy a file and a directory + container.pull_path(['/foo/bar', '/quux.txt'], '/dst') + # Destination results: /dst/bar/baz.txt, /dst/quux.txt + + Args: + source_path: A single path or list of paths to pull from the remote system. The + paths can be either a file or a directory but must be absolute paths. If + source_path is a directory, the directory base name is attached to the destination + directory - i.e. the source path directory is placed inside the destination + directory. If a source path ends with a trailing "/*" it will have its *contents* + placed inside the destination directory. + dest_dir: Local destination directory inside which the source dir/files will be + placed. + """ + if os.name == 'nt': + raise RuntimeError('Container.pull_path is not supported on Windows-based systems') + + if hasattr(source_path, '__iter__') and not isinstance(source_path, str): + source_paths = typing.cast(Iterable[StrOrPath], source_path) + else: + source_paths = typing.cast(Iterable[StrOrPath], [source_path]) + source_paths = [Path(p) for p in source_paths] + dest_dir = Path(dest_dir) + + errors = [] # type: List[Tuple[str, Exception]] + for source_path in source_paths: + try: + for info in Container._list_recursive(self.list_files, source_path): + dstpath = self._build_destpath(info.path, source_path, dest_dir) + dstpath.parent.mkdir(parents=True, exist_ok=True) + with self.pull(info.path, encoding=None) as src: + with dstpath.open(mode='wb') as dst: + shutil.copyfileobj(typing.cast(BinaryIO, src), dst) + except (OSError, pebble.Error) as err: + errors.append((str(source_path), err)) + if errors: + raise MultiPushPullError('failed to pull one or more files', errors) + + @staticmethod + def _build_fileinfo(path: StrOrPath) -> 'pebble.FileInfo': + """Constructs a FileInfo object by stat'ing a local path.""" + path = Path(path) + if path.is_symlink(): + ftype = pebble.FileType.SYMLINK + elif path.is_dir(): + ftype = pebble.FileType.DIRECTORY + elif path.is_file(): + ftype = pebble.FileType.FILE + else: + ftype = pebble.FileType.UNKNOWN + + import grp + import pwd + info = path.lstat() + return pebble.FileInfo( + path=str(path), + name=path.name, + type=ftype, + size=info.st_size, + permissions=stat.S_IMODE(info.st_mode), # type: ignore + last_modified=datetime.datetime.fromtimestamp(info.st_mtime), + user_id=info.st_uid, + user=pwd.getpwuid(info.st_uid).pw_name, + group_id=info.st_gid, + group=grp.getgrgid(info.st_gid).gr_name) + + @staticmethod + def _list_recursive(list_func: Callable[[Path], + Iterable['pebble.FileInfo']], + path: Path) -> Generator['pebble.FileInfo', None, None]: + """Recursively lists all files under path using the given list_func. + + Args: + list_func: Function taking 1 Path argument that returns a list of FileInfo objects + representing files residing directly inside the given path. + path: Filepath to recursively list. + """ + if path.name == '*': + # ignore trailing '/*' that we just use for determining how to build paths + # at destination + path = path.parent + + for info in list_func(path): + if info.type is pebble.FileType.DIRECTORY: + yield from Container._list_recursive(list_func, Path(info.path)) + elif info.type in (pebble.FileType.FILE, pebble.FileType.SYMLINK): + yield info + else: + logger.debug( + 'skipped unsupported file in Container.[push/pull]_path: %s', info.path) + + @staticmethod + def _build_destpath(file_path: StrOrPath, source_path: StrOrPath, dest_dir: StrOrPath) -> Path: + """Converts a source file and destination dir into a full destination filepath. + + file_path: + Full source-side path for the file being copied to dest_dir. + source_path + Source prefix under which the given file_path was found. + dest_dir + Destination directory to place file_path into. + """ + # select between the two following src+dst combos via trailing '/*' + # /src/* --> /dst/* + # /src --> /dst/src + file_path, source_path, dest_dir = Path(file_path), Path(source_path), Path(dest_dir) + prefix = str(source_path.parent) + if os.path.commonprefix([prefix, str(file_path)]) != prefix: + raise RuntimeError( + 'file "{}" does not have specified prefix "{}"'.format( + file_path, prefix)) + path_suffix = os.path.relpath(str(file_path), prefix) + return dest_dir / path_suffix + + def exists(self, path: str) -> bool: + """Return true if the path exists on the container filesystem.""" + try: + self._pebble.list_files(path, itself=True) + except pebble.APIError as err: + if err.code == 404: + return False + raise err + return True + + def isdir(self, path: str) -> bool: + """Return true if a directory exists at the given path on the container filesystem.""" + try: + files = self._pebble.list_files(path, itself=True) + except pebble.APIError as err: + if err.code == 404: + return False + raise err + return files[0].type == pebble.FileType.DIRECTORY + + def make_dir( + self, path: str, *, make_parents: bool = False, permissions: Optional[int] = None, + user_id: Optional[int] = None, user: Optional[str] = None, + group_id: Optional[int] = None, group: Optional[str] = None): + """Create a directory on the remote system with the given attributes. + + Args: + path: Path of the directory to create on the remote system. + make_parents: If True, create parent directories if they don't exist. + permissions: Permissions (mode) to create directory with (Pebble + default is 0o755). + user_id: User ID (UID) for directory. + user: Username for directory. User's UID must match user_id if + both are specified. + group_id: Group ID (GID) for directory. + group: Group name for directory. Group's GID must match group_id + if both are specified. + """ + # fixme: remove ignores on pebble.exec signature fix + self._pebble.make_dir(path, make_parents=make_parents, + permissions=permissions, # type: ignore + user_id=user_id, user=user, # type: ignore + group_id=group_id, group=group) # type: ignore + + def remove_path(self, path: str, *, recursive: bool = False): + """Remove a file or directory on the remote system. + + Args: + path: Path of the file or directory to delete from the remote system. + recursive: If True, recursively delete path and everything under it. + """ + self._pebble.remove_path(path, recursive=recursive) + + def exec( + self, + command: List[str], + *, + environment: Optional[Dict[str, str]] = None, + working_dir: Optional[str] = None, + timeout: Optional[float] = None, + user_id: Optional[int] = None, + user: Optional[str] = None, + group_id: Optional[int] = None, + group: Optional[str] = None, + stdin: Optional[Union[str, bytes, TextIO, BinaryIO]] = None, + stdout: Optional[Union[TextIO, BinaryIO]] = None, + stderr: Optional[Union[TextIO, BinaryIO]] = None, + encoding: str = 'utf-8', + combine_stderr: bool = False + ) -> 'pebble.ExecProcess': + """Execute the given command on the remote system. + + See :meth:`ops.pebble.Client.exec` for documentation of the parameters + and return value, as well as examples. + """ + return self._pebble.exec( + command, + # fixme: remove ignores on pebble.py typing fix + environment=environment, # type: ignore + working_dir=working_dir, # type: ignore + timeout=timeout, # type: ignore + user_id=user_id, # type: ignore + user=user, # type: ignore + group_id=group_id, # type: ignore + group=group, # type: ignore + stdin=stdin, # type: ignore + stdout=stdout, # type: ignore + stderr=stderr, # type: ignore + encoding=encoding, # type: ignore + combine_stderr=combine_stderr, # type: ignore + ) + + def send_signal(self, sig: Union[int, str], *service_names: str): + """Send the given signal to one or more services. + + Args: + sig: Name or number of signal to send, e.g., "SIGHUP", 1, or + signal.SIGHUP. + service_names: Name(s) of the service(s) to send the signal to. + + Raises: + pebble.APIError: If any of the services are not in the plan or are + not currently running. + """ + if not service_names: + raise TypeError('send_signal expected at least 1 service name, got 0') + + # fixme: remove ignore once pebble.send_signature signature is fixed + self._pebble.send_signal(sig, service_names) # type: ignore + + +class ContainerMapping(Mapping[str, Container]): + """Map of container names to Container objects. + + This is done as a mapping object rather than a plain dictionary so that we + can extend it later, and so it's not mutable. + """ + + def __init__(self, names: Iterable[str], backend: '_ModelBackend'): + self._containers = {name: Container(name, backend) for name in names} + + def __getitem__(self, key: str): + return self._containers[key] + + def __iter__(self): + return iter(self._containers) + + def __len__(self): + return len(self._containers) + + def __repr__(self): + return repr(self._containers) + + +class ServiceInfoMapping(Mapping[str, 'pebble.ServiceInfo']): + """Map of service names to :class:`ops.pebble.ServiceInfo` objects. + + This is done as a mapping object rather than a plain dictionary so that we + can extend it later, and so it's not mutable. + """ + + def __init__(self, services: Iterable['pebble.ServiceInfo']): + self._services = {s.name: s for s in services} + + def __getitem__(self, key: str): + return self._services[key] + + def __iter__(self): + return iter(self._services) + + def __len__(self): + return len(self._services) + + def __repr__(self): + return repr(self._services) + + +class CheckInfoMapping(Mapping[str, 'pebble.CheckInfo']): + """Map of check names to :class:`ops.pebble.CheckInfo` objects. + + This is done as a mapping object rather than a plain dictionary so that we + can extend it later, and so it's not mutable. + """ + + def __init__(self, checks: Iterable['pebble.CheckInfo']): + self._checks = {c.name: c for c in checks} + + def __getitem__(self, key: str): + return self._checks[key] + + def __iter__(self): + return iter(self._checks) + + def __len__(self): + return len(self._checks) + + def __repr__(self): + return repr(self._checks) + + +class ModelError(Exception): + """Base class for exceptions raised when interacting with the Model.""" + pass + + +class TooManyRelatedAppsError(ModelError): + """Raised by :meth:`Model.get_relation` if there is more than one related application.""" + + def __init__(self, relation_name: str, num_related: int, max_supported: int): + super().__init__('Too many remote applications on {} ({} > {})'.format( + relation_name, num_related, max_supported)) + self.relation_name = relation_name + self.num_related = num_related + self.max_supported = max_supported + + +class RelationDataError(ModelError): + """Raised by ``Relation.data[entity][key] = 'foo'`` if the data is invalid. + + This is raised if you're either trying to set a value to something that isn't a string, + or if you are trying to set a value in a bucket that you don't have access to. (eg, + another application/unit or setting your application data but you aren't the leader.) + """ + + +class RelationNotFoundError(ModelError): + """Backend error when querying juju for a given relation and that relation doesn't exist.""" + + +class InvalidStatusError(ModelError): + """Raised if trying to set an Application or Unit status to something invalid.""" + + +_ACTION_RESULT_KEY_REGEX = re.compile(r'^[a-z0-9](([a-z0-9-.]+)?[a-z0-9])?$') + + +def _format_action_result_dict(input: Dict[str, 'JsonObject'], + parent_key: Optional[str] = None, + output: Optional[Dict[str, str]] = None + ) -> Dict[str, str]: + """Turn a nested dictionary into a flattened dictionary, using '.' as a key seperator. + + This is used to allow nested dictionaries to be translated into the dotted format required by + the Juju `action-set` hook tool in order to set nested data on an action. + + Additionally, this method performs some validation on keys to ensure they only use permitted + characters. + + Example:: + + >>> test_dict = {'a': {'b': 1, 'c': 2}} + >>> _format_action_result_dict(test_dict) + {'a.b': 1, 'a.c': 2} + + Arguments: + input: The dictionary to flatten + parent_key: The string to prepend to dictionary's keys + output: The current dictionary to be returned, which may or may not yet be completely flat + + Returns: + A flattened dictionary with validated keys + + Raises: + ValueError: if the dict is passed with a mix of dotted/non-dotted keys that expand out to + result in duplicate keys. For example: {'a': {'b': 1}, 'a.b': 2}. Also raised if a dict + is passed with a key that fails to meet the format requirements. + """ + output_ = output or {} # type: Dict[str, str] + + for key, value in input.items(): + # Ensure the key is of a valid format, and raise a ValueError if not + if not _ACTION_RESULT_KEY_REGEX.match(key): + raise ValueError("key '{!r}' is invalid: must be similar to 'key', 'some-key2', or " + "'some.key'".format(key)) + + if parent_key: + key = "{}.{}".format(parent_key, key) + + if isinstance(value, MutableMapping): + value = typing.cast(Dict[str, 'JsonObject'], value) + output_ = _format_action_result_dict(value, key, output_) + elif key in output_: + raise ValueError("duplicate key detected in dictionary passed to 'action-set': {!r}" + .format(key)) + else: + output_[key] = value # type: ignore + + return output_ + + +class _ModelBackend: + """Represents the connection between the Model representation and talking to Juju. + + Charm authors should not directly interact with the ModelBackend, it is a private + implementation of Model. + """ + + LEASE_RENEWAL_PERIOD = datetime.timedelta(seconds=30) + _STORAGE_KEY_RE = re.compile( + r'.*^-s\s+\(=\s+(?P.*?)\)\s*?$', + re.MULTILINE | re.DOTALL + ) + + def __init__(self, unit_name: Optional[str] = None, + model_name: Optional[str] = None, + model_uuid: Optional[str] = None): + + # if JUJU_UNIT_NAME is not being passed nor in the env, something is wrong + unit_name_ = unit_name or os.getenv('JUJU_UNIT_NAME') + if unit_name_ is None: + raise ValueError('JUJU_UNIT_NAME not set') + self.unit_name = unit_name_ # type: str + + # we can cast to str because these envvars are guaranteed to be set + self.model_name = model_name or typing.cast(str, os.getenv('JUJU_MODEL_NAME')) # type: str + self.model_uuid = model_uuid or typing.cast(str, os.getenv('JUJU_MODEL_UUID')) # type: str + self.app_name = self.unit_name.split('/')[0] # type: str + + self._is_leader = None # type: Optional[bool] + self._leader_check_time = None + self._hook_is_running = '' + + def _run(self, *args: str, return_output: bool = False, use_json: bool = False + ) -> Union[str, 'JsonObject', None]: + kwargs = dict(stdout=PIPE, stderr=PIPE, check=True) + which_cmd = shutil.which(args[0]) + if which_cmd is None: + raise RuntimeError('command not found: {}'.format(args[0])) + args = (which_cmd,) + args[1:] + if use_json: + args += ('--format=json',) + try: + result = run(args, **kwargs) + except CalledProcessError as e: + raise ModelError(e.stderr) + if return_output: + if result.stdout is None: + return '' + else: + text = result.stdout.decode('utf8') + if use_json: + return json.loads(text) + else: + return text + + @staticmethod + def _is_relation_not_found(model_error: Exception) -> bool: + return 'relation not found' in str(model_error) + + def _validate_relation_access(self, relation_name: str, relations: Sequence['Relation']): + """Checks for relation usage inconsistent with the framework/backend state. + + This is used for catching Harness configuration errors and the production implementation + here should remain empty. + """ + pass + + def relation_ids(self, relation_name: str) -> List[int]: + relation_ids = self._run('relation-ids', relation_name, return_output=True, use_json=True) + relation_ids = typing.cast(Iterable[str], relation_ids) + return [int(relation_id.split(':')[-1]) for relation_id in relation_ids] + + def relation_list(self, relation_id: int) -> List[str]: + try: + rel_list = self._run('relation-list', '-r', str(relation_id), + return_output=True, use_json=True) + return typing.cast(List[str], rel_list) + except ModelError as e: + if self._is_relation_not_found(e): + raise RelationNotFoundError() from e + raise + + def relation_remote_app_name(self, relation_id: int) -> Optional[str]: + """Return remote app name for given relation ID, or None if not known.""" + if 'JUJU_RELATION_ID' in os.environ and 'JUJU_REMOTE_APP' in os.environ: + event_relation_id = int(os.environ['JUJU_RELATION_ID'].split(':')[-1]) + if relation_id == event_relation_id: + # JUJU_RELATION_ID is this relation, use JUJU_REMOTE_APP. + return os.environ['JUJU_REMOTE_APP'] + + # If caller is asking for information about another relation, use + # "relation-list --app" to get it. + try: + rel_id = self._run('relation-list', '-r', str(relation_id), '--app', + return_output=True, use_json=True) + # if it returned anything at all, it's a str. + return typing.cast(str, rel_id) + + except ModelError as e: + if self._is_relation_not_found(e): + return None + if 'option provided but not defined: --app' in str(e): + # "--app" was introduced to relation-list in Juju 2.8.1, so + # handle previous versions of Juju gracefully + return None + raise + + def relation_get(self, relation_id: int, member_name: str, is_app: bool + ) -> '_RelationDataContent_Raw': + if not isinstance(is_app, bool): # pyright: + raise TypeError('is_app parameter to relation_get must be a boolean') + + if is_app: + version = JujuVersion.from_environ() + if not version.has_app_data(): + raise RuntimeError( + 'getting application data is not supported on Juju version {}'.format(version)) + + args = ['relation-get', '-r', str(relation_id), '-', member_name] + if is_app: + args.append('--app') + + try: + raw_data_content = self._run(*args, return_output=True, use_json=True) + return typing.cast('_RelationDataContent_Raw', raw_data_content) + except ModelError as e: + if self._is_relation_not_found(e): + raise RelationNotFoundError() from e + raise + + def relation_set(self, relation_id: int, key: str, value: str, is_app: bool): + if not isinstance(is_app, bool): + raise TypeError('is_app parameter to relation_set must be a boolean') + + if is_app: + version = JujuVersion.from_environ() + if not version.has_app_data(): + raise RuntimeError( + 'setting application data is not supported on Juju version {}'.format(version)) + + args = ['relation-set', '-r', str(relation_id), '{}={}'.format(key, value)] + if is_app: + args.append('--app') + + try: + return self._run(*args) + except ModelError as e: + if self._is_relation_not_found(e): + raise RelationNotFoundError() from e + raise + + def config_get(self): + return self._run('config-get', return_output=True, use_json=True) + + def is_leader(self) -> bool: + """Obtain the current leadership status for the unit the charm code is executing on. + + The value is cached for the duration of a lease which is 30s in Juju. + """ + now = time.monotonic() + if self._leader_check_time is None: + check = True + else: + time_since_check = datetime.timedelta(seconds=now - self._leader_check_time) + check = (time_since_check > self.LEASE_RENEWAL_PERIOD or self._is_leader is None) + if check: + # Current time MUST be saved before running is-leader to ensure the cache + # is only used inside the window that is-leader itself asserts. + self._leader_check_time = now + is_leader = self._run('is-leader', return_output=True, use_json=True) + self._is_leader = typing.cast(bool, is_leader) + + # we can cast to bool now since if we're here it means we checked. + return typing.cast(bool, self._is_leader) + + def resource_get(self, resource_name: str) -> str: + out = self._run('resource-get', resource_name, return_output=True) + return typing.cast(str, out).strip() + + def pod_spec_set(self, spec: Mapping[str, 'JsonObject'], + k8s_resources: Optional[Mapping[str, 'JsonObject']] = None): + tmpdir = Path(tempfile.mkdtemp('-pod-spec-set')) + try: + spec_path = tmpdir / 'spec.yaml' + with spec_path.open("wt", encoding="utf8") as f: + yaml.safe_dump(spec, stream=f) # type: ignore + args = ['--file', str(spec_path)] + if k8s_resources: + k8s_res_path = tmpdir / 'k8s-resources.yaml' + with k8s_res_path.open("wt", encoding="utf8") as f: + yaml.safe_dump(k8s_resources, stream=f) # type: ignore + args.extend(['--k8s-resources', str(k8s_res_path)]) + self._run('pod-spec-set', *args) + finally: + shutil.rmtree(str(tmpdir)) + + def status_get(self, *, is_app: bool = False) -> '_StatusDict': + """Get a status of a unit or an application. + + Args: + is_app: A boolean indicating whether the status should be retrieved for a unit + or an application. + """ + content = self._run( + 'status-get', '--include-data', '--application={}'.format(is_app), + use_json=True, + return_output=True) + # Unit status looks like (in YAML): + # message: 'load: 0.28 0.26 0.26' + # status: active + # status-data: {} + # Application status looks like (in YAML): + # application-status: + # message: 'load: 0.28 0.26 0.26' + # status: active + # status-data: {} + # units: + # uo/0: + # message: 'load: 0.28 0.26 0.26' + # status: active + # status-data: {} + + if is_app: + content = typing.cast(Dict[str, Dict[str, str]], content) + app_status = content['application-status'] + return {'status': app_status['status'], + 'message': app_status['message']} + else: + return typing.cast('_StatusDict', content) + + def status_set(self, status: str, message: str = '', *, is_app: bool = False): + """Set a status of a unit or an application. + + Args: + status: The status to set. + message: The message to set in the status. + is_app: A boolean indicating whether the status should be set for a unit or an + application. + """ + if not isinstance(is_app, bool): + raise TypeError('is_app parameter must be boolean') + return self._run('status-set', '--application={}'.format(is_app), status, message) + + def storage_list(self, name: str): + storages = self._run('storage-list', name, return_output=True, use_json=True) + storages = typing.cast(List[str], storages) + return [int(s.split('/')[1]) for s in storages] + + def _storage_event_details(self) -> Tuple[int, str]: + output = self._run('storage-get', '--help', return_output=True) + output = typing.cast(str, output) + # Match the entire string at once instead of going line by line + match = self._STORAGE_KEY_RE.match(output) + if match is None: + raise RuntimeError('unable to find storage key in {output!r}'.format(output=output)) + key = match.groupdict()["storage_key"] + + id = int(key.split("/")[1]) + location = self.storage_get(key, "location") + return id, location + + def storage_get(self, storage_name_id: str, attribute: str) -> str: + out = self._run('storage-get', '-s', storage_name_id, attribute, + return_output=True, use_json=True) + return typing.cast(str, out) + + def storage_add(self, name: str, count: int = 1): + if not isinstance(count, int) or isinstance(count, bool): + raise TypeError('storage count must be integer, got: {} ({})'.format(count, + type(count))) + self._run('storage-add', '{}={}'.format(name, count)) + + def action_get(self): + return self._run('action-get', return_output=True, use_json=True) + + def action_set(self, results: Dict[str, 'JsonObject']): + # The Juju action-set hook tool cannot interpret nested dicts, so we use a helper to + # flatten out any nested dict structures into a dotted notation, and validate keys. + flat_results = _format_action_result_dict(results) + self._run('action-set', *["{}={}".format(k, v) for k, v in flat_results.items()]) + + def action_log(self, message: str): + self._run('action-log', message) + + def action_fail(self, message: str = ''): + self._run('action-fail', message) + + def application_version_set(self, version: str): + self._run('application-version-set', '--', version) + + @classmethod + def log_split(cls, message: str, max_len: int = MAX_LOG_LINE_LEN): + """Helper to handle log messages that are potentially too long. + + This is a generator that splits a message string into multiple chunks if it is too long + to safely pass to bash. Will only generate a single entry if the line is not too long. + """ + if len(message) > max_len: + yield "Log string greater than {}. Splitting into multiple chunks: ".format(max_len) + + while message: + yield message[:max_len] + message = message[max_len:] + + def juju_log(self, level: str, message: str): + """Pass a log message on to the juju logger.""" + for line in self.log_split(message): + self._run('juju-log', '--log-level', level, "--", line) + + def network_get(self, binding_name: str, relation_id: Optional[int] = None) -> '_NetworkDict': + """Return network info provided by network-get for a given binding. + + Args: + binding_name: A name of a binding (relation name or extra-binding name). + relation_id: An optional relation id to get network info for. + """ + cmd = ['network-get', binding_name] + if relation_id is not None: + cmd.extend(['-r', str(relation_id)]) + try: + network = self._run(*cmd, return_output=True, use_json=True) + return typing.cast('_NetworkDict', network) + except ModelError as e: + if self._is_relation_not_found(e): + raise RelationNotFoundError() from e + raise + + def add_metrics(self, metrics: Mapping[str, 'Numerical'], + labels: Optional[Mapping[str, str]] = None): + cmd = ['add-metric'] # type: List[str] + if labels: + label_args = [] # type: List[str] + for k, v in labels.items(): + _ModelBackendValidator.validate_metric_label(k) + _ModelBackendValidator.validate_label_value(k, v) + label_args.append('{}={}'.format(k, v)) + cmd.extend(['--labels', ','.join(label_args)]) + + metric_args = [] # type: List[str] + for k, v in metrics.items(): + _ModelBackendValidator.validate_metric_key(k) + metric_value = _ModelBackendValidator.format_metric_value(v) + metric_args.append('{}={}'.format(k, metric_value)) + cmd.extend(metric_args) + self._run(*cmd) + + def get_pebble(self, socket_path: str) -> 'pebble.Client': + """Create a pebble.Client instance from given socket path.""" + return pebble.Client(socket_path=socket_path) + + def planned_units(self) -> int: + """Count of "planned" units that will run this application. + + Includes the current unit in the count. + + """ + # The goal-state tool will return the information that we need. Goal state as a general + # concept is being deprecated, however, in favor of approaches such as the one that we use + # here. + app_state = self._run('goal-state', return_output=True, use_json=True) + app_state = typing.cast(Dict[str, List[str]], app_state) + # Planned units can be zero. We don't need to do error checking here. + return len(app_state.get('units', [])) + + +class _ModelBackendValidator: + """Provides facilities for validating inputs and formatting them for model backends.""" + + METRIC_KEY_REGEX = re.compile(r'^[a-zA-Z](?:[a-zA-Z0-9-_]*[a-zA-Z0-9])?$') + + @classmethod + def validate_metric_key(cls, key: str): + if cls.METRIC_KEY_REGEX.match(key) is None: + raise ModelError( + 'invalid metric key {!r}: must match {}'.format( + key, cls.METRIC_KEY_REGEX.pattern)) + + @classmethod + def validate_metric_label(cls, label_name: str): + if cls.METRIC_KEY_REGEX.match(label_name) is None: + raise ModelError( + 'invalid metric label name {!r}: must match {}'.format( + label_name, cls.METRIC_KEY_REGEX.pattern)) + + @classmethod + def format_metric_value(cls, value: 'Numerical'): + if not isinstance(value, (int, float)): # pyright: reportUnnecessaryIsInstance=false + raise ModelError('invalid metric value {!r} provided:' + ' must be a positive finite float'.format(value)) + + if math.isnan(value) or math.isinf(value) or value < 0: + raise ModelError('invalid metric value {!r} provided:' + ' must be a positive finite float'.format(value)) + return str(value) + + @classmethod + def validate_label_value(cls, label: str, value: str): + # Label values cannot be empty, contain commas or equal signs as those are + # used by add-metric as separators. + if not value: + raise ModelError( + 'metric label {} has an empty value, which is not allowed'.format(label)) + v = str(value) + if re.search('[,=]', v) is not None: + raise ModelError( + 'metric label values must not contain "," or "=": {}={!r}'.format(label, value)) diff --git a/ubuntu/venv/ops/pebble.py b/ubuntu/venv/ops/pebble.py new file mode 100644 index 0000000..f1fb06b --- /dev/null +++ b/ubuntu/venv/ops/pebble.py @@ -0,0 +1,2416 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Client for the Pebble API (HTTP over Unix socket). + +For a command-line interface for local testing, see test/pebble_cli.py. +""" + +import binascii +import cgi +import copy +import datetime +import email.parser +import enum +import http.client +import io +import json +import logging +import os +import re +import select +import shutil +import signal +import socket +import sys +import tempfile +import threading +import time +import types +import typing +import urllib.error +import urllib.parse +import urllib.request +import warnings + +from ops._private import yaml +from ops._vendor import websocket + +logger = logging.getLogger(__name__) + +_not_provided = object() + + +class _UnixSocketConnection(http.client.HTTPConnection): + """Implementation of HTTPConnection that connects to a named Unix socket.""" + + def __init__(self, host, timeout=_not_provided, socket_path=None): + if timeout is _not_provided: + super().__init__(host) + else: + super().__init__(host, timeout=timeout) + self.socket_path = socket_path + + def connect(self): + """Override connect to use Unix socket (instead of TCP socket).""" + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError('Unix sockets not supported on {}'.format(sys.platform)) + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.connect(self.socket_path) + if self.timeout is not _not_provided: + self.sock.settimeout(self.timeout) + + +class _UnixSocketHandler(urllib.request.AbstractHTTPHandler): + """Implementation of HTTPHandler that uses a named Unix socket.""" + + def __init__(self, socket_path): + super().__init__() + self.socket_path = socket_path + + def http_open(self, req): + """Override http_open to use a Unix socket connection (instead of TCP).""" + return self.do_open(_UnixSocketConnection, req, socket_path=self.socket_path) + + +# Matches yyyy-mm-ddTHH:MM:SS(.sss)ZZZ +_TIMESTAMP_RE = re.compile( + r'(\d{4})-(\d{2})-(\d{2})[Tt](\d{2}):(\d{2}):(\d{2})(\.\d+)?(.*)') + +# Matches [-+]HH:MM +_TIMEOFFSET_RE = re.compile(r'([-+])(\d{2}):(\d{2})') + + +def _parse_timestamp(s): + """Parse timestamp from Go-encoded JSON. + + This parses RFC3339 timestamps (which are a subset of ISO8601 timestamps) + that Go's encoding/json package produces for time.Time values. + + Unfortunately we can't use datetime.fromisoformat(), as that does not + support more than 6 digits for the fractional second, nor the 'Z' for UTC. + Also, it was only introduced in Python 3.7. + """ + match = _TIMESTAMP_RE.match(s) + if not match: + raise ValueError('invalid timestamp {!r}'.format(s)) + y, m, d, hh, mm, ss, sfrac, zone = match.groups() + + if zone in ('Z', 'z'): + tz = datetime.timezone.utc + else: + match = _TIMEOFFSET_RE.match(zone) + if not match: + raise ValueError('invalid timestamp {!r}'.format(s)) + sign, zh, zm = match.groups() + tz_delta = datetime.timedelta(hours=int(zh), minutes=int(zm)) + tz = datetime.timezone(tz_delta if sign == '+' else -tz_delta) + + microsecond = round(float(sfrac or '0') * 1000000) + + return datetime.datetime(int(y), int(m), int(d), int(hh), int(mm), int(ss), + microsecond=microsecond, tzinfo=tz) + + +def _format_timeout(timeout: float): + """Format timeout for use in the Pebble API. + + The format is in seconds with a millisecond resolution and an 's' suffix, + as accepted by the Pebble API (which uses Go's time.ParseDuration). + """ + return '{:.3f}s'.format(timeout) + + +def _json_loads(s: typing.Union[str, bytes]) -> typing.Dict: + """Like json.loads(), but handle str or bytes. + + This is needed because an HTTP response's read() method returns bytes on + Python 3.5, and json.load doesn't handle bytes. + """ + if isinstance(s, bytes): + s = s.decode('utf-8') + return json.loads(s) + + +def _start_thread(target, *args, **kwargs) -> threading.Thread: + """Helper to simplify starting a thread.""" + thread = threading.Thread(target=target, args=args, kwargs=kwargs) + thread.start() + return thread + + +class Error(Exception): + """Base class of most errors raised by the Pebble client.""" + + def __repr__(self): + return '<{}.{} {}>'.format(type(self).__module__, type(self).__name__, self.args) + + def name(self): + """Return a string representation of the model plus class.""" + return '<{}.{}>'.format(type(self).__module__, type(self).__name__) + + def message(self): + """Return the message passed as an argument.""" + return self.args[0] + + +class TimeoutError(TimeoutError, Error): + """Raised when a polling timeout occurs.""" + + +class ConnectionError(Error): + """Raised when the Pebble client can't connect to the socket.""" + + +class ProtocolError(Error): + """Raised when there's a higher-level protocol error talking to Pebble.""" + + +class PathError(Error): + """Raised when there's an error with a specific path.""" + + def __init__(self, kind: str, message: str): + """This shouldn't be instantiated directly.""" + self.kind = kind + self.message = message + + def __str__(self): + return '{} - {}'.format(self.kind, self.message) + + def __repr__(self): + return 'PathError({!r}, {!r})'.format(self.kind, self.message) + + +class APIError(Error): + """Raised when an HTTP API error occurs talking to the Pebble server.""" + + def __init__(self, body: typing.Dict, code: int, status: str, message: str): + """This shouldn't be instantiated directly.""" + super().__init__(message) # Makes str(e) return message + self.body = body + self.code = code + self.status = status + self.message = message + + def __repr__(self): + return 'APIError({!r}, {!r}, {!r}, {!r})'.format( + self.body, self.code, self.status, self.message) + + +class ChangeError(Error): + """Raised by actions when a change is ready but has an error. + + For example, this happens when you attempt to start an already-started + service: + + cannot perform the following tasks: + - Start service "test" (service "test" was previously started) + """ + + def __init__(self, err: str, change: 'Change'): + """This shouldn't be instantiated directly.""" + self.err = err + self.change = change + + def __str__(self): + parts = [self.err] + + # Append any task logs to the error message + for i, task in enumerate(self.change.tasks): + if not task.log: + continue + parts.append('\n----- Logs from task {} -----\n'.format(i)) + parts.append('\n'.join(task.log)) + + if len(parts) > 1: + parts.append('\n-----') + + return ''.join(parts) + + def __repr__(self): + return 'ChangeError({!r}, {!r})'.format(self.err, self.change) + + +class ExecError(Error): + """Raised when a :meth:`Client.exec` command returns a non-zero exit code. + + Attributes: + command: Command line of command being executed. + exit_code: The process's exit code. This will always be non-zero. + stdout: If :meth:`ExecProcess.wait_output` was being called, this is + the captured stdout as a str (or bytes if encoding was None). If + :meth:`ExecProcess.wait` was being called, this is None. + stderr: If :meth:`ExecProcess.wait_output` was being called and + combine_stderr was False, this is the captured stderr as a str (or + bytes if encoding was None). If :meth:`ExecProcess.wait` was being + called or combine_stderr was True, this is None. + """ + + STR_MAX_OUTPUT = 1024 + + def __init__( + self, + command: typing.List[str], + exit_code: int, + stdout: typing.Optional[typing.AnyStr], + stderr: typing.Optional[typing.AnyStr], + ): + self.command = command + self.exit_code = exit_code + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + message = 'non-zero exit code {} executing {!r}'.format( + self.exit_code, self.command) + + for name, out in [('stdout', self.stdout), ('stderr', self.stderr)]: + if out is None: + continue + truncated = ' [truncated]' if len(out) > self.STR_MAX_OUTPUT else '' + out = out[:self.STR_MAX_OUTPUT] + message = '{}, {}={!r}{}'.format(message, name, out, truncated) + + return message + + +class WarningState(enum.Enum): + """Enum of states for get_warnings() select parameter.""" + + ALL = 'all' + PENDING = 'pending' + + +class ChangeState(enum.Enum): + """Enum of states for get_changes() select parameter.""" + + ALL = 'all' + IN_PROGRESS = 'in-progress' + READY = 'ready' + + +class SystemInfo: + """System information object.""" + + def __init__(self, version: str): + self.version = version + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'SystemInfo': + """Create new SystemInfo object from dict parsed from JSON.""" + return cls(version=d['version']) + + def __repr__(self): + return 'SystemInfo(version={self.version!r})'.format(self=self) + + +class Warning: + """Warning object.""" + + def __init__( + self, + message: str, + first_added: datetime.datetime, + last_added: datetime.datetime, + last_shown: typing.Optional[datetime.datetime], + expire_after: str, + repeat_after: str, + ): + self.message = message + self.first_added = first_added + self.last_added = last_added + self.last_shown = last_shown + self.expire_after = expire_after + self.repeat_after = repeat_after + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'Warning': + """Create new Warning object from dict parsed from JSON.""" + return cls( + message=d['message'], + first_added=_parse_timestamp(d['first-added']), + last_added=_parse_timestamp(d['last-added']), + last_shown=_parse_timestamp(d['last-shown']) if d.get('last-shown') else None, + expire_after=d['expire-after'], + repeat_after=d['repeat-after'], + ) + + def __repr__(self): + return ('Warning(' + 'message={self.message!r}, ' + 'first_added={self.first_added!r}, ' + 'last_added={self.last_added!r}, ' + 'last_shown={self.last_shown!r}, ' + 'expire_after={self.expire_after!r}, ' + 'repeat_after={self.repeat_after!r})' + ).format(self=self) + + +class TaskProgress: + """Task progress object.""" + + def __init__( + self, + label: str, + done: int, + total: int, + ): + self.label = label + self.done = done + self.total = total + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'TaskProgress': + """Create new TaskProgress object from dict parsed from JSON.""" + return cls( + label=d['label'], + done=d['done'], + total=d['total'], + ) + + def __repr__(self): + return ('TaskProgress(' + 'label={self.label!r}, ' + 'done={self.done!r}, ' + 'total={self.total!r})' + ).format(self=self) + + +class TaskID(str): + """Task ID (a more strongly-typed string).""" + + def __repr__(self): + return 'TaskID({!r})'.format(str(self)) + + +class Task: + """Task object.""" + + def __init__( + self, + id: TaskID, + kind: str, + summary: str, + status: str, + log: typing.List[str], + progress: TaskProgress, + spawn_time: datetime.datetime, + ready_time: typing.Optional[datetime.datetime], + data: typing.Dict[str, typing.Any] = None, + ): + self.id = id + self.kind = kind + self.summary = summary + self.status = status + self.log = log + self.progress = progress + self.spawn_time = spawn_time + self.ready_time = ready_time + self.data = data or {} + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'Task': + """Create new Task object from dict parsed from JSON.""" + return cls( + id=TaskID(d['id']), + kind=d['kind'], + summary=d['summary'], + status=d['status'], + log=d.get('log') or [], + progress=TaskProgress.from_dict(d['progress']), + spawn_time=_parse_timestamp(d['spawn-time']), + ready_time=_parse_timestamp(d['ready-time']) if d.get('ready-time') else None, + data=d.get('data') or {}, + ) + + def __repr__(self): + return ('Task(' + 'id={self.id!r}, ' + 'kind={self.kind!r}, ' + 'summary={self.summary!r}, ' + 'status={self.status!r}, ' + 'log={self.log!r}, ' + 'progress={self.progress!r}, ' + 'spawn_time={self.spawn_time!r}, ' + 'ready_time={self.ready_time!r}, ' + 'data={self.data!r})' + ).format(self=self) + + +class ChangeID(str): + """Change ID (a more strongly-typed string).""" + + def __repr__(self): + return 'ChangeID({!r})'.format(str(self)) + + +class Change: + """Change object.""" + + def __init__( + self, + id: ChangeID, + kind: str, + summary: str, + status: str, + tasks: typing.List[Task], + ready: bool, + err: typing.Optional[str], + spawn_time: datetime.datetime, + ready_time: typing.Optional[datetime.datetime], + data: typing.Dict[str, typing.Any] = None, + ): + self.id = id + self.kind = kind + self.summary = summary + self.status = status + self.tasks = tasks + self.ready = ready + self.err = err + self.spawn_time = spawn_time + self.ready_time = ready_time + self.data = data or {} + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'Change': + """Create new Change object from dict parsed from JSON.""" + return cls( + id=ChangeID(d['id']), + kind=d['kind'], + summary=d['summary'], + status=d['status'], + tasks=[Task.from_dict(t) for t in d.get('tasks') or []], + ready=d['ready'], + err=d.get('err'), + spawn_time=_parse_timestamp(d['spawn-time']), + ready_time=_parse_timestamp(d['ready-time']) if d.get('ready-time') else None, + data=d.get('data') or {}, + ) + + def __repr__(self): + return ('Change(' + 'id={self.id!r}, ' + 'kind={self.kind!r}, ' + 'summary={self.summary!r}, ' + 'status={self.status!r}, ' + 'tasks={self.tasks!r}, ' + 'ready={self.ready!r}, ' + 'err={self.err!r}, ' + 'spawn_time={self.spawn_time!r}, ' + 'ready_time={self.ready_time!r}, ' + 'data={self.data!r})' + ).format(self=self) + + +class Plan: + """Represents the effective Pebble configuration. + + A plan is the combined layer configuration. The layer configuration is + documented at https://github.com/canonical/pebble/#layer-specification. + """ + + def __init__(self, raw: str): + d = yaml.safe_load(raw) or {} + self._raw = raw + self._services = {name: Service(name, service) + for name, service in d.get('services', {}).items()} + self._checks = {name: Check(name, check) + for name, check in d.get('checks', {}).items()} + + @property + def services(self): + """This plan's services mapping (maps service name to Service). + + This property is currently read-only. + """ + return self._services + + @property + def checks(self): + """This plan's checks mapping (maps check name to :class:`Check`). + + This property is currently read-only. + """ + return self._checks + + def to_dict(self) -> typing.Dict[str, typing.Any]: + """Convert this plan to its dict representation.""" + fields = [ + ('services', {name: service.to_dict() for name, service in self._services.items()}), + ('checks', {name: check.to_dict() for name, check in self._checks.items()}), + ] + return {name: value for name, value in fields if value} + + def to_yaml(self) -> str: + """Return this plan's YAML representation.""" + return yaml.safe_dump(self.to_dict()) + + __str__ = to_yaml + + +class Layer: + """Represents a Pebble configuration layer. + + The format of this is documented at + https://github.com/canonical/pebble/#layer-specification. + + Attributes: + summary: A summary of the purpose of this layer + description: A long form description of this layer + services: A mapping of name to :class:`Service` defined by this layer + checks: A mapping of check to :class:`Check` defined by this layer + """ + + # This is how you do type annotations, but it is not supported by Python 3.5 + # summary: str + # description: str + # services: typing.Mapping[str, 'Service'] + + def __init__(self, raw: typing.Union[str, typing.Dict] = None): + if isinstance(raw, str): + d = yaml.safe_load(raw) or {} + else: + d = raw or {} + self.summary = d.get('summary', '') + self.description = d.get('description', '') + self.services = {name: Service(name, service) + for name, service in d.get('services', {}).items()} + self.checks = {name: Check(name, check) + for name, check in d.get('checks', {}).items()} + + def to_yaml(self) -> str: + """Convert this layer to its YAML representation.""" + return yaml.safe_dump(self.to_dict()) + + def to_dict(self) -> typing.Dict[str, typing.Any]: + """Convert this layer to its dict representation.""" + fields = [ + ('summary', self.summary), + ('description', self.description), + ('services', {name: service.to_dict() for name, service in self.services.items()}), + ('checks', {name: check.to_dict() for name, check in self.checks.items()}), + ] + return {name: value for name, value in fields if value} + + def __repr__(self) -> str: + return 'Layer({!r})'.format(self.to_dict()) + + __str__ = to_yaml + + +class Service: + """Represents a service description in a Pebble configuration layer.""" + + def __init__(self, name: str, raw: typing.Dict = None): + self.name = name + raw = raw or {} + self.summary = raw.get('summary', '') + self.description = raw.get('description', '') + self.startup = raw.get('startup', '') + self.override = raw.get('override', '') + self.command = raw.get('command', '') + self.after = list(raw.get('after', [])) + self.before = list(raw.get('before', [])) + self.requires = list(raw.get('requires', [])) + self.environment = dict(raw.get('environment', {})) + self.user = raw.get('user', '') + self.user_id = raw.get('user-id') + self.group = raw.get('group', '') + self.group_id = raw.get('group-id') + self.on_success = raw.get('on-success', '') + self.on_failure = raw.get('on-failure', '') + self.on_check_failure = dict(raw.get('on-check-failure', {})) + self.backoff_delay = raw.get('backoff-delay', '') + self.backoff_factor = raw.get('backoff-factor') + self.backoff_limit = raw.get('backoff-limit', '') + + def to_dict(self) -> typing.Dict[str, typing.Any]: + """Convert this service object to its dict representation.""" + fields = [ + ('summary', self.summary), + ('description', self.description), + ('startup', self.startup), + ('override', self.override), + ('command', self.command), + ('after', self.after), + ('before', self.before), + ('requires', self.requires), + ('environment', self.environment), + ('user', self.user), + ('user-id', self.user_id), + ('group', self.group), + ('group-id', self.group_id), + ('on-success', self.on_success), + ('on-failure', self.on_failure), + ('on-check-failure', self.on_check_failure), + ('backoff-delay', self.backoff_delay), + ('backoff-factor', self.backoff_factor), + ('backoff-limit', self.backoff_limit), + ] + return {name: value for name, value in fields if value} + + def _merge(self, other: 'Service'): + """Merges this service object with another service definition. + + For attributes present in both objects, the passed in service + attributes take precedence. + """ + for name, value in other.__dict__.items(): + if not value or name == 'name': + continue + if name in ['after', 'before', 'requires']: + getattr(self, name).extend(value) + elif name in ['environment', 'on_check_failure']: + getattr(self, name).update(value) + else: + setattr(self, name, value) + + def __repr__(self) -> str: + return 'Service({!r})'.format(self.to_dict()) + + def __eq__(self, other: typing.Union[typing.Dict, 'Service']) -> bool: + """Compare this service description to another.""" + if isinstance(other, dict): + return self.to_dict() == other + elif isinstance(other, Service): + return self.to_dict() == other.to_dict() + else: + raise ValueError( + "Cannot compare pebble.Service to {}".format(type(other)) + ) + + +class ServiceStartup(enum.Enum): + """Enum of service startup options.""" + + ENABLED = 'enabled' + DISABLED = 'disabled' + + +class ServiceStatus(enum.Enum): + """Enum of service statuses.""" + + ACTIVE = 'active' + INACTIVE = 'inactive' + ERROR = 'error' + + +class ServiceInfo: + """Service status information.""" + + def __init__( + self, + name: str, + startup: typing.Union[ServiceStartup, str], + current: typing.Union[ServiceStatus, str], + ): + self.name = name + self.startup = startup + self.current = current + + def is_running(self) -> bool: + """Return True if this service is running (in the active state).""" + return self.current == ServiceStatus.ACTIVE + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'ServiceInfo': + """Create new ServiceInfo object from dict parsed from JSON.""" + try: + startup = ServiceStartup(d['startup']) + except ValueError: + startup = d['startup'] + try: + current = ServiceStatus(d['current']) + except ValueError: + current = d['current'] + return cls( + name=d['name'], + startup=startup, + current=current, + ) + + def __repr__(self): + return ('ServiceInfo(' + 'name={self.name!r}, ' + 'startup={self.startup}, ' + 'current={self.current})' + ).format(self=self) + + +class Check: + """Represents a check in a Pebble configuration layer.""" + + def __init__(self, name: str, raw: typing.Dict = None): + self.name = name + raw = raw or {} + self.override = raw.get('override', '') + try: + self.level = CheckLevel(raw.get('level', '')) + except ValueError: + self.level = raw.get('level') + self.period = raw.get('period', '') + self.timeout = raw.get('timeout', '') + self.threshold = raw.get('threshold') + + http = raw.get('http') + if http is not None: + http = copy.deepcopy(http) + self.http = http + + tcp = raw.get('tcp') + if tcp is not None: + tcp = copy.deepcopy(tcp) + self.tcp = tcp + + exec_ = raw.get('exec') + if exec_ is not None: + exec_ = copy.deepcopy(exec_) + self.exec = exec_ + + def to_dict(self) -> typing.Dict[str, typing.Any]: + """Convert this check object to its dict representation.""" + fields = [ + ('override', self.override), + ('level', self.level.value), + ('period', self.period), + ('timeout', self.timeout), + ('threshold', self.threshold), + ('http', self.http), + ('tcp', self.tcp), + ('exec', self.exec), + ] + return {name: value for name, value in fields if value} + + def __repr__(self) -> str: + return 'Check({!r})'.format(self.to_dict()) + + def __eq__(self, other: typing.Union[typing.Dict, 'Check']) -> bool: + """Compare this check configuration to another.""" + if isinstance(other, dict): + return self.to_dict() == other + elif isinstance(other, Check): + return self.to_dict() == other.to_dict() + else: + raise ValueError("Cannot compare pebble.Check to {}".format(type(other))) + + +class CheckLevel(enum.Enum): + """Enum of check levels.""" + + UNSET = '' + ALIVE = 'alive' + READY = 'ready' + + +class CheckStatus(enum.Enum): + """Enum of check statuses.""" + + UP = 'up' + DOWN = 'down' + + +class FileType(enum.Enum): + """Enum of file types.""" + + FILE = 'file' + DIRECTORY = 'directory' + SYMLINK = 'symlink' + SOCKET = 'socket' + NAMED_PIPE = 'named-pipe' + DEVICE = 'device' + UNKNOWN = 'unknown' + + +class FileInfo: + """Stat-like information about a single file or directory.""" + + def __init__( + self, + path: str, + name: str, + type: typing.Union['FileType', str], + size: typing.Optional[int], + permissions: int, + last_modified: datetime.datetime, + user_id: typing.Optional[int], + user: typing.Optional[str], + group_id: typing.Optional[int], + group: typing.Optional[str], + ): + self.path = path + self.name = name + self.type = type + self.size = size + self.permissions = permissions + self.last_modified = last_modified + self.user_id = user_id + self.user = user + self.group_id = group_id + self.group = group + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'FileInfo': + """Create new FileInfo object from dict parsed from JSON.""" + try: + file_type = FileType(d['type']) + except ValueError: + file_type = d['type'] + return cls( + path=d['path'], + name=d['name'], + type=file_type, + size=d.get('size'), + permissions=int(d['permissions'], 8), + last_modified=_parse_timestamp(d['last-modified']), + user_id=d.get('user-id'), + user=d.get('user'), + group_id=d.get('group-id'), + group=d.get('group'), + ) + + def __repr__(self): + return ('FileInfo(' + 'path={self.path!r}, ' + 'name={self.name!r}, ' + 'type={self.type}, ' + 'size={self.size}, ' + 'permissions=0o{self.permissions:o}, ' + 'last_modified={self.last_modified!r}, ' + 'user_id={self.user_id}, ' + 'user={self.user!r}, ' + 'group_id={self.group_id}, ' + 'group={self.group!r})' + ).format(self=self) + + +class CheckInfo: + """Check status information. + + A list of these objects is returned from :meth:`Client.get_checks`. + + Attributes: + name: The name of the check. + level: The check level: :attr:`CheckLevel.ALIVE`, + :attr:`CheckLevel.READY`, or None (level not set). + status: The status of the check: :attr:`CheckStatus.UP` means the + check is healthy (the number of failures is less than the + threshold), :attr:`CheckStatus.DOWN` means the check is unhealthy + (the number of failures has reached the threshold). + failures: The number of failures since the check last succeeded (reset + to zero if the check succeeds). + threshold: The failure threshold, that is, how many consecutive + failures for the check to be considered "down". + """ + + def __init__( + self, + name: str, + level: typing.Optional[typing.Union[CheckLevel, str]], + status: typing.Union[CheckStatus, str], + failures: int = 0, + threshold: int = 0, + ): + self.name = name + self.level = level + self.status = status + self.failures = failures + self.threshold = threshold + + @classmethod + def from_dict(cls, d: typing.Dict) -> 'CheckInfo': + """Create new :class:`CheckInfo` object from dict parsed from JSON.""" + try: + level = CheckLevel(d.get('level', '')) + except ValueError: + level = d.get('level') + try: + status = CheckStatus(d['status']) + except ValueError: + status = d['status'] + return cls( + name=d['name'], + level=level, + status=status, + failures=d.get('failures', 0), + threshold=d['threshold'], + ) + + def __repr__(self): + return ('CheckInfo(' + 'name={self.name!r}, ' + 'level={self.level!r}, ' + 'status={self.status}, ' + 'failures={self.failures}, ' + 'threshold={self.threshold!r})' + ).format(self=self) + + +class ExecProcess: + """Represents a process started by :meth:`Client.exec`. + + To avoid deadlocks, most users should use :meth:`wait_output` instead of + reading and writing the :attr:`stdin`, :attr:`stdout`, and :attr:`stderr` + attributes directly. Alternatively, users can pass stdin/stdout/stderr to + :meth:`Client.exec`. + + This class should not be instantiated directly, only via + :meth:`Client.exec`. + + Attributes: + stdin: If the stdin argument was not passed to :meth:`Client.exec`, + this is a writable file-like object the caller can use to stream + input to the process. It is None if stdin was passed to + :meth:`Client.exec`. + stdout: If the stdout argument was not passed to :meth:`Client.exec`, + this is a readable file-like object the caller can use to stream + output from the process. It is None if stdout was passed to + :meth:`Client.exec`. + stderr: If the stderr argument was not passed to :meth:`Client.exec` + and combine_stderr was False, this is a readable file-like object + the caller can use to stream error output from the process. It is + None if stderr was passed to :meth:`Client.exec` or combine_stderr + was True. + """ + + def __init__( + self, + stdin: typing.Optional[typing.Union[typing.TextIO, typing.BinaryIO]], + stdout: typing.Optional[typing.Union[typing.TextIO, typing.BinaryIO]], + stderr: typing.Optional[typing.Union[typing.TextIO, typing.BinaryIO]], + client: 'Client', + timeout: typing.Optional[float], + control_ws: websocket.WebSocket, + stdio_ws: websocket.WebSocket, + stderr_ws: websocket.WebSocket, + command: typing.List[str], + encoding: typing.Optional[str], + change_id: ChangeID, + cancel_stdin: typing.Callable[[], None], + cancel_reader: typing.Optional[int], + threads: typing.List[threading.Thread], + ): + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self._client = client + self._timeout = timeout + self._control_ws = control_ws + self._stdio_ws = stdio_ws + self._stderr_ws = stderr_ws + self._command = command + self._encoding = encoding + self._change_id = change_id + self._cancel_stdin = cancel_stdin + self._cancel_reader = cancel_reader + self._threads = threads + self._waited = False + + def __del__(self): + if not self._waited: + msg = 'ExecProcess instance garbage collected without call to wait() or wait_output()' + warnings.warn(msg, ResourceWarning) + + def wait(self): + """Wait for the process to finish. + + If a timeout was specified to the :meth:`Client.exec` call, this waits + at most that duration. + + Raises: + ChangeError: if there was an error starting or running the process. + ExecError: if the process exits with a non-zero exit code. + """ + exit_code = self._wait() + if exit_code != 0: + raise ExecError(self._command, exit_code, None, None) + + def _wait(self): + self._waited = True + timeout = self._timeout + if timeout is not None: + # A bit more than the command timeout to ensure that happens first + timeout += 1 + change = self._client.wait_change(self._change_id, timeout=timeout) + + # If stdin reader thread is running, stop it + if self._cancel_stdin is not None: + self._cancel_stdin() + + # Wait for all threads to finish (e.g., message barrier sent) + for thread in self._threads: + thread.join() + + # If we opened a cancel_reader pipe, close the read side now (write + # side was already closed by _cancel_stdin(). + if self._cancel_reader is not None: + os.close(self._cancel_reader) + + # Close websockets (shutdown doesn't send CLOSE message or wait for response). + self._control_ws.shutdown() + self._stdio_ws.shutdown() + if self._stderr_ws is not None: + self._stderr_ws.shutdown() + + if change.err: + raise ChangeError(change.err, change) + + exit_code = -1 + if change.tasks: + exit_code = change.tasks[0].data.get('exit-code', -1) + return exit_code + + def wait_output(self) -> typing.Tuple[typing.AnyStr, typing.AnyStr]: + """Wait for the process to finish and return tuple of (stdout, stderr). + + If a timeout was specified to the :meth:`Client.exec` call, this waits + at most that duration. If combine_stderr was True, stdout will include + the process's standard error, and stderr will be None. + + Raises: + ChangeError: if there was an error starting or running the process. + ExecError: if the process exits with a non-zero exit code. + """ + if self._encoding is not None: + out = io.StringIO() + err = io.StringIO() if self.stderr is not None else None + else: + out = io.BytesIO() + err = io.BytesIO() if self.stderr is not None else None + + t = _start_thread(shutil.copyfileobj, self.stdout, out) + self._threads.append(t) + + if self.stderr is not None: + t = _start_thread(shutil.copyfileobj, self.stderr, err) + self._threads.append(t) + + exit_code = self._wait() + + out_value = out.getvalue() + err_value = err.getvalue() if err is not None else None + if exit_code != 0: + raise ExecError(self._command, exit_code, out_value, err_value) + + return (out_value, err_value) + + def send_signal(self, sig: typing.Union[int, str]): + """Send the given signal to the running process. + + Args: + sig: Name or number of signal to send, e.g., "SIGHUP", 1, or + signal.SIGHUP. + """ + if isinstance(sig, int): + sig = signal.Signals(sig).name + payload = { + 'command': 'signal', + 'signal': {'name': sig}, + } + msg = json.dumps(payload, sort_keys=True) + self._control_ws.send(msg) + + +def _has_fileno(f): + """Return True if the file-like object has a valid fileno() method.""" + try: + f.fileno() + return True + except Exception: + # Some types define a fileno method that raises io.UnsupportedOperation, + # but just catching all exceptions here won't hurt. + return False + + +def _reader_to_websocket(reader, ws, encoding, cancel_reader=None, bufsize=16 * 1024): + """Read reader through to EOF and send each chunk read to the websocket.""" + while True: + if cancel_reader is not None: + # Wait for either a read to be ready or the caller to cancel stdin + result = select.select([cancel_reader, reader], [], []) + if cancel_reader in result[0]: + break + + chunk = reader.read(bufsize) + if not chunk: + break + if isinstance(chunk, str): + chunk = chunk.encode(encoding) + ws.send_binary(chunk) + + ws.send('{"command":"end"}') # Send "end" command as TEXT frame to signal EOF + + +def _websocket_to_writer(ws, writer, encoding): + """Receive messages from websocket (until end signal) and write to writer.""" + while True: + chunk = ws.recv() + + if isinstance(chunk, str): + try: + payload = json.loads(chunk) + except ValueError: + # Garbage sent, try to keep going + logger.warning('Cannot decode I/O command (invalid JSON)') + continue + command = payload.get('command') + if command != 'end': + # A command we don't recognize, keep going + logger.warning('Invalid I/O command {!r}'.format(command)) + continue + # Received "end" command (EOF signal), stop thread + break + + if encoding is not None: + chunk = chunk.decode(encoding) + writer.write(chunk) + + +class _WebsocketWriter(io.BufferedIOBase): + """A writable file-like object that sends what's written to it to a websocket.""" + + def __init__(self, ws): + self.ws = ws + + def writable(self): + """Denote this file-like object as writable.""" + return True + + def write(self, chunk): + """Write chunk to the websocket.""" + if not isinstance(chunk, bytes): + raise TypeError('value to write must be bytes, not {}'.format(type(chunk).__name__)) + self.ws.send_binary(chunk) + return len(chunk) + + def close(self): + """Send end-of-file message to websocket.""" + self.ws.send('{"command":"end"}') + + +class _WebsocketReader(io.BufferedIOBase): + """A readable file-like object whose reads come from a websocket.""" + + def __init__(self, ws): + self.ws = ws + self.remaining = b'' + self.eof = False + + def readable(self): + """Denote this file-like object as readable.""" + return True + + def read(self, n=-1): + """Read up to n bytes from the websocket (or one message if n<0).""" + if self.eof: + # Calling read() multiple times after EOF should still return EOF + return b'' + + while not self.remaining: + chunk = self.ws.recv() + + if isinstance(chunk, str): + try: + payload = json.loads(chunk) + except ValueError: + # Garbage sent, try to keep going + logger.warning('Cannot decode I/O command (invalid JSON)') + continue + command = payload.get('command') + if command != 'end': + # A command we don't recognize, keep going + logger.warning('Invalid I/O command {!r}'.format(command)) + continue + # Received "end" command, return EOF designator + self.eof = True + return b'' + + self.remaining = chunk + + if n < 0: + n = len(self.remaining) + result = self.remaining[:n] + self.remaining = self.remaining[n:] + return result + + def read1(self, n=-1): + """An alias for read.""" + return self.read(n) + + +class Client: + """Pebble API client.""" + + _chunk_size = 8192 + + def __init__(self, socket_path=None, opener=None, base_url='http://localhost', timeout=5.0): + """Initialize a client instance. + + Defaults to using a Unix socket at socket_path (which must be specified + unless a custom opener is provided). + """ + if opener is None: + if socket_path is None: + raise ValueError('no socket path provided') + opener = self._get_default_opener(socket_path) + self.socket_path = socket_path + self.opener = opener + self.base_url = base_url + self.timeout = timeout + + @classmethod + def _get_default_opener(cls, socket_path): + """Build the default opener to use for requests (HTTP over Unix socket).""" + opener = urllib.request.OpenerDirector() + opener.add_handler(_UnixSocketHandler(socket_path)) + opener.add_handler(urllib.request.HTTPDefaultErrorHandler()) + opener.add_handler(urllib.request.HTTPRedirectHandler()) + opener.add_handler(urllib.request.HTTPErrorProcessor()) + return opener + + def _request( + self, method: str, path: str, query: typing.Dict = None, body: typing.Dict = None, + ) -> typing.Dict: + """Make a JSON request to the Pebble server with the given HTTP method and path. + + If query dict is provided, it is encoded and appended as a query string + to the URL. If body dict is provided, it is serialied as JSON and used + as the HTTP body (with Content-Type: "application/json"). The resulting + body is decoded from JSON. + """ + headers = {'Accept': 'application/json'} + data = None + if body is not None: + data = json.dumps(body).encode('utf-8') + headers['Content-Type'] = 'application/json' + + response = self._request_raw(method, path, query, headers, data) + self._ensure_content_type(response.headers, 'application/json') + return _json_loads(response.read()) + + @staticmethod + def _ensure_content_type(headers, expected): + """Parse Content-Type header from headers and ensure it's equal to expected. + + Return a dict of any options in the header, e.g., {'boundary': ...}. + """ + ctype, options = cgi.parse_header(headers.get('Content-Type', '')) + if ctype != expected: + raise ProtocolError('expected Content-Type {!r}, got {!r}'.format(expected, ctype)) + return options + + def _request_raw( + self, method: str, path: str, query: typing.Dict = None, headers: typing.Dict = None, + data: bytes = None, + ) -> http.client.HTTPResponse: + """Make a request to the Pebble server; return the raw HTTPResponse object.""" + url = self.base_url + path + if query: + url = url + '?' + urllib.parse.urlencode(query, doseq=True) + + # python 3.5 urllib requests require their data to be a bytes object - + # generators won't work. + if sys.version_info[:2] < (3, 6) and isinstance(data, types.GeneratorType): + data = b''.join(data) + + if headers is None: + headers = {} + request = urllib.request.Request(url, method=method, data=data, headers=headers) + + try: + response = self.opener.open(request, timeout=self.timeout) + except urllib.error.HTTPError as e: + code = e.code + status = e.reason + try: + body = _json_loads(e.read()) + message = body['result']['message'] + except (IOError, ValueError, KeyError) as e2: + # Will only happen on read error or if Pebble sends invalid JSON. + body = {} + message = '{} - {}'.format(type(e2).__name__, e2) + raise APIError(body, code, status, message) + except urllib.error.URLError as e: + raise ConnectionError(e.reason) + + return response + + def get_system_info(self) -> SystemInfo: + """Get system info.""" + resp = self._request('GET', '/v1/system-info') + return SystemInfo.from_dict(resp['result']) + + def get_warnings(self, select: WarningState = WarningState.PENDING) -> typing.List[Warning]: + """Get list of warnings in given state (pending or all).""" + query = {'select': select.value} + resp = self._request('GET', '/v1/warnings', query) + return [Warning.from_dict(w) for w in resp['result']] + + def ack_warnings(self, timestamp: datetime.datetime) -> int: + """Acknowledge warnings up to given timestamp, return number acknowledged.""" + body = {'action': 'okay', 'timestamp': timestamp.isoformat()} + resp = self._request('POST', '/v1/warnings', body=body) + return resp['result'] + + def get_changes( + self, select: ChangeState = ChangeState.IN_PROGRESS, service: str = None, + ) -> typing.List[Change]: + """Get list of changes in given state, filter by service name if given.""" + query = {'select': select.value} + if service is not None: + query['for'] = service + resp = self._request('GET', '/v1/changes', query) + return [Change.from_dict(c) for c in resp['result']] + + def get_change(self, change_id: ChangeID) -> Change: + """Get single change by ID.""" + resp = self._request('GET', '/v1/changes/{}'.format(change_id)) + return Change.from_dict(resp['result']) + + def abort_change(self, change_id: ChangeID) -> Change: + """Abort change with given ID.""" + body = {'action': 'abort'} + resp = self._request('POST', '/v1/changes/{}'.format(change_id), body=body) + return Change.from_dict(resp['result']) + + def autostart_services(self, timeout: float = 30.0, delay: float = 0.1) -> ChangeID: + """Start the startup-enabled services and wait (poll) for them to be started. + + Args: + timeout: Seconds before autostart change is considered timed out (float). + delay: Seconds before executing the autostart change (float). + + Returns: + ChangeID of the autostart change. + + Raises: + ChangeError: if one or more of the services didn't start. If + timeout is 0, submit the action but don't wait; just return the change + ID immediately. + """ + return self._services_action('autostart', [], timeout, delay) + + def replan_services(self, timeout: float = 30.0, delay: float = 0.1) -> ChangeID: + """Replan by (re)starting changed and startup-enabled services and wait for them to start. + + Args: + timeout: Seconds before replan change is considered timed out (float). + delay: Seconds before executing the replan change (float). + + Returns: + ChangeID of the replan change. + + Raises: + ChangeError: if one or more of the services didn't stop/start. If + timeout is 0, submit the action but don't wait; just return the change + ID immediately. + """ + return self._services_action('replan', [], timeout, delay) + + def start_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> ChangeID: + """Start services by name and wait (poll) for them to be started. + + Args: + services: Non-empty list of services to start. + timeout: Seconds before start change is considered timed out (float). + delay: Seconds before executing the start change (float). + + Returns: + ChangeID of the start change. + + Raises: + ChangeError: if one or more of the services didn't stop/start. If + timeout is 0, submit the action but don't wait; just return the change + ID immediately. + """ + return self._services_action('start', services, timeout, delay) + + def stop_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> ChangeID: + """Stop services by name and wait (poll) for them to be started. + + Args: + services: Non-empty list of services to stop. + timeout: Seconds before stop change is considered timed out (float). + delay: Seconds before executing the stop change (float). + + Returns: + ChangeID of the stop change. + + Raises: + ChangeError: if one or more of the services didn't stop/start. If + timeout is 0, submit the action but don't wait; just return the change + ID immediately. + """ + return self._services_action('stop', services, timeout, delay) + + def restart_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> ChangeID: + """Restart services by name and wait (poll) for them to be started. + + Args: + services: Non-empty list of services to restart. + timeout: Seconds before restart change is considered timed out (float). + delay: Seconds before executing the restart change (float). + + Returns: + ChangeID of the restart change. + + Raises: + ChangeError: if one or more of the services didn't stop/start. If + timeout is 0, submit the action but don't wait; just return the change + ID immediately. + """ + return self._services_action('restart', services, timeout, delay) + + def _services_action( + self, action: str, services: typing.Iterable[str], timeout: float, delay: float, + ) -> ChangeID: + if not isinstance(services, (list, tuple)): + raise TypeError('services must be a list of str, not {}'.format( + type(services).__name__)) + for s in services: + if not isinstance(s, str): + raise TypeError('service names must be str, not {}'.format(type(s).__name__)) + + body = {'action': action, 'services': services} + resp = self._request('POST', '/v1/services', body=body) + change_id = ChangeID(resp['change']) + if timeout: + change = self.wait_change(change_id, timeout=timeout, delay=delay) + if change.err: + raise ChangeError(change.err, change) + return change_id + + def wait_change( + self, change_id: ChangeID, timeout: float = 30.0, delay: float = 0.1, + ) -> Change: + """Wait for the given change to be ready. + + If the Pebble server supports the /v1/changes/{id}/wait API endpoint, + use that to avoid polling, otherwise poll /v1/changes/{id} every delay + seconds. + + Args: + change_id: Change ID of change to wait for. + timeout: Maximum time in seconds to wait for the change to be + ready. May be None, in which case wait_change never times out. + delay: If polling, this is the delay in seconds between attempts. + + Returns: + The Change object being waited on. + + Raises: + TimeoutError: If the maximum timeout is reached. + """ + try: + return self._wait_change_using_wait(change_id, timeout) + except NotImplementedError: + # Pebble server doesn't support wait endpoint, fall back to polling + return self._wait_change_using_polling(change_id, timeout, delay) + + def _wait_change_using_wait(self, change_id, timeout): + """Wait for a change to be ready using the wait-change API.""" + deadline = time.time() + timeout if timeout is not None else None + + # Hit the wait endpoint every Client.timeout-1 seconds to avoid long + # requests (the -1 is to ensure it wakes up before the socket timeout) + while True: + this_timeout = max(self.timeout - 1, 1) # minimum of 1 second + if timeout is not None: + time_remaining = deadline - time.time() + if time_remaining <= 0: + break + # Wait the lesser of the time remaining and Client.timeout-1 + this_timeout = min(time_remaining, this_timeout) + + try: + return self._wait_change(change_id, this_timeout) + except TimeoutError: + # Catch timeout from wait endpoint and loop to check deadline + pass + + raise TimeoutError('timed out waiting for change {} ({} seconds)'.format( + change_id, timeout)) + + def _wait_change(self, change_id: ChangeID, timeout: float = None) -> Change: + """Call the wait-change API endpoint directly.""" + query = {} + if timeout is not None: + query['timeout'] = _format_timeout(timeout) + + try: + resp = self._request('GET', '/v1/changes/{}/wait'.format(change_id), query) + except APIError as e: + if e.code == 404: + raise NotImplementedError('server does not implement wait-change endpoint') + if e.code == 504: + raise TimeoutError('timed out waiting for change {} ({} seconds)'.format( + change_id, timeout)) + raise + + return Change.from_dict(resp['result']) + + def _wait_change_using_polling(self, change_id, timeout, delay): + """Wait for a change to be ready by polling the get-change API.""" + deadline = time.time() + timeout if timeout is not None else None + + while timeout is None or time.time() < deadline: + change = self.get_change(change_id) + if change.ready: + return change + + time.sleep(delay) + + raise TimeoutError('timed out waiting for change {} ({} seconds)'.format( + change_id, timeout)) + + def add_layer( + self, label: str, layer: typing.Union[str, dict, Layer], *, combine: bool = False): + """Dynamically add a new layer onto the Pebble configuration layers. + + If combine is False (the default), append the new layer as the top + layer with the given label. If combine is True and the label already + exists, the two layers are combined into a single one considering the + layer override rules; if the layer doesn't exist, it is added as usual. + """ + if not isinstance(label, str): + raise TypeError('label must be a str, not {}'.format(type(label).__name__)) + + if isinstance(layer, str): + layer_yaml = layer + elif isinstance(layer, dict): + layer_yaml = Layer(layer).to_yaml() + elif isinstance(layer, Layer): + layer_yaml = layer.to_yaml() + else: + raise TypeError('layer must be str, dict, or pebble.Layer, not {}'.format( + type(layer).__name__)) + + body = { + 'action': 'add', + 'combine': combine, + 'label': label, + 'format': 'yaml', + 'layer': layer_yaml, + } + self._request('POST', '/v1/layers', body=body) + + def get_plan(self) -> Plan: + """Get the Pebble plan (contains combined layer configuration).""" + resp = self._request('GET', '/v1/plan', {'format': 'yaml'}) + return Plan(resp['result']) + + def get_services(self, names: typing.List[str] = None) -> typing.List[ServiceInfo]: + """Get the service status for the configured services. + + If names is specified, only fetch the service status for the services + named. + """ + query = None + if names is not None: + query = {'names': ','.join(names)} + resp = self._request('GET', '/v1/services', query) + return [ServiceInfo.from_dict(info) for info in resp['result']] + + def pull(self, + path: str, + *, + encoding: typing.Optional[str] = 'utf-8') -> typing.Union[typing.BinaryIO, + typing.TextIO]: + """Read a file's content from the remote system. + + Args: + path: Path of the file to read from the remote system. + encoding: Encoding to use for decoding the file's bytes to str, + or None to specify no decoding. + + Returns: + A readable file-like object, whose read() method will return str + objects decoded according to the specified encoding, or bytes if + encoding is None. + """ + query = { + 'action': 'read', + 'path': path, + } + headers = {'Accept': 'multipart/form-data'} + response = self._request_raw('GET', '/v1/files', query, headers) + + options = self._ensure_content_type(response.headers, 'multipart/form-data') + boundary = options.get('boundary', '') + if not boundary: + raise ProtocolError('invalid boundary {!r}'.format(boundary)) + + parser = _FilesParser(boundary) + + while True: + chunk = response.read(self._chunk_size) + if not chunk: + break + parser.feed(chunk) + + resp = parser.get_response() + if resp is None: + raise ProtocolError('no "response" field in multipart body') + self._raise_on_path_error(resp, path) + + filenames = parser.filenames() + if not filenames: + raise ProtocolError('no file content in multipart response') + elif len(filenames) > 1: + raise ProtocolError('single file request resulted in a multi-file response') + + filename = filenames[0] + if filename != path: + raise ProtocolError('path not expected: {!r}'.format(filename)) + + f = parser.get_file(path, encoding) + + # The opened file remains usable after removing/unlinking on posix + # platforms until it is closed. This prevents callers from + # interacting with it on disk. But e.g. Windows doesn't allow + # removing opened files, and so we use the tempfile lib's + # helper class to auto-delete on close/gc for us. + if os.name != 'posix' or sys.platform == 'cygwin': + return tempfile._TemporaryFileWrapper(f, f.name, delete=True) + parser.remove_files() + return f + + @staticmethod + def _raise_on_path_error(resp, path): + result = resp['result'] or [] # in case it's null instead of [] + paths = {item['path']: item for item in result} + if path not in paths: + raise ProtocolError('path not found in response metadata: {}'.format(resp)) + error = paths[path].get('error') + if error: + raise PathError(error['kind'], error['message']) + + def push( + self, path: str, source: typing.Union[bytes, str, typing.BinaryIO, typing.TextIO], *, + encoding: str = 'utf-8', make_dirs: bool = False, permissions: int = None, + user_id: int = None, user: str = None, group_id: int = None, group: str = None): + """Write content to a given file path on the remote system. + + Args: + path: Path of the file to write to on the remote system. + source: Source of data to write. This is either a concrete str or + bytes instance, or a readable file-like object. + encoding: Encoding to use for encoding source str to bytes, or + strings read from source if it is a TextIO type. Ignored if + source is bytes or BinaryIO. + make_dirs: If True, create parent directories if they don't exist. + permissions: Permissions (mode) to create file with (Pebble default + is 0o644). + user_id: User ID (UID) for file. + user: Username for file. User's UID must match user_id if both are + specified. + group_id: Group ID (GID) for file. + group: Group name for file. Group's GID must match group_id if + both are specified. + """ + info = self._make_auth_dict(permissions, user_id, user, group_id, group) + info['path'] = path + if make_dirs: + info['make-dirs'] = True + metadata = { + 'action': 'write', + 'files': [info], + } + + data, content_type = self._encode_multipart(metadata, path, source, encoding) + + headers = { + 'Accept': 'application/json', + 'Content-Type': content_type, + } + response = self._request_raw('POST', '/v1/files', None, headers, data) + self._ensure_content_type(response.headers, 'application/json') + resp = _json_loads(response.read()) + self._raise_on_path_error(resp, path) + + @staticmethod + def _make_auth_dict(permissions, user_id, user, group_id, group) -> typing.Dict: + d = {} + if permissions is not None: + d['permissions'] = format(permissions, '03o') + if user_id is not None: + d['user-id'] = user_id + if user is not None: + d['user'] = user + if group_id is not None: + d['group-id'] = group_id + if group is not None: + d['group'] = group + return d + + def _encode_multipart(self, metadata, path, source, encoding): + # Python's stdlib mime/multipart handling is screwy and doesn't handle + # binary properly, so roll our own. + + if isinstance(source, str): + source = io.StringIO(source) + elif isinstance(source, bytes): + source = io.BytesIO(source) + + boundary = binascii.hexlify(os.urandom(16)) + path_escaped = path.replace('"', '\\"').encode('utf-8') # NOQA: test_quote_backslashes + content_type = 'multipart/form-data; boundary="' + boundary.decode('utf-8') + '"' + + def generator(): + yield b''.join([ + b'--', boundary, b'\r\n', + b'Content-Type: application/json\r\n', + b'Content-Disposition: form-data; name="request"\r\n', + b'\r\n', + json.dumps(metadata).encode('utf-8'), b'\r\n', + b'--', boundary, b'\r\n', + b'Content-Type: application/octet-stream\r\n', + b'Content-Disposition: form-data; name="files"; filename="', + path_escaped, b'"\r\n', + b'\r\n', + ]) + + content = source.read(self._chunk_size) + while content: + if isinstance(content, str): + content = content.encode(encoding) + yield content + content = source.read(self._chunk_size) + + yield b''.join([ + b'\r\n', + b'--', boundary, b'--\r\n', + ]) + + return generator(), content_type + + def list_files(self, path: str, *, pattern: str = None, + itself: bool = False) -> typing.List[FileInfo]: + """Return list of directory entries from given path on remote system. + + Despite the name, this method returns a list of files *and* + directories, similar to :func:`os.listdir` or :func:`os.scandir`. + + Args: + path: Path of the directory to list, or path of the file to return + information about. + pattern: If specified, filter the list to just the files that match, + for example ``*.txt``. + itself: If path refers to a directory, return information about the + directory itself, rather than its contents. + """ + query = { + 'action': 'list', + 'path': path, + } + if pattern: + query['pattern'] = pattern + if itself: + query['itself'] = 'true' + resp = self._request('GET', '/v1/files', query) + result = resp['result'] or [] # in case it's null instead of [] + return [FileInfo.from_dict(d) for d in result] + + def make_dir( + self, path: str, *, make_parents: bool = False, permissions: int = None, + user_id: int = None, user: str = None, group_id: int = None, group: str = None): + """Create a directory on the remote system with the given attributes. + + Args: + path: Path of the directory to create on the remote system. + make_parents: If True, create parent directories if they don't exist. + permissions: Permissions (mode) to create directory with (Pebble + default is 0o755). + user_id: User ID (UID) for directory. + user: Username for directory. User's UID must match user_id if + both are specified. + group_id: Group ID (GID) for directory. + group: Group name for directory. Group's GID must match group_id + if both are specified. + """ + info = self._make_auth_dict(permissions, user_id, user, group_id, group) + info['path'] = path + if make_parents: + info['make-parents'] = True + body = { + 'action': 'make-dirs', + 'dirs': [info], + } + resp = self._request('POST', '/v1/files', None, body) + self._raise_on_path_error(resp, path) + + def remove_path(self, path: str, *, recursive: bool = False): + """Remove a file or directory on the remote system. + + Args: + path: Path of the file or directory to delete from the remote system. + recursive: If True, and path is a directory recursively deletes it and + everything under it. If the path is a file, delete the file and + do nothing if the file is non-existent. Behaviourally similar + to `rm -rf ` + + """ + info = {'path': path} + if recursive: + info['recursive'] = True + body = { + 'action': 'remove', + 'paths': [info], + } + resp = self._request('POST', '/v1/files', None, body) + self._raise_on_path_error(resp, path) + + def exec( + self, + command: typing.List[str], + *, + environment: typing.Dict[str, str] = None, + working_dir: str = None, + timeout: float = None, + user_id: int = None, + user: str = None, + group_id: int = None, + group: str = None, + stdin: typing.Union[str, bytes, typing.TextIO, typing.BinaryIO] = None, + stdout: typing.Union[typing.TextIO, typing.BinaryIO] = None, + stderr: typing.Union[typing.TextIO, typing.BinaryIO] = None, + encoding: str = 'utf-8', + combine_stderr: bool = False + ) -> ExecProcess: + r"""Execute the given command on the remote system. + + Most of the parameters are explained in the "Parameters" section + below, however, input/output handling is a bit more complex. Some + examples are shown below:: + + # Simple command with no output; just check exit code + >>> process = client.exec(['send-emails']) + >>> process.wait() + + # Fetch output as string + >>> process = client.exec(['python3', '--version']) + >>> version, _ = process.wait_output() + >>> print(version) + Python 3.8.10 + + # Fetch both stdout and stderr as strings + >>> process = client.exec(['pg_dump', '-s', ...]) + >>> schema, logs = process.wait_output() + + # Stream input from a string and write output to files + >>> stdin = 'foo\nbar\n' + >>> with open('out.txt', 'w') as out, open('err.txt', 'w') as err: + ... process = client.exec(['awk', '{ print toupper($0) }'], + ... stdin=stdin, stdout=out, stderr=err) + ... process.wait() + >>> open('out.txt').read() + 'FOO\nBAR\n' + >>> open('err.txt').read() + '' + + # Real-time streaming using ExecProcess.stdin and ExecProcess.stdout + >>> process = client.exec(['cat']) + >>> def stdin_thread(): + ... for line in ['one\n', '2\n', 'THREE\n']: + ... process.stdin.write(line) + ... process.stdin.flush() + ... time.sleep(1) + ... process.stdin.close() + ... + >>> threading.Thread(target=stdin_thread).start() + >>> for line in process.stdout: + ... print(datetime.datetime.now().strftime('%H:%M:%S'), repr(line)) + ... + 16:20:26 'one\n' + 16:20:27 '2\n' + 16:20:28 'THREE\n' + >>> process.wait() # will return immediately as stdin was closed + + # Show exception raised for non-zero return code + >>> process = client.exec(['ls', 'notexist']) + >>> out, err = process.wait_output() + Traceback (most recent call last): + ... + ExecError: "ls" returned exit code 2 + >>> exc = sys.last_value + >>> exc.exit_code + 2 + >>> exc.stdout + '' + >>> exc.stderr + "ls: cannot access 'notfound': No such file or directory\n" + + Args: + command: Command to execute: the first item is the name (or path) + of the executable, the rest of the items are the arguments. + environment: Environment variables to pass to the process. + working_dir: Working directory to run the command in. If not set, + Pebble uses the target user's $HOME directory (and if the user + argument is not set, $HOME of the user Pebble is running as). + timeout: Timeout in seconds for the command execution, after which + the process will be terminated. If not specified, the + execution never times out. + user_id: User ID (UID) to run the process as. + user: Username to run the process as. User's UID must match + user_id if both are specified. + group_id: Group ID (GID) to run the process as. + group: Group name to run the process as. Group's GID must match + group_id if both are specified. + stdin: A string or readable file-like object that is sent to the + process's standard input. If not set, the caller can write + input to :attr:`ExecProcess.stdin` to stream input to the + process. + stdout: A writable file-like object that the process's standard + output is written to. If not set, the caller can use + :meth:`ExecProcess.wait_output` to capture output as a string, + or read from :meth:`ExecProcess.stdout` to stream output from + the process. + stderr: A writable file-like object that the process's standard + error is written to. If not set, the caller can use + :meth:`ExecProcess.wait_output` to capture error output as a + string, or read from :meth:`ExecProcess.stderr` to stream + error output from the process. Must be None if combine_stderr + is True. + encoding: If encoding is set (the default is UTF-8), the types + read or written to stdin/stdout/stderr are str, and encoding + is used to encode them to bytes. If encoding is None, the + types read or written are raw bytes. + combine_stderr: If True, process's stderr output is combined into + its stdout (the stderr argument must be None). If False, + separate streams are used for stdout and stderr. + + Returns: + A Process object representing the state of the running process. + To wait for the command to finish, the caller will typically call + :meth:`ExecProcess.wait` if stdout/stderr were provided as + arguments to :meth:`exec`, or :meth:`ExecProcess.wait_output` if + not. + """ + if not isinstance(command, list) or not all(isinstance(s, str) for s in command): + raise TypeError('command must be a list of str, not {}'.format( + type(command).__name__)) + if len(command) < 1: + raise ValueError('command must contain at least one item') + + if stdin is not None: + if isinstance(stdin, str): + if encoding is None: + raise ValueError('encoding must be set if stdin is str') + stdin = io.BytesIO(stdin.encode(encoding)) + elif isinstance(stdin, bytes): + if encoding is not None: + raise ValueError('encoding must be None if stdin is bytes') + stdin = io.BytesIO(stdin) + elif not hasattr(stdin, 'read'): + raise TypeError('stdin must be str, bytes, or a readable file-like object') + + if combine_stderr and stderr is not None: + raise ValueError('stderr must be None if combine_stderr is True') + + body = { + 'command': command, + 'environment': environment or {}, + 'working-dir': working_dir, + 'timeout': _format_timeout(timeout) if timeout is not None else None, + 'user-id': user_id, + 'user': user, + 'group-id': group_id, + 'group': group, + 'split-stderr': not combine_stderr, + } + resp = self._request('POST', '/v1/exec', body=body) + change_id = resp['change'] + task_id = resp['result']['task-id'] + + stderr_ws = None + try: + control_ws = self._connect_websocket(task_id, 'control') + stdio_ws = self._connect_websocket(task_id, 'stdio') + if not combine_stderr: + stderr_ws = self._connect_websocket(task_id, 'stderr') + except websocket.WebSocketException as e: + # Error connecting to websockets, probably due to the exec/change + # finishing early with an error. Call wait_change to pick that up. + change = self.wait_change(ChangeID(change_id)) + if change.err: + raise ChangeError(change.err, change) + raise ConnectionError('unexpected error connecting to websockets: {}'.format(e)) + + cancel_stdin = None + cancel_reader = None + threads = [] + + if stdin is not None: + if _has_fileno(stdin): + if sys.platform == 'win32': + raise NotImplementedError('file-based stdin not supported on Windows') + + # Create a pipe so _reader_to_websocket can select() on the + # reader as well as this cancel_reader; when we write anything + # to cancel_writer it'll trigger the select and end the thread. + cancel_reader, cancel_writer = os.pipe() + + def cancel_stdin(): + os.write(cancel_writer, b'x') # doesn't matter what we write + os.close(cancel_writer) + + t = _start_thread(_reader_to_websocket, stdin, stdio_ws, encoding, cancel_reader) + threads.append(t) + process_stdin = None + else: + process_stdin = _WebsocketWriter(stdio_ws) + if encoding is not None: + process_stdin = io.TextIOWrapper(process_stdin, encoding=encoding, newline='') + + if stdout is not None: + t = _start_thread(_websocket_to_writer, stdio_ws, stdout, encoding) + threads.append(t) + process_stdout = None + else: + process_stdout = _WebsocketReader(stdio_ws) + if encoding is not None: + process_stdout = io.TextIOWrapper(process_stdout, encoding=encoding, newline='') + + process_stderr = None + if not combine_stderr: + if stderr is not None: + t = _start_thread(_websocket_to_writer, stderr_ws, stderr, encoding) + threads.append(t) + else: + process_stderr = _WebsocketReader(stderr_ws) + if encoding is not None: + process_stderr = io.TextIOWrapper( + process_stderr, encoding=encoding, newline='') + + process = ExecProcess( + stdin=process_stdin, + stdout=process_stdout, + stderr=process_stderr, + client=self, + timeout=timeout, + stdio_ws=stdio_ws, + stderr_ws=stderr_ws, + control_ws=control_ws, + command=command, + encoding=encoding, + change_id=ChangeID(change_id), + cancel_stdin=cancel_stdin, + cancel_reader=cancel_reader, + threads=threads, + ) + return process + + def _connect_websocket(self, task_id: str, websocket_id: str) -> websocket.WebSocket: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(self.socket_path) + url = self._websocket_url(task_id, websocket_id) + ws = websocket.WebSocket(skip_utf8_validation=True) + ws.connect(url, socket=sock) + return ws + + def _websocket_url(self, task_id: str, websocket_id: str) -> str: + base_url = self.base_url.replace('http://', 'ws://') + url = '{}/v1/tasks/{}/websocket/{}'.format(base_url, task_id, websocket_id) + return url + + def send_signal(self, sig: typing.Union[int, str], services: typing.List[str]): + """Send the given signal to the list of services named. + + Args: + sig: Name or number of signal to send, e.g., "SIGHUP", 1, or + signal.SIGHUP. + services: Non-empty list of service names to send the signal to. + + Raises: + APIError: If any of the services are not in the plan or are not + currently running. + """ + if not isinstance(services, (list, tuple)): + raise TypeError('services must be a list of str, not {}'.format( + type(services).__name__)) + for s in services: + if not isinstance(s, str): + raise TypeError('service names must be str, not {}'.format(type(s).__name__)) + + if isinstance(sig, int): + sig = signal.Signals(sig).name + body = { + 'signal': sig, + 'services': services, + } + self._request('POST', '/v1/signals', body=body) + + def get_checks( + self, + level: CheckLevel = None, + names: typing.List[str] = None + ) -> typing.List[CheckInfo]: + """Get the check status for the configured checks. + + Args: + level: Optional check level to query for (default is to fetch + checks with any level). + names: Optional list of check names to query for (default is to + fetch checks with any name). + + Returns: + List of :class:`CheckInfo` objects. + """ + query = {} + if level is not None: + query['level'] = level.value + if names: + query['names'] = names + resp = self._request('GET', '/v1/checks', query) + return [CheckInfo.from_dict(info) for info in resp['result']] + + +class _FilesParser: + """A limited purpose multi-part parser backed by files for memory efficiency.""" + + def __init__(self, boundary: typing.Union[bytes, str]): + self._response = None + self._files = {} + + # Prepare the MIME multipart boundary line patterns. + if isinstance(boundary, str): + boundary = boundary.encode() + + # State vars, as we may enter the feed() function multiple times. + self._headers = None + self._part_type = None + self._response_data = bytearray() + + self._max_lookahead = 8 * 1024 * 1024 + + self._parser = _MultipartParser( + boundary, + self._process_header, + self._process_body, + max_lookahead=self._max_lookahead) + + # RFC 2046 says that the boundary string needs to be preceded by a CRLF. + # Unfortunately, the request library's header parsing logic strips off one of + # these, so we'll prime the parser buffer with that missing sequence. + self._parser.feed(b'\r\n') + + def _process_header(self, data: bytes): + parser = email.parser.BytesFeedParser() + parser.feed(data) + self._headers = parser.close() + + content_disposition = self._headers.get_content_disposition() + if content_disposition != 'form-data': + raise ProtocolError( + 'unexpected content disposition: {!r}'.format(content_disposition)) + + name = self._headers.get_param('name', header='content-disposition') + if name == 'files': + filename = self._headers.get_filename() + if filename is None: + raise ProtocolError('multipart "files" part missing filename') + self._prepare_tempfile(filename) + elif name != 'response': + raise ProtocolError( + 'unexpected name in content-disposition header: {!r}'.format(name)) + + self._part_type = name + + def _process_body(self, data: bytes, done=False): + if self._part_type == 'response': + self._response_data.extend(data) + if done: + if len(self._response_data) > self._max_lookahead: + raise ProtocolError('response end marker not found') + self._response = json.loads(self._response_data.decode()) + self._response_data = bytearray() + elif self._part_type == 'files': + if done: + # This is the final write. + outfile = self._get_open_tempfile() + outfile.write(data) + outfile.close() + self._headers = None + else: + # Not the end of file data yet. Don't open/close file for intermediate writes + outfile = self._get_open_tempfile() + outfile.write(data) + + def remove_files(self): + """Remove all temporary files on disk.""" + for file in self._files.values(): + os.unlink(file.name) + self._files.clear() + + def feed(self, data: bytes): + """Provide more data to the running parser.""" + self._parser.feed(data) + + def _prepare_tempfile(self, filename): + tf = tempfile.NamedTemporaryFile(delete=False) + self._files[filename] = tf + self.current_filename = filename + + def _get_open_tempfile(self): + return self._files[self.current_filename] + + def get_response(self): + """Return the deserialized JSON object from the multipart "response" field.""" + return self._response + + def filenames(self): + """Return a list of filenames from the "files" parts of the response.""" + return list(self._files.keys()) + + def get_file(self, path, encoding): + """Return an open file object containing the data.""" + mode = 'r' if encoding else 'rb' + # We're using text-based file I/O purely for file encoding purposes, not for + # newline normalization. newline='' serves the line endings as-is. + newline = '' if encoding else None + return open(self._files[path].name, mode, encoding=encoding, newline=newline) + + +class _MultipartParser: + def __init__( + self, + marker: bytes, + handle_header, + handle_body, + max_lookahead: int = 0, + max_boundary_length: int = 0): + r"""Configures a parser for mime multipart messages. + + Args: + marker: the multipart boundary marker (i.e. in "\r\n----\r\n") + + handle_header(data): called once with the entire contents of a part + header as encountered in data fed to the parser + + handle_body(data, done=False): called incrementally as part body + data is fed into the parser - its "done" parameter is set to true when + the body is complete. + + max_lookahead: maximum amount of bytes to buffer when searching for a complete header. + + max_boundary_length: maximum number of bytes that can make up a part + boundary (e.g. \r\n----\r\n") + """ + self._marker = marker + self._handle_header = handle_header + self._handle_body = handle_body + self._max_lookahead = max_lookahead + self._max_boundary_length = max_boundary_length + + self._buf = bytearray() + self._pos = 0 # current position in buf + self._done = False # whether we have found the terminal boundary and are done parsing + self._header_terminator = b'\r\n\r\n' + + # RFC 2046 notes optional "linear whitespace" (e.g. [ \t]+) after the boundary pattern + # and the optional "--" suffix. The boundaries strings can be constructed as follows: + # + # boundary = \r\n--[ \t]+\r\n + # terminal_boundary = \r\n----[ \t]+\r\n + # + # 99 is arbitrarily chosen to represent a max number of linear + # whitespace characters to help avoid wrongly writing boundary + # characters into a (temporary) file. + if not max_boundary_length: + self._max_boundary_length = len(b'\r\n--' + marker + b'--\r\n') + 99 + + def feed(self, data: bytes): + """Feeds data incrementally into the parser.""" + if self._done: + return + self._buf.extend(data) + + while True: + # seek to a boundary if we aren't already on one + i, n, self._done = _next_part_boundary(self._buf, self._marker) + if i == -1 or self._done: + return # waiting for more data or terminal boundary reached + + if self._pos == 0: + # parse the part header + if self._max_lookahead and len(self._buf) - self._pos > self._max_lookahead: + raise ProtocolError('header terminator not found') + term_index = self._buf.find(self._header_terminator) + if term_index == -1: + return # waiting for more data + + start = i + n + # data includes the double CRLF at the end of the header. + end = term_index + len(self._header_terminator) + + self._handle_header(self._buf[start:end]) + self._pos = end + else: + # parse the part body + ii, nn, self._done = _next_part_boundary(self._buf, self._marker, start=self._pos) + safe_bound = max(0, len(self._buf) - self._max_boundary_length) + if ii != -1: + # part body is finished + self._handle_body(self._buf[self._pos:ii], done=True) + self._buf = self._buf[ii:] + self._pos = 0 + if self._done: + return # terminal boundary reached + elif safe_bound > self._pos: + # write partial body data + data = self._buf[self._pos:safe_bound] + self._pos = safe_bound + self._handle_body(data) + return # waiting for more data + else: + return # waiting for more data + + +def _next_part_boundary(buf, marker, start=0): + """Returns the index of the next boundary marker in buf beginning at start. + + Returns: + (index, length, is_terminal) or (-1, -1, False) if no boundary is found. + """ + prefix = b'\r\n--' + marker + suffix = b'\r\n' + terminal_midfix = b'--' + + i = buf.find(prefix, start) + if i == -1: + return -1, -1, False + + pos = i + len(prefix) + is_terminal = False + if buf[pos:].startswith(terminal_midfix): + is_terminal = True + pos += len(terminal_midfix) + + # Note: RFC 2046 notes optional "linear whitespace" (e.g. [ \t]) after the boundary pattern + # and the optional "--" suffix. + tail = buf[pos:] + for c in tail: + if c not in b' \t': + break + pos += 1 + + if buf[pos:].startswith(suffix): + pos += len(suffix) + return i, pos - i, is_terminal + return -1, -1, False diff --git a/ubuntu/venv/ops/py.typed b/ubuntu/venv/ops/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/ops/storage.py b/ubuntu/venv/ops/storage.py new file mode 100644 index 0000000..52f2ec5 --- /dev/null +++ b/ubuntu/venv/ops/storage.py @@ -0,0 +1,390 @@ +# Copyright 2019-2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Structures to offer storage to the charm (through Juju or locally).""" + +import pickle +import shutil +import sqlite3 +import subprocess +import typing +from datetime import timedelta +from typing import Any, Callable, Generator, List, Optional, Tuple, Type, Union + +import yaml + +if typing.TYPE_CHECKING: + from pathlib import Path + + # _Notice = Tuple[event_path, observer_path, method_name] + _Notice = Tuple[str, str, str] + _Notices = List[_Notice] + + # This is a function that takes a Tuple and returns a yaml node. + # it replaces a method, so the first argument passed to the function + # (Any) is 'self'. + _TupleRepresenterType = Callable[[Any, Tuple[Any, ...]], yaml.Node] + _NoticeGenerator = Generator['_Notice', None, None] + + +def _run(args: List[str], **kw: Any): + cmd = shutil.which(args[0]) # type: Optional[str] + if cmd is None: + raise FileNotFoundError(args[0]) + return subprocess.run([cmd, *args[1:]], **kw) + + +class SQLiteStorage: + """Storage using SQLite backend.""" + + DB_LOCK_TIMEOUT = timedelta(hours=1) + + def __init__(self, filename: Union['Path', str]): + # The isolation_level argument is set to None such that the implicit + # transaction management behavior of the sqlite3 module is disabled. + self._db = sqlite3.connect(str(filename), + isolation_level=None, + timeout=self.DB_LOCK_TIMEOUT.total_seconds()) + self._setup() + + def _setup(self): + """Make the database ready to be used as storage.""" + # Make sure that the database is locked until the connection is closed, + # not until the transaction ends. + self._db.execute("PRAGMA locking_mode=EXCLUSIVE") + c = self._db.execute("BEGIN") + c.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='snapshot'") + if c.fetchone()[0] == 0: + # Keep in mind what might happen if the process dies somewhere below. + # The system must not be rendered permanently broken by that. + self._db.execute("CREATE TABLE snapshot (handle TEXT PRIMARY KEY, data BLOB)") + self._db.execute(''' + CREATE TABLE notice ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_path TEXT, + observer_path TEXT, + method_name TEXT) + ''') + self._db.commit() + + def close(self): + """Part of the Storage API, close the storage backend.""" + self._db.close() + + def commit(self): + """Part of the Storage API, commit latest changes in the storage backend.""" + self._db.commit() + + # There's commit but no rollback. For abort to be supported, we'll need logic that + # can rollback decisions made by third-party code in terms of the internal state + # of objects that have been snapshotted, and hooks to let them know about it and + # take the needed actions to undo their logic until the last snapshot. + # This is doable but will increase significantly the chances for mistakes. + + def save_snapshot(self, handle_path: str, snapshot_data: Any) -> None: + """Part of the Storage API, persist a snapshot data under the given handle. + + Args: + handle_path: The string identifying the snapshot. + snapshot_data: The data to be persisted. (as returned by Object.snapshot()). This + might be a dict/tuple/int, but must only contain 'simple' python types. + """ + # Use pickle for serialization, so the value remains portable. + raw_data = pickle.dumps(snapshot_data) + self._db.execute("REPLACE INTO snapshot VALUES (?, ?)", (handle_path, raw_data)) + + def load_snapshot(self, handle_path: str) -> Any: + """Part of the Storage API, retrieve a snapshot that was previously saved. + + Args: + handle_path: The string identifying the snapshot. + + Raises: + NoSnapshotError: if there is no snapshot for the given handle_path. + """ + c = self._db.cursor() + c.execute("SELECT data FROM snapshot WHERE handle=?", (handle_path,)) + row = c.fetchone() + if row: + return pickle.loads(row[0]) + raise NoSnapshotError(handle_path) + + def drop_snapshot(self, handle_path: str): + """Part of the Storage API, remove a snapshot that was previously saved. + + Dropping a snapshot that doesn't exist is treated as a no-op. + """ + self._db.execute("DELETE FROM snapshot WHERE handle=?", (handle_path,)) + + def list_snapshots(self) -> Generator[str, None, None]: + """Return the name of all snapshots that are currently saved.""" + c = self._db.cursor() + c.execute("SELECT handle FROM snapshot") + while True: + rows = c.fetchmany() + if not rows: + break + for row in rows: + yield row[0] + + def save_notice(self, event_path: str, observer_path: str, method_name: str) -> None: + """Part of the Storage API, record an notice (event and observer).""" + self._db.execute('INSERT INTO notice VALUES (NULL, ?, ?, ?)', + (event_path, observer_path, method_name)) + + def drop_notice(self, event_path: str, observer_path: str, method_name: str) -> None: + """Part of the Storage API, remove a notice that was previously recorded.""" + self._db.execute(''' + DELETE FROM notice + WHERE event_path=? + AND observer_path=? + AND method_name=? + ''', (event_path, observer_path, method_name)) + + def notices(self, event_path: Optional[str] = None) -> '_NoticeGenerator': + """Part of the Storage API, return all notices that begin with event_path. + + Args: + event_path: If supplied, will only yield events that match event_path. If not + supplied (or None/'') will return all events. + + Returns: + Iterable of (event_path, observer_path, method_name) tuples + """ + if event_path: + c = self._db.execute(''' + SELECT event_path, observer_path, method_name + FROM notice + WHERE event_path=? + ORDER BY sequence + ''', (event_path,)) + else: + c = self._db.execute(''' + SELECT event_path, observer_path, method_name + FROM notice + ORDER BY sequence + ''') + while True: + rows = c.fetchmany() + if not rows: + break + for row in rows: + yield tuple(row) + + +class JujuStorage: + """Storing the content tracked by the Framework in Juju. + + This uses :class:`_JujuStorageBackend` to interact with state-get/state-set + as the way to store state for the framework and for components. + """ + + NOTICE_KEY = "#notices#" + + def __init__(self, backend: Optional['_JujuStorageBackend'] = None): + self._backend = backend or _JujuStorageBackend() # type: _JujuStorageBackend + + def close(self): + """Part of the Storage API, close the storage backend. + + Nothing to be done for Juju backend, as it's transactional. + """ + + def commit(self): + """Part of the Storage API, commit latest changes in the storage backend. + + Nothing to be done for Juju backend, as it's transactional. + """ + + def save_snapshot(self, handle_path: str, snapshot_data: Any) -> None: + """Part of the Storage API, persist a snapshot data under the given handle. + + Args: + handle_path: The string identifying the snapshot. + snapshot_data: The data to be persisted. (as returned by Object.snapshot()). This + might be a dict/tuple/int, but must only contain 'simple' python types. + """ + self._backend.set(handle_path, snapshot_data) + + def load_snapshot(self, handle_path: str): + """Part of the Storage API, retrieve a snapshot that was previously saved. + + Args: + handle_path: The string identifying the snapshot. + + Raises: + NoSnapshotError: if there is no snapshot for the given handle_path. + """ + try: + content = self._backend.get(handle_path) + except KeyError: + raise NoSnapshotError(handle_path) + return content + + def drop_snapshot(self, handle_path: str): + """Part of the Storage API, remove a snapshot that was previously saved. + + Dropping a snapshot that doesn't exist is treated as a no-op. + """ + self._backend.delete(handle_path) + + def save_notice(self, event_path: str, observer_path: str, method_name: str): + """Part of the Storage API, record a notice (event and observer).""" + notice_list = self._load_notice_list() + notice_list.append((event_path, observer_path, method_name)) + self._save_notice_list(notice_list) + + def drop_notice(self, event_path: str, observer_path: str, method_name: str): + """Part of the Storage API, remove a notice that was previously recorded.""" + notice_list = self._load_notice_list() + notice_list.remove((event_path, observer_path, method_name)) + self._save_notice_list(notice_list) + + def notices(self, event_path: Optional[str] = None): + """Part of the Storage API, return all notices that begin with event_path. + + Args: + event_path: If supplied, will only yield events that match event_path. If not + supplied (or None/'') will return all events. + + Returns: + Iterable of (event_path, observer_path, method_name) tuples + """ + notice_list = self._load_notice_list() + for row in notice_list: + if event_path and row[0] != event_path: + continue + yield tuple(row) + + def _load_notice_list(self) -> '_Notices': + """Load a notice list from current key. + + Returns: + List of (event_path, observer_path, method_name) tuples; empty if no key or is None. + """ + try: + notice_list = self._backend.get(self.NOTICE_KEY) + except KeyError: + return [] + if notice_list is None: + return [] + return notice_list + + def _save_notice_list(self, notices: '_Notices') -> None: + """Save a notice list under current key. + + Args: + notices: List of (event_path, observer_path, method_name) tuples. + """ + self._backend.set(self.NOTICE_KEY, notices) + + +# we load yaml.CSafeX if available, falling back to slower yaml.SafeX. +_BaseDumper = getattr(yaml, 'CSafeDumper', yaml.SafeDumper) # type: Type[yaml.SafeDumper] +_BaseLoader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader) # type: Type[yaml.SafeLoader] + + +class _SimpleLoader(_BaseLoader): + """Handle a couple basic python types. + + yaml.SafeLoader can handle all the basic int/float/dict/set/etc that we want. The only one + that it *doesn't* handle is tuples. We don't want to support arbitrary types, so we just + subclass SafeLoader and add tuples back in. + """ + # Taken from the example at: + # https://stackoverflow.com/questions/9169025/how-can-i-add-a-python-tuple-to-a-yaml-file-using-pyyaml + + construct_python_tuple = yaml.Loader.construct_python_tuple # type: ignore + + +_SimpleLoader.add_constructor( # type: ignore + 'tag:yaml.org,2002:python/tuple', + _SimpleLoader.construct_python_tuple) # type: ignore + + +class _SimpleDumper(_BaseDumper): + """Add types supported by 'marshal'. + + YAML can support arbitrary types, but that is generally considered unsafe (like pickle). So + we want to only support dumping out types that are safe to load. + """ + represent_tuple = yaml.Dumper.represent_tuple # type: _TupleRepresenterType + + +_SimpleDumper.add_representer(tuple, _SimpleDumper.represent_tuple) + + +def juju_backend_available() -> bool: + """Check if Juju state storage is available.""" + p = shutil.which('state-get') + return p is not None + + +class _JujuStorageBackend: + """Implements the interface from the Operator framework to Juju's state-get/set/etc.""" + + def set(self, key: str, value: Any) -> None: + """Set a key to a given value. + + Args: + key: The string key that will be used to find the value later + value: Arbitrary content that will be returned by get(). + + Raises: + CalledProcessError: if 'state-set' returns an error code. + """ + # default_flow_style=None means that it can use Block for + # complex types (types that have nested types) but use flow + # for simple types (like an array). Not all versions of PyYAML + # have the same default style. + encoded_value = yaml.dump(value, Dumper=_SimpleDumper, default_flow_style=None) + content = yaml.dump( + {key: encoded_value}, encoding='utf8', default_style='|', + default_flow_style=False, + Dumper=_SimpleDumper) + _run(["state-set", "--file", "-"], input=content, check=True) + + def get(self, key: str) -> Any: + """Get the bytes value associated with a given key. + + Args: + key: The string key that will be used to find the value + Raises: + CalledProcessError: if 'state-get' returns an error code. + """ + # We don't capture stderr here so it can end up in debug logs. + p = _run(["state-get", key], stdout=subprocess.PIPE, check=True, universal_newlines=True) + if p.stdout == '' or p.stdout == '\n': + raise KeyError(key) + return yaml.load(p.stdout, Loader=_SimpleLoader) # type: ignore + + def delete(self, key: str) -> None: + """Remove a key from being tracked. + + Args: + key: The key to stop storing + Raises: + CalledProcessError: if 'state-delete' returns an error code. + """ + _run(["state-delete", key], check=True) + + +class NoSnapshotError(Exception): + """Exception to flag that there is no snapshot for the given handle_path.""" + + def __init__(self, handle_path: str): + self.handle_path = handle_path + + def __str__(self): + return 'no snapshot data found for {} object'.format(self.handle_path) diff --git a/ubuntu/venv/ops/testing.py b/ubuntu/venv/ops/testing.py new file mode 100644 index 0000000..bf6eda3 --- /dev/null +++ b/ubuntu/venv/ops/testing.py @@ -0,0 +1,2153 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Infrastructure to build unittests for Charms using the Operator Framework. + +Global Variables: + + SIMULATE_CAN_CONNECT: This enables can_connect simulation for the test + harness. It should be set *before* you create Harness instances and not + changed after. You *should* set this to true - it will help your tests be + more accurate! This causes all containers' can_connect states initially + be False rather than True and causes the testing with the harness to model + and track can_connect state for containers accurately. This means that + calls that require communication with the container API (e.g. + Container.push, Container.get_plan, Container.add_layer, etc.) will only + succeed if Container.can_connect() returns True and will raise exceptions + otherwise. can_connect state evolves automatically to track with events + associated with container state, (e.g. calling container_pebble_ready). + If SIMULATE_CAN_CONNECT is True, can_connect state for containers can also + be manually controlled using Harness.set_can_connect. +""" + + +import datetime +import fnmatch +import inspect +import os +import pathlib +import random +import signal +import tempfile +import typing +import warnings +from contextlib import contextmanager +from io import BytesIO, StringIO +from textwrap import dedent + +from ops import charm, framework, model, pebble, storage +from ops._private import yaml + +# Toggles Container.can_connect simulation globally for all harness instances. +# For this to work, it must be set *before* Harness instances are created. +SIMULATE_CAN_CONNECT = False + +# OptionalYAML is something like metadata.yaml or actions.yaml. You can +# pass in a file-like object or the string directly. +OptionalYAML = typing.Optional[typing.Union[str, typing.TextIO]] + + +# An instance of an Application or Unit, or the name of either. +# This is done here to avoid a scoping issue with the `model` property +# of the Harness class below. +AppUnitOrName = typing.Union[str, model.Application, model.Unit] + + +# CharmType represents user charms that are derived from CharmBase. +CharmType = typing.TypeVar('CharmType', bound=charm.CharmBase) + + +# noinspection PyProtectedMember +class Harness(typing.Generic[CharmType]): + """This class represents a way to build up the model that will drive a test suite. + + The model that is created is from the viewpoint of the charm that you are testing. + + Example:: + + harness = Harness(MyCharm) + # Do initial setup here + relation_id = harness.add_relation('db', 'postgresql') + # Now instantiate the charm to see events as the model changes + harness.begin() + harness.add_relation_unit(relation_id, 'postgresql/0') + harness.update_relation_data(relation_id, 'postgresql/0', {'key': 'val'}) + # Check that charm has properly handled the relation_joined event for postgresql/0 + self.assertEqual(harness.charm. ...) + + Args: + charm_cls: The Charm class that you'll be testing. + meta: charm.CharmBase is a A string or file-like object containing the contents of + metadata.yaml. If not supplied, we will look for a 'metadata.yaml' file in the + parent directory of the Charm, and if not found fall back to a trivial + 'name: test-charm' metadata. + actions: A string or file-like object containing the contents of + actions.yaml. If not supplied, we will look for a 'actions.yaml' file in the + parent directory of the Charm. + config: A string or file-like object containing the contents of + config.yaml. If not supplied, we will look for a 'config.yaml' file in the + parent directory of the Charm. + """ + + def __init__( + self, + charm_cls: typing.Type[CharmType], + *, + meta: OptionalYAML = None, + actions: OptionalYAML = None, + config: OptionalYAML = None): + self._charm_cls = charm_cls + self._charm = None + self._charm_dir = 'no-disk-path' # this may be updated by _create_meta + self._meta = self._create_meta(meta, actions) + self._unit_name = self._meta.name + '/0' + self._framework = None + self._hooks_enabled = True + self._relation_id_counter = 0 + self._backend = _TestingModelBackend(self._unit_name, self._meta) + self._model = model.Model(self._meta, self._backend) + self._storage = storage.SQLiteStorage(':memory:') + self._oci_resources = {} + self._framework = framework.Framework( + self._storage, self._charm_dir, self._meta, self._model) + self._defaults = self._load_config_defaults(config) + self._update_config(key_values=self._defaults) + + # TODO: If/when we decide to allow breaking changes for a release, + # change SIMULATE_CAN_CONNECT default value to True and remove the + # warning message below. This warning was added 2022-03-22 + if not SIMULATE_CAN_CONNECT: + warnings.warn( + 'Please set ops.testing.SIMULATE_CAN_CONNECT=True.' + 'See https://juju.is/docs/sdk/testing#heading--simulate-can-connect for details.') + + def set_can_connect(self, container: typing.Union[str, model.Container], val: bool): + """Change the simulated can_connect status of a container's underlying pebble client. + + Calling this method raises an exception if SIMULATE_CAN_CONNECT is False. + """ + if isinstance(container, str): + container = self.model.unit.get_container(container) + self._backend._set_can_connect(container._pebble, val) + + @property + def charm(self) -> CharmType: + """Return the instance of the charm class that was passed to __init__. + + Note that the Charm is not instantiated until you have called + :meth:`.begin()`. + """ + return self._charm + + @property + def model(self) -> model.Model: + """Return the :class:`~ops.model.Model` that is being driven by this Harness.""" + return self._model + + @property + def framework(self) -> framework.Framework: + """Return the Framework that is being driven by this Harness.""" + return self._framework + + def begin(self) -> None: + """Instantiate the Charm and start handling events. + + Before calling :meth:`begin`, there is no Charm instance, so changes to the Model won't + emit events. You must call :meth:`.begin` before :attr:`.charm` is valid. + """ + if self._charm is not None: + raise RuntimeError('cannot call the begin method on the harness more than once') + + # The Framework adds attributes to class objects for events, etc. As such, we can't re-use + # the original class against multiple Frameworks. So create a locally defined class + # and register it. + # TODO: jam 2020-03-16 We are looking to changes this to Instance attributes instead of + # Class attributes which should clean up this ugliness. The API can stay the same + class TestEvents(self._charm_cls.on.__class__): + pass + + TestEvents.__name__ = self._charm_cls.on.__class__.__name__ + + class TestCharm(self._charm_cls): + on = TestEvents() + + # Note: jam 2020-03-01 This is so that errors in testing say MyCharm has no attribute foo, + # rather than TestCharm has no attribute foo. + TestCharm.__name__ = self._charm_cls.__name__ + self._charm = TestCharm(self._framework) + + def begin_with_initial_hooks(self) -> None: + """Called when you want the Harness to fire the same hooks that Juju would fire at startup. + + This triggers install, relation-created, config-changed, start, and any relation-joined + hooks based on what relations have been defined+added before you called begin. This does + NOT trigger a pebble-ready hook. Note that all of these are fired before returning control + to the test suite, so if you want to introspect what happens at each step, you need to fire + them directly (e.g. Charm.on.install.emit()). In your hook callback functions, you should + not assume that workload containers are active; guard such code with checks to + Container.can_connect(). You are encouraged to test this by setting the global + SIMULATE_CAN_CONNECT variable to True. + + To use this with all the normal hooks, you should instantiate the harness, setup any + relations that you want active when the charm starts, and then call this method. This + method will automatically create and add peer relations that are specified in + metadata.yaml. + + Example:: + + harness = Harness(MyCharm) + # Do initial setup here + # Add storage if needed before begin_with_initial_hooks() is called + storage_ids = harness.add_storage('data', count=1)[0] + storage_id = storage_id[0] # we only added one storage instance + relation_id = harness.add_relation('db', 'postgresql') + harness.add_relation_unit(relation_id, 'postgresql/0') + harness.update_relation_data(relation_id, 'postgresql/0', {'key': 'val'}) + harness.set_leader(True) + harness.update_config({'initial': 'config'}) + harness.begin_with_initial_hooks() + # This will cause + # install, db-relation-created('postgresql'), leader-elected, config-changed, start + # db-relation-joined('postrgesql/0'), db-relation-changed('postgresql/0') + # To be fired. + """ + self.begin() + + # Checking if disks have been added + # storage-attached events happen before install + for storage_name in self._meta.storages: + for storage_index in self._backend.storage_list(storage_name): + storage_name = storage_name.replace('-', '_') + # Storage device(s) detected, emit storage-attached event(s) + self._charm.on[storage_name].storage_attached.emit( + model.Storage(storage_name, storage_index, self._backend)) + # Storage done, emit install event + self._charm.on.install.emit() + # Juju itself iterates what relation to fire based on a map[int]relation, so it doesn't + # guarantee a stable ordering between relation events. It *does* give a stable ordering + # of joined units for a given relation. + items = list(self._meta.relations.items()) + random.shuffle(items) + this_app_name = self._meta.name + for relname, rel_meta in items: + if rel_meta.role == charm.RelationRole.peer: + # If the user has directly added a relation, leave it be, but otherwise ensure + # that peer relations are always established at before leader-elected. + rel_ids = self._backend._relation_ids_map.get(relname) + if rel_ids is None: + self.add_relation(relname, self._meta.name) + else: + random.shuffle(rel_ids) + for rel_id in rel_ids: + self._emit_relation_created(relname, rel_id, this_app_name) + else: + rel_ids = self._backend._relation_ids_map.get(relname, []) + random.shuffle(rel_ids) + for rel_id in rel_ids: + app_name = self._backend._relation_app_and_units[rel_id]["app"] + self._emit_relation_created(relname, rel_id, app_name) + if self._backend._is_leader: + self._charm.on.leader_elected.emit() + else: + self._charm.on.leader_settings_changed.emit() + self._charm.on.config_changed.emit() + self._charm.on.start.emit() + # If the initial hooks do not set a unit status, the Juju controller will switch + # the unit status from "Maintenance" to "Unknown". See gh#726 + post_setup_sts = self._backend.status_get() + if post_setup_sts.get("status") == "maintenance" and not post_setup_sts.get("message"): + self._backend.status_set("unknown", "", is_app=False) + all_ids = list(self._backend._relation_names.items()) + random.shuffle(all_ids) + for rel_id, rel_name in all_ids: + rel_app_and_units = self._backend._relation_app_and_units[rel_id] + app_name = rel_app_and_units["app"] + # Note: Juju *does* fire relation events for a given relation in the sorted order of + # the unit names. It also always fires relation-changed immediately after + # relation-joined for the same unit. + # Juju only fires relation-changed (app) if there is data for the related application + relation = self._model.get_relation(rel_name, rel_id) + if self._backend._relation_data[rel_id].get(app_name): + app = self._model.get_app(app_name) + self._charm.on[rel_name].relation_changed.emit( + relation, app, None) + for unit_name in sorted(rel_app_and_units["units"]): + remote_unit = self._model.get_unit(unit_name) + self._charm.on[rel_name].relation_joined.emit( + relation, remote_unit.app, remote_unit) + self._charm.on[rel_name].relation_changed.emit( + relation, remote_unit.app, remote_unit) + + def cleanup(self) -> None: + """Called by your test infrastructure to cleanup any temporary directories/files/etc. + + Currently this only needs to be called if you test with resources. But it is reasonable + to always include a `testcase.addCleanup(harness.cleanup)` just in case. + """ + self._backend._cleanup() + + def _create_meta(self, charm_metadata, action_metadata): + """Create a CharmMeta object. + + Handle the cases where a user doesn't supply explicit metadata snippets. + """ + filename = inspect.getfile(self._charm_cls) + charm_dir = pathlib.Path(filename).parents[1] + + if charm_metadata is None: + metadata_path = charm_dir / 'metadata.yaml' + if metadata_path.is_file(): + charm_metadata = metadata_path.read_text() + self._charm_dir = charm_dir + else: + # The simplest of metadata that the framework can support + charm_metadata = 'name: test-charm' + elif isinstance(charm_metadata, str): + charm_metadata = dedent(charm_metadata) + + if action_metadata is None: + actions_path = charm_dir / 'actions.yaml' + if actions_path.is_file(): + action_metadata = actions_path.read_text() + self._charm_dir = charm_dir + elif isinstance(action_metadata, str): + action_metadata = dedent(action_metadata) + + return charm.CharmMeta.from_yaml(charm_metadata, action_metadata) + + def _load_config_defaults(self, charm_config): + """Load default values from config.yaml. + + Handle the case where a user doesn't supply explicit config snippets. + """ + filename = inspect.getfile(self._charm_cls) + charm_dir = pathlib.Path(filename).parents[1] + + if charm_config is None: + config_path = charm_dir / 'config.yaml' + if config_path.is_file(): + charm_config = config_path.read_text() + self._charm_dir = charm_dir + else: + # The simplest of config that the framework can support + charm_config = '{}' + elif isinstance(charm_config, str): + charm_config = dedent(charm_config) + charm_config = yaml.safe_load(charm_config) + charm_config = charm_config.get('options', {}) + return {key: value.get('default', None) for key, value in charm_config.items()} + + def add_oci_resource(self, resource_name: str, + contents: typing.Mapping[str, str] = None) -> None: + """Add oci resources to the backend. + + This will register an oci resource and create a temporary file for processing metadata + about the resource. A default set of values will be used for all the file contents + unless a specific contents dict is provided. + + Args: + resource_name: Name of the resource to add custom contents to. + contents: Optional custom dict to write for the named resource. + """ + if not contents: + contents = {'registrypath': 'registrypath', + 'username': 'username', + 'password': 'password', + } + if resource_name not in self._meta.resources.keys(): + raise RuntimeError('Resource {} is not a defined resources'.format(resource_name)) + if self._meta.resources[resource_name].type != "oci-image": + raise RuntimeError('Resource {} is not an OCI Image'.format(resource_name)) + + as_yaml = yaml.safe_dump(contents) + self._backend._resources_map[resource_name] = ('contents.yaml', as_yaml) + + def add_resource(self, resource_name: str, content: typing.AnyStr) -> None: + """Add content for a resource to the backend. + + This will register the content, so that a call to `Model.resources.fetch(resource_name)` + will return a path to a file containing that content. + + Args: + resource_name: The name of the resource being added + content: Either string or bytes content, which will be the content of the filename + returned by resource-get. If contents is a string, it will be encoded in utf-8 + """ + if resource_name not in self._meta.resources.keys(): + raise RuntimeError('Resource {} is not a defined resources'.format(resource_name)) + record = self._meta.resources[resource_name] + if record.type != "file": + raise RuntimeError( + 'Resource {} is not a file, but actually {}'.format(resource_name, record.type)) + filename = record.filename + if filename is None: + filename = resource_name + + self._backend._resources_map[resource_name] = (filename, content) + + def populate_oci_resources(self) -> None: + """Populate all OCI resources.""" + for name, data in self._meta.resources.items(): + if data.type == "oci-image": + self.add_oci_resource(name) + + def disable_hooks(self) -> None: + """Stop emitting hook events when the model changes. + + This can be used by developers to stop changes to the model from emitting events that + the charm will react to. Call :meth:`.enable_hooks` + to re-enable them. + """ + self._hooks_enabled = False + + def enable_hooks(self) -> None: + """Re-enable hook events from charm.on when the model is changed. + + By default hook events are enabled once you call :meth:`.begin`, + but if you have used :meth:`.disable_hooks`, this can be used to + enable them again. + """ + self._hooks_enabled = True + + @contextmanager + def hooks_disabled(self): + """A context manager to run code with hooks disabled. + + Example:: + + with harness.hooks_disabled(): + # things in here don't fire events + harness.set_leader(True) + harness.update_config(unset=['foo', 'bar']) + # things here will again fire events + """ + if self._hooks_enabled: + self.disable_hooks() + try: + yield None + finally: + self.enable_hooks() + else: + yield None + + def _next_relation_id(self): + rel_id = self._relation_id_counter + self._relation_id_counter += 1 + return rel_id + + def add_storage(self, storage_name: str, count: int = 1) -> typing.List[str]: + """Declare a new storage device attached to this unit. + + To have repeatable tests, each device will be initialized with + location set to /N, where N is the counter and + will be a number from [0,total_num_disks-1] + + Args: + storage_name: The storage backend name on the Charm + count: Number of disks being added + + Return: + A list of storage IDs, e.g. ["my-storage/1", "my-storage/2"]. + """ + if storage_name not in self._meta.storages: + raise RuntimeError( + "the key '{}' is not specified as a storage key in metadata".format(storage_name)) + storage_indices = self._backend.storage_add(storage_name, count) + + # Reset associated cached value in the storage mappings. If we don't do this, + # Model._storages won't return Storage objects for subsequently-added storage. + self._model._storages._invalidate(storage_name) + + ids = [] + for storage_index in storage_indices: + s = model.Storage(storage_name, storage_index, self._backend) + ids.append(s.full_id) + if self.charm is not None and self._hooks_enabled: + self.charm.on[storage_name].storage_attached.emit(s) + return ids + + def detach_storage(self, storage_id: str) -> None: + """Detach a storage device. + + The intent of this function is to simulate a "juju detach-storage" call. + It will trigger a storage-detaching hook if the storage unit in question exists + and is presently marked as attached. + + Args: + storage_id: The full storage ID of the storage unit being detached, including the + storage key, e.g. my-storage/0. + """ + if self.charm is None: + raise RuntimeError('cannot detach storage before Harness is initialised') + storage_name, storage_index = storage_id.split('/', 1) + storage_index = int(storage_index) + if self._backend._storage_is_attached(storage_name, storage_index) and self._hooks_enabled: + self.charm.on[storage_name].storage_detaching.emit( + model.Storage(storage_name, storage_index, self._backend)) + self._backend._storage_detach(storage_id) + + def attach_storage(self, storage_id: str) -> None: + """Attach a storage device. + + The intent of this function is to simulate a "juju attach-storage" call. + It will trigger a storage-attached hook if the storage unit in question exists + and is presently marked as detached. + + Args: + storage_id: The full storage ID of the storage unit being attached, including the + storage key, e.g. my-storage/0. + """ + if self._backend._storage_attach(storage_id) and self._hooks_enabled: + storage_name, storage_index = storage_id.split('/', 1) + storage_index = int(storage_index) + self.charm.on[storage_name].storage_attached.emit( + model.Storage(storage_name, storage_index, self._backend)) + + def remove_storage(self, storage_id: str) -> None: + """Detach a storage device. + + The intent of this function is to simulate a "juju remove-storage" call. + It will trigger a storage-detaching hook if the storage unit in question exists + and is presently marked as attached. Then it will remove the storage + unit from the testing backend. + + Args: + storage_id: The full storage ID of the storage unit being removed, including the + storage key, e.g. my-storage/0. + """ + storage_name, storage_index = storage_id.split('/', 1) + storage_index = int(storage_index) + if storage_name not in self._meta.storages: + raise RuntimeError( + "the key '{}' is not specified as a storage key in metadata".format(storage_name)) + is_attached = self._backend._storage_is_attached(storage_name, storage_index) + if self.charm is not None and self._hooks_enabled and is_attached: + self.charm.on[storage_name].storage_detaching.emit( + model.Storage(storage_name, storage_index, self._backend)) + self._backend._storage_remove(storage_id) + + def add_relation(self, relation_name: str, remote_app: str) -> int: + """Declare that there is a new relation between this app and `remote_app`. + + In the case of adding peer relations, `remote_app` is *this* app. This function creates a + relation with an application and will trigger a relation-created hook. To relate units (and + trigger relation-joined and relation-changed hooks), you should also call + :meth:`.add_relation_unit`. + + Args: + relation_name: The relation on Charm that is being related to + remote_app: The name of the application that is being related to + + Return: + The relation_id created by this add_relation. + """ + rel_id = self._next_relation_id() + self._backend._relation_ids_map.setdefault(relation_name, []).append(rel_id) + self._backend._relation_names[rel_id] = relation_name + self._backend._relation_list_map[rel_id] = [] + self._backend._relation_data[rel_id] = { + remote_app: {}, + self._backend.unit_name: {}, + self._backend.app_name: {}, + } + self._backend._relation_app_and_units[rel_id] = { + "app": remote_app, + "units": [], + } + # Reload the relation_ids list + if self._model is not None: + self._model.relations._invalidate(relation_name) + self._emit_relation_created(relation_name, rel_id, remote_app) + return rel_id + + def remove_relation(self, relation_id: int) -> None: + """Remove a relation. + + Args: + relation_id: The relation ID for the relation to be removed. + + Raises: + RelationNotFoundError: if relation id is not valid + """ + try: + relation_name = self._backend._relation_names[relation_id] + remote_app = self._backend.relation_remote_app_name(relation_id) + except KeyError as e: + raise model.RelationNotFoundError from e + + for unit_name in self._backend._relation_list_map[relation_id].copy(): + self.remove_relation_unit(relation_id, unit_name) + + self._emit_relation_broken(relation_name, relation_id, remote_app) + if self._model is not None: + self._model.relations._invalidate(relation_name) + + self._backend._relation_app_and_units.pop(relation_id) + self._backend._relation_data.pop(relation_id) + self._backend._relation_list_map.pop(relation_id) + self._backend._relation_ids_map[relation_name].remove(relation_id) + self._backend._relation_names.pop(relation_id) + + def _emit_relation_created(self, relation_name: str, relation_id: int, + remote_app: str) -> None: + """Trigger relation-created for a given relation with a given remote application.""" + if self._charm is None or not self._hooks_enabled: + return + relation = self._model.get_relation(relation_name, relation_id) + app = self._model.get_app(remote_app) + self._charm.on[relation_name].relation_created.emit( + relation, app) + + def _emit_relation_broken(self, relation_name: str, relation_id: int, + remote_app: str) -> None: + """Trigger relation-broken for a given relation with a given remote application.""" + if self._charm is None or not self._hooks_enabled: + return + relation = self._model.get_relation(relation_name, relation_id) + app = self._model.get_app(remote_app) + self._charm.on[relation_name].relation_broken.emit( + relation, app) + + def add_relation_unit(self, relation_id: int, remote_unit_name: str) -> None: + """Add a new unit to a relation. + + Example:: + + rel_id = harness.add_relation('db', 'postgresql') + harness.add_relation_unit(rel_id, 'postgresql/0') + + This will trigger a `relation_joined` event. This would naturally be + followed by a `relation_changed` event, which you can trigger with + :meth:`.update_relation_data`. This separation is artificial in the + sense that Juju will always fire the two, but is intended to make + testing relations and their data bags slightly more natural. + + Args: + relation_id: The integer relation identifier (as returned by add_relation). + remote_unit_name: A string representing the remote unit that is being added. + + Return: + None + """ + self._backend._relation_list_map[relation_id].append(remote_unit_name) + self._backend._relation_data[relation_id][remote_unit_name] = {} + # TODO: jam 2020-08-03 This is where we could assert that the unit name matches the + # application name (eg you don't have a relation to 'foo' but add units of 'bar/0' + self._backend._relation_app_and_units[relation_id]["units"].append(remote_unit_name) + relation_name = self._backend._relation_names[relation_id] + # Make sure that the Model reloads the relation_list for this relation_id, as well as + # reloading the relation data for this unit. + remote_unit = self._model.get_unit(remote_unit_name) + relation = self._model.get_relation(relation_name, relation_id) + unit_cache = relation.data.get(remote_unit, None) + if unit_cache is not None: + unit_cache._invalidate() + self._model.relations._invalidate(relation_name) + if self._charm is None or not self._hooks_enabled: + return + self._charm.on[relation_name].relation_joined.emit( + relation, remote_unit.app, remote_unit) + + def remove_relation_unit(self, relation_id: int, remote_unit_name: str) -> None: + """Remove a unit from a relation. + + Example:: + + rel_id = harness.add_relation('db', 'postgresql') + harness.add_relation_unit(rel_id, 'postgresql/0') + ... + harness.remove_relation_unit(rel_id, 'postgresql/0') + + This will trigger a `relation_departed` event. This would + normally be followed by a `relation_changed` event triggered + by Juju. However when using the test harness a + `relation_changed` event must be triggererd using + :meth:`.update_relation_data`. This deviation from normal Juju + behaviour, facilitates testing by making each step in the + charm life cycle explicit. + + Args: + relation_id: The integer relation identifier (as returned by add_relation). + remote_unit_name: A string representing the remote unit that is being added. + + Raises: + KeyError: if relation_id or remote_unit_name is not valid + ValueError: if remote_unit_name is not valid + """ + relation_name = self._backend._relation_names[relation_id] + + # gather data to invalidate cache later + remote_unit = self._model.get_unit(remote_unit_name) + relation = self._model.get_relation(relation_name, relation_id) + unit_cache = relation.data.get(remote_unit, None) + + # remove the unit from the list of units in the relation + relation.units.remove(remote_unit) + + self._emit_relation_departed(relation_id, remote_unit_name) + # remove the relation data for the departed unit now that the event has happened + self._backend._relation_list_map[relation_id].remove(remote_unit_name) + self._backend._relation_app_and_units[relation_id]["units"].remove(remote_unit_name) + self._backend._relation_data[relation_id].pop(remote_unit_name) + self.model._relations._invalidate(relation_name=relation.name) + + if unit_cache is not None: + unit_cache._invalidate() + + def _emit_relation_departed(self, relation_id, unit_name): + """Trigger relation-departed event for a given relation id and unit.""" + if self._charm is None or not self._hooks_enabled: + return + rel_name = self._backend._relation_names[relation_id] + relation = self.model.get_relation(rel_name, relation_id) + if '/' in unit_name: + app_name = unit_name.split('/')[0] + app = self.model.get_app(app_name) + unit = self.model.get_unit(unit_name) + else: + raise ValueError('Invalid Unit Name') + self._charm.on[rel_name].relation_departed.emit(relation, app, unit) + + def get_relation_data(self, relation_id: int, app_or_unit: AppUnitOrName) -> typing.Mapping: + """Get the relation data bucket for a single app or unit in a given relation. + + This ignores all of the safety checks of who can and can't see data in relations (eg, + non-leaders can't read their own application's relation data because there are no events + that keep that data up-to-date for the unit). + + Args: + relation_id: The relation whose content we want to look at. + app_or_unit: An Application or Unit instance, or its name, whose data we want to read + Return: + A dict containing the relation data for `app_or_unit` or None. + + Raises: + KeyError: if relation_id doesn't exist + """ + if hasattr(app_or_unit, 'name'): + app_or_unit = app_or_unit.name + return self._backend._relation_data[relation_id].get(app_or_unit, None) + + def get_pod_spec(self) -> (typing.Mapping, typing.Mapping): + """Return the content of the pod spec as last set by the charm. + + This returns both the pod spec and any k8s_resources that were supplied. + See the signature of Model.pod.set_spec + """ + return self._backend._pod_spec + + def get_container_pebble_plan( + self, container_name: str + ) -> pebble.Plan: + """Return the current Plan that pebble is executing for the given container. + + Args: + container_name: The simple name of the associated container + Return: + The pebble.Plan for this container. You can use :meth:`ops.pebble.Plan.to_yaml` to get + a string form for the content. Will raise KeyError if no pebble client exists + for that container name. (should only happen if container is not present in + metadata.yaml) + """ + client = self._backend._pebble_clients.get(container_name) + if client is None: + raise KeyError('no known pebble client for container "{}"'.format(container_name)) + return client.get_plan() + + def container_pebble_ready(self, container_name: str): + """Fire the pebble_ready hook for the associated container. + + This will do nothing if the begin() has not been called. If + SIMULATE_CAN_CONNECT is True, this will switch the given + container's can_connect state to True before the hook + function is called. + """ + if self.charm is None: + return + container = self.model.unit.get_container(container_name) + if SIMULATE_CAN_CONNECT: + self.set_can_connect(container, True) + self.charm.on[container_name].pebble_ready.emit(container) + + def get_workload_version(self) -> str: + """Read the workload version that was set by the unit.""" + return self._backend._workload_version + + def set_model_info(self, name: str = None, uuid: str = None) -> None: + """Set the name and uuid of the Model that this is representing. + + This cannot be called once begin() has been called. But it lets you set the value that + will be returned by Model.name and Model.uuid. + + This is a convenience method to invoke both Harness.set_model_name + and Harness.set_model_uuid at once. + """ + self.set_model_name(name) + self.set_model_uuid(uuid) + + def set_model_name(self, name: str) -> None: + """Set the name of the Model that this is representing. + + This cannot be called once begin() has been called. But it lets you set the value that + will be returned by Model.name. + """ + if self._charm is not None: + raise RuntimeError('cannot set the Model name after begin()') + self._backend.model_name = name + + def set_model_uuid(self, uuid: str) -> None: + """Set the uuid of the Model that this is representing. + + This cannot be called once begin() has been called. But it lets you set the value that + will be returned by Model.uuid. + """ + if self._charm is not None: + raise RuntimeError('cannot set the Model uuid after begin()') + self._backend.model_uuid = uuid + + def update_relation_data( + self, + relation_id: int, + app_or_unit: str, + key_values: typing.Mapping, + ) -> None: + """Update the relation data for a given unit or application in a given relation. + + This also triggers the `relation_changed` event for this relation_id. + + Args: + relation_id: The integer relation_id representing this relation. + app_or_unit: The unit or application name that is being updated. + This can be the local or remote application. + key_values: Each key/value will be updated in the relation data. + """ + relation_name = self._backend._relation_names[relation_id] + relation = self._model.get_relation(relation_name, relation_id) + if '/' in app_or_unit: + entity = self._model.get_unit(app_or_unit) + else: + entity = self._model.get_app(app_or_unit) + rel_data = relation.data.get(entity, None) + if rel_data is not None: + # rel_data may have cached now-stale data, so _invalidate() it. + # Note, this won't cause the data to be loaded if it wasn't already. + rel_data._invalidate() + + new_values = self._backend._relation_data[relation_id][app_or_unit].copy() + values_have_changed = False + for k, v in key_values.items(): + if v == '': + if new_values.pop(k, None) != v: + values_have_changed = True + else: + if k not in new_values or new_values[k] != v: + new_values[k] = v + values_have_changed = True + + # Update the relation data in any case to avoid spurious references + # by an test to an updated value to be invalidated by a lack of assignment + self._backend._relation_data[relation_id][app_or_unit] = new_values + + if not values_have_changed: + # Do not issue a relation changed event if the data bags have not changed + return + + if app_or_unit == self._model.unit.name: + # No events for our own unit + return + if app_or_unit == self._model.app.name: + # updating our own app only generates an event if it is a peer relation and we + # aren't the leader + is_peer = self._meta.relations[relation_name].role.is_peer() + if not is_peer: + return + if self._model.unit.is_leader(): + return + self._emit_relation_changed(relation_id, app_or_unit) + + def _emit_relation_changed(self, relation_id, app_or_unit): + if self._charm is None or not self._hooks_enabled: + return + rel_name = self._backend._relation_names[relation_id] + relation = self.model.get_relation(rel_name, relation_id) + if '/' in app_or_unit: + app_name = app_or_unit.split('/')[0] + unit_name = app_or_unit + app = self.model.get_app(app_name) + unit = self.model.get_unit(unit_name) + args = (relation, app, unit) + else: + app_name = app_or_unit + app = self.model.get_app(app_name) + args = (relation, app) + self._charm.on[rel_name].relation_changed.emit(*args) + + def _update_config( + self, + key_values: typing.Mapping[str, str] = None, + unset: typing.Iterable[str] = (), + ) -> None: + """Update the config as seen by the charm. + + This will *not* trigger a `config_changed` event, and is intended for internal use. + + Note that the `key_values` mapping will only add or update configuration items. + To remove existing ones, see the `unset` parameter. + + Args: + key_values: A Mapping of key:value pairs to update in config. + unset: An iterable of keys to remove from config. + """ + # NOTE: jam 2020-03-01 Note that this sort of works "by accident". Config + # is a LazyMapping, but its _load returns a dict and this method mutates + # the dict that Config is caching. Arguably we should be doing some sort + # of charm.framework.model.config._invalidate() + config = self._backend._config + if key_values is not None: + for key, value in key_values.items(): + if key in self._defaults: + if value is not None: + config[key] = value + else: + raise ValueError("unknown config option: '{}'".format(key)) + + for key in unset: + # When the key is unset, revert to the default if one exists + default = self._defaults.get(key, None) + if default is not None: + config[key] = default + else: + config.pop(key, None) + + def update_config( + self, + key_values: typing.Mapping[str, str] = None, + unset: typing.Iterable[str] = (), + ) -> None: + """Update the config as seen by the charm. + + This will trigger a `config_changed` event. + + Note that the `key_values` mapping will only add or update configuration items. + To remove existing ones, see the `unset` parameter. + + Args: + key_values: A Mapping of key:value pairs to update in config. + unset: An iterable of keys to remove from Config. (Note that this does + not currently reset the config values to the default defined in config.yaml.) + """ + self._update_config(key_values, unset) + if self._charm is None or not self._hooks_enabled: + return + self._charm.on.config_changed.emit() + + def set_leader(self, is_leader: bool = True) -> None: + """Set whether this unit is the leader or not. + + If this charm becomes a leader then `leader_elected` will be triggered. If Harness.begin() + has already been called, then the charm's peer relation should usually be added *prior* to + calling this method (i.e. with Harness.add_relation) to properly initialize and make + available relation data that leader elected hooks may want to access. + + Args: + is_leader: True/False as to whether this unit is the leader. + """ + self._backend._is_leader = is_leader + + # Note: jam 2020-03-01 currently is_leader is cached at the ModelBackend level, not in + # the Model objects, so this automatically gets noticed. + if is_leader and self._charm is not None and self._hooks_enabled: + self._charm.on.leader_elected.emit() + + def set_planned_units(self, num_units: int) -> None: + """Set the number of "planned" units that "Application.planned_units" should return. + + In real world circumstances, this number will be the number of units in the + application. E.g., this number will be the number of peers this unit has, plus one, as we + count our own unit in the total. + + A change to the return from planned_units will not generate an event. Typically, a charm + author would check planned units during a config or install hook, or after receiving a peer + relation joined event. + + """ + if num_units < 0: + raise TypeError("num_units must be 0 or a positive integer.") + self._backend._planned_units = num_units + + def reset_planned_units(self): + """Reset the planned units override. + + This allows the harness to fall through to the built in methods that will try to + guess at a value for planned units, based on the number of peer relations that + have been setup in the testing harness. + + """ + self._backend._planned_units = None + + def _get_backend_calls(self, reset: bool = True) -> list: + """Return the calls that we have made to the TestingModelBackend. + + This is useful mostly for testing the framework itself, so that we can assert that we + do/don't trigger extra calls. + + Args: + reset: If True, reset the calls list back to empty, if false, the call list is + preserved. + + Return: + ``[(call1, args...), (call2, args...)]`` + """ + calls = self._backend._calls.copy() + if reset: + self._backend._calls.clear() + return calls + + +def _record_calls(cls): + """Replace methods on cls with methods that record that they have been called. + + Iterate all attributes of cls, and for public methods, replace them with a wrapped method + that records the method called along with the arguments and keyword arguments. + """ + for meth_name, orig_method in cls.__dict__.items(): + if meth_name.startswith('_'): + continue + + def decorator(orig_method): + def wrapped(self, *args, **kwargs): + full_args = (orig_method.__name__,) + args + if kwargs: + full_args = full_args + (kwargs,) + self._calls.append(full_args) + return orig_method(self, *args, **kwargs) + return wrapped + + setattr(cls, meth_name, decorator(orig_method)) + return cls + + +def _copy_docstrings(source_cls): + """Copy the docstrings from source_cls to target_cls. + + Use this as: + @_copy_docstrings(source_class) + class TargetClass: + + And for any public method that exists on both classes, it will copy the + __doc__ for that method. + """ + def decorator(target_cls): + for meth_name, orig_method in target_cls.__dict__.items(): + if meth_name.startswith('_'): + continue + source_method = source_cls.__dict__.get(meth_name) + if source_method is not None and source_method.__doc__: + target_cls.__dict__[meth_name].__doc__ = source_method.__doc__ + return target_cls + return decorator + + +class _ResourceEntry: + """Tracks the contents of a Resource.""" + + def __init__(self, resource_name): + self.name = resource_name + + +@_copy_docstrings(model._ModelBackend) +@_record_calls +class _TestingModelBackend: + """This conforms to the interface for ModelBackend but provides canned data. + + DO NOT use this class directly, it is used by `Harness`_ to drive the model. + `Harness`_ is responsible for maintaining the internal consistency of the values here, + as the only public methods of this type are for implementing ModelBackend. + """ + + def __init__(self, unit_name, meta): + self.unit_name = unit_name + self.app_name = self.unit_name.split('/')[0] + self.model_name = None + self.model_uuid = 'f2c1b2a6-e006-11eb-ba80-0242ac130004' + + self._harness_tmp_dir = tempfile.TemporaryDirectory(prefix='ops-harness-') + self._calls = [] + self._meta = meta + self._relation_ids_map = {} # relation name to [relation_ids,...] + self._relation_names = {} # reverse map from relation_id to relation_name + self._relation_list_map = {} # relation_id: [unit_name,...] + self._relation_data = {} # {relation_id: {name: data}} + # {relation_id: {"app": app_name, "units": ["app/0",...]} + self._relation_app_and_units = {} + self._config = {} + self._is_leader = False + self._resources_map = {} # {resource_name: resource_content} + self._pod_spec = None + self._app_status = {'status': 'unknown', 'message': ''} + self._unit_status = {'status': 'maintenance', 'message': ''} + self._workload_version = None + self._resource_dir = None + # Format: + # { "storage_name": {"": { }, ... } + # : device id that is key for given storage_name + # Initialize the _storage_list with values present on metadata.yaml + self._storage_list = {k: {} for k in self._meta.storages} + + self._storage_detached = {k: set() for k in self._meta.storages} + self._storage_index_counter = 0 + # {container_name : _TestingPebbleClient} + self._pebble_clients = {} # type: {str: _TestingPebbleClient} + self._pebble_clients_can_connect = {} # type: {_TestingPebbleClient: bool} + self._planned_units = None + self._hook_is_running = '' + + def _validate_relation_access(self, relation_name, relations): + """Ensures that the named relation exists/has been added. + + This is called whenever relation data is accessed via model.get_relation(...). + """ + if len(relations) > 0: + return + + relations = list(self._meta.peers.keys()) + relations.extend(self._meta.requires.keys()) + relations.extend(self._meta.provides.keys()) + if self._hook_is_running == 'leader_elected' and relation_name in relations: + raise RuntimeError( + 'cannot access relation data without first adding the relation: ' + 'use Harness.add_relation({!r}, ) before calling set_leader' + .format(relation_name)) + + def _can_connect(self, pebble_client) -> bool: + """Returns whether the mock client is active and can support API calls with no errors.""" + return self._pebble_clients_can_connect[pebble_client] + + def _set_can_connect(self, pebble_client, val): + """Manually sets the can_connect state for the given mock client.""" + if not SIMULATE_CAN_CONNECT: + raise RuntimeError('must set SIMULATE_CAN_CONNECT=True before using set_can_connect') + if pebble_client not in self._pebble_clients_can_connect: + msg = 'cannot set can_connect for the client - are you running a "real" pebble test?' + raise RuntimeError(msg) + self._pebble_clients_can_connect[pebble_client] = val + + def _cleanup(self): + if self._resource_dir is not None: + self._resource_dir.cleanup() + self._resource_dir = None + + def _get_resource_dir(self) -> pathlib.Path: + if self._resource_dir is None: + # In actual Juju, the resource path for a charm's resource is + # $AGENT_DIR/resources/$RESOURCE_NAME/$RESOURCE_FILENAME + # However, charms shouldn't depend on this. + self._resource_dir = tempfile.TemporaryDirectory(prefix='tmp-ops-test-resource-') + return pathlib.Path(self._resource_dir.name) + + def relation_ids(self, relation_name): + try: + return self._relation_ids_map[relation_name] + except KeyError as e: + if relation_name not in self._meta.relations: + raise model.ModelError('{} is not a known relation'.format(relation_name)) from e + return [] + + def relation_list(self, relation_id): + try: + return self._relation_list_map[relation_id] + except KeyError as e: + raise model.RelationNotFoundError from e + + def relation_remote_app_name(self, relation_id: int) -> typing.Optional[str]: + if relation_id not in self._relation_app_and_units: + # Non-existent or dead relation + return None + return self._relation_app_and_units[relation_id]['app'] + + def relation_get(self, relation_id, member_name, is_app): + if is_app and '/' in member_name: + member_name = member_name.split('/')[0] + if relation_id not in self._relation_data: + raise model.RelationNotFoundError() + return self._relation_data[relation_id][member_name].copy() + + def relation_set(self, relation_id, key, value, is_app): + relation = self._relation_data[relation_id] + if is_app: + bucket_key = self.app_name + else: + bucket_key = self.unit_name + if bucket_key not in relation: + relation[bucket_key] = {} + bucket = relation[bucket_key] + if value == '': + bucket.pop(key, None) + else: + bucket[key] = value + + def config_get(self): + return self._config + + def is_leader(self): + return self._is_leader + + def application_version_set(self, version): + self._workload_version = version + + def resource_get(self, resource_name): + if resource_name not in self._resources_map: + raise model.ModelError( + "ERROR could not download resource: HTTP request failed: " + "Get https://.../units/unit-{}/resources/{}: resource#{}/{} not found".format( + self.unit_name.replace('/', '-'), resource_name, self.app_name, resource_name + )) + filename, contents = self._resources_map[resource_name] + resource_dir = self._get_resource_dir() + resource_filename = resource_dir / resource_name / filename + if not resource_filename.exists(): + if isinstance(contents, bytes): + mode = 'wb' + else: + mode = 'wt' + resource_filename.parent.mkdir(exist_ok=True) + with resource_filename.open(mode=mode) as resource_file: + resource_file.write(contents) + return resource_filename + + def pod_spec_set(self, spec, k8s_resources): + self._pod_spec = (spec, k8s_resources) + + def status_get(self, *, is_app=False): + if is_app: + return self._app_status + else: + return self._unit_status + + def status_set(self, status, message='', *, is_app=False): + if is_app: + self._app_status = {'status': status, 'message': message} + else: + self._unit_status = {'status': status, 'message': message} + + def storage_list(self, name): + return list(index for index in self._storage_list[name] + if self._storage_is_attached(name, index)) + + def storage_get(self, storage_name_id, attribute): + name, index = storage_name_id.split("/", 1) + index = int(index) + try: + if index in self._storage_detached[name]: + raise KeyError() # Pretend the key isn't there + else: + return self._storage_list[name][index][attribute] + except KeyError: + raise model.ModelError( + 'ERROR invalid value "{}/{}" for option -s: storage not found'.format(name, index)) + + def storage_add(self, name: str, count: int = 1) -> typing.List[int]: + if '/' in name: + raise model.ModelError('storage name cannot contain "/"') + + if name not in self._storage_list: + self._storage_list[name] = {} + result = [] + for i in range(count): + index = self._storage_index_counter + self._storage_index_counter += 1 + self._storage_list[name][index] = { + 'location': os.path.join(self._harness_tmp_dir.name, name, str(index)), + } + result.append(index) + return result + + def _storage_detach(self, storage_id: str): + # NOTE: This is an extra function for _TestingModelBackend to simulate + # detachment of a storage unit. This is not present in ops.model._ModelBackend. + name, index = storage_id.split('/', 1) + index = int(index) + + for client in self._pebble_clients.values(): + client._fs.remove_mount(name) + + if self._storage_is_attached(name, index): + self._storage_detached[name].add(index) + + def _storage_attach(self, storage_id: str): + # NOTE: This is an extra function for _TestingModelBackend to simulate + # re-attachment of a storage unit. This is not present in + # ops.model._ModelBackend. + name, index = storage_id.split('/', 1) + + for container, client in self._pebble_clients.items(): + for _, mount in self._meta.containers[container].mounts.items(): + if mount.storage != name: + continue + for index, store in self._storage_list[mount.storage].items(): + client._fs.add_mount(mount.storage, mount.location, store['location']) + + index = int(index) + if not self._storage_is_attached(name, index): + self._storage_detached[name].remove(index) + return True + return False + + def _storage_is_attached(self, storage_name, storage_index): + return storage_index not in self._storage_detached[storage_name] + + def _storage_remove(self, storage_id: str): + # NOTE: This is an extra function for _TestingModelBackend to simulate + # full removal of a storage unit. This is not present in + # ops.model._ModelBackend. + self._storage_detach(storage_id) + name, index = storage_id.split('/', 1) + index = int(index) + self._storage_list[name].pop(index, None) + + def action_get(self): + raise NotImplementedError(self.action_get) + + def action_set(self, results): + raise NotImplementedError(self.action_set) + + def action_log(self, message): + raise NotImplementedError(self.action_log) + + def action_fail(self, message=''): + raise NotImplementedError(self.action_fail) + + def network_get(self, endpoint_name, relation_id=None): + raise NotImplementedError(self.network_get) + + def add_metrics(self, metrics, labels=None): + raise NotImplementedError(self.add_metrics) + + @classmethod + def log_split(cls, message, max_len): + raise NotImplementedError(cls.log_split) + + def juju_log(self, level, msg): + raise NotImplementedError(self.juju_log) + + def get_pebble(self, socket_path: str) -> 'pebble.Client': + container = socket_path.split('/')[3] # /charm/containers//pebble.socket + client = self._pebble_clients.get(container, None) + if client is None: + client = _TestingPebbleClient(self) + self._pebble_clients[container] = client + + # we need to know which container a new pebble client belongs to + # so we can figure out which storage mounts must be simulated on + # this pebble client's mock file systems when storage is + # attached/detached later. + self._pebble_clients[container] = client + + self._pebble_clients_can_connect[client] = not SIMULATE_CAN_CONNECT + return client + + def planned_units(self): + """Simulate fetching the number of planned application units from the model. + + If self._planned_units is None, then we simulate what the Juju controller will do, which is + to report the number of peers, plus one (we include this unit in the count). This can be + overridden for testing purposes: a charm author can set the number of planned units + explicitly by calling `Harness.set_planned_units` + """ + if self._planned_units is not None: + return self._planned_units + + units = [] + peer_names = set(self._meta.peers.keys()) + for peer_id, peer_name in self._relation_names.items(): + if peer_name not in peer_names: + continue + peer_units = self._relation_list_map[peer_id] + units += peer_units + + count = len(set(units)) # de-dupe and get length. + + return count + 1 # Account for this unit. + + +@_copy_docstrings(pebble.Client) +class _TestingPebbleClient: + """This conforms to the interface for pebble.Client but provides canned data. + + DO NOT use this class directly, it is used by `Harness`_ to run interactions with Pebble. + `Harness`_ is responsible for maintaining the internal consistency of the values here, + as the only public methods of this type are for implementing Client. + """ + + def __init__(self, backend: _TestingModelBackend): + self._backend = _TestingModelBackend + self._layers = {} + # Has a service been started/stopped? + self._service_status = {} + self._fs = _TestingFilesystem() + self._backend = backend + + def _check_connection(self): + if not self._backend._can_connect(self): + raise pebble.ConnectionError('cannot connect to pebble') + + def get_system_info(self) -> pebble.SystemInfo: + self._check_connection() + return pebble.SystemInfo(version='1.0.0') + + def get_warnings( + self, select: pebble.WarningState = pebble.WarningState.PENDING, + ) -> typing.List['pebble.Warning']: + raise NotImplementedError(self.get_warnings) + + def ack_warnings(self, timestamp: datetime.datetime) -> int: + raise NotImplementedError(self.ack_warnings) + + def get_changes( + self, select: pebble.ChangeState = pebble.ChangeState.IN_PROGRESS, service: str = None, + ) -> typing.List[pebble.Change]: + raise NotImplementedError(self.get_changes) + + def get_change(self, change_id: pebble.ChangeID) -> pebble.Change: + raise NotImplementedError(self.get_change) + + def abort_change(self, change_id: pebble.ChangeID) -> pebble.Change: + raise NotImplementedError(self.abort_change) + + def autostart_services(self, timeout: float = 30.0, delay: float = 0.1) -> pebble.ChangeID: + self._check_connection() + for name, service in self._render_services().items(): + # TODO: jam 2021-04-20 This feels awkward that Service.startup might be a string or + # might be an enum. Probably should make Service.startup a property rather than an + # attribute. + if service.startup == '': + startup = pebble.ServiceStartup.DISABLED + else: + startup = pebble.ServiceStartup(service.startup) + if startup == pebble.ServiceStartup.ENABLED: + self._service_status[name] = pebble.ServiceStatus.ACTIVE + + def replan_services(self, timeout: float = 30.0, delay: float = 0.1) -> pebble.ChangeID: + return self.autostart_services(timeout, delay) + + def start_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> pebble.ChangeID: + # A common mistake is to pass just the name of a service, rather than a list of services, + # so trap that so it is caught quickly. + if isinstance(services, str): + raise TypeError('start_services should take a list of names, not just "{}"'.format( + services)) + + self._check_connection() + + # Note: jam 2021-04-20 We don't implement ChangeID, but the default caller of this is + # Container.start() which currently ignores the return value + known_services = self._render_services() + # Names appear to be validated before any are activated, so do two passes + for name in services: + if name not in known_services: + # TODO: jam 2021-04-20 This needs a better error type + raise RuntimeError('400 Bad Request: service "{}" does not exist'.format(name)) + current = self._service_status.get(name, pebble.ServiceStatus.INACTIVE) + if current == pebble.ServiceStatus.ACTIVE: + # TODO: jam 2021-04-20 I believe pebble actually validates all the service names + # can be started before starting any, and gives a list of things that couldn't + # be done, but this is good enough for now + raise pebble.ChangeError('''\ +cannot perform the following tasks: +- Start service "{}" (service "{}" was previously started) +'''.format(name, name), change=1234) # the change id is not respected + for name in services: + # If you try to start a service which is started, you get a ChangeError: + # $ PYTHONPATH=. python3 ./test/pebble_cli.py start serv + # ChangeError: cannot perform the following tasks: + # - Start service "serv" (service "serv" was previously started) + self._service_status[name] = pebble.ServiceStatus.ACTIVE + + def stop_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> pebble.ChangeID: + # handle a common mistake of passing just a name rather than a list of names + if isinstance(services, str): + raise TypeError('stop_services should take a list of names, not just "{}"'.format( + services)) + + self._check_connection() + + # TODO: handle invalid names + # Note: jam 2021-04-20 We don't implement ChangeID, but the default caller of this is + # Container.stop() which currently ignores the return value + known_services = self._render_services() + for name in services: + if name not in known_services: + # TODO: jam 2021-04-20 This needs a better error type + # 400 Bad Request: service "bal" does not exist + raise RuntimeError('400 Bad Request: service "{}" does not exist'.format(name)) + current = self._service_status.get(name, pebble.ServiceStatus.INACTIVE) + if current != pebble.ServiceStatus.ACTIVE: + # TODO: jam 2021-04-20 I believe pebble actually validates all the service names + # can be started before starting any, and gives a list of things that couldn't + # be done, but this is good enough for now + raise pebble.ChangeError('''\ +ChangeError: cannot perform the following tasks: +- Stop service "{}" (service "{}" is not active) +'''.format(name, name), change=1234) # the change id is not respected + for name in services: + self._service_status[name] = pebble.ServiceStatus.INACTIVE + + def restart_services( + self, services: typing.List[str], timeout: float = 30.0, delay: float = 0.1, + ) -> pebble.ChangeID: + # handle a common mistake of passing just a name rather than a list of names + if isinstance(services, str): + raise TypeError('restart_services should take a list of names, not just "{}"'.format( + services)) + + self._check_connection() + + # TODO: handle invalid names + # Note: jam 2021-04-20 We don't implement ChangeID, but the default caller of this is + # Container.restart() which currently ignores the return value + known_services = self._render_services() + for name in services: + if name not in known_services: + # TODO: jam 2021-04-20 This needs a better error type + # 400 Bad Request: service "bal" does not exist + raise RuntimeError('400 Bad Request: service "{}" does not exist'.format(name)) + for name in services: + self._service_status[name] = pebble.ServiceStatus.ACTIVE + + def wait_change( + self, change_id: pebble.ChangeID, timeout: float = 30.0, delay: float = 0.1, + ) -> pebble.Change: + raise NotImplementedError(self.wait_change) + + def add_layer( + self, label: str, layer: typing.Union[str, dict, pebble.Layer], *, + combine: bool = False): + # I wish we could combine some of this helpful object corralling with the actual backend, + # rather than having to re-implement it. Maybe we could subclass + if not isinstance(label, str): + raise TypeError('label must be a str, not {}'.format(type(label).__name__)) + + if isinstance(layer, (str, dict)): + layer_obj = pebble.Layer(layer) + elif isinstance(layer, pebble.Layer): + layer_obj = layer + else: + raise TypeError('layer must be str, dict, or pebble.Layer, not {}'.format( + type(layer).__name__)) + + self._check_connection() + + if label in self._layers: + # TODO: jam 2021-04-19 These should not be RuntimeErrors but should be proper error + # types. https://github.com/canonical/operator/issues/514 + if not combine: + raise RuntimeError('400 Bad Request: layer "{}" already exists'.format(label)) + layer = self._layers[label] + for name, service in layer_obj.services.items(): + # 'override' is actually single quoted in the real error, but + # it shouldn't be, hopefully that gets cleaned up. + if not service.override: + raise RuntimeError('500 Internal Server Error: layer "{}" must define' + '"override" for service "{}"'.format(label, name)) + if service.override not in ('merge', 'replace'): + raise RuntimeError('500 Internal Server Error: layer "{}" has invalid ' + '"override" value on service "{}"'.format(label, name)) + elif service.override == 'replace': + layer.services[name] = service + elif service.override == 'merge': + if combine and name in layer.services: + s = layer.services[name] + s._merge(service) + else: + layer.services[name] = service + + else: + self._layers[label] = layer_obj + + def _render_services(self) -> typing.Mapping[str, pebble.Service]: + services = {} + for key in sorted(self._layers.keys()): + layer = self._layers[key] + for name, service in layer.services.items(): + # TODO: (jam) 2021-04-07 have a way to merge existing services + services[name] = service + return services + + def get_plan(self) -> pebble.Plan: + self._check_connection() + plan = pebble.Plan('{}') + services = self._render_services() + if not services: + return plan + for name in sorted(services.keys()): + plan.services[name] = services[name] + return plan + + def get_services(self, names: typing.List[str] = None) -> typing.List[pebble.ServiceInfo]: + if isinstance(names, str): + raise TypeError('start_services should take a list of names, not just "{}"'.format( + names)) + + self._check_connection() + services = self._render_services() + infos = [] + if names is None: + names = sorted(services.keys()) + for name in sorted(names): + try: + service = services[name] + except KeyError: + # in pebble, it just returns "nothing matched" if there are 0 matches, + # but it ignores services it doesn't recognize + continue + status = self._service_status.get(name, pebble.ServiceStatus.INACTIVE) + if service.startup == '': + startup = pebble.ServiceStartup.DISABLED + else: + startup = pebble.ServiceStartup(service.startup) + info = pebble.ServiceInfo(name, + startup=startup, + current=pebble.ServiceStatus(status)) + infos.append(info) + return infos + + def pull(self, path: str, *, encoding: str = 'utf-8') -> typing.Union[typing.BinaryIO, + typing.TextIO]: + self._check_connection() + return self._fs.open(path, encoding=encoding) + + def push( + self, path: str, source: typing.Union[bytes, str, typing.BinaryIO, typing.TextIO], *, + encoding: str = 'utf-8', make_dirs: bool = False, permissions: int = None, + user_id: int = None, user: str = None, group_id: int = None, group: str = None): + self._check_connection() + if permissions is not None and not (0 <= permissions <= 0o777): + raise pebble.PathError( + 'generic-file-error', + 'permissions not within 0o000 to 0o777: {:#o}'.format(permissions)) + try: + self._fs.create_file( + path, source, encoding=encoding, make_dirs=make_dirs, permissions=permissions, + user_id=user_id, user=user, group_id=group_id, group=group) + except FileNotFoundError as e: + raise pebble.PathError( + 'not-found', 'parent directory not found: {}'.format(e.args[0])) + except NonAbsolutePathError as e: + raise pebble.PathError( + 'generic-file-error', + 'paths must be absolute, got {!r}'.format(e.args[0]) + ) + + def list_files(self, path: str, *, pattern: str = None, + itself: bool = False) -> typing.List[pebble.FileInfo]: + self._check_connection() + try: + files = [self._fs.get_path(path)] + except FileNotFoundError: + # conform with the real pebble api + raise pebble.APIError( + body={}, code=404, status='Not Found', + message="stat {}: no such file or directory".format(path)) + + if not itself: + try: + files = self._fs.list_dir(path) + except NotADirectoryError: + pass + + if pattern is not None: + files = [file for file in files if fnmatch.fnmatch(file.name, pattern)] + + type_mappings = { + _File: pebble.FileType.FILE, + _Directory: pebble.FileType.DIRECTORY, + } + return [ + pebble.FileInfo( + path=str(file.path), + name=file.name, + type=type_mappings.get(type(file)), + size=file.size if isinstance(file, _File) else None, + permissions=file.kwargs.get('permissions'), + last_modified=file.last_modified, + user_id=file.kwargs.get('user_id'), + user=file.kwargs.get('user'), + group_id=file.kwargs.get('group_id'), + group=file.kwargs.get('group'), + ) + for file in files + ] + + def make_dir( + self, path: str, *, make_parents: bool = False, permissions: int = None, + user_id: int = None, user: str = None, group_id: int = None, group: str = None): + self._check_connection() + if permissions is not None and not (0 <= permissions <= 0o777): + raise pebble.PathError( + 'generic-file-error', + 'permissions not within 0o000 to 0o777: {:#o}'.format(permissions)) + try: + self._fs.create_dir( + path, make_parents=make_parents, permissions=permissions, + user_id=user_id, user=user, group_id=group_id, group=group) + except FileNotFoundError as e: + # Parent directory doesn't exist and make_parents is False + raise pebble.PathError( + 'not-found', 'parent directory not found: {}'.format(e.args[0])) + except NotADirectoryError as e: + # Attempted to create a subdirectory of a file + raise pebble.PathError('generic-file-error', 'not a directory: {}'.format(e.args[0])) + except NonAbsolutePathError as e: + raise pebble.PathError( + 'generic-file-error', + 'paths must be absolute, got {!r}'.format(e.args[0]) + ) + + def remove_path(self, path: str, *, recursive: bool = False): + self._check_connection() + try: + file_or_dir = self._fs.get_path(path) + except FileNotFoundError: + if recursive: + # Pebble doesn't give not-found error when recursive is specified + return + raise pebble.PathError( + 'not-found', 'remove {}: no such file or directory'.format(path)) + + if isinstance(file_or_dir, _Directory) and len(file_or_dir) > 0 and not recursive: + raise pebble.PathError( + 'generic-file-error', 'cannot remove non-empty directory without recursive=True') + self._fs.delete_path(path) + + def exec(self, command, **kwargs): + raise NotImplementedError(self.exec) + + def send_signal(self, sig: typing.Union[int, str], *service_names: str): + if not service_names: + raise TypeError('send_signal expected at least 1 service name, got 0') + self._check_connection() + + # Convert signal to str + if isinstance(sig, int): + sig = signal.Signals(sig).name + + # pebble first validates the service name, and then the signal name + + plan = self.get_plan() + for service in service_names: + if service not in plan.services or not self.get_services([service])[0].is_running(): + # conform with the real pebble api + message = 'cannot send signal to "{}": service is not running'.format(service) + body = {'type': 'error', 'status-code': 500, 'status': 'Internal Server Error', + 'result': {'message': message}} + raise pebble.APIError( + body=body, code=500, status='Internal Server Error', message=message + ) + + # Check if signal name is valid + try: + signal.Signals[sig] + except KeyError: + # conform with the real pebble api + message = 'cannot send signal to "{}": invalid signal name "{}"'.format( + service_names[0], + sig) + body = {'type': 'error', 'status-code': 500, 'status': 'Internal Server Error', + 'result': {'message': message}} + raise pebble.APIError( + body=body, + code=500, + status='Internal Server Error', + message=message) + + def get_checks(self, level=None, names=None): + raise NotImplementedError(self.get_checks) + + +class NonAbsolutePathError(Exception): + """Error raised by _TestingFilesystem. + + This error is raised when an absolute path is required but the code instead encountered a + relative path. + """ + + +class _TestingStorageMount: + """Simulates a filesystem backend for storage mounts.""" + + def __init__(self, location: pathlib.PurePosixPath, src: pathlib.Path): + """Creates a new simulated storage mount. + + Args: + location: The path within simulated filesystem at which this storage will be mounted. + src: The temporary on-disk location where the simulated storage will live. + """ + self._src = src + self._location = location + + src.mkdir(exist_ok=True, parents=True) + + def contains(self, path: typing.Union[str, pathlib.PurePosixPath]) -> bool: + """Returns true whether path resides within this simulated storage mount's location.""" + try: + pathlib.PurePosixPath(path).relative_to(self._location) + return True + except Exception: + return False + + def check_contains(self, path: typing.Union[str, + pathlib.PurePosixPath]) -> pathlib.PurePosixPath: + """Raises if path does not reside within this simulated storage mount's location.""" + if not self.contains(path): + msg = 'the provided path "{!s}" does not reside within the mount location "{!s}"' \ + .format(path, self._location) + raise RuntimeError(msg) + return pathlib.PurePosixPath(path) + + def _srcpath(self, path: pathlib.PurePosixPath) -> pathlib.Path: + """Returns the disk-backed path where the simulated path will actually be stored.""" + suffix = path.relative_to(self._location) + return self._src / suffix + + def create_dir( + self, + path: pathlib.PurePosixPath, + make_parents: bool = False, + **kwargs) -> '_Directory': + if not pathlib.PurePosixPath(path).is_absolute(): + raise NonAbsolutePathError(str(path)) + path = self.check_contains(path) + srcpath = self._srcpath(path) + + if srcpath.exists() and srcpath.is_dir() and make_parents: + return _Directory(path, **kwargs) # nothing to do + if srcpath.exists(): + raise FileExistsError(str(path)) + + dirname = srcpath.parent + if not dirname.exists(): + if not make_parents: + raise FileNotFoundError(str(path.parent)) + dirname.mkdir(parents=True, exist_ok=True) + srcpath.mkdir(exist_ok=True) + return _Directory(path, **kwargs) + + def create_file( + self, + path: pathlib.PurePosixPath, + data: typing.Union[bytes, str, typing.BinaryIO, typing.TextIO], + encoding: str = 'utf-8', + make_dirs: bool = False, + **kwargs + ) -> '_File': + path = self.check_contains(path) + srcpath = self._srcpath(path) + + dirname = srcpath.parent + if not dirname.exists(): + if not make_dirs: + raise FileNotFoundError(str(path.parent)) + dirname.mkdir(parents=True, exist_ok=True) + + if isinstance(data, str): + data = data.encode(encoding=encoding) + elif not isinstance(data, bytes): + data = data.getvalue() + if isinstance(data, str): + data = data.encode() + + with srcpath.open('wb') as f: + f.write(data) + + return _File(path, data, encoding=encoding, **kwargs) + + def list_dir(self, path: pathlib.PurePosixPath) -> typing.List['_File']: + path = self.check_contains(path) + srcpath = self._srcpath(path) + + results = [] + if not srcpath.exists(): + raise FileNotFoundError(str(path)) + if not srcpath.is_dir(): + raise NotADirectoryError(str(path)) + for fpath in srcpath.iterdir(): + mountpath = path / fpath.name + if fpath.is_dir(): + results.append(_Directory(mountpath)) + elif fpath.is_file(): + with fpath.open('rb') as f: + results.append(_File(mountpath, f.read())) + else: + raise RuntimeError('unsupported file type at path {}'.format(fpath)) + return results + + def open( + self, + path: typing.Union[str, pathlib.PurePosixPath], + encoding: typing.Optional[str] = 'utf-8', + ) -> typing.Union[typing.BinaryIO, typing.TextIO]: + path = self.check_contains(path) + + file = self.get_path(path) + if isinstance(file, _Directory): + raise IsADirectoryError(str(file.path)) + return file.open(encoding=encoding) + + def get_path(self, path: typing.Union[str, pathlib.PurePosixPath] + ) -> typing.Union['_Directory', '_File']: + path = self.check_contains(path) + srcpath = self._srcpath(path) + if srcpath.is_dir(): + return _Directory(path) + if not srcpath.exists(): + raise FileNotFoundError(str(path)) + with srcpath.open('rb') as f: + return _File(path, f.read()) + + def delete_path(self, path: typing.Union[str, pathlib.PurePosixPath]) -> None: + path = self.check_contains(path) + srcpath = self._srcpath(path) + if srcpath.exists(): + srcpath.unlink() + else: + raise FileNotFoundError(str(path)) + + +class _TestingFilesystem: + r"""An in-memory mock of a pebble-controlled container's filesystem. + + For now, the filesystem is assumed to be a POSIX-style filesytem; Windows-style directories + (e.g. \, \foo\bar, C:\foo\bar) are not supported. + """ + + def __init__(self): + self.root = _Directory(pathlib.PurePosixPath('/')) + self._mounts = {} + + def add_mount(self, name, mount_path, backing_src_path): + self._mounts[name] = _TestingStorageMount( + pathlib.PurePosixPath(mount_path), pathlib.Path(backing_src_path)) + + def remove_mount(self, name): + if name in self._mounts: + del self._mounts[name] + + def create_dir(self, path: str, make_parents: bool = False, **kwargs) -> '_Directory': + if not path.startswith('/'): + raise NonAbsolutePathError(path) + for mount in self._mounts.values(): + if mount.contains(path): + return mount.create_dir(path, make_parents, **kwargs) + current_dir = self.root + tokens = pathlib.PurePosixPath(path).parts[1:] + for token in tokens[:-1]: + if token in current_dir: + current_dir = current_dir[token] + else: + if make_parents: + # NOTE: other parameters (e.g. ownership, permissions) only get applied to the + # final directory. + # (At the time of writing, Pebble defaults to 0o755 permissions and root:root + # ownership.) + current_dir = current_dir.create_dir(token) + else: + raise FileNotFoundError(str(current_dir.path / token)) + if isinstance(current_dir, _File): + raise NotADirectoryError(str(current_dir.path)) + + # Current backend will always raise an error if the final directory component + # already exists. + token = tokens[-1] + if token not in current_dir: + current_dir = current_dir.create_dir(token, **kwargs) + else: + # If 'make_parents' is specified, behave like 'mkdir -p' and ignore if the dir already + # exists. + if make_parents: + current_dir = _Directory(current_dir.path / token) + else: + raise FileExistsError(str(current_dir.path / token)) + return current_dir + + def create_file( + self, + path: str, + data: typing.Union[bytes, str, typing.BinaryIO, typing.TextIO], + encoding: typing.Optional[str] = 'utf-8', + make_dirs: bool = False, + **kwargs + ) -> '_File': + if not path.startswith('/'): + raise NonAbsolutePathError(path) + for mount in self._mounts.values(): + if mount.contains(path): + return mount.create_file(path, data, encoding, make_dirs, **kwargs) + path_obj = pathlib.PurePosixPath(path) + try: + dir_ = self.get_path(path_obj.parent) + except FileNotFoundError: + if make_dirs: + dir_ = self.create_dir(str(path_obj.parent), make_parents=make_dirs) + # NOTE: other parameters (e.g. ownership, permissions) only get applied to the + # final directory. + # (At the time of writing, Pebble defaults to the specified permissions and + # root:root ownership, which is inconsistent with the push function's + # behavior for parent directories.) + else: + raise + if not isinstance(dir_, _Directory): + raise pebble.PathError( + 'generic-file-error', 'parent is not a directory: {}'.format(str(dir_))) + return dir_.create_file(path_obj.name, data, encoding=encoding, **kwargs) + + def list_dir(self, path) -> typing.List['_File']: + for mount in self._mounts.values(): + if mount.contains(path): + return mount.list_dir(path) + current_dir = self.root + tokens = pathlib.PurePosixPath(path).parts[1:] + for token in tokens: + try: + current_dir = current_dir[token] + except KeyError: + raise FileNotFoundError(str(current_dir.path / token)) + if isinstance(current_dir, _File): + raise NotADirectoryError(str(current_dir.path)) + if not isinstance(current_dir, _Directory): + # For now, ignoring other possible cases besides File and Directory (e.g. Symlink). + raise NotImplementedError() + + return [child for child in current_dir] + + def open( + self, + path: typing.Union[str, pathlib.PurePosixPath], + encoding: typing.Optional[str] = 'utf-8', + ) -> typing.Union[typing.BinaryIO, typing.TextIO]: + for mount in self._mounts.values(): + if mount.contains(path): + return mount.open(path, encoding) + path = pathlib.PurePosixPath(path) + file = self.get_path(path) # warning: no check re: directories + if isinstance(file, _Directory): + raise IsADirectoryError(str(file.path)) + return file.open(encoding=encoding) + + def get_path(self, path: typing.Union[str, pathlib.PurePosixPath]) \ + -> typing.Union['_Directory', '_File']: + for mount in self._mounts.values(): + if mount.contains(path): + return mount.get_path(path) + path = pathlib.PurePosixPath(path) + tokens = path.parts[1:] + current_object = self.root + for token in tokens: + # ASSUMPTION / TESTME: object might be file + if token in current_object: + current_object = current_object[token] + else: + raise FileNotFoundError(str(current_object.path / token)) + return current_object + + def delete_path(self, path: typing.Union[str, pathlib.PurePosixPath]) -> None: + for mount in self._mounts.values(): + if mount.contains(path): + return mount.delete_path(path) + path = pathlib.PurePosixPath(path) + parent_dir = self.get_path(path.parent) + del parent_dir[path.name] + + +class _Directory: + def __init__(self, path: pathlib.PurePosixPath, **kwargs): + self.path = path + self._children = {} + self.last_modified = datetime.datetime.now() + self.kwargs = kwargs + + @property + def name(self) -> str: + # Need to handle special case for root. + # pathlib.PurePosixPath('/').name is '', but pebble returns '/'. + return self.path.name if self.path.name else '/' + + def __contains__(self, child: str) -> bool: + return child in self._children + + def __iter__(self) -> typing.Iterator[typing.Union['_File', '_Directory']]: + return (value for value in self._children.values()) + + def __getitem__(self, key: str) -> typing.Union['_File', '_Directory']: + return self._children[key] + + def __delitem__(self, key: str) -> None: + try: + del self._children[key] + except KeyError: + raise FileNotFoundError(str(self.path / key)) + + def __len__(self): + return len(self._children) + + def create_dir(self, name: str, **kwargs) -> '_Directory': + self._children[name] = _Directory(self.path / name, **kwargs) + return self._children[name] + + def create_file( + self, + name: str, + data: typing.Union[bytes, str, typing.BinaryIO, typing.TextIO], + encoding: typing.Optional[str] = 'utf-8', + **kwargs + ) -> '_File': + self._children[name] = _File(self.path / name, data, encoding=encoding, **kwargs) + return self._children[name] + + +class _File: + def __init__( + self, + path: pathlib.PurePosixPath, + data: typing.Union[str, bytes, typing.BinaryIO, typing.TextIO], + encoding: typing.Optional[str] = 'utf-8', + **kwargs): + + if hasattr(data, 'read'): + data = data.read() + if isinstance(data, str): + data = data.encode(encoding) + data_size = len(data) + + self.path = path + self.data = data + self.size = data_size + self.last_modified = datetime.datetime.now() + self.kwargs = kwargs + + @property + def name(self) -> str: + return self.path.name + + def open( + self, + encoding: typing.Optional[str] = 'utf-8', + ) -> typing.Union[typing.TextIO, typing.BinaryIO]: + if encoding is None: + return BytesIO(self.data) + else: + return StringIO(self.data.decode(encoding)) diff --git a/ubuntu/venv/ops/version.py b/ubuntu/venv/ops/version.py new file mode 100644 index 0000000..5994965 --- /dev/null +++ b/ubuntu/venv/ops/version.py @@ -0,0 +1,3 @@ +# this is a generated file + +version = '1.5.0' diff --git a/ubuntu/venv/pip-20.0.2.dist-info/INSTALLER b/ubuntu/venv/pip-20.0.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/ubuntu/venv/pip-20.0.2.dist-info/LICENSE.txt b/ubuntu/venv/pip-20.0.2.dist-info/LICENSE.txt new file mode 100644 index 0000000..737fec5 --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/ubuntu/venv/pip-20.0.2.dist-info/METADATA b/ubuntu/venv/pip-20.0.2.dist-info/METADATA new file mode 100644 index 0000000..5183c4e --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/METADATA @@ -0,0 +1,84 @@ +Metadata-Version: 2.1 +Name: pip +Version: 20.0.2 +Summary: The PyPA recommended tool for installing Python packages. +Home-page: https://pip.pypa.io/ +Author: The pip developers +Author-email: pypa-dev@groups.google.com +License: MIT +Project-URL: Documentation, https://pip.pypa.io +Project-URL: Source, https://github.com/pypa/pip +Keywords: distutils easy_install egg setuptools wheel virtualenv +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Topic :: Software Development :: Build Tools +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.* + +pip - The Python Package Installer +================================== + +.. image:: https://img.shields.io/pypi/v/pip.svg + :target: https://pypi.org/project/pip/ + +.. image:: https://readthedocs.org/projects/pip/badge/?version=latest + :target: https://pip.pypa.io/en/latest + +pip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes. + +Please take a look at our documentation for how to install and use pip: + +* `Installation`_ +* `Usage`_ + +Updates are released regularly, with a new version every 3 months. More details can be found in our documentation: + +* `Release notes`_ +* `Release process`_ + +If you find bugs, need help, or want to talk to the developers please use our mailing lists or chat rooms: + +* `Issue tracking`_ +* `Discourse channel`_ +* `User IRC`_ + +If you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms: + +* `GitHub page`_ +* `Dev documentation`_ +* `Dev mailing list`_ +* `Dev IRC`_ + +Code of Conduct +--------------- + +Everyone interacting in the pip project's codebases, issue trackers, chat +rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. + +.. _package installer: https://packaging.python.org/guides/tool-recommendations/ +.. _Python Package Index: https://pypi.org +.. _Installation: https://pip.pypa.io/en/stable/installing.html +.. _Usage: https://pip.pypa.io/en/stable/ +.. _Release notes: https://pip.pypa.io/en/stable/news.html +.. _Release process: https://pip.pypa.io/en/latest/development/release-process/ +.. _GitHub page: https://github.com/pypa/pip +.. _Dev documentation: https://pip.pypa.io/en/latest/development +.. _Issue tracking: https://github.com/pypa/pip/issues +.. _Discourse channel: https://discuss.python.org/c/packaging +.. _Dev mailing list: https://groups.google.com/forum/#!forum/pypa-dev +.. _User IRC: https://webchat.freenode.net/?channels=%23pypa +.. _Dev IRC: https://webchat.freenode.net/?channels=%23pypa-dev +.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ + + diff --git a/ubuntu/venv/pip-20.0.2.dist-info/RECORD b/ubuntu/venv/pip-20.0.2.dist-info/RECORD new file mode 100644 index 0000000..0a562ab --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/RECORD @@ -0,0 +1,246 @@ +../../../bin/pip,sha256=6patQSzNPq1pkffvPTdySieJtz6j0CAu1RO-RGfHAqQ,253 +../../../bin/pip3,sha256=6patQSzNPq1pkffvPTdySieJtz6j0CAu1RO-RGfHAqQ,253 +../../../bin/pip3.8,sha256=6patQSzNPq1pkffvPTdySieJtz6j0CAu1RO-RGfHAqQ,253 +pip-20.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pip-20.0.2.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090 +pip-20.0.2.dist-info/METADATA,sha256=MSgjT2JTt8usp4Hopp5AGEmc-7sKR2Jd7HTMJqCoRhw,3352 +pip-20.0.2.dist-info/RECORD,, +pip-20.0.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 +pip-20.0.2.dist-info/entry_points.txt,sha256=HtfDOwpUlr9s73jqLQ6wF9V0_0qvUXJwCBz7Vwx0Ue0,125 +pip-20.0.2.dist-info/top_level.txt,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pip/__init__.py,sha256=U1AM82iShMaw90K6Yq0Q2-AZ1EsOcqQLQRB-rxwFtII,455 +pip/__main__.py,sha256=NM95x7KuQr-lwPoTjAC0d_QzLJsJjpmAoxZg0mP8s98,632 +pip/__pycache__/__init__.cpython-38.pyc,, +pip/__pycache__/__main__.cpython-38.pyc,, +pip/_internal/__init__.py,sha256=j5fiII6yCeZjpW7_7wAVRMM4DwE-gyARGVU4yAADDeE,517 +pip/_internal/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/__pycache__/build_env.cpython-38.pyc,, +pip/_internal/__pycache__/cache.cpython-38.pyc,, +pip/_internal/__pycache__/configuration.cpython-38.pyc,, +pip/_internal/__pycache__/exceptions.cpython-38.pyc,, +pip/_internal/__pycache__/legacy_resolve.cpython-38.pyc,, +pip/_internal/__pycache__/locations.cpython-38.pyc,, +pip/_internal/__pycache__/main.cpython-38.pyc,, +pip/_internal/__pycache__/pep425tags.cpython-38.pyc,, +pip/_internal/__pycache__/pyproject.cpython-38.pyc,, +pip/_internal/__pycache__/self_outdated_check.cpython-38.pyc,, +pip/_internal/__pycache__/wheel_builder.cpython-38.pyc,, +pip/_internal/build_env.py,sha256=--aNgzIdYrCOclHMwoAdpclCpfdFE_jooRuCy5gczwg,7532 +pip/_internal/cache.py,sha256=16GrnDRLBQNlfKWIuIF6Sa-EFS78kez_w1WEjT3ykTI,11605 +pip/_internal/cli/__init__.py,sha256=FkHBgpxxb-_gd6r1FjnNhfMOzAUYyXoXKJ6abijfcFU,132 +pip/_internal/cli/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/cli/__pycache__/autocompletion.cpython-38.pyc,, +pip/_internal/cli/__pycache__/base_command.cpython-38.pyc,, +pip/_internal/cli/__pycache__/cmdoptions.cpython-38.pyc,, +pip/_internal/cli/__pycache__/command_context.cpython-38.pyc,, +pip/_internal/cli/__pycache__/main.cpython-38.pyc,, +pip/_internal/cli/__pycache__/main_parser.cpython-38.pyc,, +pip/_internal/cli/__pycache__/parser.cpython-38.pyc,, +pip/_internal/cli/__pycache__/req_command.cpython-38.pyc,, +pip/_internal/cli/__pycache__/status_codes.cpython-38.pyc,, +pip/_internal/cli/autocompletion.py,sha256=ekGNtcDI0p7rFVc-7s4T9Tbss4Jgb7vsB649XJIblRg,6547 +pip/_internal/cli/base_command.py,sha256=v6yl5XNRqye8BT9ep8wvpMu6lylP_Hu6D95r_HqbpbQ,7948 +pip/_internal/cli/cmdoptions.py,sha256=f1TVHuu_fR3lLlMo6b367H_GsWFv26tLI9cAS-kZfE0,28114 +pip/_internal/cli/command_context.py,sha256=ygMVoTy2jpNilKT-6416gFSQpaBtrKRBbVbi2fy__EU,975 +pip/_internal/cli/main.py,sha256=8iq3bHe5lxJTB2EvKOqZ38NS0MmoS79_S1kgj4QuH8A,2610 +pip/_internal/cli/main_parser.py,sha256=W9OWeryh7ZkqELohaFh0Ko9sB98ZkSeDmnYbOZ1imBc,2819 +pip/_internal/cli/parser.py,sha256=O9djTuYQuSfObiY-NU6p4MJCfWsRUnDpE2YGA_fwols,9487 +pip/_internal/cli/req_command.py,sha256=pAUAglpTn0mUA6lRs7KN71yOm1KDabD0ySVTQTqWTSA,12463 +pip/_internal/cli/status_codes.py,sha256=F6uDG6Gj7RNKQJUDnd87QKqI16Us-t-B0wPF_4QMpWc,156 +pip/_internal/commands/__init__.py,sha256=uTSj58QlrSKeXqCUSdL-eAf_APzx5BHy1ABxb0j5ZNE,3714 +pip/_internal/commands/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/commands/__pycache__/check.cpython-38.pyc,, +pip/_internal/commands/__pycache__/completion.cpython-38.pyc,, +pip/_internal/commands/__pycache__/configuration.cpython-38.pyc,, +pip/_internal/commands/__pycache__/debug.cpython-38.pyc,, +pip/_internal/commands/__pycache__/download.cpython-38.pyc,, +pip/_internal/commands/__pycache__/freeze.cpython-38.pyc,, +pip/_internal/commands/__pycache__/hash.cpython-38.pyc,, +pip/_internal/commands/__pycache__/help.cpython-38.pyc,, +pip/_internal/commands/__pycache__/install.cpython-38.pyc,, +pip/_internal/commands/__pycache__/list.cpython-38.pyc,, +pip/_internal/commands/__pycache__/search.cpython-38.pyc,, +pip/_internal/commands/__pycache__/show.cpython-38.pyc,, +pip/_internal/commands/__pycache__/uninstall.cpython-38.pyc,, +pip/_internal/commands/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/commands/check.py,sha256=mgLNYT3bd6Kmynwh4zzcBmVlFZ-urMo40jTgk6U405E,1505 +pip/_internal/commands/completion.py,sha256=UFQvq0Q4_B96z1bvnQyMOq82aPSu05RejbLmqeTZjC0,2975 +pip/_internal/commands/configuration.py,sha256=6riioZjMhsNSEct7dE-X8SobGodk3WERKJvuyjBje4Q,7226 +pip/_internal/commands/debug.py,sha256=a8llax2hRkxgK-tvwdJgaCaZCYPIx0fDvrlMDoYr8bQ,4209 +pip/_internal/commands/download.py,sha256=zX_0-IeFb4C8dxSmGHxk-6H5kehtyTSsdWpjNpAhSww,5007 +pip/_internal/commands/freeze.py,sha256=QS-4ib8jbKJ2wrDaDbTuyaB3Y_iJ5CQC2gAVHuAv9QU,3481 +pip/_internal/commands/hash.py,sha256=47teimfAPhpkaVbSDaafck51BT3XXYuL83lAqc5lOcE,1735 +pip/_internal/commands/help.py,sha256=Nhecq--ydFn80Gm1Zvbf9943EcRJfO0TnXUhsF0RO7s,1181 +pip/_internal/commands/install.py,sha256=T4P3J1rw7CQrZX4OUamtcoWMkTrJBfUe6gWpTfZW1bQ,27286 +pip/_internal/commands/list.py,sha256=2l0JiqHxjxDHNTCb2HZOjwwdo4duS1R0MsqZb6HSMKk,10660 +pip/_internal/commands/search.py,sha256=7Il8nKZ9mM7qF5jlnBoPvSIFY9f-0-5IbYoX3miTuZY,5148 +pip/_internal/commands/show.py,sha256=Vzsj2oX0JBl94MPyF3LV8YoMcigl8B2UsMM8zp0pH2s,6792 +pip/_internal/commands/uninstall.py,sha256=8mldFbrQecSoWDZRqxBgJkrlvx6Y9Iy7cs-2BIgtXt4,2983 +pip/_internal/commands/wheel.py,sha256=TMU5ZhjLo7BIZQApGPsYfoCsbGTnvP-N9jkgPJXhj1Y,7170 +pip/_internal/configuration.py,sha256=MgKrLFBJBkF3t2VJM4tvlnEspfSuS4scp_LhHWh53nY,14222 +pip/_internal/distributions/__init__.py,sha256=ECBUW5Gtu9TjJwyFLvim-i6kUMYVuikNh9I5asL6tbA,959 +pip/_internal/distributions/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/distributions/__pycache__/base.cpython-38.pyc,, +pip/_internal/distributions/__pycache__/installed.cpython-38.pyc,, +pip/_internal/distributions/__pycache__/sdist.cpython-38.pyc,, +pip/_internal/distributions/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/distributions/base.py,sha256=ruprpM_L2T2HNi3KLUHlbHimZ1sWVw-3Q0Lb8O7TDAI,1425 +pip/_internal/distributions/installed.py,sha256=YqlkBKr6TVP1MAYS6SG8ojud21wVOYLMZ8jMLJe9MSU,760 +pip/_internal/distributions/sdist.py,sha256=D4XTMlCwgPlK69l62GLYkNSVTVe99fR5iAcVt2EbGok,4086 +pip/_internal/distributions/wheel.py,sha256=95uD-TfaYoq3KiKBdzk9YMN4RRqJ28LNoSTS2K46gek,1294 +pip/_internal/exceptions.py,sha256=6YRuwXAK6F1iyUWKIkCIpWWN2khkAn1sZOgrFA9S8Ro,10247 +pip/_internal/index/__init__.py,sha256=vpt-JeTZefh8a-FC22ZeBSXFVbuBcXSGiILhQZJaNpQ,30 +pip/_internal/index/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/index/__pycache__/collector.cpython-38.pyc,, +pip/_internal/index/__pycache__/package_finder.cpython-38.pyc,, +pip/_internal/index/collector.py,sha256=YS7Ix4oylU7ZbPTPFugh-244GSRqMvdHsGUG6nmz2gE,17892 +pip/_internal/index/package_finder.py,sha256=2Rg75AOpLj8BN1jyL8EI-Iw-Hv6ibJkrYVARCht3bX8,37542 +pip/_internal/legacy_resolve.py,sha256=L7R72I7CjVgJlPTggmA1j4b-H8NmxNu_dKVhrpGXGps,16277 +pip/_internal/locations.py,sha256=VifFEqhc7FWFV8QGoEM3CpECRY8Doq7kTytytxsEgx0,6734 +pip/_internal/main.py,sha256=IVBnUQ-FG7DK6617uEXRB5_QJqspAsBFmTmTesYkbdQ,437 +pip/_internal/models/__init__.py,sha256=3DHUd_qxpPozfzouoqa9g9ts1Czr5qaHfFxbnxriepM,63 +pip/_internal/models/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/models/__pycache__/candidate.cpython-38.pyc,, +pip/_internal/models/__pycache__/format_control.cpython-38.pyc,, +pip/_internal/models/__pycache__/index.cpython-38.pyc,, +pip/_internal/models/__pycache__/link.cpython-38.pyc,, +pip/_internal/models/__pycache__/scheme.cpython-38.pyc,, +pip/_internal/models/__pycache__/search_scope.cpython-38.pyc,, +pip/_internal/models/__pycache__/selection_prefs.cpython-38.pyc,, +pip/_internal/models/__pycache__/target_python.cpython-38.pyc,, +pip/_internal/models/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/models/candidate.py,sha256=Y58Bcm6oXUj0iS-yhmerlGo5CQJI2p0Ww9h6hR9zQDw,1150 +pip/_internal/models/format_control.py,sha256=ICzVjjGwfZYdX-eLLKHjMHLutEJlAGpfj09OG_eMqac,2673 +pip/_internal/models/index.py,sha256=K59A8-hVhBM20Xkahr4dTwP7OjkJyEqXH11UwHFVgqM,1060 +pip/_internal/models/link.py,sha256=y0H2ZOk0P6d1lfGUL2Pl09xFgZcRt5HwN2LElMifOpI,6827 +pip/_internal/models/scheme.py,sha256=vvhBrrno7eVDXcdKHiZWwxhPHf4VG5uSCEkC0QDR2RU,679 +pip/_internal/models/search_scope.py,sha256=2LXbU4wV8LwqdtXQXNXFYKv-IxiDI_QwSz9ZgbwtAfk,3898 +pip/_internal/models/selection_prefs.py,sha256=rPeif2KKjhTPXeMoQYffjqh10oWpXhdkxRDaPT1HO8k,1908 +pip/_internal/models/target_python.py,sha256=c-cFi6zCuo5HYbXNS3rVVpKRaHVh5yQlYEjEW23SidQ,3799 +pip/_internal/models/wheel.py,sha256=6KLuLKH5b0C5goWQXGSISRaq2UZtkHUEAU1y1Zsrwms,2766 +pip/_internal/network/__init__.py,sha256=jf6Tt5nV_7zkARBrKojIXItgejvoegVJVKUbhAa5Ioc,50 +pip/_internal/network/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/network/__pycache__/auth.cpython-38.pyc,, +pip/_internal/network/__pycache__/cache.cpython-38.pyc,, +pip/_internal/network/__pycache__/download.cpython-38.pyc,, +pip/_internal/network/__pycache__/session.cpython-38.pyc,, +pip/_internal/network/__pycache__/utils.cpython-38.pyc,, +pip/_internal/network/__pycache__/xmlrpc.cpython-38.pyc,, +pip/_internal/network/auth.py,sha256=K3G1ukKb3PiH8w_UnpXTz8qQsTULO-qdbfOE9zTo1fE,11119 +pip/_internal/network/cache.py,sha256=51CExcRkXWrgMZ7WsrZ6cmijKfViD5tVgKbBvJHO1IE,2394 +pip/_internal/network/download.py,sha256=3D9vdJmVwmCUMxzC-TaVI_GvVOpQna3BLEYNPCSx3Fc,6260 +pip/_internal/network/session.py,sha256=u1IXQfv21R1xv86ulyiB58-be4sYm90eFB0Wp8fVMYw,14702 +pip/_internal/network/utils.py,sha256=iiixo1OeaQ3niUWiBjg59PN6f1w7vvTww1vFriTD_IU,1959 +pip/_internal/network/xmlrpc.py,sha256=AL115M3vFJ8xiHVJneb8Hi0ZFeRvdPhblC89w25OG5s,1597 +pip/_internal/operations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pip/_internal/operations/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/operations/__pycache__/check.cpython-38.pyc,, +pip/_internal/operations/__pycache__/freeze.cpython-38.pyc,, +pip/_internal/operations/__pycache__/prepare.cpython-38.pyc,, +pip/_internal/operations/build/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pip/_internal/operations/build/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/operations/build/__pycache__/metadata.cpython-38.pyc,, +pip/_internal/operations/build/__pycache__/metadata_legacy.cpython-38.pyc,, +pip/_internal/operations/build/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/operations/build/__pycache__/wheel_legacy.cpython-38.pyc,, +pip/_internal/operations/build/metadata.py,sha256=yHMi5gHYXcXyHcvUPWHdO-UyOo3McFWljn_nHfM1O9c,1307 +pip/_internal/operations/build/metadata_legacy.py,sha256=4n6N7BTysqVmEpITzT2UVClyt0Peij_Im8Qm965IWB4,3957 +pip/_internal/operations/build/wheel.py,sha256=ntltdNP6D2Tpr4V0agssu6rE0F9LaBpJkYT6zSdhEbw,1469 +pip/_internal/operations/build/wheel_legacy.py,sha256=DYSxQKutwSZnmNvWkwsl2HzE2XQBxV0i0wTphjtUe90,3349 +pip/_internal/operations/check.py,sha256=a6uHG0daoWpmSPCdL7iYJaGQYZ-CRvPvTnCv2PnIIs0,5353 +pip/_internal/operations/freeze.py,sha256=td4BeRnW10EXFTZrx6VgygO3CrjqD5B9f0BGzjQm-Ew,10180 +pip/_internal/operations/install/__init__.py,sha256=mX7hyD2GNBO2mFGokDQ30r_GXv7Y_PLdtxcUv144e-s,51 +pip/_internal/operations/install/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/operations/install/__pycache__/editable_legacy.cpython-38.pyc,, +pip/_internal/operations/install/__pycache__/legacy.cpython-38.pyc,, +pip/_internal/operations/install/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/operations/install/editable_legacy.py,sha256=rJ_xs2qtDUjpY2-n6eYlVyZiNoKbOtZXZrYrcnIELt4,1488 +pip/_internal/operations/install/legacy.py,sha256=eBV8gHbO9sBlBc-4nuR3Sd2nikHgEcnC9khfeLiypio,4566 +pip/_internal/operations/install/wheel.py,sha256=xdCjH6uIUyg39Pf8tUaMFUN4a7eozJAFMb_wKcgQlsY,23012 +pip/_internal/operations/prepare.py,sha256=ro2teBlbBpkRJhBKraP9CoJgVLpueSk62ziWhRToXww,20942 +pip/_internal/pep425tags.py,sha256=SlIQokevkoKnXhoK3PZvXiDoj8hFKoJ7thDifDtga3k,5490 +pip/_internal/pyproject.py,sha256=VJKsrXORGiGoDPVKCQhuu4tWlQSTOhoiRlVLRNu4rx4,7400 +pip/_internal/req/__init__.py,sha256=UVaYPlHZVGRBQQPjvGC_6jJDQtewXm0ws-8Lxhg_TiY,2671 +pip/_internal/req/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/req/__pycache__/constructors.cpython-38.pyc,, +pip/_internal/req/__pycache__/req_file.cpython-38.pyc,, +pip/_internal/req/__pycache__/req_install.cpython-38.pyc,, +pip/_internal/req/__pycache__/req_set.cpython-38.pyc,, +pip/_internal/req/__pycache__/req_tracker.cpython-38.pyc,, +pip/_internal/req/__pycache__/req_uninstall.cpython-38.pyc,, +pip/_internal/req/constructors.py,sha256=w5-kWWVCqlSqcIBitw86yq7XGMPpKrHDfQZSE2mJ_xc,14388 +pip/_internal/req/req_file.py,sha256=ECqRUicCw5Y08R1YynZAAp8dSKQhDXoc1Q-mY3a9b6I,18485 +pip/_internal/req/req_install.py,sha256=wjsIr4lDpbVSLqANKJI9mXwRVHaRxcnj8q30UiHoLRA,30442 +pip/_internal/req/req_set.py,sha256=GsrKmupRKhNMhjkofVfCEHEHfgEvYBxClaQH5xLBQHg,8066 +pip/_internal/req/req_tracker.py,sha256=27fvVG8Y2MJS1KpU2rBMnQyUEMHG4lkHT_bzbzQK-c0,4723 +pip/_internal/req/req_uninstall.py,sha256=DWnOsuyYGju6-sylyoCm7GtUNevn9qMAVhjAGLcdXUE,23609 +pip/_internal/self_outdated_check.py,sha256=3KO1pTJUuYaiV9X0t87I9PimkGL82HbhLWbocqKZpBU,8009 +pip/_internal/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pip/_internal/utils/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/utils/__pycache__/appdirs.cpython-38.pyc,, +pip/_internal/utils/__pycache__/compat.cpython-38.pyc,, +pip/_internal/utils/__pycache__/deprecation.cpython-38.pyc,, +pip/_internal/utils/__pycache__/distutils_args.cpython-38.pyc,, +pip/_internal/utils/__pycache__/encoding.cpython-38.pyc,, +pip/_internal/utils/__pycache__/entrypoints.cpython-38.pyc,, +pip/_internal/utils/__pycache__/filesystem.cpython-38.pyc,, +pip/_internal/utils/__pycache__/filetypes.cpython-38.pyc,, +pip/_internal/utils/__pycache__/glibc.cpython-38.pyc,, +pip/_internal/utils/__pycache__/hashes.cpython-38.pyc,, +pip/_internal/utils/__pycache__/inject_securetransport.cpython-38.pyc,, +pip/_internal/utils/__pycache__/logging.cpython-38.pyc,, +pip/_internal/utils/__pycache__/marker_files.cpython-38.pyc,, +pip/_internal/utils/__pycache__/misc.cpython-38.pyc,, +pip/_internal/utils/__pycache__/models.cpython-38.pyc,, +pip/_internal/utils/__pycache__/packaging.cpython-38.pyc,, +pip/_internal/utils/__pycache__/pkg_resources.cpython-38.pyc,, +pip/_internal/utils/__pycache__/setuptools_build.cpython-38.pyc,, +pip/_internal/utils/__pycache__/subprocess.cpython-38.pyc,, +pip/_internal/utils/__pycache__/temp_dir.cpython-38.pyc,, +pip/_internal/utils/__pycache__/typing.cpython-38.pyc,, +pip/_internal/utils/__pycache__/ui.cpython-38.pyc,, +pip/_internal/utils/__pycache__/unpacking.cpython-38.pyc,, +pip/_internal/utils/__pycache__/urls.cpython-38.pyc,, +pip/_internal/utils/__pycache__/virtualenv.cpython-38.pyc,, +pip/_internal/utils/__pycache__/wheel.cpython-38.pyc,, +pip/_internal/utils/appdirs.py,sha256=PVo_7-IQWHa9qNuNbWSFiF2QGqeLbSAR4eLcYYhQ9ek,1307 +pip/_internal/utils/compat.py,sha256=D7FKGLBdQwWH-dHIGaoWMawDZWBYApvtJVL1kFPJ930,8869 +pip/_internal/utils/deprecation.py,sha256=pBnNogoA4UGTxa_JDnPXBRRYpKMbExAhXpBwAwklOBs,3318 +pip/_internal/utils/distutils_args.py,sha256=a56mblNxk9BGifbpEETG61mmBrqhjtjRkJ4HYn-oOEE,1350 +pip/_internal/utils/encoding.py,sha256=hxZz0t3Whw3d4MHQEiofxalTlfKwxFdLc8fpeGfhKo8,1320 +pip/_internal/utils/entrypoints.py,sha256=vHcNpnksCv6mllihU6hfifdsKPEjwcaJ1aLIXEaynaU,1152 +pip/_internal/utils/filesystem.py,sha256=PXa3vMcz4mbEKtkD0joFI8pBwddLQxhfPFOkVH5xjfE,5255 +pip/_internal/utils/filetypes.py,sha256=R2FwzoeX7b-rZALOXx5cuO8VPPMhUQ4ne7wm3n3IcWA,571 +pip/_internal/utils/glibc.py,sha256=LOeNGgawCKS-4ke9fii78fwXD73dtNav3uxz1Bf-Ab8,3297 +pip/_internal/utils/hashes.py,sha256=my-wSnAWEDvl_8rQaOQcVIWjwh1-f_QiEvGy9TPf53U,3942 +pip/_internal/utils/inject_securetransport.py,sha256=M17ZlFVY66ApgeASVjKKLKNz0LAfk-SyU0HZ4ZB6MmI,810 +pip/_internal/utils/logging.py,sha256=aJL7NldPhS5KGFof6Qt3o3MG5cjm5TOoo7bGRu9_wsg,13033 +pip/_internal/utils/marker_files.py,sha256=CO5djQlrPIozJpJybViH_insoAaBGY1aqEt6-cC-iW0,741 +pip/_internal/utils/misc.py,sha256=uIb58Hiu_g2HRORo2aMcgnW_7R5d-5wUAuoW0fA2ZME,26085 +pip/_internal/utils/models.py,sha256=IA0hw_T4awQzui0kqfIEASm5yLtgZAB08ag59Nip5G8,1148 +pip/_internal/utils/packaging.py,sha256=VtiwcAAL7LBi7tGL2je7LeW4bE11KMHGCsJ1NZY5XtM,3035 +pip/_internal/utils/pkg_resources.py,sha256=ZX-k7V5q_aNWyDse92nN7orN1aCpRLsaxzpkBZ1XKzU,1254 +pip/_internal/utils/setuptools_build.py,sha256=DouaVolV9olDDFIIN9IszaL-FHdNaZt10ufOZFH9ZAU,5070 +pip/_internal/utils/subprocess.py,sha256=Ph3x5eHQBxFotyGhpZN8asSMBud-BBkmgaNfARG-di8,9922 +pip/_internal/utils/temp_dir.py,sha256=87Ib8aNic_hoSDEmUYJHTQIn5-prL2AYL5u_yZ3s4sI,7768 +pip/_internal/utils/typing.py,sha256=xkYwOeHlf4zsHXBDC4310HtEqwhQcYXFPq2h35Tcrl0,1401 +pip/_internal/utils/ui.py,sha256=0FNxXlGtbpPtTviv2oXS9t8bQG_NBdfUgP4GbubhS9U,13911 +pip/_internal/utils/unpacking.py,sha256=M944JTSiapBOSKLWu7lbawpVHSE7flfzZTEr3TAG7v8,9438 +pip/_internal/utils/urls.py,sha256=aNV9wq5ClUmrz6sG-al7hEWJ4ToitOy7l82CmFGFNW8,1481 +pip/_internal/utils/virtualenv.py,sha256=Q3S1WPlI7JWpGOT2jUVJ8l2chm_k7VPJ9cHA_cUluEU,3396 +pip/_internal/utils/wheel.py,sha256=grTRwZtMQwApwbbSPmRVLtac6FKy6SVKeCXNkWyyePA,7302 +pip/_internal/vcs/__init__.py,sha256=viJxJRqRE_mVScum85bgQIXAd6o0ozFt18VpC-qIJrM,617 +pip/_internal/vcs/__pycache__/__init__.cpython-38.pyc,, +pip/_internal/vcs/__pycache__/bazaar.cpython-38.pyc,, +pip/_internal/vcs/__pycache__/git.cpython-38.pyc,, +pip/_internal/vcs/__pycache__/mercurial.cpython-38.pyc,, +pip/_internal/vcs/__pycache__/subversion.cpython-38.pyc,, +pip/_internal/vcs/__pycache__/versioncontrol.cpython-38.pyc,, +pip/_internal/vcs/bazaar.py,sha256=84q1-kj1_nJ9AMzMu8RmMp-riRZu81M7K9kowcYgi3U,3957 +pip/_internal/vcs/git.py,sha256=CdLz3DTsZsLMLPZpEuUwiS40npvDaVB1CNRzoXgcuJQ,14352 +pip/_internal/vcs/mercurial.py,sha256=2mg7BdYI_Fe00fF6omaNccFQLPHBsDBG5CAEzvqn5sA,5110 +pip/_internal/vcs/subversion.py,sha256=Fpwy71AmuqXnoKi6h1SrXRtPjEMn8fieuM1O4j01IBg,12292 +pip/_internal/vcs/versioncontrol.py,sha256=nqoaM1_rzx24WnHtihXA8RcPpnUae0sV2sR_LS_5HFA,22600 +pip/_internal/wheel_builder.py,sha256=gr9jE14W5ZuYblpldo-tpRuyG0e0AVmHLttImuAvXlE,9441 +pip/_vendor/__init__.py,sha256=RcHf8jwLPL0ZEaa6uMhTSfyCrA_TpWgDWAW5br9xD7Y,4975 +pip/_vendor/__pycache__/__init__.cpython-38.pyc,, diff --git a/ubuntu/venv/pip-20.0.2.dist-info/WHEEL b/ubuntu/venv/pip-20.0.2.dist-info/WHEEL new file mode 100644 index 0000000..ef99c6c --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/ubuntu/venv/pip-20.0.2.dist-info/entry_points.txt b/ubuntu/venv/pip-20.0.2.dist-info/entry_points.txt new file mode 100644 index 0000000..d48bd8a --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/entry_points.txt @@ -0,0 +1,5 @@ +[console_scripts] +pip = pip._internal.cli.main:main +pip3 = pip._internal.cli.main:main +pip3.8 = pip._internal.cli.main:main + diff --git a/ubuntu/venv/pip-20.0.2.dist-info/top_level.txt b/ubuntu/venv/pip-20.0.2.dist-info/top_level.txt new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/ubuntu/venv/pip-20.0.2.dist-info/top_level.txt @@ -0,0 +1 @@ +pip diff --git a/ubuntu/venv/pip/__init__.py b/ubuntu/venv/pip/__init__.py new file mode 100644 index 0000000..827a4e2 --- /dev/null +++ b/ubuntu/venv/pip/__init__.py @@ -0,0 +1,18 @@ +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + + +__version__ = "20.0.2" + + +def main(args=None): + # type: (Optional[List[str]]) -> int + """This is an internal API only meant for use by pip's own console scripts. + + For additional details, see https://github.com/pypa/pip/issues/7498. + """ + from pip._internal.utils.entrypoints import _wrapper + + return _wrapper(args) diff --git a/ubuntu/venv/pip/__main__.py b/ubuntu/venv/pip/__main__.py new file mode 100644 index 0000000..e83b9e0 --- /dev/null +++ b/ubuntu/venv/pip/__main__.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import os +import sys + +# If we are running from a wheel, add the wheel to sys.path +# This allows the usage python pip-*.whl/pip install pip-*.whl +if __package__ == '': + # __file__ is pip-*.whl/pip/__main__.py + # first dirname call strips of '/__main__.py', second strips off '/pip' + # Resulting path is the name of the wheel itself + # Add that to sys.path so we can import pip + path = os.path.dirname(os.path.dirname(__file__)) + sys.path.insert(0, path) + +from pip._internal.cli.main import main as _main # isort:skip # noqa + +if __name__ == '__main__': + sys.exit(_main()) diff --git a/ubuntu/venv/pip/_internal/__init__.py b/ubuntu/venv/pip/_internal/__init__.py new file mode 100644 index 0000000..3aa8a46 --- /dev/null +++ b/ubuntu/venv/pip/_internal/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import pip._internal.utils.inject_securetransport # noqa +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, List + + +def main(args=None): + # type: (Optional[List[str]]) -> int + """This is preserved for old console scripts that may still be referencing + it. + + For additional details, see https://github.com/pypa/pip/issues/7498. + """ + from pip._internal.utils.entrypoints import _wrapper + + return _wrapper(args) diff --git a/ubuntu/venv/pip/_internal/build_env.py b/ubuntu/venv/pip/_internal/build_env.py new file mode 100644 index 0000000..f55f0e6 --- /dev/null +++ b/ubuntu/venv/pip/_internal/build_env.py @@ -0,0 +1,221 @@ +"""Build Environment used for isolation during sdist building +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +import logging +import os +import sys +import textwrap +from collections import OrderedDict +from distutils.sysconfig import get_python_lib +from sysconfig import get_paths + +from pip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet + +from pip import __file__ as pip_location +from pip._internal.utils.subprocess import call_subprocess +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import open_spinner + +if MYPY_CHECK_RUNNING: + from typing import Tuple, Set, Iterable, Optional, List + from pip._internal.index.package_finder import PackageFinder + +logger = logging.getLogger(__name__) + + +class _Prefix: + + def __init__(self, path): + # type: (str) -> None + self.path = path + self.setup = False + self.bin_dir = get_paths( + 'nt' if os.name == 'nt' else 'posix_prefix', + vars={'base': path, 'platbase': path} + )['scripts'] + # Note: prefer distutils' sysconfig to get the + # library paths so PyPy is correctly supported. + purelib = get_python_lib(plat_specific=False, prefix=path) + platlib = get_python_lib(plat_specific=True, prefix=path) + if purelib == platlib: + self.lib_dirs = [purelib] + else: + self.lib_dirs = [purelib, platlib] + + +class BuildEnvironment(object): + """Creates and manages an isolated environment to install build deps + """ + + def __init__(self): + # type: () -> None + self._temp_dir = TempDirectory(kind="build-env") + + self._prefixes = OrderedDict(( + (name, _Prefix(os.path.join(self._temp_dir.path, name))) + for name in ('normal', 'overlay') + )) + + self._bin_dirs = [] # type: List[str] + self._lib_dirs = [] # type: List[str] + for prefix in reversed(list(self._prefixes.values())): + self._bin_dirs.append(prefix.bin_dir) + self._lib_dirs.extend(prefix.lib_dirs) + + # Customize site to: + # - ensure .pth files are honored + # - prevent access to system site packages + system_sites = { + os.path.normcase(site) for site in ( + get_python_lib(plat_specific=False), + get_python_lib(plat_specific=True), + ) + } + self._site_dir = os.path.join(self._temp_dir.path, 'site') + if not os.path.exists(self._site_dir): + os.mkdir(self._site_dir) + with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: + fp.write(textwrap.dedent( + ''' + import os, site, sys + + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in {system_sites!r}: + site.addsitedir(path, known_paths=known_paths) + system_paths = set( + os.path.normcase(path) + for path in sys.path[len(original_sys_path):] + ) + original_sys_path = [ + path for path in original_sys_path + if os.path.normcase(path) not in system_paths + ] + sys.path = original_sys_path + + # Second, add lib directories. + # ensuring .pth file are processed. + for path in {lib_dirs!r}: + assert not path in sys.path + site.addsitedir(path) + ''' + ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) + + def __enter__(self): + self._save_env = { + name: os.environ.get(name, None) + for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') + } + + path = self._bin_dirs[:] + old_path = self._save_env['PATH'] + if old_path: + path.extend(old_path.split(os.pathsep)) + + pythonpath = [self._site_dir] + + os.environ.update({ + 'PATH': os.pathsep.join(path), + 'PYTHONNOUSERSITE': '1', + 'PYTHONPATH': os.pathsep.join(pythonpath), + }) + + def __exit__(self, exc_type, exc_val, exc_tb): + for varname, old_value in self._save_env.items(): + if old_value is None: + os.environ.pop(varname, None) + else: + os.environ[varname] = old_value + + def cleanup(self): + # type: () -> None + self._temp_dir.cleanup() + + def check_requirements(self, reqs): + # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] + """Return 2 sets: + - conflicting requirements: set of (installed, wanted) reqs tuples + - missing requirements: set of reqs + """ + missing = set() + conflicting = set() + if reqs: + ws = WorkingSet(self._lib_dirs) + for req in reqs: + try: + if ws.find(Requirement.parse(req)) is None: + missing.add(req) + except VersionConflict as e: + conflicting.add((str(e.args[0].as_requirement()), + str(e.args[1]))) + return conflicting, missing + + def install_requirements( + self, + finder, # type: PackageFinder + requirements, # type: Iterable[str] + prefix_as_string, # type: str + message # type: Optional[str] + ): + # type: (...) -> None + prefix = self._prefixes[prefix_as_string] + assert not prefix.setup + prefix.setup = True + if not requirements: + return + args = [ + sys.executable, os.path.dirname(pip_location), 'install', + '--ignore-installed', '--no-user', '--prefix', prefix.path, + '--no-warn-script-location', + ] # type: List[str] + if logger.getEffectiveLevel() <= logging.DEBUG: + args.append('-v') + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) + + index_urls = finder.index_urls + if index_urls: + args.extend(['-i', index_urls[0]]) + for extra_index in index_urls[1:]: + args.extend(['--extra-index-url', extra_index]) + else: + args.append('--no-index') + for link in finder.find_links: + args.extend(['--find-links', link]) + + for host in finder.trusted_hosts: + args.extend(['--trusted-host', host]) + if finder.allow_all_prereleases: + args.append('--pre') + args.append('--') + args.extend(requirements) + with open_spinner(message) as spinner: + call_subprocess(args, spinner=spinner) + + +class NoOpBuildEnvironment(BuildEnvironment): + """A no-op drop-in replacement for BuildEnvironment + """ + + def __init__(self): + pass + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + def cleanup(self): + pass + + def install_requirements(self, finder, requirements, prefix, message): + raise NotImplementedError() diff --git a/ubuntu/venv/pip/_internal/cache.py b/ubuntu/venv/pip/_internal/cache.py new file mode 100644 index 0000000..abecd78 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cache.py @@ -0,0 +1,329 @@ +"""Cache Management +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import hashlib +import json +import logging +import os + +from pip._vendor.packaging.tags import interpreter_name, interpreter_version +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.exceptions import InvalidWheelFilename +from pip._internal.models.link import Link +from pip._internal.models.wheel import Wheel +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url + +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, List, Any, Dict + + from pip._vendor.packaging.tags import Tag + + from pip._internal.models.format_control import FormatControl + +logger = logging.getLogger(__name__) + + +def _hash_dict(d): + # type: (Dict[str, str]) -> str + """Return a stable sha224 of a dictionary.""" + s = json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha224(s.encode("ascii")).hexdigest() + + +class Cache(object): + """An abstract class - provides cache directories for data from links + + + :param cache_dir: The root of the cache. + :param format_control: An object of FormatControl class to limit + binaries being read from the cache. + :param allowed_formats: which formats of files the cache should store. + ('binary' and 'source' are the only allowed values) + """ + + def __init__(self, cache_dir, format_control, allowed_formats): + # type: (str, FormatControl, Set[str]) -> None + super(Cache, self).__init__() + assert not cache_dir or os.path.isabs(cache_dir) + self.cache_dir = cache_dir or None + self.format_control = format_control + self.allowed_formats = allowed_formats + + _valid_formats = {"source", "binary"} + assert self.allowed_formats.union(_valid_formats) == _valid_formats + + def _get_cache_path_parts_legacy(self, link): + # type: (Link) -> List[str] + """Get parts of part that must be os.path.joined with cache_dir + + Legacy cache key (pip < 20) for compatibility with older caches. + """ + + # We want to generate an url to use as our cache key, we don't want to + # just re-use the URL because it might have other items in the fragment + # and we don't care about those. + key_parts = [link.url_without_fragment] + if link.hash_name is not None and link.hash is not None: + key_parts.append("=".join([link.hash_name, link.hash])) + key_url = "#".join(key_parts) + + # Encode our key url with sha224, we'll use this because it has similar + # security properties to sha256, but with a shorter total output (and + # thus less secure). However the differences don't make a lot of + # difference for our use case here. + hashed = hashlib.sha224(key_url.encode()).hexdigest() + + # We want to nest the directories some to prevent having a ton of top + # level directories where we might run out of sub directories on some + # FS. + parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] + + return parts + + def _get_cache_path_parts(self, link): + # type: (Link) -> List[str] + """Get parts of part that must be os.path.joined with cache_dir + """ + + # We want to generate an url to use as our cache key, we don't want to + # just re-use the URL because it might have other items in the fragment + # and we don't care about those. + key_parts = {"url": link.url_without_fragment} + if link.hash_name is not None and link.hash is not None: + key_parts[link.hash_name] = link.hash + if link.subdirectory_fragment: + key_parts["subdirectory"] = link.subdirectory_fragment + + # Include interpreter name, major and minor version in cache key + # to cope with ill-behaved sdists that build a different wheel + # depending on the python version their setup.py is being run on, + # and don't encode the difference in compatibility tags. + # https://github.com/pypa/pip/issues/7296 + key_parts["interpreter_name"] = interpreter_name() + key_parts["interpreter_version"] = interpreter_version() + + # Encode our key url with sha224, we'll use this because it has similar + # security properties to sha256, but with a shorter total output (and + # thus less secure). However the differences don't make a lot of + # difference for our use case here. + hashed = _hash_dict(key_parts) + + # We want to nest the directories some to prevent having a ton of top + # level directories where we might run out of sub directories on some + # FS. + parts = [hashed[:2], hashed[2:4], hashed[4:6], hashed[6:]] + + return parts + + def _get_candidates(self, link, canonical_package_name): + # type: (Link, Optional[str]) -> List[Any] + can_not_cache = ( + not self.cache_dir or + not canonical_package_name or + not link + ) + if can_not_cache: + return [] + + formats = self.format_control.get_allowed_formats( + canonical_package_name + ) + if not self.allowed_formats.intersection(formats): + return [] + + candidates = [] + path = self.get_path_for_link(link) + if os.path.isdir(path): + for candidate in os.listdir(path): + candidates.append((candidate, path)) + # TODO remove legacy path lookup in pip>=21 + legacy_path = self.get_path_for_link_legacy(link) + if os.path.isdir(legacy_path): + for candidate in os.listdir(legacy_path): + candidates.append((candidate, legacy_path)) + return candidates + + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + raise NotImplementedError() + + def get_path_for_link(self, link): + # type: (Link) -> str + """Return a directory to store cached items in for link. + """ + raise NotImplementedError() + + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Tag] + ): + # type: (...) -> Link + """Returns a link to a cached item if it exists, otherwise returns the + passed link. + """ + raise NotImplementedError() + + def cleanup(self): + # type: () -> None + pass + + +class SimpleWheelCache(Cache): + """A cache of wheels for future installs. + """ + + def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None + super(SimpleWheelCache, self).__init__( + cache_dir, format_control, {"binary"} + ) + + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + parts = self._get_cache_path_parts_legacy(link) + return os.path.join(self.cache_dir, "wheels", *parts) + + def get_path_for_link(self, link): + # type: (Link) -> str + """Return a directory to store cached wheels for link + + Because there are M wheels for any one sdist, we provide a directory + to cache them in, and then consult that directory when looking up + cache hits. + + We only insert things into the cache if they have plausible version + numbers, so that we don't contaminate the cache with things that were + not unique. E.g. ./package might have dozens of installs done for it + and build a version of 0.0...and if we built and cached a wheel, we'd + end up using the same wheel even if the source has been edited. + + :param link: The link of the sdist for which this will cache wheels. + """ + parts = self._get_cache_path_parts(link) + + # Store wheels within the root cache_dir + return os.path.join(self.cache_dir, "wheels", *parts) + + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Tag] + ): + # type: (...) -> Link + candidates = [] + + if not package_name: + return link + + canonical_package_name = canonicalize_name(package_name) + for wheel_name, wheel_dir in self._get_candidates( + link, canonical_package_name + ): + try: + wheel = Wheel(wheel_name) + except InvalidWheelFilename: + continue + if canonicalize_name(wheel.name) != canonical_package_name: + logger.debug( + "Ignoring cached wheel {} for {} as it " + "does not match the expected distribution name {}.".format( + wheel_name, link, package_name + ) + ) + continue + if not wheel.supported(supported_tags): + # Built for a different python/arch/etc + continue + candidates.append( + ( + wheel.support_index_min(supported_tags), + wheel_name, + wheel_dir, + ) + ) + + if not candidates: + return link + + _, wheel_name, wheel_dir = min(candidates) + return Link(path_to_url(os.path.join(wheel_dir, wheel_name))) + + +class EphemWheelCache(SimpleWheelCache): + """A SimpleWheelCache that creates it's own temporary cache directory + """ + + def __init__(self, format_control): + # type: (FormatControl) -> None + self._temp_dir = TempDirectory(kind="ephem-wheel-cache") + + super(EphemWheelCache, self).__init__( + self._temp_dir.path, format_control + ) + + def cleanup(self): + # type: () -> None + self._temp_dir.cleanup() + + +class WheelCache(Cache): + """Wraps EphemWheelCache and SimpleWheelCache into a single Cache + + This Cache allows for gracefully degradation, using the ephem wheel cache + when a certain link is not found in the simple wheel cache first. + """ + + def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None + super(WheelCache, self).__init__( + cache_dir, format_control, {'binary'} + ) + self._wheel_cache = SimpleWheelCache(cache_dir, format_control) + self._ephem_cache = EphemWheelCache(format_control) + + def get_path_for_link_legacy(self, link): + # type: (Link) -> str + return self._wheel_cache.get_path_for_link_legacy(link) + + def get_path_for_link(self, link): + # type: (Link) -> str + return self._wheel_cache.get_path_for_link(link) + + def get_ephem_path_for_link(self, link): + # type: (Link) -> str + return self._ephem_cache.get_path_for_link(link) + + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Tag] + ): + # type: (...) -> Link + retval = self._wheel_cache.get( + link=link, + package_name=package_name, + supported_tags=supported_tags, + ) + if retval is not link: + return retval + + return self._ephem_cache.get( + link=link, + package_name=package_name, + supported_tags=supported_tags, + ) + + def cleanup(self): + # type: () -> None + self._wheel_cache.cleanup() + self._ephem_cache.cleanup() diff --git a/ubuntu/venv/pip/_internal/cli/__init__.py b/ubuntu/venv/pip/_internal/cli/__init__.py new file mode 100644 index 0000000..e589bb9 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of pip's command line interface related code +""" + +# This file intentionally does not import submodules diff --git a/ubuntu/venv/pip/_internal/cli/autocompletion.py b/ubuntu/venv/pip/_internal/cli/autocompletion.py new file mode 100644 index 0000000..329de60 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/autocompletion.py @@ -0,0 +1,164 @@ +"""Logic that powers autocompletion installed by ``pip completion``. +""" + +import optparse +import os +import sys +from itertools import chain + +from pip._internal.cli.main_parser import create_main_parser +from pip._internal.commands import commands_dict, create_command +from pip._internal.utils.misc import get_installed_distributions +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Iterable, List, Optional + + +def autocomplete(): + # type: () -> None + """Entry Point for completion of main and subcommand options. + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword - 1] + except IndexError: + current = '' + + parser = create_main_parser() + subcommands = list(commands_dict) + options = [] + + # subcommand + subcommand_name = None # type: Optional[str] + for word in cwords: + if word in subcommands: + subcommand_name = word + break + # subcommand options + if subcommand_name is not None: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for show and uninstall + should_list_installed = ( + subcommand_name in ['show', 'uninstall'] and + not current.startswith('-') + ) + if should_list_installed: + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print(dist) + sys.exit(1) + + subcommand = create_command(subcommand_name) + + for opt in subcommand.parser.option_list_all: + if opt.help != optparse.SUPPRESS_HELP: + for opt_str in opt._long_opts + opt._short_opts: + options.append((opt_str, opt.nargs)) + + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] + options = [(x, v) for (x, v) in options if x not in prev_opts] + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + # get completion type given cwords and available subcommand options + completion_type = get_path_completion_type( + cwords, cword, subcommand.parser.option_list_all, + ) + # get completion files and directories if ``completion_type`` is + # ````, ```` or ```` + if completion_type: + paths = auto_complete_paths(current, completion_type) + options = [(path, 0) for path in paths] + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1] and option[0][:2] == "--": + opt_label += '=' + print(opt_label) + else: + # show main parser options only when necessary + + opts = [i.option_list for i in parser.option_groups] + opts.append(parser.option_list) + flattened_opts = chain.from_iterable(opts) + if current.startswith('-'): + for opt in flattened_opts: + if opt.help != optparse.SUPPRESS_HELP: + subcommands += opt._long_opts + opt._short_opts + else: + # get completion type given cwords and all available options + completion_type = get_path_completion_type(cwords, cword, + flattened_opts) + if completion_type: + subcommands = list(auto_complete_paths(current, + completion_type)) + + print(' '.join([x for x in subcommands if x.startswith(current)])) + sys.exit(1) + + +def get_path_completion_type(cwords, cword, opts): + # type: (List[str], int, Iterable[Any]) -> Optional[str] + """Get the type of path completion (``file``, ``dir``, ``path`` or None) + + :param cwords: same as the environmental variable ``COMP_WORDS`` + :param cword: same as the environmental variable ``COMP_CWORD`` + :param opts: The available options to check + :return: path completion type (``file``, ``dir``, ``path`` or None) + """ + if cword < 2 or not cwords[cword - 2].startswith('-'): + return None + for opt in opts: + if opt.help == optparse.SUPPRESS_HELP: + continue + for o in str(opt).split('/'): + if cwords[cword - 2].split('=')[0] == o: + if not opt.metavar or any( + x in ('path', 'file', 'dir') + for x in opt.metavar.split('/')): + return opt.metavar + return None + + +def auto_complete_paths(current, completion_type): + # type: (str, str) -> Iterable[str] + """If ``completion_type`` is ``file`` or ``path``, list all regular files + and directories starting with ``current``; otherwise only list directories + starting with ``current``. + + :param current: The word to be completed + :param completion_type: path completion type(`file`, `path` or `dir`)i + :return: A generator of regular files and/or directories + """ + directory, filename = os.path.split(current) + current_path = os.path.abspath(directory) + # Don't complete paths if they can't be accessed + if not os.access(current_path, os.R_OK): + return + filename = os.path.normcase(filename) + # list all files that start with ``filename`` + file_list = (x for x in os.listdir(current_path) + if os.path.normcase(x).startswith(filename)) + for f in file_list: + opt = os.path.join(current_path, f) + comp_file = os.path.normcase(os.path.join(directory, f)) + # complete regular files when there is not ```` after option + # complete directories when there is ````, ```` or + # ````after option + if completion_type != 'dir' and os.path.isfile(opt): + yield comp_file + elif os.path.isdir(opt): + yield os.path.join(comp_file, '') diff --git a/ubuntu/venv/pip/_internal/cli/base_command.py b/ubuntu/venv/pip/_internal/cli/base_command.py new file mode 100644 index 0000000..628faa3 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/base_command.py @@ -0,0 +1,226 @@ +"""Base Command class, and related routines""" + +from __future__ import absolute_import, print_function + +import logging +import logging.config +import optparse +import os +import platform +import sys +import traceback + +from pip._internal.cli import cmdoptions +from pip._internal.cli.command_context import CommandContextMixIn +from pip._internal.cli.parser import ( + ConfigOptionParser, + UpdatingDefaultsHelpFormatter, +) +from pip._internal.cli.status_codes import ( + ERROR, + PREVIOUS_BUILD_DIR_ERROR, + SUCCESS, + UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, +) +from pip._internal.exceptions import ( + BadCommand, + CommandError, + InstallationError, + PreviousBuildDirError, + UninstallationError, +) +from pip._internal.utils.deprecation import deprecated +from pip._internal.utils.filesystem import check_path_owner +from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging +from pip._internal.utils.misc import get_prog, normalize_path +from pip._internal.utils.temp_dir import global_tempdir_manager +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.virtualenv import running_under_virtualenv + +if MYPY_CHECK_RUNNING: + from typing import List, Tuple, Any + from optparse import Values + +__all__ = ['Command'] + +logger = logging.getLogger(__name__) + + +class Command(CommandContextMixIn): + usage = None # type: str + ignore_require_venv = False # type: bool + + def __init__(self, name, summary, isolated=False): + # type: (str, str, bool) -> None + super(Command, self).__init__() + parser_kw = { + 'usage': self.usage, + 'prog': '%s %s' % (get_prog(), name), + 'formatter': UpdatingDefaultsHelpFormatter(), + 'add_help_option': False, + 'name': name, + 'description': self.__doc__, + 'isolated': isolated, + } + + self.name = name + self.summary = summary + self.parser = ConfigOptionParser(**parser_kw) + + # Commands should add options to this option group + optgroup_name = '%s Options' % self.name.capitalize() + self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name) + + # Add the general options + gen_opts = cmdoptions.make_option_group( + cmdoptions.general_group, + self.parser, + ) + self.parser.add_option_group(gen_opts) + + def handle_pip_version_check(self, options): + # type: (Values) -> None + """ + This is a no-op so that commands by default do not do the pip version + check. + """ + # Make sure we do the pip version check if the index_group options + # are present. + assert not hasattr(options, 'no_index') + + def run(self, options, args): + # type: (Values, List[Any]) -> Any + raise NotImplementedError + + def parse_args(self, args): + # type: (List[str]) -> Tuple[Any, Any] + # factored out for testability + return self.parser.parse_args(args) + + def main(self, args): + # type: (List[str]) -> int + try: + with self.main_context(): + return self._main(args) + finally: + logging.shutdown() + + def _main(self, args): + # type: (List[str]) -> int + # Intentionally set as early as possible so globally-managed temporary + # directories are available to the rest of the code. + self.enter_context(global_tempdir_manager()) + + options, args = self.parse_args(args) + + # Set verbosity so that it can be used elsewhere. + self.verbosity = options.verbose - options.quiet + + level_number = setup_logging( + verbosity=self.verbosity, + no_color=options.no_color, + user_log_file=options.log, + ) + + if ( + sys.version_info[:2] == (2, 7) and + not options.no_python_version_warning + ): + message = ( + "A future version of pip will drop support for Python 2.7. " + "More details about Python 2 support in pip, can be found at " + "https://pip.pypa.io/en/latest/development/release-process/#python-2-support" # noqa + ) + if platform.python_implementation() == "CPython": + message = ( + "Python 2.7 reached the end of its life on January " + "1st, 2020. Please upgrade your Python as Python 2.7 " + "is no longer maintained. " + ) + message + deprecated(message, replacement=None, gone_in=None) + + if options.skip_requirements_regex: + deprecated( + "--skip-requirements-regex is unsupported and will be removed", + replacement=( + "manage requirements/constraints files explicitly, " + "possibly generating them from metadata" + ), + gone_in="20.1", + issue=7297, + ) + + # TODO: Try to get these passing down from the command? + # without resorting to os.environ to hold these. + # This also affects isolated builds and it should. + + if options.no_input: + os.environ['PIP_NO_INPUT'] = '1' + + if options.exists_action: + os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action) + + if options.require_venv and not self.ignore_require_venv: + # If a venv is required check if it can really be found + if not running_under_virtualenv(): + logger.critical( + 'Could not find an activated virtualenv (required).' + ) + sys.exit(VIRTUALENV_NOT_FOUND) + + if options.cache_dir: + options.cache_dir = normalize_path(options.cache_dir) + if not check_path_owner(options.cache_dir): + logger.warning( + "The directory '%s' or its parent directory is not owned " + "or is not writable by the current user. The cache " + "has been disabled. Check the permissions and owner of " + "that directory. If executing pip with sudo, you may want " + "sudo's -H flag.", + options.cache_dir, + ) + options.cache_dir = None + + try: + status = self.run(options, args) + # FIXME: all commands should return an exit status + # and when it is done, isinstance is not needed anymore + if isinstance(status, int): + return status + except PreviousBuildDirError as exc: + logger.critical(str(exc)) + logger.debug('Exception information:', exc_info=True) + + return PREVIOUS_BUILD_DIR_ERROR + except (InstallationError, UninstallationError, BadCommand) as exc: + logger.critical(str(exc)) + logger.debug('Exception information:', exc_info=True) + + return ERROR + except CommandError as exc: + logger.critical('%s', exc) + logger.debug('Exception information:', exc_info=True) + + return ERROR + except BrokenStdoutLoggingError: + # Bypass our logger and write any remaining messages to stderr + # because stdout no longer works. + print('ERROR: Pipe to stdout was broken', file=sys.stderr) + if level_number <= logging.DEBUG: + traceback.print_exc(file=sys.stderr) + + return ERROR + except KeyboardInterrupt: + logger.critical('Operation cancelled by user') + logger.debug('Exception information:', exc_info=True) + + return ERROR + except BaseException: + logger.critical('Exception:', exc_info=True) + + return UNKNOWN_ERROR + finally: + self.handle_pip_version_check(options) + + return SUCCESS diff --git a/ubuntu/venv/pip/_internal/cli/cmdoptions.py b/ubuntu/venv/pip/_internal/cli/cmdoptions.py new file mode 100644 index 0000000..447f319 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/cmdoptions.py @@ -0,0 +1,957 @@ +""" +shared options and groups + +The principle here is to define options once, but *not* instantiate them +globally. One reason being that options with action='append' can carry state +between parses. pip parses general options twice internally, and shouldn't +pass on state. To be consistent, all options will follow this design. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import os +import textwrap +import warnings +from distutils.util import strtobool +from functools import partial +from optparse import SUPPRESS_HELP, Option, OptionGroup +from textwrap import dedent + +from pip._internal.exceptions import CommandError +from pip._internal.locations import USER_CACHE_DIR, get_src_prefix +from pip._internal.models.format_control import FormatControl +from pip._internal.models.index import PyPI +from pip._internal.models.target_python import TargetPython +from pip._internal.utils.hashes import STRONG_HASHES +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import BAR_TYPES + +if MYPY_CHECK_RUNNING: + from typing import Any, Callable, Dict, Optional, Tuple + from optparse import OptionParser, Values + from pip._internal.cli.parser import ConfigOptionParser + +logger = logging.getLogger(__name__) + + +def raise_option_error(parser, option, msg): + # type: (OptionParser, Option, str) -> None + """ + Raise an option parsing error using parser.error(). + + Args: + parser: an OptionParser instance. + option: an Option instance. + msg: the error text. + """ + msg = '{} error: {}'.format(option, msg) + msg = textwrap.fill(' '.join(msg.split())) + parser.error(msg) + + +def make_option_group(group, parser): + # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup + """ + Return an OptionGroup object + group -- assumed to be dict with 'name' and 'options' keys + parser -- an optparse Parser + """ + option_group = OptionGroup(parser, group['name']) + for option in group['options']: + option_group.add_option(option()) + return option_group + + +def check_install_build_global(options, check_options=None): + # type: (Values, Optional[Values]) -> None + """Disable wheels if per-setup.py call options are set. + + :param options: The OptionParser options to update. + :param check_options: The options to check, if not supplied defaults to + options. + """ + if check_options is None: + check_options = options + + def getname(n): + # type: (str) -> Optional[Any] + return getattr(check_options, n, None) + names = ["build_options", "global_options", "install_options"] + if any(map(getname, names)): + control = options.format_control + control.disallow_binaries() + warnings.warn( + 'Disabling all use of wheels due to the use of --build-option ' + '/ --global-option / --install-option.', stacklevel=2, + ) + + +def check_dist_restriction(options, check_target=False): + # type: (Values, bool) -> None + """Function for determining if custom platform options are allowed. + + :param options: The OptionParser options. + :param check_target: Whether or not to check if --target is being used. + """ + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + + binary_only = FormatControl(set(), {':all:'}) + sdist_dependencies_allowed = ( + options.format_control != binary_only and + not options.ignore_dependencies + ) + + # Installations or downloads using dist restrictions must not combine + # source distributions and dist-specific wheels, as they are not + # guaranteed to be locally compatible. + if dist_restriction_set and sdist_dependencies_allowed: + raise CommandError( + "When restricting platform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + "either --no-deps must be set, or --only-binary=:all: must be " + "set and --no-binary must not be set (or must be set to " + ":none:)." + ) + + if check_target: + if dist_restriction_set and not options.target_dir: + raise CommandError( + "Can not use any platform or abi specific options unless " + "installing via '--target'" + ) + + +def _path_option_check(option, opt, value): + # type: (Option, str, str) -> str + return os.path.expanduser(value) + + +class PipOption(Option): + TYPES = Option.TYPES + ("path",) + TYPE_CHECKER = Option.TYPE_CHECKER.copy() + TYPE_CHECKER["path"] = _path_option_check + + +########### +# options # +########### + +help_ = partial( + Option, + '-h', '--help', + dest='help', + action='help', + help='Show help.', +) # type: Callable[..., Option] + +isolated_mode = partial( + Option, + "--isolated", + dest="isolated_mode", + action="store_true", + default=False, + help=( + "Run pip in an isolated mode, ignoring environment variables and user " + "configuration." + ), +) # type: Callable[..., Option] + +require_virtualenv = partial( + Option, + # Run only if inside a virtualenv, bail if not. + '--require-virtualenv', '--require-venv', + dest='require_venv', + action='store_true', + default=False, + help=SUPPRESS_HELP +) # type: Callable[..., Option] + +verbose = partial( + Option, + '-v', '--verbose', + dest='verbose', + action='count', + default=0, + help='Give more output. Option is additive, and can be used up to 3 times.' +) # type: Callable[..., Option] + +no_color = partial( + Option, + '--no-color', + dest='no_color', + action='store_true', + default=False, + help="Suppress colored output", +) # type: Callable[..., Option] + +version = partial( + Option, + '-V', '--version', + dest='version', + action='store_true', + help='Show version and exit.', +) # type: Callable[..., Option] + +quiet = partial( + Option, + '-q', '--quiet', + dest='quiet', + action='count', + default=0, + help=( + 'Give less output. Option is additive, and can be used up to 3' + ' times (corresponding to WARNING, ERROR, and CRITICAL logging' + ' levels).' + ), +) # type: Callable[..., Option] + +progress_bar = partial( + Option, + '--progress-bar', + dest='progress_bar', + type='choice', + choices=list(BAR_TYPES.keys()), + default='on', + help=( + 'Specify type of progress to be displayed [' + + '|'.join(BAR_TYPES.keys()) + '] (default: %default)' + ), +) # type: Callable[..., Option] + +log = partial( + PipOption, + "--log", "--log-file", "--local-log", + dest="log", + metavar="path", + type="path", + help="Path to a verbose appending log." +) # type: Callable[..., Option] + +no_input = partial( + Option, + # Don't ask for input + '--no-input', + dest='no_input', + action='store_true', + default=False, + help=SUPPRESS_HELP +) # type: Callable[..., Option] + +proxy = partial( + Option, + '--proxy', + dest='proxy', + type='str', + default='', + help="Specify a proxy in the form [user:passwd@]proxy.server:port." +) # type: Callable[..., Option] + +retries = partial( + Option, + '--retries', + dest='retries', + type='int', + default=5, + help="Maximum number of retries each connection should attempt " + "(default %default times).", +) # type: Callable[..., Option] + +timeout = partial( + Option, + '--timeout', '--default-timeout', + metavar='sec', + dest='timeout', + type='float', + default=15, + help='Set the socket timeout (default %default seconds).', +) # type: Callable[..., Option] + +skip_requirements_regex = partial( + Option, + # A regex to be used to skip requirements + '--skip-requirements-regex', + dest='skip_requirements_regex', + type='str', + default='', + help=SUPPRESS_HELP, +) # type: Callable[..., Option] + + +def exists_action(): + # type: () -> Option + return Option( + # Option when path already exist + '--exists-action', + dest='exists_action', + type='choice', + choices=['s', 'i', 'w', 'b', 'a'], + default=[], + action='append', + metavar='action', + help="Default action when a path already exists: " + "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.", + ) + + +cert = partial( + PipOption, + '--cert', + dest='cert', + type='path', + metavar='path', + help="Path to alternate CA bundle.", +) # type: Callable[..., Option] + +client_cert = partial( + PipOption, + '--client-cert', + dest='client_cert', + type='path', + default=None, + metavar='path', + help="Path to SSL client certificate, a single file containing the " + "private key and the certificate in PEM format.", +) # type: Callable[..., Option] + +index_url = partial( + Option, + '-i', '--index-url', '--pypi-url', + dest='index_url', + metavar='URL', + default=PyPI.simple_url, + help="Base URL of the Python Package Index (default %default). " + "This should point to a repository compliant with PEP 503 " + "(the simple repository API) or a local directory laid out " + "in the same format.", +) # type: Callable[..., Option] + + +def extra_index_url(): + # type: () -> Option + return Option( + '--extra-index-url', + dest='extra_index_urls', + metavar='URL', + action='append', + default=[], + help="Extra URLs of package indexes to use in addition to " + "--index-url. Should follow the same rules as " + "--index-url.", + ) + + +no_index = partial( + Option, + '--no-index', + dest='no_index', + action='store_true', + default=False, + help='Ignore package index (only looking at --find-links URLs instead).', +) # type: Callable[..., Option] + + +def find_links(): + # type: () -> Option + return Option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='url', + help="If a url or path to an html file, then parse for links to " + "archives. If a local path or file:// url that's a directory, " + "then look for archives in the directory listing.", + ) + + +def trusted_host(): + # type: () -> Option + return Option( + "--trusted-host", + dest="trusted_hosts", + action="append", + metavar="HOSTNAME", + default=[], + help="Mark this host or host:port pair as trusted, even though it " + "does not have valid or any HTTPS.", + ) + + +def constraints(): + # type: () -> Option + return Option( + '-c', '--constraint', + dest='constraints', + action='append', + default=[], + metavar='file', + help='Constrain versions using the given constraints file. ' + 'This option can be used multiple times.' + ) + + +def requirements(): + # type: () -> Option + return Option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='file', + help='Install from the given requirements file. ' + 'This option can be used multiple times.' + ) + + +def editable(): + # type: () -> Option + return Option( + '-e', '--editable', + dest='editables', + action='append', + default=[], + metavar='path/url', + help=('Install a project in editable mode (i.e. setuptools ' + '"develop mode") from a local project path or a VCS url.'), + ) + + +def _handle_src(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + value = os.path.abspath(value) + setattr(parser.values, option.dest, value) + + +src = partial( + PipOption, + '--src', '--source', '--source-dir', '--source-directory', + dest='src_dir', + type='path', + metavar='dir', + default=get_src_prefix(), + action='callback', + callback=_handle_src, + help='Directory to check out editable projects into. ' + 'The default in a virtualenv is "/src". ' + 'The default for global installs is "/src".' +) # type: Callable[..., Option] + + +def _get_format_control(values, option): + # type: (Values, Option) -> Any + """Get a format_control object.""" + return getattr(values, option.dest) + + +def _handle_no_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( + value, existing.no_binary, existing.only_binary, + ) + + +def _handle_only_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( + value, existing.only_binary, existing.no_binary, + ) + + +def no_binary(): + # type: () -> Option + format_control = FormatControl(set(), set()) + return Option( + "--no-binary", dest="format_control", action="callback", + callback=_handle_no_binary, type="str", + default=format_control, + help="Do not use binary packages. Can be supplied multiple times, and " + "each time adds to the existing value. Accepts either :all: to " + "disable all binary packages, :none: to empty the set, or one or " + "more package names with commas between them (no colons). Note " + "that some packages are tricky to compile and may fail to " + "install when this option is used on them.", + ) + + +def only_binary(): + # type: () -> Option + format_control = FormatControl(set(), set()) + return Option( + "--only-binary", dest="format_control", action="callback", + callback=_handle_only_binary, type="str", + default=format_control, + help="Do not use source packages. Can be supplied multiple times, and " + "each time adds to the existing value. Accepts either :all: to " + "disable all source packages, :none: to empty the set, or one or " + "more package names with commas between them. Packages without " + "binary distributions will fail to install when this option is " + "used on them.", + ) + + +platform = partial( + Option, + '--platform', + dest='platform', + metavar='platform', + default=None, + help=("Only use wheels compatible with . " + "Defaults to the platform of the running system."), +) # type: Callable[..., Option] + + +# This was made a separate function for unit-testing purposes. +def _convert_python_version(value): + # type: (str) -> Tuple[Tuple[int, ...], Optional[str]] + """ + Convert a version string like "3", "37", or "3.7.3" into a tuple of ints. + + :return: A 2-tuple (version_info, error_msg), where `error_msg` is + non-None if and only if there was a parsing error. + """ + if not value: + # The empty string is the same as not providing a value. + return (None, None) + + parts = value.split('.') + if len(parts) > 3: + return ((), 'at most three version parts are allowed') + + if len(parts) == 1: + # Then we are in the case of "3" or "37". + value = parts[0] + if len(value) > 1: + parts = [value[0], value[1:]] + + try: + version_info = tuple(int(part) for part in parts) + except ValueError: + return ((), 'each version part must be an integer') + + return (version_info, None) + + +def _handle_python_version(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Handle a provided --python-version value. + """ + version_info, error_msg = _convert_python_version(value) + if error_msg is not None: + msg = ( + 'invalid --python-version value: {!r}: {}'.format( + value, error_msg, + ) + ) + raise_option_error(parser, option=option, msg=msg) + + parser.values.python_version = version_info + + +python_version = partial( + Option, + '--python-version', + dest='python_version', + metavar='python_version', + action='callback', + callback=_handle_python_version, type='str', + default=None, + help=dedent("""\ + The Python interpreter version to use for wheel and "Requires-Python" + compatibility checks. Defaults to a version derived from the running + interpreter. The version can be specified using up to three dot-separated + integers (e.g. "3" for 3.0.0, "3.7" for 3.7.0, or "3.7.3"). A major-minor + version can also be given as a string without dots (e.g. "37" for 3.7.0). + """), +) # type: Callable[..., Option] + + +implementation = partial( + Option, + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only use wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), +) # type: Callable[..., Option] + + +abi = partial( + Option, + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only use wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), +) # type: Callable[..., Option] + + +def add_target_python_options(cmd_opts): + # type: (OptionGroup) -> None + cmd_opts.add_option(platform()) + cmd_opts.add_option(python_version()) + cmd_opts.add_option(implementation()) + cmd_opts.add_option(abi()) + + +def make_target_python(options): + # type: (Values) -> TargetPython + target_python = TargetPython( + platform=options.platform, + py_version_info=options.python_version, + abi=options.abi, + implementation=options.implementation, + ) + + return target_python + + +def prefer_binary(): + # type: () -> Option + return Option( + "--prefer-binary", + dest="prefer_binary", + action="store_true", + default=False, + help="Prefer older binary packages over newer source packages." + ) + + +cache_dir = partial( + PipOption, + "--cache-dir", + dest="cache_dir", + default=USER_CACHE_DIR, + metavar="dir", + type='path', + help="Store the cache data in ." +) # type: Callable[..., Option] + + +def _handle_no_cache_dir(option, opt, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Process a value provided for the --no-cache-dir option. + + This is an optparse.Option callback for the --no-cache-dir option. + """ + # The value argument will be None if --no-cache-dir is passed via the + # command-line, since the option doesn't accept arguments. However, + # the value can be non-None if the option is triggered e.g. by an + # environment variable, like PIP_NO_CACHE_DIR=true. + if value is not None: + # Then parse the string value to get argument error-checking. + try: + strtobool(value) + except ValueError as exc: + raise_option_error(parser, option=option, msg=str(exc)) + + # Originally, setting PIP_NO_CACHE_DIR to a value that strtobool() + # converted to 0 (like "false" or "no") caused cache_dir to be disabled + # rather than enabled (logic would say the latter). Thus, we disable + # the cache directory not just on values that parse to True, but (for + # backwards compatibility reasons) also on values that parse to False. + # In other words, always set it to False if the option is provided in + # some (valid) form. + parser.values.cache_dir = False + + +no_cache = partial( + Option, + "--no-cache-dir", + dest="cache_dir", + action="callback", + callback=_handle_no_cache_dir, + help="Disable the cache.", +) # type: Callable[..., Option] + +no_deps = partial( + Option, + '--no-deps', '--no-dependencies', + dest='ignore_dependencies', + action='store_true', + default=False, + help="Don't install package dependencies.", +) # type: Callable[..., Option] + + +def _handle_build_dir(option, opt, value, parser): + # type: (Option, str, str, OptionParser) -> None + if value: + value = os.path.abspath(value) + setattr(parser.values, option.dest, value) + + +build_dir = partial( + PipOption, + '-b', '--build', '--build-dir', '--build-directory', + dest='build_dir', + type='path', + metavar='dir', + action='callback', + callback=_handle_build_dir, + help='Directory to unpack packages into and build in. Note that ' + 'an initial build still takes place in a temporary directory. ' + 'The location of temporary directories can be controlled by setting ' + 'the TMPDIR environment variable (TEMP on Windows) appropriately. ' + 'When passed, build directories are not cleaned in case of failures.' +) # type: Callable[..., Option] + +ignore_requires_python = partial( + Option, + '--ignore-requires-python', + dest='ignore_requires_python', + action='store_true', + help='Ignore the Requires-Python information.' +) # type: Callable[..., Option] + +no_build_isolation = partial( + Option, + '--no-build-isolation', + dest='build_isolation', + action='store_false', + default=True, + help='Disable isolation when building a modern source distribution. ' + 'Build dependencies specified by PEP 518 must be already installed ' + 'if this option is used.' +) # type: Callable[..., Option] + + +def _handle_no_use_pep517(option, opt, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Process a value provided for the --no-use-pep517 option. + + This is an optparse.Option callback for the no_use_pep517 option. + """ + # Since --no-use-pep517 doesn't accept arguments, the value argument + # will be None if --no-use-pep517 is passed via the command-line. + # However, the value can be non-None if the option is triggered e.g. + # by an environment variable, for example "PIP_NO_USE_PEP517=true". + if value is not None: + msg = """A value was passed for --no-use-pep517, + probably using either the PIP_NO_USE_PEP517 environment variable + or the "no-use-pep517" config file option. Use an appropriate value + of the PIP_USE_PEP517 environment variable or the "use-pep517" + config file option instead. + """ + raise_option_error(parser, option=option, msg=msg) + + # Otherwise, --no-use-pep517 was passed via the command-line. + parser.values.use_pep517 = False + + +use_pep517 = partial( + Option, + '--use-pep517', + dest='use_pep517', + action='store_true', + default=None, + help='Use PEP 517 for building source distributions ' + '(use --no-use-pep517 to force legacy behaviour).' +) # type: Any + +no_use_pep517 = partial( + Option, + '--no-use-pep517', + dest='use_pep517', + action='callback', + callback=_handle_no_use_pep517, + default=None, + help=SUPPRESS_HELP +) # type: Any + +install_options = partial( + Option, + '--install-option', + dest='install_options', + action='append', + metavar='options', + help="Extra arguments to be supplied to the setup.py install " + "command (use like --install-option=\"--install-scripts=/usr/local/" + "bin\"). Use multiple --install-option options to pass multiple " + "options to setup.py install. If you are using an option with a " + "directory path, be sure to use absolute path.", +) # type: Callable[..., Option] + +global_options = partial( + Option, + '--global-option', + dest='global_options', + action='append', + metavar='options', + help="Extra global options to be supplied to the setup.py " + "call before the install command.", +) # type: Callable[..., Option] + +no_clean = partial( + Option, + '--no-clean', + action='store_true', + default=False, + help="Don't clean up build directories." +) # type: Callable[..., Option] + +pre = partial( + Option, + '--pre', + action='store_true', + default=False, + help="Include pre-release and development versions. By default, " + "pip only finds stable versions.", +) # type: Callable[..., Option] + +disable_pip_version_check = partial( + Option, + "--disable-pip-version-check", + dest="disable_pip_version_check", + action="store_true", + default=True, + help="Don't periodically check PyPI to determine whether a new version " + "of pip is available for download. Implied with --no-index.", +) # type: Callable[..., Option] + + +# Deprecated, Remove later +always_unzip = partial( + Option, + '-Z', '--always-unzip', + dest='always_unzip', + action='store_true', + help=SUPPRESS_HELP, +) # type: Callable[..., Option] + + +def _handle_merge_hash(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + """Given a value spelled "algo:digest", append the digest to a list + pointed to in a dict by the algo name.""" + if not parser.values.hashes: + parser.values.hashes = {} + try: + algo, digest = value.split(':', 1) + except ValueError: + parser.error('Arguments to %s must be a hash name ' + 'followed by a value, like --hash=sha256:abcde...' % + opt_str) + if algo not in STRONG_HASHES: + parser.error('Allowed hash algorithms for %s are %s.' % + (opt_str, ', '.join(STRONG_HASHES))) + parser.values.hashes.setdefault(algo, []).append(digest) + + +hash = partial( + Option, + '--hash', + # Hash values eventually end up in InstallRequirement.hashes due to + # __dict__ copying in process_line(). + dest='hashes', + action='callback', + callback=_handle_merge_hash, + type='string', + help="Verify that the package's archive matches this " + 'hash before installing. Example: --hash=sha256:abcdef...', +) # type: Callable[..., Option] + + +require_hashes = partial( + Option, + '--require-hashes', + dest='require_hashes', + action='store_true', + default=False, + help='Require a hash to check each requirement against, for ' + 'repeatable installs. This option is implied when any package in a ' + 'requirements file has a --hash option.', +) # type: Callable[..., Option] + + +list_path = partial( + PipOption, + '--path', + dest='path', + type='path', + action='append', + help='Restrict to the specified installation path for listing ' + 'packages (can be used multiple times).' +) # type: Callable[..., Option] + + +def check_list_path_option(options): + # type: (Values) -> None + if options.path and (options.user or options.local): + raise CommandError( + "Cannot combine '--path' with '--user' or '--local'" + ) + + +no_python_version_warning = partial( + Option, + '--no-python-version-warning', + dest='no_python_version_warning', + action='store_true', + default=False, + help='Silence deprecation warnings for upcoming unsupported Pythons.', +) # type: Callable[..., Option] + + +########## +# groups # +########## + +general_group = { + 'name': 'General Options', + 'options': [ + help_, + isolated_mode, + require_virtualenv, + verbose, + version, + quiet, + log, + no_input, + proxy, + retries, + timeout, + skip_requirements_regex, + exists_action, + trusted_host, + cert, + client_cert, + cache_dir, + no_cache, + disable_pip_version_check, + no_color, + no_python_version_warning, + ] +} # type: Dict[str, Any] + +index_group = { + 'name': 'Package Index Options', + 'options': [ + index_url, + extra_index_url, + no_index, + find_links, + ] +} # type: Dict[str, Any] diff --git a/ubuntu/venv/pip/_internal/cli/command_context.py b/ubuntu/venv/pip/_internal/cli/command_context.py new file mode 100644 index 0000000..d1a64a7 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/command_context.py @@ -0,0 +1,36 @@ +from contextlib import contextmanager + +from pip._vendor.contextlib2 import ExitStack + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterator, ContextManager, TypeVar + + _T = TypeVar('_T', covariant=True) + + +class CommandContextMixIn(object): + def __init__(self): + # type: () -> None + super(CommandContextMixIn, self).__init__() + self._in_main_context = False + self._main_context = ExitStack() + + @contextmanager + def main_context(self): + # type: () -> Iterator[None] + assert not self._in_main_context + + self._in_main_context = True + try: + with self._main_context: + yield + finally: + self._in_main_context = False + + def enter_context(self, context_provider): + # type: (ContextManager[_T]) -> _T + assert self._in_main_context + + return self._main_context.enter_context(context_provider) diff --git a/ubuntu/venv/pip/_internal/cli/main.py b/ubuntu/venv/pip/_internal/cli/main.py new file mode 100644 index 0000000..5e97a51 --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/main.py @@ -0,0 +1,75 @@ +"""Primary application entrypoint. +""" +from __future__ import absolute_import + +import locale +import logging +import os +import sys + +from pip._internal.cli.autocompletion import autocomplete +from pip._internal.cli.main_parser import parse_command +from pip._internal.commands import create_command +from pip._internal.exceptions import PipError +from pip._internal.utils import deprecation +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + +logger = logging.getLogger(__name__) + + +# Do not import and use main() directly! Using it directly is actively +# discouraged by pip's maintainers. The name, location and behavior of +# this function is subject to change, so calling it directly is not +# portable across different pip versions. + +# In addition, running pip in-process is unsupported and unsafe. This is +# elaborated in detail at +# https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program. +# That document also provides suggestions that should work for nearly +# all users that are considering importing and using main() directly. + +# However, we know that certain users will still want to invoke pip +# in-process. If you understand and accept the implications of using pip +# in an unsupported manner, the best approach is to use runpy to avoid +# depending on the exact location of this entry point. + +# The following example shows how to use runpy to invoke pip in that +# case: +# +# sys.argv = ["pip", your, args, here] +# runpy.run_module("pip", run_name="__main__") +# +# Note that this will exit the process after running, unlike a direct +# call to main. As it is not safe to do any processing after calling +# main, this should not be an issue in practice. + +def main(args=None): + # type: (Optional[List[str]]) -> int + if args is None: + args = sys.argv[1:] + + # Configure our deprecation warnings to be sent through loggers + deprecation.install_warning_logger() + + autocomplete() + + try: + cmd_name, cmd_args = parse_command(args) + except PipError as exc: + sys.stderr.write("ERROR: %s" % exc) + sys.stderr.write(os.linesep) + sys.exit(1) + + # Needed for locale.getpreferredencoding(False) to work + # in pip._internal.utils.encoding.auto_decode + try: + locale.setlocale(locale.LC_ALL, '') + except locale.Error as e: + # setlocale can apparently crash if locale are uninitialized + logger.debug("Ignoring error %s when setting locale", e) + command = create_command(cmd_name, isolated=("--isolated" in cmd_args)) + + return command.main(cmd_args) diff --git a/ubuntu/venv/pip/_internal/cli/main_parser.py b/ubuntu/venv/pip/_internal/cli/main_parser.py new file mode 100644 index 0000000..a89821d --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/main_parser.py @@ -0,0 +1,99 @@ +"""A single place for constructing and exposing the main parser +""" + +import os +import sys + +from pip._internal.cli import cmdoptions +from pip._internal.cli.parser import ( + ConfigOptionParser, + UpdatingDefaultsHelpFormatter, +) +from pip._internal.commands import commands_dict, get_similar_commands +from pip._internal.exceptions import CommandError +from pip._internal.utils.misc import get_pip_version, get_prog +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, List + + +__all__ = ["create_main_parser", "parse_command"] + + +def create_main_parser(): + # type: () -> ConfigOptionParser + """Creates and returns the main parser for pip's CLI + """ + + parser_kw = { + 'usage': '\n%prog [options]', + 'add_help_option': False, + 'formatter': UpdatingDefaultsHelpFormatter(), + 'name': 'global', + 'prog': get_prog(), + } + + parser = ConfigOptionParser(**parser_kw) + parser.disable_interspersed_args() + + parser.version = get_pip_version() + + # add the general options + gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) + parser.add_option_group(gen_opts) + + # so the help formatter knows + parser.main = True # type: ignore + + # create command listing for description + description = [''] + [ + '%-27s %s' % (name, command_info.summary) + for name, command_info in commands_dict.items() + ] + parser.description = '\n'.join(description) + + return parser + + +def parse_command(args): + # type: (List[str]) -> Tuple[str, List[str]] + parser = create_main_parser() + + # Note: parser calls disable_interspersed_args(), so the result of this + # call is to split the initial args into the general options before the + # subcommand and everything else. + # For example: + # args: ['--timeout=5', 'install', '--user', 'INITools'] + # general_options: ['--timeout==5'] + # args_else: ['install', '--user', 'INITools'] + general_options, args_else = parser.parse_args(args) + + # --version + if general_options.version: + sys.stdout.write(parser.version) # type: ignore + sys.stdout.write(os.linesep) + sys.exit() + + # pip || pip help -> print_help() + if not args_else or (args_else[0] == 'help' and len(args_else) == 1): + parser.print_help() + sys.exit() + + # the subcommand name + cmd_name = args_else[0] + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + # all the args without the subcommand + cmd_args = args[:] + cmd_args.remove(cmd_name) + + return cmd_name, cmd_args diff --git a/ubuntu/venv/pip/_internal/cli/parser.py b/ubuntu/venv/pip/_internal/cli/parser.py new file mode 100644 index 0000000..c99456b --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/parser.py @@ -0,0 +1,265 @@ +"""Base option parser setup""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import optparse +import sys +import textwrap +from distutils.util import strtobool + +from pip._vendor.six import string_types + +from pip._internal.cli.status_codes import UNKNOWN_ERROR +from pip._internal.configuration import Configuration, ConfigurationError +from pip._internal.utils.compat import get_terminal_size + +logger = logging.getLogger(__name__) + + +class PrettyHelpFormatter(optparse.IndentedHelpFormatter): + """A prettier/less verbose help formatter for optparse.""" + + def __init__(self, *args, **kwargs): + # help position must be aligned with __init__.parseopts.description + kwargs['max_help_position'] = 30 + kwargs['indent_increment'] = 1 + kwargs['width'] = get_terminal_size()[0] - 2 + optparse.IndentedHelpFormatter.__init__(self, *args, **kwargs) + + def format_option_strings(self, option): + return self._format_option_strings(option, ' <%s>', ', ') + + def _format_option_strings(self, option, mvarfmt=' <%s>', optsep=', '): + """ + Return a comma-separated list of option strings and metavars. + + :param option: tuple of (short opt, long opt), e.g: ('-f', '--format') + :param mvarfmt: metavar format string - evaluated as mvarfmt % metavar + :param optsep: separator + """ + opts = [] + + if option._short_opts: + opts.append(option._short_opts[0]) + if option._long_opts: + opts.append(option._long_opts[0]) + if len(opts) > 1: + opts.insert(1, optsep) + + if option.takes_value(): + metavar = option.metavar or option.dest.lower() + opts.append(mvarfmt % metavar.lower()) + + return ''.join(opts) + + def format_heading(self, heading): + if heading == 'Options': + return '' + return heading + ':\n' + + def format_usage(self, usage): + """ + Ensure there is only one newline between usage and the first heading + if there is no description. + """ + msg = '\nUsage: %s\n' % self.indent_lines(textwrap.dedent(usage), " ") + return msg + + def format_description(self, description): + # leave full control over description to us + if description: + if hasattr(self.parser, 'main'): + label = 'Commands' + else: + label = 'Description' + # some doc strings have initial newlines, some don't + description = description.lstrip('\n') + # some doc strings have final newlines and spaces, some don't + description = description.rstrip() + # dedent, then reindent + description = self.indent_lines(textwrap.dedent(description), " ") + description = '%s:\n%s\n' % (label, description) + return description + else: + return '' + + def format_epilog(self, epilog): + # leave full control over epilog to us + if epilog: + return epilog + else: + return '' + + def indent_lines(self, text, indent): + new_lines = [indent + line for line in text.split('\n')] + return "\n".join(new_lines) + + +class UpdatingDefaultsHelpFormatter(PrettyHelpFormatter): + """Custom help formatter for use in ConfigOptionParser. + + This is updates the defaults before expanding them, allowing + them to show up correctly in the help listing. + """ + + def expand_default(self, option): + if self.parser is not None: + self.parser._update_defaults(self.parser.defaults) + return optparse.IndentedHelpFormatter.expand_default(self, option) + + +class CustomOptionParser(optparse.OptionParser): + + def insert_option_group(self, idx, *args, **kwargs): + """Insert an OptionGroup at a given position.""" + group = self.add_option_group(*args, **kwargs) + + self.option_groups.pop() + self.option_groups.insert(idx, group) + + return group + + @property + def option_list_all(self): + """Get a list of all options, including those in option groups.""" + res = self.option_list[:] + for i in self.option_groups: + res.extend(i.option_list) + + return res + + +class ConfigOptionParser(CustomOptionParser): + """Custom option parser which updates its defaults by checking the + configuration files and environmental variables""" + + def __init__(self, *args, **kwargs): + self.name = kwargs.pop('name') + + isolated = kwargs.pop("isolated", False) + self.config = Configuration(isolated) + + assert self.name + optparse.OptionParser.__init__(self, *args, **kwargs) + + def check_default(self, option, key, val): + try: + return option.check_value(key, val) + except optparse.OptionValueError as exc: + print("An error occurred during configuration: %s" % exc) + sys.exit(3) + + def _get_ordered_configuration_items(self): + # Configuration gives keys in an unordered manner. Order them. + override_order = ["global", self.name, ":env:"] + + # Pool the options into different groups + section_items = {name: [] for name in override_order} + for section_key, val in self.config.items(): + # ignore empty values + if not val: + logger.debug( + "Ignoring configuration key '%s' as it's value is empty.", + section_key + ) + continue + + section, key = section_key.split(".", 1) + if section in override_order: + section_items[section].append((key, val)) + + # Yield each group in their override order + for section in override_order: + for key, val in section_items[section]: + yield key, val + + def _update_defaults(self, defaults): + """Updates the given defaults with values from the config files and + the environ. Does a little special handling for certain types of + options (lists).""" + + # Accumulate complex default state. + self.values = optparse.Values(self.defaults) + late_eval = set() + # Then set the options with those values + for key, val in self._get_ordered_configuration_items(): + # '--' because configuration supports only long names + option = self.get_option('--' + key) + + # Ignore options not present in this parser. E.g. non-globals put + # in [global] by users that want them to apply to all applicable + # commands. + if option is None: + continue + + if option.action in ('store_true', 'store_false', 'count'): + try: + val = strtobool(val) + except ValueError: + error_msg = invalid_config_error_message( + option.action, key, val + ) + self.error(error_msg) + + elif option.action == 'append': + val = val.split() + val = [self.check_default(option, key, v) for v in val] + elif option.action == 'callback': + late_eval.add(option.dest) + opt_str = option.get_opt_string() + val = option.convert_value(opt_str, val) + # From take_action + args = option.callback_args or () + kwargs = option.callback_kwargs or {} + option.callback(option, opt_str, val, self, *args, **kwargs) + else: + val = self.check_default(option, key, val) + + defaults[option.dest] = val + + for key in late_eval: + defaults[key] = getattr(self.values, key) + self.values = None + return defaults + + def get_default_values(self): + """Overriding to make updating the defaults after instantiation of + the option parser possible, _update_defaults() does the dirty work.""" + if not self.process_default_values: + # Old, pre-Optik 1.5 behaviour. + return optparse.Values(self.defaults) + + # Load the configuration, or error out in case of an error + try: + self.config.load() + except ConfigurationError as err: + self.exit(UNKNOWN_ERROR, str(err)) + + defaults = self._update_defaults(self.defaults.copy()) # ours + for option in self._get_all_options(): + default = defaults.get(option.dest) + if isinstance(default, string_types): + opt_str = option.get_opt_string() + defaults[option.dest] = option.check_value(opt_str, default) + return optparse.Values(defaults) + + def error(self, msg): + self.print_usage(sys.stderr) + self.exit(UNKNOWN_ERROR, "%s\n" % msg) + + +def invalid_config_error_message(action, key, val): + """Returns a better error message when invalid configuration option + is provided.""" + if action in ('store_true', 'store_false'): + return ("{0} is not a valid value for {1} option, " + "please specify a boolean value like yes/no, " + "true/false or 1/0 instead.").format(val, key) + + return ("{0} is not a valid value for {1} option, " + "please specify a numerical value like 1/0 " + "instead.").format(val, key) diff --git a/ubuntu/venv/pip/_internal/cli/req_command.py b/ubuntu/venv/pip/_internal/cli/req_command.py new file mode 100644 index 0000000..9383b3b --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/req_command.py @@ -0,0 +1,333 @@ +"""Contains the Command base classes that depend on PipSession. + +The classes in this module are in a separate module so the commands not +needing download / PackageFinder capability don't unnecessarily import the +PackageFinder machinery and all its vendored dependencies, etc. +""" + +import logging +import os +from functools import partial + +from pip._internal.cli.base_command import Command +from pip._internal.cli.command_context import CommandContextMixIn +from pip._internal.exceptions import CommandError +from pip._internal.index.package_finder import PackageFinder +from pip._internal.legacy_resolve import Resolver +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.network.download import Downloader +from pip._internal.network.session import PipSession +from pip._internal.operations.prepare import RequirementPreparer +from pip._internal.req.constructors import ( + install_req_from_editable, + install_req_from_line, + install_req_from_req_string, +) +from pip._internal.req.req_file import parse_requirements +from pip._internal.self_outdated_check import ( + make_link_collector, + pip_self_version_check, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import List, Optional, Tuple + from pip._internal.cache import WheelCache + from pip._internal.models.target_python import TargetPython + from pip._internal.req.req_set import RequirementSet + from pip._internal.req.req_tracker import RequirementTracker + from pip._internal.utils.temp_dir import TempDirectory + +logger = logging.getLogger(__name__) + + +class SessionCommandMixin(CommandContextMixIn): + + """ + A class mixin for command classes needing _build_session(). + """ + def __init__(self): + # type: () -> None + super(SessionCommandMixin, self).__init__() + self._session = None # Optional[PipSession] + + @classmethod + def _get_index_urls(cls, options): + # type: (Values) -> Optional[List[str]] + """Return a list of index urls from user-provided options.""" + index_urls = [] + if not getattr(options, "no_index", False): + url = getattr(options, "index_url", None) + if url: + index_urls.append(url) + urls = getattr(options, "extra_index_urls", None) + if urls: + index_urls.extend(urls) + # Return None rather than an empty list + return index_urls or None + + def get_default_session(self, options): + # type: (Values) -> PipSession + """Get a default-managed session.""" + if self._session is None: + self._session = self.enter_context(self._build_session(options)) + # there's no type annotation on requests.Session, so it's + # automatically ContextManager[Any] and self._session becomes Any, + # then https://github.com/python/mypy/issues/7696 kicks in + assert self._session is not None + return self._session + + def _build_session(self, options, retries=None, timeout=None): + # type: (Values, Optional[int], Optional[int]) -> PipSession + assert not options.cache_dir or os.path.isabs(options.cache_dir) + session = PipSession( + cache=( + os.path.join(options.cache_dir, "http") + if options.cache_dir else None + ), + retries=retries if retries is not None else options.retries, + trusted_hosts=options.trusted_hosts, + index_urls=self._get_index_urls(options), + ) + + # Handle custom ca-bundles from the user + if options.cert: + session.verify = options.cert + + # Handle SSL client certificate + if options.client_cert: + session.cert = options.client_cert + + # Handle timeouts + if options.timeout or timeout: + session.timeout = ( + timeout if timeout is not None else options.timeout + ) + + # Handle configured proxies + if options.proxy: + session.proxies = { + "http": options.proxy, + "https": options.proxy, + } + + # Determine if we can prompt the user for authentication or not + session.auth.prompting = not options.no_input + + return session + + +class IndexGroupCommand(Command, SessionCommandMixin): + + """ + Abstract base class for commands with the index_group options. + + This also corresponds to the commands that permit the pip version check. + """ + + def handle_pip_version_check(self, options): + # type: (Values) -> None + """ + Do the pip version check if not disabled. + + This overrides the default behavior of not doing the check. + """ + # Make sure the index_group options are present. + assert hasattr(options, 'no_index') + + if options.disable_pip_version_check or options.no_index: + return + + # Otherwise, check if we're using the latest version of pip available. + session = self._build_session( + options, + retries=0, + timeout=min(5, options.timeout) + ) + with session: + pip_self_version_check(session, options) + + +class RequirementCommand(IndexGroupCommand): + + @staticmethod + def make_requirement_preparer( + temp_build_dir, # type: TempDirectory + options, # type: Values + req_tracker, # type: RequirementTracker + session, # type: PipSession + finder, # type: PackageFinder + use_user_site, # type: bool + download_dir=None, # type: str + wheel_download_dir=None, # type: str + ): + # type: (...) -> RequirementPreparer + """ + Create a RequirementPreparer instance for the given parameters. + """ + downloader = Downloader(session, progress_bar=options.progress_bar) + + temp_build_dir_path = temp_build_dir.path + assert temp_build_dir_path is not None + + return RequirementPreparer( + build_dir=temp_build_dir_path, + src_dir=options.src_dir, + download_dir=download_dir, + wheel_download_dir=wheel_download_dir, + build_isolation=options.build_isolation, + req_tracker=req_tracker, + downloader=downloader, + finder=finder, + require_hashes=options.require_hashes, + use_user_site=use_user_site, + ) + + @staticmethod + def make_resolver( + preparer, # type: RequirementPreparer + finder, # type: PackageFinder + options, # type: Values + wheel_cache=None, # type: Optional[WheelCache] + use_user_site=False, # type: bool + ignore_installed=True, # type: bool + ignore_requires_python=False, # type: bool + force_reinstall=False, # type: bool + upgrade_strategy="to-satisfy-only", # type: str + use_pep517=None, # type: Optional[bool] + py_version_info=None # type: Optional[Tuple[int, ...]] + ): + # type: (...) -> Resolver + """ + Create a Resolver instance for the given parameters. + """ + make_install_req = partial( + install_req_from_req_string, + isolated=options.isolated_mode, + wheel_cache=wheel_cache, + use_pep517=use_pep517, + ) + return Resolver( + preparer=preparer, + finder=finder, + make_install_req=make_install_req, + use_user_site=use_user_site, + ignore_dependencies=options.ignore_dependencies, + ignore_installed=ignore_installed, + ignore_requires_python=ignore_requires_python, + force_reinstall=force_reinstall, + upgrade_strategy=upgrade_strategy, + py_version_info=py_version_info, + ) + + def populate_requirement_set( + self, + requirement_set, # type: RequirementSet + args, # type: List[str] + options, # type: Values + finder, # type: PackageFinder + session, # type: PipSession + wheel_cache, # type: Optional[WheelCache] + ): + # type: (...) -> None + """ + Marshal cmd line args into a requirement set. + """ + for filename in options.constraints: + for req_to_add in parse_requirements( + filename, + constraint=True, finder=finder, options=options, + session=session, wheel_cache=wheel_cache): + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + for req in args: + req_to_add = install_req_from_line( + req, None, isolated=options.isolated_mode, + use_pep517=options.use_pep517, + wheel_cache=wheel_cache + ) + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + for req in options.editables: + req_to_add = install_req_from_editable( + req, + isolated=options.isolated_mode, + use_pep517=options.use_pep517, + wheel_cache=wheel_cache + ) + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + # NOTE: options.require_hashes may be set if --require-hashes is True + for filename in options.requirements: + for req_to_add in parse_requirements( + filename, + finder=finder, options=options, session=session, + wheel_cache=wheel_cache, + use_pep517=options.use_pep517): + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + # If any requirement has hash options, enable hash checking. + requirements = ( + requirement_set.unnamed_requirements + + list(requirement_set.requirements.values()) + ) + if any(req.has_hash_options for req in requirements): + options.require_hashes = True + + if not (args or options.editables or options.requirements): + opts = {'name': self.name} + if options.find_links: + raise CommandError( + 'You must give at least one requirement to %(name)s ' + '(maybe you meant "pip %(name)s %(links)s"?)' % + dict(opts, links=' '.join(options.find_links))) + else: + raise CommandError( + 'You must give at least one requirement to %(name)s ' + '(see "pip help %(name)s")' % opts) + + @staticmethod + def trace_basic_info(finder): + # type: (PackageFinder) -> None + """ + Trace basic information about the provided objects. + """ + # Display where finder is looking for packages + search_scope = finder.search_scope + locations = search_scope.get_formatted_locations() + if locations: + logger.info(locations) + + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + target_python=None, # type: Optional[TargetPython] + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> PackageFinder + """ + Create a package finder appropriate to this requirement command. + + :param ignore_requires_python: Whether to ignore incompatible + "Requires-Python" values in links. Defaults to False. + """ + link_collector = make_link_collector(session, options=options) + selection_prefs = SelectionPreferences( + allow_yanked=True, + format_control=options.format_control, + allow_all_prereleases=options.pre, + prefer_binary=options.prefer_binary, + ignore_requires_python=ignore_requires_python, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + target_python=target_python, + ) diff --git a/ubuntu/venv/pip/_internal/cli/status_codes.py b/ubuntu/venv/pip/_internal/cli/status_codes.py new file mode 100644 index 0000000..275360a --- /dev/null +++ b/ubuntu/venv/pip/_internal/cli/status_codes.py @@ -0,0 +1,8 @@ +from __future__ import absolute_import + +SUCCESS = 0 +ERROR = 1 +UNKNOWN_ERROR = 2 +VIRTUALENV_NOT_FOUND = 3 +PREVIOUS_BUILD_DIR_ERROR = 4 +NO_MATCHES_FOUND = 23 diff --git a/ubuntu/venv/pip/_internal/commands/__init__.py b/ubuntu/venv/pip/_internal/commands/__init__.py new file mode 100644 index 0000000..2a311f8 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/__init__.py @@ -0,0 +1,114 @@ +""" +Package containing all pip commands +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import importlib +from collections import OrderedDict, namedtuple + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any + from pip._internal.cli.base_command import Command + + +CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary') + +# The ordering matters for help display. +# Also, even though the module path starts with the same +# "pip._internal.commands" prefix in each case, we include the full path +# because it makes testing easier (specifically when modifying commands_dict +# in test setup / teardown by adding info for a FakeCommand class defined +# in a test-related module). +# Finally, we need to pass an iterable of pairs here rather than a dict +# so that the ordering won't be lost when using Python 2.7. +commands_dict = OrderedDict([ + ('install', CommandInfo( + 'pip._internal.commands.install', 'InstallCommand', + 'Install packages.', + )), + ('download', CommandInfo( + 'pip._internal.commands.download', 'DownloadCommand', + 'Download packages.', + )), + ('uninstall', CommandInfo( + 'pip._internal.commands.uninstall', 'UninstallCommand', + 'Uninstall packages.', + )), + ('freeze', CommandInfo( + 'pip._internal.commands.freeze', 'FreezeCommand', + 'Output installed packages in requirements format.', + )), + ('list', CommandInfo( + 'pip._internal.commands.list', 'ListCommand', + 'List installed packages.', + )), + ('show', CommandInfo( + 'pip._internal.commands.show', 'ShowCommand', + 'Show information about installed packages.', + )), + ('check', CommandInfo( + 'pip._internal.commands.check', 'CheckCommand', + 'Verify installed packages have compatible dependencies.', + )), + ('config', CommandInfo( + 'pip._internal.commands.configuration', 'ConfigurationCommand', + 'Manage local and global configuration.', + )), + ('search', CommandInfo( + 'pip._internal.commands.search', 'SearchCommand', + 'Search PyPI for packages.', + )), + ('wheel', CommandInfo( + 'pip._internal.commands.wheel', 'WheelCommand', + 'Build wheels from your requirements.', + )), + ('hash', CommandInfo( + 'pip._internal.commands.hash', 'HashCommand', + 'Compute hashes of package archives.', + )), + ('completion', CommandInfo( + 'pip._internal.commands.completion', 'CompletionCommand', + 'A helper command used for command completion.', + )), + ('debug', CommandInfo( + 'pip._internal.commands.debug', 'DebugCommand', + 'Show information useful for debugging.', + )), + ('help', CommandInfo( + 'pip._internal.commands.help', 'HelpCommand', + 'Show help for commands.', + )), +]) # type: OrderedDict[str, CommandInfo] + + +def create_command(name, **kwargs): + # type: (str, **Any) -> Command + """ + Create an instance of the Command class with the given name. + """ + module_path, class_name, summary = commands_dict[name] + module = importlib.import_module(module_path) + command_class = getattr(module, class_name) + command = command_class(name=name, summary=summary, **kwargs) + + return command + + +def get_similar_commands(name): + """Command name auto-correct.""" + from difflib import get_close_matches + + name = name.lower() + + close_commands = get_close_matches(name, commands_dict.keys()) + + if close_commands: + return close_commands[0] + else: + return False diff --git a/ubuntu/venv/pip/_internal/commands/check.py b/ubuntu/venv/pip/_internal/commands/check.py new file mode 100644 index 0000000..9689446 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/check.py @@ -0,0 +1,45 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pip._internal.cli.base_command import Command +from pip._internal.operations.check import ( + check_package_set, + create_package_set_from_installed, +) +from pip._internal.utils.misc import write_output + +logger = logging.getLogger(__name__) + + +class CheckCommand(Command): + """Verify installed packages have compatible dependencies.""" + + usage = """ + %prog [options]""" + + def run(self, options, args): + package_set, parsing_probs = create_package_set_from_installed() + missing, conflicting = check_package_set(package_set) + + for project_name in missing: + version = package_set[project_name].version + for dependency in missing[project_name]: + write_output( + "%s %s requires %s, which is not installed.", + project_name, version, dependency[0], + ) + + for project_name in conflicting: + version = package_set[project_name].version + for dep_name, dep_version, req in conflicting[project_name]: + write_output( + "%s %s has requirement %s, but you have %s %s.", + project_name, version, req, dep_name, dep_version, + ) + + if missing or conflicting or parsing_probs: + return 1 + else: + write_output("No broken requirements found.") diff --git a/ubuntu/venv/pip/_internal/commands/completion.py b/ubuntu/venv/pip/_internal/commands/completion.py new file mode 100644 index 0000000..c532806 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/completion.py @@ -0,0 +1,96 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import sys +import textwrap + +from pip._internal.cli.base_command import Command +from pip._internal.utils.misc import get_prog + +BASE_COMPLETION = """ +# pip %(shell)s completion start%(script)s# pip %(shell)s completion end +""" + +COMPLETION_SCRIPTS = { + 'bash': """ + _pip_completion() + { + COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ + COMP_CWORD=$COMP_CWORD \\ + PIP_AUTO_COMPLETE=1 $1 2>/dev/null ) ) + } + complete -o default -F _pip_completion %(prog)s + """, + 'zsh': """ + function _pip_completion { + local words cword + read -Ac words + read -cn cword + reply=( $( COMP_WORDS="$words[*]" \\ + COMP_CWORD=$(( cword-1 )) \\ + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) + } + compctl -K _pip_completion %(prog)s + """, + 'fish': """ + function __fish_complete_pip + set -lx COMP_WORDS (commandline -o) "" + set -lx COMP_CWORD ( \\ + math (contains -i -- (commandline -t) $COMP_WORDS)-1 \\ + ) + set -lx PIP_AUTO_COMPLETE 1 + string split \\ -- (eval $COMP_WORDS[1]) + end + complete -fa "(__fish_complete_pip)" -c %(prog)s + """, +} + + +class CompletionCommand(Command): + """A helper command to be used for command completion.""" + + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(CompletionCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + + cmd_opts.add_option( + '--bash', '-b', + action='store_const', + const='bash', + dest='shell', + help='Emit completion code for bash') + cmd_opts.add_option( + '--zsh', '-z', + action='store_const', + const='zsh', + dest='shell', + help='Emit completion code for zsh') + cmd_opts.add_option( + '--fish', '-f', + action='store_const', + const='fish', + dest='shell', + help='Emit completion code for fish') + + self.parser.insert_option_group(0, cmd_opts) + + def run(self, options, args): + """Prints the completion code of the given shell""" + shells = COMPLETION_SCRIPTS.keys() + shell_options = ['--' + shell for shell in sorted(shells)] + if options.shell in shells: + script = textwrap.dedent( + COMPLETION_SCRIPTS.get(options.shell, '') % { + 'prog': get_prog(), + } + ) + print(BASE_COMPLETION % {'script': script, 'shell': options.shell}) + else: + sys.stderr.write( + 'ERROR: You must pass %s\n' % ' or '.join(shell_options) + ) diff --git a/ubuntu/venv/pip/_internal/commands/configuration.py b/ubuntu/venv/pip/_internal/commands/configuration.py new file mode 100644 index 0000000..efcf5bb --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/configuration.py @@ -0,0 +1,233 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging +import os +import subprocess + +from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.configuration import ( + Configuration, + get_configuration_files, + kinds, +) +from pip._internal.exceptions import PipError +from pip._internal.utils.misc import get_prog, write_output + +logger = logging.getLogger(__name__) + + +class ConfigurationCommand(Command): + """Manage local and global configuration. + + Subcommands: + + list: List the active configuration (or from the file specified) + edit: Edit the configuration file in an editor + get: Get the value associated with name + set: Set the name=value + unset: Unset the value associated with name + + If none of --user, --global and --site are passed, a virtual + environment configuration file is used if one is active and the file + exists. Otherwise, all modifications happen on the to the user file by + default. + """ + + ignore_require_venv = True + usage = """ + %prog [] list + %prog [] [--editor ] edit + + %prog [] get name + %prog [] set name value + %prog [] unset name + """ + + def __init__(self, *args, **kwargs): + super(ConfigurationCommand, self).__init__(*args, **kwargs) + + self.configuration = None + + self.cmd_opts.add_option( + '--editor', + dest='editor', + action='store', + default=None, + help=( + 'Editor to use to edit the file. Uses VISUAL or EDITOR ' + 'environment variables if not provided.' + ) + ) + + self.cmd_opts.add_option( + '--global', + dest='global_file', + action='store_true', + default=False, + help='Use the system-wide configuration file only' + ) + + self.cmd_opts.add_option( + '--user', + dest='user_file', + action='store_true', + default=False, + help='Use the user configuration file only' + ) + + self.cmd_opts.add_option( + '--site', + dest='site_file', + action='store_true', + default=False, + help='Use the current environment configuration file only' + ) + + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + handlers = { + "list": self.list_values, + "edit": self.open_in_editor, + "get": self.get_name, + "set": self.set_name_value, + "unset": self.unset_name + } + + # Determine action + if not args or args[0] not in handlers: + logger.error("Need an action ({}) to perform.".format( + ", ".join(sorted(handlers))) + ) + return ERROR + + action = args[0] + + # Determine which configuration files are to be loaded + # Depends on whether the command is modifying. + try: + load_only = self._determine_file( + options, need_value=(action in ["get", "set", "unset", "edit"]) + ) + except PipError as e: + logger.error(e.args[0]) + return ERROR + + # Load a new configuration + self.configuration = Configuration( + isolated=options.isolated_mode, load_only=load_only + ) + self.configuration.load() + + # Error handling happens here, not in the action-handlers. + try: + handlers[action](options, args[1:]) + except PipError as e: + logger.error(e.args[0]) + return ERROR + + return SUCCESS + + def _determine_file(self, options, need_value): + file_options = [key for key, value in ( + (kinds.USER, options.user_file), + (kinds.GLOBAL, options.global_file), + (kinds.SITE, options.site_file), + ) if value] + + if not file_options: + if not need_value: + return None + # Default to user, unless there's a site file. + elif any( + os.path.exists(site_config_file) + for site_config_file in get_configuration_files()[kinds.SITE] + ): + return kinds.SITE + else: + return kinds.USER + elif len(file_options) == 1: + return file_options[0] + + raise PipError( + "Need exactly one file to operate upon " + "(--user, --site, --global) to perform." + ) + + def list_values(self, options, args): + self._get_n_args(args, "list", n=0) + + for key, value in sorted(self.configuration.items()): + write_output("%s=%r", key, value) + + def get_name(self, options, args): + key = self._get_n_args(args, "get [name]", n=1) + value = self.configuration.get_value(key) + + write_output("%s", value) + + def set_name_value(self, options, args): + key, value = self._get_n_args(args, "set [name] [value]", n=2) + self.configuration.set_value(key, value) + + self._save_configuration() + + def unset_name(self, options, args): + key = self._get_n_args(args, "unset [name]", n=1) + self.configuration.unset_value(key) + + self._save_configuration() + + def open_in_editor(self, options, args): + editor = self._determine_editor(options) + + fname = self.configuration.get_file_to_edit() + if fname is None: + raise PipError("Could not determine appropriate file.") + + try: + subprocess.check_call([editor, fname]) + except subprocess.CalledProcessError as e: + raise PipError( + "Editor Subprocess exited with exit code {}" + .format(e.returncode) + ) + + def _get_n_args(self, args, example, n): + """Helper to make sure the command got the right number of arguments + """ + if len(args) != n: + msg = ( + 'Got unexpected number of arguments, expected {}. ' + '(example: "{} config {}")' + ).format(n, get_prog(), example) + raise PipError(msg) + + if n == 1: + return args[0] + else: + return args + + def _save_configuration(self): + # We successfully ran a modifying command. Need to save the + # configuration. + try: + self.configuration.save() + except Exception: + logger.error( + "Unable to save configuration. Please report this as a bug.", + exc_info=1 + ) + raise PipError("Internal Error.") + + def _determine_editor(self, options): + if options.editor is not None: + return options.editor + elif "VISUAL" in os.environ: + return os.environ["VISUAL"] + elif "EDITOR" in os.environ: + return os.environ["EDITOR"] + else: + raise PipError("Could not determine editor to use.") diff --git a/ubuntu/venv/pip/_internal/commands/debug.py b/ubuntu/venv/pip/_internal/commands/debug.py new file mode 100644 index 0000000..fe93b3a --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/debug.py @@ -0,0 +1,142 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import locale +import logging +import os +import sys + +from pip._vendor.certifi import where + +from pip._internal.cli import cmdoptions +from pip._internal.cli.base_command import Command +from pip._internal.cli.cmdoptions import make_target_python +from pip._internal.cli.status_codes import SUCCESS +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import get_pip_version +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, List, Optional + from optparse import Values + +logger = logging.getLogger(__name__) + + +def show_value(name, value): + # type: (str, Optional[str]) -> None + logger.info('{}: {}'.format(name, value)) + + +def show_sys_implementation(): + # type: () -> None + logger.info('sys.implementation:') + if hasattr(sys, 'implementation'): + implementation = sys.implementation # type: ignore + implementation_name = implementation.name + else: + implementation_name = '' + + with indent_log(): + show_value('name', implementation_name) + + +def show_tags(options): + # type: (Values) -> None + tag_limit = 10 + + target_python = make_target_python(options) + tags = target_python.get_tags() + + # Display the target options that were explicitly provided. + formatted_target = target_python.format_given() + suffix = '' + if formatted_target: + suffix = ' (target: {})'.format(formatted_target) + + msg = 'Compatible tags: {}{}'.format(len(tags), suffix) + logger.info(msg) + + if options.verbose < 1 and len(tags) > tag_limit: + tags_limited = True + tags = tags[:tag_limit] + else: + tags_limited = False + + with indent_log(): + for tag in tags: + logger.info(str(tag)) + + if tags_limited: + msg = ( + '...\n' + '[First {tag_limit} tags shown. Pass --verbose to show all.]' + ).format(tag_limit=tag_limit) + logger.info(msg) + + +def ca_bundle_info(config): + levels = set() + for key, value in config.items(): + levels.add(key.split('.')[0]) + + if not levels: + return "Not specified" + + levels_that_override_global = ['install', 'wheel', 'download'] + global_overriding_level = [ + level for level in levels if level in levels_that_override_global + ] + if not global_overriding_level: + return 'global' + + levels.remove('global') + return ", ".join(levels) + + +class DebugCommand(Command): + """ + Display debug information. + """ + + usage = """ + %prog """ + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(DebugCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + cmdoptions.add_target_python_options(cmd_opts) + self.parser.insert_option_group(0, cmd_opts) + self.parser.config.load() + + def run(self, options, args): + # type: (Values, List[Any]) -> int + logger.warning( + "This command is only meant for debugging. " + "Do not use this with automation for parsing and getting these " + "details, since the output and options of this command may " + "change without notice." + ) + show_value('pip version', get_pip_version()) + show_value('sys.version', sys.version) + show_value('sys.executable', sys.executable) + show_value('sys.getdefaultencoding', sys.getdefaultencoding()) + show_value('sys.getfilesystemencoding', sys.getfilesystemencoding()) + show_value( + 'locale.getpreferredencoding', locale.getpreferredencoding(), + ) + show_value('sys.platform', sys.platform) + show_sys_implementation() + + show_value("'cert' config value", ca_bundle_info(self.parser.config)) + show_value("REQUESTS_CA_BUNDLE", os.environ.get('REQUESTS_CA_BUNDLE')) + show_value("CURL_CA_BUNDLE", os.environ.get('CURL_CA_BUNDLE')) + show_value("pip._vendor.certifi.where()", where()) + + show_tags(options) + + return SUCCESS diff --git a/ubuntu/venv/pip/_internal/commands/download.py b/ubuntu/venv/pip/_internal/commands/download.py new file mode 100644 index 0000000..24da3eb --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/download.py @@ -0,0 +1,147 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._internal.cli import cmdoptions +from pip._internal.cli.cmdoptions import make_target_python +from pip._internal.cli.req_command import RequirementCommand +from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.misc import ensure_dir, normalize_path, write_output +from pip._internal.utils.temp_dir import TempDirectory + +logger = logging.getLogger(__name__) + + +class DownloadCommand(RequirementCommand): + """ + Download packages from: + + - PyPI (and other indexes) using requirement specifiers. + - VCS project urls. + - Local project directories. + - Local or remote source archives. + + pip also supports downloading from "requirements files", which provide + an easy way to specify a whole environment to be downloaded. + """ + + usage = """ + %prog [options] [package-index-options] ... + %prog [options] -r [package-index-options] ... + %prog [options] ... + %prog [options] ... + %prog [options] ...""" + + def __init__(self, *args, **kw): + super(DownloadCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + + cmd_opts.add_option(cmdoptions.constraints()) + cmd_opts.add_option(cmdoptions.requirements()) + cmd_opts.add_option(cmdoptions.build_dir()) + cmd_opts.add_option(cmdoptions.no_deps()) + cmd_opts.add_option(cmdoptions.global_options()) + cmd_opts.add_option(cmdoptions.no_binary()) + cmd_opts.add_option(cmdoptions.only_binary()) + cmd_opts.add_option(cmdoptions.prefer_binary()) + cmd_opts.add_option(cmdoptions.src()) + cmd_opts.add_option(cmdoptions.pre()) + cmd_opts.add_option(cmdoptions.no_clean()) + cmd_opts.add_option(cmdoptions.require_hashes()) + cmd_opts.add_option(cmdoptions.progress_bar()) + cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) + + cmd_opts.add_option( + '-d', '--dest', '--destination-dir', '--destination-directory', + dest='download_dir', + metavar='dir', + default=os.curdir, + help=("Download packages into ."), + ) + + cmdoptions.add_target_python_options(cmd_opts) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, cmd_opts) + + def run(self, options, args): + options.ignore_installed = True + # editable doesn't really make sense for `pip download`, but the bowels + # of the RequirementSet code require that property. + options.editables = [] + + cmdoptions.check_dist_restriction(options) + + options.download_dir = normalize_path(options.download_dir) + + ensure_dir(options.download_dir) + + session = self.get_default_session(options) + + target_python = make_target_python(options) + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ) + build_delete = (not (options.no_clean or options.build_dir)) + + with get_requirement_tracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="download" + ) as directory: + + requirement_set = RequirementSet() + self.populate_requirement_set( + requirement_set, + args, + options, + finder, + session, + None + ) + + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + session=session, + finder=finder, + download_dir=options.download_dir, + use_user_site=False, + ) + + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + options=options, + py_version_info=options.python_version, + ) + + self.trace_basic_info(finder) + + resolver.resolve(requirement_set) + + downloaded = ' '.join([ + req.name for req in requirement_set.successfully_downloaded + ]) + if downloaded: + write_output('Successfully downloaded %s', downloaded) + + # Clean up + if not options.no_clean: + requirement_set.cleanup_files() + + return requirement_set diff --git a/ubuntu/venv/pip/_internal/commands/freeze.py b/ubuntu/venv/pip/_internal/commands/freeze.py new file mode 100644 index 0000000..e96c083 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/freeze.py @@ -0,0 +1,103 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import sys + +from pip._internal.cache import WheelCache +from pip._internal.cli import cmdoptions +from pip._internal.cli.base_command import Command +from pip._internal.models.format_control import FormatControl +from pip._internal.operations.freeze import freeze +from pip._internal.utils.compat import stdlib_pkgs + +DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel', 'pkg-resources'} + + +class FreezeCommand(Command): + """ + Output installed packages in requirements format. + + packages are listed in a case-insensitive sorted order. + """ + + usage = """ + %prog [options]""" + log_streams = ("ext://sys.stderr", "ext://sys.stderr") + + def __init__(self, *args, **kw): + super(FreezeCommand, self).__init__(*args, **kw) + + self.cmd_opts.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='file', + help="Use the order in the given requirements file and its " + "comments when generating output. This option can be " + "used multiple times.") + self.cmd_opts.add_option( + '-f', '--find-links', + dest='find_links', + action='append', + default=[], + metavar='URL', + help='URL for finding packages, which will be added to the ' + 'output.') + self.cmd_opts.add_option( + '-l', '--local', + dest='local', + action='store_true', + default=False, + help='If in a virtualenv that has global access, do not output ' + 'globally-installed packages.') + self.cmd_opts.add_option( + '--user', + dest='user', + action='store_true', + default=False, + help='Only output packages installed in user-site.') + self.cmd_opts.add_option(cmdoptions.list_path()) + self.cmd_opts.add_option( + '--all', + dest='freeze_all', + action='store_true', + help='Do not skip these packages in the output:' + ' %s' % ', '.join(DEV_PKGS)) + self.cmd_opts.add_option( + '--exclude-editable', + dest='exclude_editable', + action='store_true', + help='Exclude editable package from output.') + + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + format_control = FormatControl(set(), set()) + wheel_cache = WheelCache(options.cache_dir, format_control) + skip = set(stdlib_pkgs) + if not options.freeze_all: + skip.update(DEV_PKGS) + + cmdoptions.check_list_path_option(options) + + freeze_kwargs = dict( + requirement=options.requirements, + find_links=options.find_links, + local_only=options.local, + user_only=options.user, + paths=options.path, + skip_regex=options.skip_requirements_regex, + isolated=options.isolated_mode, + wheel_cache=wheel_cache, + skip=skip, + exclude_editable=options.exclude_editable, + ) + + try: + for line in freeze(**freeze_kwargs): + sys.stdout.write(line + '\n') + finally: + wheel_cache.cleanup() diff --git a/ubuntu/venv/pip/_internal/commands/hash.py b/ubuntu/venv/pip/_internal/commands/hash.py new file mode 100644 index 0000000..1dc7fb0 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/hash.py @@ -0,0 +1,58 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import hashlib +import logging +import sys + +from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import ERROR +from pip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES +from pip._internal.utils.misc import read_chunks, write_output + +logger = logging.getLogger(__name__) + + +class HashCommand(Command): + """ + Compute a hash of a local package archive. + + These can be used with --hash in a requirements file to do repeatable + installs. + """ + + usage = '%prog [options] ...' + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(HashCommand, self).__init__(*args, **kw) + self.cmd_opts.add_option( + '-a', '--algorithm', + dest='algorithm', + choices=STRONG_HASHES, + action='store', + default=FAVORITE_HASH, + help='The hash algorithm to use: one of %s' % + ', '.join(STRONG_HASHES)) + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + if not args: + self.parser.print_usage(sys.stderr) + return ERROR + + algorithm = options.algorithm + for path in args: + write_output('%s:\n--hash=%s:%s', + path, algorithm, _hash_of_file(path, algorithm)) + + +def _hash_of_file(path, algorithm): + """Return the hash digest of a file.""" + with open(path, 'rb') as archive: + hash = hashlib.new(algorithm) + for chunk in read_chunks(archive): + hash.update(chunk) + return hash.hexdigest() diff --git a/ubuntu/venv/pip/_internal/commands/help.py b/ubuntu/venv/pip/_internal/commands/help.py new file mode 100644 index 0000000..75af999 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/help.py @@ -0,0 +1,41 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import SUCCESS +from pip._internal.exceptions import CommandError + + +class HelpCommand(Command): + """Show help for commands""" + + usage = """ + %prog """ + ignore_require_venv = True + + def run(self, options, args): + from pip._internal.commands import ( + commands_dict, create_command, get_similar_commands, + ) + + try: + # 'pip help' with no args is handled by pip.__init__.parseopt() + cmd_name = args[0] # the command we need help for + except IndexError: + return SUCCESS + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + command = create_command(cmd_name) + command.parser.print_help() + + return SUCCESS diff --git a/ubuntu/venv/pip/_internal/commands/install.py b/ubuntu/venv/pip/_internal/commands/install.py new file mode 100644 index 0000000..cb2fb28 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/install.py @@ -0,0 +1,727 @@ +# The following comment should be removed at some point in the future. +# It's included for now because without it InstallCommand.run() has a +# couple errors where we have to know req.name is str rather than +# Optional[str] for the InstallRequirement req. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import errno +import logging +import operator +import os +import shutil +import site +from optparse import SUPPRESS_HELP + +from pip._vendor import pkg_resources +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.cache import WheelCache +from pip._internal.cli import cmdoptions +from pip._internal.cli.cmdoptions import make_target_python +from pip._internal.cli.req_command import RequirementCommand +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.exceptions import ( + CommandError, + InstallationError, + PreviousBuildDirError, +) +from pip._internal.locations import distutils_scheme +from pip._internal.operations.check import check_install_conflicts +from pip._internal.req import RequirementSet, install_given_reqs +from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.deprecation import deprecated +from pip._internal.utils.distutils_args import parse_distutils_args +from pip._internal.utils.filesystem import test_writable_dir +from pip._internal.utils.misc import ( + ensure_dir, + get_installed_version, + protect_pip_from_modification_on_windows, + write_output, +) +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.virtualenv import virtualenv_no_global +from pip._internal.wheel_builder import build, should_build_for_install_command + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Any, Iterable, List, Optional + + from pip._internal.models.format_control import FormatControl + from pip._internal.req.req_install import InstallRequirement + from pip._internal.wheel_builder import BinaryAllowedPredicate + +from pip._internal.locations import running_under_virtualenv + +logger = logging.getLogger(__name__) + + +def get_check_binary_allowed(format_control): + # type: (FormatControl) -> BinaryAllowedPredicate + def check_binary_allowed(req): + # type: (InstallRequirement) -> bool + if req.use_pep517: + return True + canonical_name = canonicalize_name(req.name) + allowed_formats = format_control.get_allowed_formats(canonical_name) + return "binary" in allowed_formats + + return check_binary_allowed + + +class InstallCommand(RequirementCommand): + """ + Install packages from: + + - PyPI (and other indexes) using requirement specifiers. + - VCS project urls. + - Local project directories. + - Local or remote source archives. + + pip also supports installing from "requirements files", which provide + an easy way to specify a whole environment to be installed. + """ + + usage = """ + %prog [options] [package-index-options] ... + %prog [options] -r [package-index-options] ... + %prog [options] [-e] ... + %prog [options] [-e] ... + %prog [options] ...""" + + def __init__(self, *args, **kw): + super(InstallCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + + cmd_opts.add_option(cmdoptions.requirements()) + cmd_opts.add_option(cmdoptions.constraints()) + cmd_opts.add_option(cmdoptions.no_deps()) + cmd_opts.add_option(cmdoptions.pre()) + + cmd_opts.add_option(cmdoptions.editable()) + cmd_opts.add_option( + '-t', '--target', + dest='target_dir', + metavar='dir', + default=None, + help='Install packages into . ' + 'By default this will not replace existing files/folders in ' + '. Use --upgrade to replace existing packages in ' + 'with new versions.' + ) + cmdoptions.add_target_python_options(cmd_opts) + + cmd_opts.add_option( + '--user', + dest='use_user_site', + action='store_true', + help="Install to the Python user install directory for your " + "platform. Typically ~/.local/, or %APPDATA%\\Python on " + "Windows. (See the Python documentation for site.USER_BASE " + "for full details.) On Debian systems, this is the " + "default when running outside of a virtual environment " + "and not as root.") + + cmd_opts.add_option( + '--no-user', + dest='use_system_location', + action='store_true', + help=SUPPRESS_HELP) + cmd_opts.add_option( + '--root', + dest='root_path', + metavar='dir', + default=None, + help="Install everything relative to this alternate root " + "directory.") + cmd_opts.add_option( + '--prefix', + dest='prefix_path', + metavar='dir', + default=None, + help="Installation prefix where lib, bin and other top-level " + "folders are placed") + + cmd_opts.add_option( + '--system', + dest='use_system_location', + action='store_true', + help="Install using the system scheme (overrides --user on " + "Debian systems)") + + cmd_opts.add_option(cmdoptions.build_dir()) + + cmd_opts.add_option(cmdoptions.src()) + + cmd_opts.add_option( + '-U', '--upgrade', + dest='upgrade', + action='store_true', + help='Upgrade all specified packages to the newest available ' + 'version. The handling of dependencies depends on the ' + 'upgrade-strategy used.' + ) + + cmd_opts.add_option( + '--upgrade-strategy', + dest='upgrade_strategy', + default='only-if-needed', + choices=['only-if-needed', 'eager'], + help='Determines how dependency upgrading should be handled ' + '[default: %default]. ' + '"eager" - dependencies are upgraded regardless of ' + 'whether the currently installed version satisfies the ' + 'requirements of the upgraded package(s). ' + '"only-if-needed" - are upgraded only when they do not ' + 'satisfy the requirements of the upgraded package(s).' + ) + + cmd_opts.add_option( + '--force-reinstall', + dest='force_reinstall', + action='store_true', + help='Reinstall all packages even if they are already ' + 'up-to-date.') + + cmd_opts.add_option( + '-I', '--ignore-installed', + dest='ignore_installed', + action='store_true', + help='Ignore the installed packages, overwriting them. ' + 'This can break your system if the existing package ' + 'is of a different version or was installed ' + 'with a different package manager!' + ) + + cmd_opts.add_option(cmdoptions.ignore_requires_python()) + cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) + + cmd_opts.add_option(cmdoptions.install_options()) + cmd_opts.add_option(cmdoptions.global_options()) + + cmd_opts.add_option( + "--compile", + action="store_true", + dest="compile", + default=True, + help="Compile Python source files to bytecode", + ) + + cmd_opts.add_option( + "--no-compile", + action="store_false", + dest="compile", + help="Do not compile Python source files to bytecode", + ) + + cmd_opts.add_option( + "--no-warn-script-location", + action="store_false", + dest="warn_script_location", + default=True, + help="Do not warn when installing scripts outside PATH", + ) + cmd_opts.add_option( + "--no-warn-conflicts", + action="store_false", + dest="warn_about_conflicts", + default=True, + help="Do not warn about broken dependencies", + ) + + cmd_opts.add_option(cmdoptions.no_binary()) + cmd_opts.add_option(cmdoptions.only_binary()) + cmd_opts.add_option(cmdoptions.prefer_binary()) + cmd_opts.add_option(cmdoptions.no_clean()) + cmd_opts.add_option(cmdoptions.require_hashes()) + cmd_opts.add_option(cmdoptions.progress_bar()) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, cmd_opts) + + def run(self, options, args): + # type: (Values, List[Any]) -> int + cmdoptions.check_install_build_global(options) + upgrade_strategy = "to-satisfy-only" + if options.upgrade: + upgrade_strategy = options.upgrade_strategy + + cmdoptions.check_dist_restriction(options, check_target=True) + + if options.python_version: + python_versions = [options.python_version] + else: + python_versions = None + + # compute install location defaults + if (not options.use_user_site and not options.prefix_path and not + options.target_dir and not options.use_system_location): + if not running_under_virtualenv() and os.geteuid() != 0: + options.use_user_site = True + + if options.use_system_location: + options.use_user_site = False + + options.src_dir = os.path.abspath(options.src_dir) + install_options = options.install_options or [] + + options.use_user_site = decide_user_install( + options.use_user_site, + prefix_path=options.prefix_path, + target_dir=options.target_dir, + root_path=options.root_path, + isolated_mode=options.isolated_mode, + ) + + target_temp_dir = None # type: Optional[TempDirectory] + target_temp_dir_path = None # type: Optional[str] + if options.target_dir: + options.ignore_installed = True + options.target_dir = os.path.abspath(options.target_dir) + if (os.path.exists(options.target_dir) and not + os.path.isdir(options.target_dir)): + raise CommandError( + "Target path exists but is not a directory, will not " + "continue." + ) + + # Create a target directory for using with the target option + target_temp_dir = TempDirectory(kind="target") + target_temp_dir_path = target_temp_dir.path + + global_options = options.global_options or [] + + session = self.get_default_session(options) + + target_python = make_target_python(options) + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + build_delete = (not (options.no_clean or options.build_dir)) + wheel_cache = WheelCache(options.cache_dir, options.format_control) + + with get_requirement_tracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="install" + ) as directory: + requirement_set = RequirementSet( + check_supported_wheels=not options.target_dir, + ) + + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + wheel_cache + ) + + warn_deprecated_install_options( + requirement_set, options.install_options + ) + + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + session=session, + finder=finder, + use_user_site=options.use_user_site, + ) + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + options=options, + wheel_cache=wheel_cache, + use_user_site=options.use_user_site, + ignore_installed=options.ignore_installed, + ignore_requires_python=options.ignore_requires_python, + force_reinstall=options.force_reinstall, + upgrade_strategy=upgrade_strategy, + use_pep517=options.use_pep517, + ) + + self.trace_basic_info(finder) + + resolver.resolve(requirement_set) + + try: + pip_req = requirement_set.get_requirement("pip") + except KeyError: + modifying_pip = None + else: + # If we're not replacing an already installed pip, + # we're not modifying it. + modifying_pip = pip_req.satisfied_by is None + protect_pip_from_modification_on_windows( + modifying_pip=modifying_pip + ) + + check_binary_allowed = get_check_binary_allowed( + finder.format_control + ) + + reqs_to_build = [ + r for r in requirement_set.requirements.values() + if should_build_for_install_command( + r, check_binary_allowed + ) + ] + + _, build_failures = build( + reqs_to_build, + wheel_cache=wheel_cache, + build_options=[], + global_options=[], + ) + + # If we're using PEP 517, we cannot do a direct install + # so we fail here. + # We don't care about failures building legacy + # requirements, as we'll fall through to a direct + # install for those. + pep517_build_failures = [ + r for r in build_failures if r.use_pep517 + ] + if pep517_build_failures: + raise InstallationError( + "Could not build wheels for {} which use" + " PEP 517 and cannot be installed directly".format( + ", ".join(r.name for r in pep517_build_failures))) + + to_install = resolver.get_installation_order( + requirement_set + ) + + # Consistency Checking of the package set we're installing. + should_warn_about_conflicts = ( + not options.ignore_dependencies and + options.warn_about_conflicts + ) + if should_warn_about_conflicts: + self._warn_about_conflicts(to_install) + + # Don't warn about script install locations if + # --target has been specified + warn_script_location = options.warn_script_location + if options.target_dir: + warn_script_location = False + + installed = install_given_reqs( + to_install, + install_options, + global_options, + root=options.root_path, + home=target_temp_dir_path, + prefix=options.prefix_path, + pycompile=options.compile, + warn_script_location=warn_script_location, + use_user_site=options.use_user_site, + ) + + lib_locations = get_lib_location_guesses( + user=options.use_user_site, + home=target_temp_dir_path, + root=options.root_path, + prefix=options.prefix_path, + isolated=options.isolated_mode, + ) + working_set = pkg_resources.WorkingSet(lib_locations) + + installed.sort(key=operator.attrgetter('name')) + items = [] + for result in installed: + item = result.name + try: + installed_version = get_installed_version( + result.name, working_set=working_set + ) + if installed_version: + item += '-' + installed_version + except Exception: + pass + items.append(item) + installed_desc = ' '.join(items) + if installed_desc: + write_output( + 'Successfully installed %s', installed_desc, + ) + except EnvironmentError as error: + show_traceback = (self.verbosity >= 1) + + message = create_env_error_message( + error, show_traceback, options.use_user_site, + ) + logger.error(message, exc_info=show_traceback) + + return ERROR + except PreviousBuildDirError: + options.no_clean = True + raise + finally: + # Clean up + if not options.no_clean: + requirement_set.cleanup_files() + wheel_cache.cleanup() + + if options.target_dir: + self._handle_target_dir( + options.target_dir, target_temp_dir, options.upgrade + ) + + return SUCCESS + + def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): + ensure_dir(target_dir) + + # Checking both purelib and platlib directories for installed + # packages to be moved to target directory + lib_dir_list = [] + + with target_temp_dir: + # Checking both purelib and platlib directories for installed + # packages to be moved to target directory + scheme = distutils_scheme('', home=target_temp_dir.path) + purelib_dir = scheme['purelib'] + platlib_dir = scheme['platlib'] + data_dir = scheme['data'] + + if os.path.exists(purelib_dir): + lib_dir_list.append(purelib_dir) + if os.path.exists(platlib_dir) and platlib_dir != purelib_dir: + lib_dir_list.append(platlib_dir) + if os.path.exists(data_dir): + lib_dir_list.append(data_dir) + + for lib_dir in lib_dir_list: + for item in os.listdir(lib_dir): + if lib_dir == data_dir: + ddir = os.path.join(data_dir, item) + if any(s.startswith(ddir) for s in lib_dir_list[:-1]): + continue + target_item_dir = os.path.join(target_dir, item) + if os.path.exists(target_item_dir): + if not upgrade: + logger.warning( + 'Target directory %s already exists. Specify ' + '--upgrade to force replacement.', + target_item_dir + ) + continue + if os.path.islink(target_item_dir): + logger.warning( + 'Target directory %s already exists and is ' + 'a link. Pip will not automatically replace ' + 'links, please remove if replacement is ' + 'desired.', + target_item_dir + ) + continue + if os.path.isdir(target_item_dir): + shutil.rmtree(target_item_dir) + else: + os.remove(target_item_dir) + + shutil.move( + os.path.join(lib_dir, item), + target_item_dir + ) + + def _warn_about_conflicts(self, to_install): + try: + package_set, _dep_info = check_install_conflicts(to_install) + except Exception: + logger.error("Error checking for conflicts.", exc_info=True) + return + missing, conflicting = _dep_info + + # NOTE: There is some duplication here from pip check + for project_name in missing: + version = package_set[project_name][0] + for dependency in missing[project_name]: + logger.critical( + "%s %s requires %s, which is not installed.", + project_name, version, dependency[1], + ) + + for project_name in conflicting: + version = package_set[project_name][0] + for dep_name, dep_version, req in conflicting[project_name]: + logger.critical( + "%s %s has requirement %s, but you'll have %s %s which is " + "incompatible.", + project_name, version, req, dep_name, dep_version, + ) + + +def get_lib_location_guesses(*args, **kwargs): + scheme = distutils_scheme('', *args, **kwargs) + return [scheme['purelib'], scheme['platlib']] + + +def site_packages_writable(**kwargs): + return all( + test_writable_dir(d) for d in set(get_lib_location_guesses(**kwargs)) + ) + + +def decide_user_install( + use_user_site, # type: Optional[bool] + prefix_path=None, # type: Optional[str] + target_dir=None, # type: Optional[str] + root_path=None, # type: Optional[str] + isolated_mode=False, # type: bool +): + # type: (...) -> bool + """Determine whether to do a user install based on the input options. + + If use_user_site is False, no additional checks are done. + If use_user_site is True, it is checked for compatibility with other + options. + If use_user_site is None, the default behaviour depends on the environment, + which is provided by the other arguments. + """ + # In some cases (config from tox), use_user_site can be set to an integer + # rather than a bool, which 'use_user_site is False' wouldn't catch. + if (use_user_site is not None) and (not use_user_site): + logger.debug("Non-user install by explicit request") + return False + + if use_user_site: + if prefix_path: + raise CommandError( + "Can not combine '--user' and '--prefix' as they imply " + "different installation locations" + ) + if virtualenv_no_global(): + raise InstallationError( + "Can not perform a '--user' install. User site-packages " + "are not visible in this virtualenv." + ) + logger.debug("User install by explicit request") + return True + + # If we are here, user installs have not been explicitly requested/avoided + assert use_user_site is None + + # user install incompatible with --prefix/--target + if prefix_path or target_dir: + logger.debug("Non-user install due to --prefix or --target option") + return False + + # If user installs are not enabled, choose a non-user install + if not site.ENABLE_USER_SITE: + logger.debug("Non-user install because user site-packages disabled") + return False + + # If we have permission for a non-user install, do that, + # otherwise do a user install. + if site_packages_writable(root=root_path, isolated=isolated_mode): + logger.debug("Non-user install because site-packages writeable") + return False + + logger.info("Defaulting to user installation because normal site-packages " + "is not writeable") + return True + + +def warn_deprecated_install_options(requirement_set, options): + # type: (RequirementSet, Optional[List[str]]) -> None + """If any location-changing --install-option arguments were passed for + requirements or on the command-line, then show a deprecation warning. + """ + def format_options(option_names): + # type: (Iterable[str]) -> List[str] + return ["--{}".format(name.replace("_", "-")) for name in option_names] + + requirements = ( + requirement_set.unnamed_requirements + + list(requirement_set.requirements.values()) + ) + + offenders = [] + + for requirement in requirements: + install_options = requirement.options.get("install_options", []) + location_options = parse_distutils_args(install_options) + if location_options: + offenders.append( + "{!r} from {}".format( + format_options(location_options.keys()), requirement + ) + ) + + if options: + location_options = parse_distutils_args(options) + if location_options: + offenders.append( + "{!r} from command line".format( + format_options(location_options.keys()) + ) + ) + + if not offenders: + return + + deprecated( + reason=( + "Location-changing options found in --install-option: {}. " + "This configuration may cause unexpected behavior and is " + "unsupported.".format( + "; ".join(offenders) + ) + ), + replacement=( + "using pip-level options like --user, --prefix, --root, and " + "--target" + ), + gone_in="20.2", + issue=7309, + ) + + +def create_env_error_message(error, show_traceback, using_user_site): + """Format an error message for an EnvironmentError + + It may occur anytime during the execution of the install command. + """ + parts = [] + + # Mention the error if we are not going to show a traceback + parts.append("Could not install packages due to an EnvironmentError") + if not show_traceback: + parts.append(": ") + parts.append(str(error)) + else: + parts.append(".") + + # Spilt the error indication from a helper message (if any) + parts[-1] += "\n" + + # Suggest useful actions to the user: + # (1) using user site-packages or (2) verifying the permissions + if error.errno == errno.EACCES: + user_option_part = "Consider using the `--user` option" + permissions_part = "Check the permissions" + + if not using_user_site: + parts.extend([ + user_option_part, " or ", + permissions_part.lower(), + ]) + else: + parts.append(permissions_part) + parts.append(".\n") + + return "".join(parts).strip() + "\n" diff --git a/ubuntu/venv/pip/_internal/commands/list.py b/ubuntu/venv/pip/_internal/commands/list.py new file mode 100644 index 0000000..d006206 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/list.py @@ -0,0 +1,315 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import json +import logging + +from pip._vendor import six +from pip._vendor.six.moves import zip_longest + +from pip._internal.cli import cmdoptions +from pip._internal.cli.req_command import IndexGroupCommand +from pip._internal.exceptions import CommandError +from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.self_outdated_check import make_link_collector +from pip._internal.utils.misc import ( + dist_is_editable, + get_installed_distributions, + write_output, +) +from pip._internal.utils.packaging import get_installer + +from pip._vendor.packaging.version import parse + +logger = logging.getLogger(__name__) + + +class ListCommand(IndexGroupCommand): + """ + List installed packages, including editables. + + Packages are listed in a case-insensitive sorted order. + """ + + usage = """ + %prog [options]""" + + def __init__(self, *args, **kw): + super(ListCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + + cmd_opts.add_option( + '-o', '--outdated', + action='store_true', + default=False, + help='List outdated packages') + cmd_opts.add_option( + '-u', '--uptodate', + action='store_true', + default=False, + help='List uptodate packages') + cmd_opts.add_option( + '-e', '--editable', + action='store_true', + default=False, + help='List editable projects.') + cmd_opts.add_option( + '-l', '--local', + action='store_true', + default=False, + help=('If in a virtualenv that has global access, do not list ' + 'globally-installed packages.'), + ) + self.cmd_opts.add_option( + '--user', + dest='user', + action='store_true', + default=False, + help='Only output packages installed in user-site.') + cmd_opts.add_option(cmdoptions.list_path()) + cmd_opts.add_option( + '--pre', + action='store_true', + default=False, + help=("Include pre-release and development versions. By default, " + "pip only finds stable versions."), + ) + + cmd_opts.add_option( + '--format', + action='store', + dest='list_format', + default="columns", + choices=('columns', 'freeze', 'json'), + help="Select the output format among: columns (default), freeze, " + "or json", + ) + + cmd_opts.add_option( + '--not-required', + action='store_true', + dest='not_required', + help="List packages that are not dependencies of " + "installed packages.", + ) + + cmd_opts.add_option( + '--exclude-editable', + action='store_false', + dest='include_editable', + help='Exclude editable package from output.', + ) + cmd_opts.add_option( + '--include-editable', + action='store_true', + dest='include_editable', + help='Include editable package from output.', + default=True, + ) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, self.parser + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, cmd_opts) + + def _build_package_finder(self, options, session): + """ + Create a package finder appropriate to this list command. + """ + link_collector = make_link_collector(session, options=options) + + # Pass allow_yanked=False to ignore yanked versions. + selection_prefs = SelectionPreferences( + allow_yanked=False, + allow_all_prereleases=options.pre, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + ) + + def run(self, options, args): + if options.outdated and options.uptodate: + raise CommandError( + "Options --outdated and --uptodate cannot be combined.") + + cmdoptions.check_list_path_option(options) + + packages = get_installed_distributions( + local_only=options.local, + user_only=options.user, + editables_only=options.editable, + include_editables=options.include_editable, + paths=options.path, + ) + + # get_not_required must be called firstly in order to find and + # filter out all dependencies correctly. Otherwise a package + # can't be identified as requirement because some parent packages + # could be filtered out before. + if options.not_required: + packages = self.get_not_required(packages, options) + + if options.outdated: + packages = self.get_outdated(packages, options) + elif options.uptodate: + packages = self.get_uptodate(packages, options) + + self.output_package_listing(packages, options) + + def get_outdated(self, packages, options): + return [ + dist for dist in self.iter_packages_latest_infos(packages, options) + if parse(str(dist.latest_version)) > parse(str(dist.parsed_version)) + ] + + def get_uptodate(self, packages, options): + return [ + dist for dist in self.iter_packages_latest_infos(packages, options) + if parse(str(dist.latest_version)) == parse(str(dist.parsed_version)) + ] + + def get_not_required(self, packages, options): + dep_keys = set() + for dist in packages: + dep_keys.update(requirement.key for requirement in dist.requires()) + return {pkg for pkg in packages if pkg.key not in dep_keys} + + def iter_packages_latest_infos(self, packages, options): + with self._build_session(options) as session: + finder = self._build_package_finder(options, session) + + for dist in packages: + typ = 'unknown' + all_candidates = finder.find_all_candidates(dist.key) + if not options.pre: + # Remove prereleases + all_candidates = [candidate for candidate in all_candidates + if not candidate.version.is_prerelease] + + evaluator = finder.make_candidate_evaluator( + project_name=dist.project_name, + ) + best_candidate = evaluator.sort_best_candidate(all_candidates) + if best_candidate is None: + continue + + remote_version = best_candidate.version + if best_candidate.link.is_wheel: + typ = 'wheel' + else: + typ = 'sdist' + # This is dirty but makes the rest of the code much cleaner + dist.latest_version = remote_version + dist.latest_filetype = typ + yield dist + + def output_package_listing(self, packages, options): + packages = sorted( + packages, + key=lambda dist: dist.project_name.lower(), + ) + if options.list_format == 'columns' and packages: + data, header = format_for_columns(packages, options) + self.output_package_listing_columns(data, header) + elif options.list_format == 'freeze': + for dist in packages: + if options.verbose >= 1: + write_output("%s==%s (%s)", dist.project_name, + dist.version, dist.location) + else: + write_output("%s==%s", dist.project_name, dist.version) + elif options.list_format == 'json': + write_output(format_for_json(packages, options)) + + def output_package_listing_columns(self, data, header): + # insert the header first: we need to know the size of column names + if len(data) > 0: + data.insert(0, header) + + pkg_strings, sizes = tabulate(data) + + # Create and add a separator. + if len(data) > 0: + pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes))) + + for val in pkg_strings: + write_output(val) + + +def tabulate(vals): + # From pfmoore on GitHub: + # https://github.com/pypa/pip/issues/3651#issuecomment-216932564 + assert len(vals) > 0 + + sizes = [0] * max(len(x) for x in vals) + for row in vals: + sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)] + + result = [] + for row in vals: + display = " ".join([str(c).ljust(s) if c is not None else '' + for s, c in zip_longest(sizes, row)]) + result.append(display) + + return result, sizes + + +def format_for_columns(pkgs, options): + """ + Convert the package data into something usable + by output_package_listing_columns. + """ + running_outdated = options.outdated + # Adjust the header for the `pip list --outdated` case. + if running_outdated: + header = ["Package", "Version", "Latest", "Type"] + else: + header = ["Package", "Version"] + + data = [] + if options.verbose >= 1 or any(dist_is_editable(x) for x in pkgs): + header.append("Location") + if options.verbose >= 1: + header.append("Installer") + + for proj in pkgs: + # if we're working on the 'outdated' list, separate out the + # latest_version and type + row = [proj.project_name, proj.version] + + if running_outdated: + row.append(proj.latest_version) + row.append(proj.latest_filetype) + + if options.verbose >= 1 or dist_is_editable(proj): + row.append(proj.location) + if options.verbose >= 1: + row.append(get_installer(proj)) + + data.append(row) + + return data, header + + +def format_for_json(packages, options): + data = [] + for dist in packages: + info = { + 'name': dist.project_name, + 'version': six.text_type(dist.version), + } + if options.verbose >= 1: + info['location'] = dist.location + info['installer'] = get_installer(dist) + if options.outdated: + info['latest_version'] = six.text_type(dist.latest_version) + info['latest_filetype'] = dist.latest_filetype + data.append(info) + return json.dumps(data) diff --git a/ubuntu/venv/pip/_internal/commands/search.py b/ubuntu/venv/pip/_internal/commands/search.py new file mode 100644 index 0000000..2e880ee --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/search.py @@ -0,0 +1,145 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import sys +import textwrap +from collections import OrderedDict + +from pip._vendor import pkg_resources +from pip._vendor.packaging.version import parse as parse_version +# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import +from pip._vendor.six.moves import xmlrpc_client # type: ignore + +from pip._internal.cli.base_command import Command +from pip._internal.cli.req_command import SessionCommandMixin +from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS +from pip._internal.exceptions import CommandError +from pip._internal.models.index import PyPI +from pip._internal.network.xmlrpc import PipXmlrpcTransport +from pip._internal.utils.compat import get_terminal_size +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import write_output + +logger = logging.getLogger(__name__) + + +class SearchCommand(Command, SessionCommandMixin): + """Search for PyPI packages whose name or summary contains .""" + + usage = """ + %prog [options] """ + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(SearchCommand, self).__init__(*args, **kw) + self.cmd_opts.add_option( + '-i', '--index', + dest='index', + metavar='URL', + default=PyPI.pypi_url, + help='Base URL of Python Package Index (default %default)') + + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + if not args: + raise CommandError('Missing required argument (search query).') + query = args + pypi_hits = self.search(query, options) + hits = transform_hits(pypi_hits) + + terminal_width = None + if sys.stdout.isatty(): + terminal_width = get_terminal_size()[0] + + print_results(hits, terminal_width=terminal_width) + if pypi_hits: + return SUCCESS + return NO_MATCHES_FOUND + + def search(self, query, options): + index_url = options.index + + session = self.get_default_session(options) + + transport = PipXmlrpcTransport(index_url, session) + pypi = xmlrpc_client.ServerProxy(index_url, transport) + hits = pypi.search({'name': query, 'summary': query}, 'or') + return hits + + +def transform_hits(hits): + """ + The list from pypi is really a list of versions. We want a list of + packages with the list of versions stored inline. This converts the + list from pypi into one we can use. + """ + packages = OrderedDict() + for hit in hits: + name = hit['name'] + summary = hit['summary'] + version = hit['version'] + + if name not in packages.keys(): + packages[name] = { + 'name': name, + 'summary': summary, + 'versions': [version], + } + else: + packages[name]['versions'].append(version) + + # if this is the highest version, replace summary and score + if version == highest_version(packages[name]['versions']): + packages[name]['summary'] = summary + + return list(packages.values()) + + +def print_results(hits, name_column_width=None, terminal_width=None): + if not hits: + return + if name_column_width is None: + name_column_width = max([ + len(hit['name']) + len(highest_version(hit.get('versions', ['-']))) + for hit in hits + ]) + 4 + + installed_packages = [p.project_name for p in pkg_resources.working_set] + for hit in hits: + name = hit['name'] + summary = hit['summary'] or '' + latest = highest_version(hit.get('versions', ['-'])) + if terminal_width is not None: + target_width = terminal_width - name_column_width - 5 + if target_width > 10: + # wrap and indent summary to fit terminal + summary = textwrap.wrap(summary, target_width) + summary = ('\n' + ' ' * (name_column_width + 3)).join(summary) + + line = '%-*s - %s' % (name_column_width, + '%s (%s)' % (name, latest), summary) + try: + write_output(line) + if name in installed_packages: + dist = pkg_resources.get_distribution(name) + with indent_log(): + if dist.version == latest: + write_output('INSTALLED: %s (latest)', dist.version) + else: + write_output('INSTALLED: %s', dist.version) + if parse_version(latest).pre: + write_output('LATEST: %s (pre-release; install' + ' with "pip install --pre")', latest) + else: + write_output('LATEST: %s', latest) + except UnicodeEncodeError: + pass + + +def highest_version(versions): + return max(versions, key=parse_version) diff --git a/ubuntu/venv/pip/_internal/commands/show.py b/ubuntu/venv/pip/_internal/commands/show.py new file mode 100644 index 0000000..a46b08e --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/show.py @@ -0,0 +1,180 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os +from email.parser import FeedParser + +from pip._vendor import pkg_resources +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.cli.base_command import Command +from pip._internal.cli.status_codes import ERROR, SUCCESS +from pip._internal.utils.misc import write_output + +logger = logging.getLogger(__name__) + + +class ShowCommand(Command): + """ + Show information about one or more installed packages. + + The output is in RFC-compliant mail header format. + """ + + usage = """ + %prog [options] ...""" + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(ShowCommand, self).__init__(*args, **kw) + self.cmd_opts.add_option( + '-f', '--files', + dest='files', + action='store_true', + default=False, + help='Show the full list of installed files for each package.') + + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + if not args: + logger.warning('ERROR: Please provide a package name or names.') + return ERROR + query = args + + results = search_packages_info(query) + if not print_results( + results, list_files=options.files, verbose=options.verbose): + return ERROR + return SUCCESS + + +def search_packages_info(query): + """ + Gather details from installed distributions. Print distribution name, + version, location, and installed files. Installed files requires a + pip generated 'installed-files.txt' in the distributions '.egg-info' + directory. + """ + installed = {} + for p in pkg_resources.working_set: + installed[canonicalize_name(p.project_name)] = p + + query_names = [canonicalize_name(name) for name in query] + missing = sorted( + [name for name, pkg in zip(query, query_names) if pkg not in installed] + ) + if missing: + logger.warning('Package(s) not found: %s', ', '.join(missing)) + + def get_requiring_packages(package_name): + canonical_name = canonicalize_name(package_name) + return [ + pkg.project_name for pkg in pkg_resources.working_set + if canonical_name in + [canonicalize_name(required.name) for required in + pkg.requires()] + ] + + for dist in [installed[pkg] for pkg in query_names if pkg in installed]: + package = { + 'name': dist.project_name, + 'version': dist.version, + 'location': dist.location, + 'requires': [dep.project_name for dep in dist.requires()], + 'required_by': get_requiring_packages(dist.project_name) + } + file_list = None + metadata = None + if isinstance(dist, pkg_resources.DistInfoDistribution): + # RECORDs should be part of .dist-info metadatas + if dist.has_metadata('RECORD'): + lines = dist.get_metadata_lines('RECORD') + paths = [l.split(',')[0] for l in lines] + paths = [os.path.join(dist.location, p) for p in paths] + file_list = [os.path.relpath(p, dist.location) for p in paths] + + if dist.has_metadata('METADATA'): + metadata = dist.get_metadata('METADATA') + else: + # Otherwise use pip's log for .egg-info's + if dist.has_metadata('installed-files.txt'): + paths = dist.get_metadata_lines('installed-files.txt') + paths = [os.path.join(dist.egg_info, p) for p in paths] + file_list = [os.path.relpath(p, dist.location) for p in paths] + + if dist.has_metadata('PKG-INFO'): + metadata = dist.get_metadata('PKG-INFO') + + if dist.has_metadata('entry_points.txt'): + entry_points = dist.get_metadata_lines('entry_points.txt') + package['entry_points'] = entry_points + + if dist.has_metadata('INSTALLER'): + for line in dist.get_metadata_lines('INSTALLER'): + if line.strip(): + package['installer'] = line.strip() + break + + # @todo: Should pkg_resources.Distribution have a + # `get_pkg_info` method? + feed_parser = FeedParser() + feed_parser.feed(metadata) + pkg_info_dict = feed_parser.close() + for key in ('metadata-version', 'summary', + 'home-page', 'author', 'author-email', 'license'): + package[key] = pkg_info_dict.get(key) + + # It looks like FeedParser cannot deal with repeated headers + classifiers = [] + for line in metadata.splitlines(): + if line.startswith('Classifier: '): + classifiers.append(line[len('Classifier: '):]) + package['classifiers'] = classifiers + + if file_list: + package['files'] = sorted(file_list) + yield package + + +def print_results(distributions, list_files=False, verbose=False): + """ + Print the informations from installed distributions found. + """ + results_printed = False + for i, dist in enumerate(distributions): + results_printed = True + if i > 0: + write_output("---") + + write_output("Name: %s", dist.get('name', '')) + write_output("Version: %s", dist.get('version', '')) + write_output("Summary: %s", dist.get('summary', '')) + write_output("Home-page: %s", dist.get('home-page', '')) + write_output("Author: %s", dist.get('author', '')) + write_output("Author-email: %s", dist.get('author-email', '')) + write_output("License: %s", dist.get('license', '')) + write_output("Location: %s", dist.get('location', '')) + write_output("Requires: %s", ', '.join(dist.get('requires', []))) + write_output("Required-by: %s", ', '.join(dist.get('required_by', []))) + + if verbose: + write_output("Metadata-Version: %s", + dist.get('metadata-version', '')) + write_output("Installer: %s", dist.get('installer', '')) + write_output("Classifiers:") + for classifier in dist.get('classifiers', []): + write_output(" %s", classifier) + write_output("Entry-points:") + for entry in dist.get('entry_points', []): + write_output(" %s", entry.strip()) + if list_files: + write_output("Files:") + for line in dist.get('files', []): + write_output(" %s", line.strip()) + if "files" not in dist: + write_output("Cannot locate installed-files.txt") + return results_printed diff --git a/ubuntu/venv/pip/_internal/commands/uninstall.py b/ubuntu/venv/pip/_internal/commands/uninstall.py new file mode 100644 index 0000000..1bde414 --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/uninstall.py @@ -0,0 +1,82 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.cli.base_command import Command +from pip._internal.cli.req_command import SessionCommandMixin +from pip._internal.exceptions import InstallationError +from pip._internal.req import parse_requirements +from pip._internal.req.constructors import install_req_from_line +from pip._internal.utils.misc import protect_pip_from_modification_on_windows + + +class UninstallCommand(Command, SessionCommandMixin): + """ + Uninstall packages. + + pip is able to uninstall most installed packages. Known exceptions are: + + - Pure distutils packages installed with ``python setup.py install``, which + leave behind no metadata to determine what files were installed. + - Script wrappers installed by ``python setup.py develop``. + """ + + usage = """ + %prog [options] ... + %prog [options] -r ...""" + + def __init__(self, *args, **kw): + super(UninstallCommand, self).__init__(*args, **kw) + self.cmd_opts.add_option( + '-r', '--requirement', + dest='requirements', + action='append', + default=[], + metavar='file', + help='Uninstall all the packages listed in the given requirements ' + 'file. This option can be used multiple times.', + ) + self.cmd_opts.add_option( + '-y', '--yes', + dest='yes', + action='store_true', + help="Don't ask for confirmation of uninstall deletions.") + + self.parser.insert_option_group(0, self.cmd_opts) + + def run(self, options, args): + session = self.get_default_session(options) + + reqs_to_uninstall = {} + for name in args: + req = install_req_from_line( + name, isolated=options.isolated_mode, + ) + if req.name: + reqs_to_uninstall[canonicalize_name(req.name)] = req + for filename in options.requirements: + for req in parse_requirements( + filename, + options=options, + session=session): + if req.name: + reqs_to_uninstall[canonicalize_name(req.name)] = req + if not reqs_to_uninstall: + raise InstallationError( + 'You must give at least one requirement to %(name)s (see ' + '"pip help %(name)s")' % dict(name=self.name) + ) + + protect_pip_from_modification_on_windows( + modifying_pip="pip" in reqs_to_uninstall + ) + + for req in reqs_to_uninstall.values(): + uninstall_pathset = req.uninstall( + auto_confirm=options.yes, verbose=self.verbosity > 0, + ) + if uninstall_pathset: + uninstall_pathset.commit() diff --git a/ubuntu/venv/pip/_internal/commands/wheel.py b/ubuntu/venv/pip/_internal/commands/wheel.py new file mode 100644 index 0000000..eb44bce --- /dev/null +++ b/ubuntu/venv/pip/_internal/commands/wheel.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os +import shutil + +from pip._internal.cache import WheelCache +from pip._internal.cli import cmdoptions +from pip._internal.cli.req_command import RequirementCommand +from pip._internal.exceptions import CommandError, PreviousBuildDirError +from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.utils.misc import ensure_dir, normalize_path +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.wheel_builder import build, should_build_for_wheel_command + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Any, List + + +logger = logging.getLogger(__name__) + + +class WheelCommand(RequirementCommand): + """ + Build Wheel archives for your requirements and dependencies. + + Wheel is a built-package format, and offers the advantage of not + recompiling your software during every install. For more details, see the + wheel docs: https://wheel.readthedocs.io/en/latest/ + + Requirements: setuptools>=0.8, and wheel. + + 'pip wheel' uses the bdist_wheel setuptools extension from the wheel + package to build individual wheels. + + """ + + usage = """ + %prog [options] ... + %prog [options] -r ... + %prog [options] [-e] ... + %prog [options] [-e] ... + %prog [options] ...""" + + def __init__(self, *args, **kw): + super(WheelCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + + cmd_opts.add_option( + '-w', '--wheel-dir', + dest='wheel_dir', + metavar='dir', + default=os.curdir, + help=("Build wheels into , where the default is the " + "current working directory."), + ) + cmd_opts.add_option(cmdoptions.no_binary()) + cmd_opts.add_option(cmdoptions.only_binary()) + cmd_opts.add_option(cmdoptions.prefer_binary()) + cmd_opts.add_option( + '--build-option', + dest='build_options', + metavar='options', + action='append', + help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", + ) + cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) + cmd_opts.add_option(cmdoptions.constraints()) + cmd_opts.add_option(cmdoptions.editable()) + cmd_opts.add_option(cmdoptions.requirements()) + cmd_opts.add_option(cmdoptions.src()) + cmd_opts.add_option(cmdoptions.ignore_requires_python()) + cmd_opts.add_option(cmdoptions.no_deps()) + cmd_opts.add_option(cmdoptions.build_dir()) + cmd_opts.add_option(cmdoptions.progress_bar()) + + cmd_opts.add_option( + '--global-option', + dest='global_options', + action='append', + metavar='options', + help="Extra global options to be supplied to the setup.py " + "call before the 'bdist_wheel' command.") + + cmd_opts.add_option( + '--pre', + action='store_true', + default=False, + help=("Include pre-release and development versions. By default, " + "pip only finds stable versions."), + ) + + cmd_opts.add_option(cmdoptions.no_clean()) + cmd_opts.add_option(cmdoptions.require_hashes()) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, cmd_opts) + + def run(self, options, args): + # type: (Values, List[Any]) -> None + cmdoptions.check_install_build_global(options) + + session = self.get_default_session(options) + + finder = self._build_package_finder(options, session) + build_delete = (not (options.no_clean or options.build_dir)) + wheel_cache = WheelCache(options.cache_dir, options.format_control) + + options.wheel_dir = normalize_path(options.wheel_dir) + ensure_dir(options.wheel_dir) + + with get_requirement_tracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="wheel" + ) as directory: + + requirement_set = RequirementSet() + + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + wheel_cache + ) + + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + session=session, + finder=finder, + wheel_download_dir=options.wheel_dir, + use_user_site=False, + ) + + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + options=options, + wheel_cache=wheel_cache, + ignore_requires_python=options.ignore_requires_python, + use_pep517=options.use_pep517, + ) + + self.trace_basic_info(finder) + + resolver.resolve(requirement_set) + + reqs_to_build = [ + r for r in requirement_set.requirements.values() + if should_build_for_wheel_command(r) + ] + + # build wheels + build_successes, build_failures = build( + reqs_to_build, + wheel_cache=wheel_cache, + build_options=options.build_options or [], + global_options=options.global_options or [], + ) + for req in build_successes: + assert req.link and req.link.is_wheel + assert req.local_file_path + # copy from cache to target directory + try: + shutil.copy(req.local_file_path, options.wheel_dir) + except OSError as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + build_failures.append(req) + if len(build_failures) != 0: + raise CommandError( + "Failed to build one or more wheels" + ) + except PreviousBuildDirError: + options.no_clean = True + raise + finally: + if not options.no_clean: + requirement_set.cleanup_files() + wheel_cache.cleanup() diff --git a/ubuntu/venv/pip/_internal/configuration.py b/ubuntu/venv/pip/_internal/configuration.py new file mode 100644 index 0000000..f09a1ae --- /dev/null +++ b/ubuntu/venv/pip/_internal/configuration.py @@ -0,0 +1,422 @@ +"""Configuration management setup + +Some terminology: +- name + As written in config files. +- value + Value associated with a name +- key + Name combined with it's section (section.name) +- variant + A single word describing where the configuration key-value pair came from +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import locale +import logging +import os +import sys + +from pip._vendor.six.moves import configparser + +from pip._internal.exceptions import ( + ConfigurationError, + ConfigurationFileCouldNotBeLoaded, +) +from pip._internal.utils import appdirs +from pip._internal.utils.compat import WINDOWS, expanduser +from pip._internal.utils.misc import ensure_dir, enum +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, NewType, Optional, Tuple + ) + + RawConfigParser = configparser.RawConfigParser # Shorthand + Kind = NewType("Kind", str) + +logger = logging.getLogger(__name__) + + +# NOTE: Maybe use the optionx attribute to normalize keynames. +def _normalize_name(name): + # type: (str) -> str + """Make a name consistent regardless of source (environment or file) + """ + name = name.lower().replace('_', '-') + if name.startswith('--'): + name = name[2:] # only prefer long opts + return name + + +def _disassemble_key(name): + # type: (str) -> List[str] + if "." not in name: + error_message = ( + "Key does not contain dot separated section and key. " + "Perhaps you wanted to use 'global.{}' instead?" + ).format(name) + raise ConfigurationError(error_message) + return name.split(".", 1) + + +# The kinds of configurations there are. +kinds = enum( + USER="user", # User Specific + GLOBAL="global", # System Wide + SITE="site", # [Virtual] Environment Specific + ENV="env", # from PIP_CONFIG_FILE + ENV_VAR="env-var", # from Environment Variables +) + + +CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' + + +def get_configuration_files(): + # type: () -> Dict[Kind, List[str]] + global_config_files = [ + os.path.join(path, CONFIG_BASENAME) + for path in appdirs.site_config_dirs('pip') + ] + + site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME) + legacy_config_file = os.path.join( + expanduser('~'), + 'pip' if WINDOWS else '.pip', + CONFIG_BASENAME, + ) + new_config_file = os.path.join( + appdirs.user_config_dir("pip"), CONFIG_BASENAME + ) + return { + kinds.GLOBAL: global_config_files, + kinds.SITE: [site_config_file], + kinds.USER: [legacy_config_file, new_config_file], + } + + +class Configuration(object): + """Handles management of configuration. + + Provides an interface to accessing and managing configuration files. + + This class converts provides an API that takes "section.key-name" style + keys and stores the value associated with it as "key-name" under the + section "section". + + This allows for a clean interface wherein the both the section and the + key-name are preserved in an easy to manage form in the configuration files + and the data stored is also nice. + """ + + def __init__(self, isolated, load_only=None): + # type: (bool, Kind) -> None + super(Configuration, self).__init__() + + _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] + if load_only not in _valid_load_only: + raise ConfigurationError( + "Got invalid value for load_only - should be one of {}".format( + ", ".join(map(repr, _valid_load_only[:-1])) + ) + ) + self.isolated = isolated # type: bool + self.load_only = load_only # type: Optional[Kind] + + # The order here determines the override order. + self._override_order = [ + kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR + ] + + self._ignore_env_names = ["version", "help"] + + # Because we keep track of where we got the data from + self._parsers = { + variant: [] for variant in self._override_order + } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] + self._config = { + variant: {} for variant in self._override_order + } # type: Dict[Kind, Dict[str, Any]] + self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] + + def load(self): + # type: () -> None + """Loads configuration from configuration files and environment + """ + self._load_config_files() + if not self.isolated: + self._load_environment_vars() + + def get_file_to_edit(self): + # type: () -> Optional[str] + """Returns the file with highest priority in configuration + """ + assert self.load_only is not None, \ + "Need to be specified a file to be editing" + + try: + return self._get_parser_to_modify()[0] + except IndexError: + return None + + def items(self): + # type: () -> Iterable[Tuple[str, Any]] + """Returns key-value pairs like dict.items() representing the loaded + configuration + """ + return self._dictionary.items() + + def get_value(self, key): + # type: (str) -> Any + """Get a value from the configuration. + """ + try: + return self._dictionary[key] + except KeyError: + raise ConfigurationError("No such key - {}".format(key)) + + def set_value(self, key, value): + # type: (str, Any) -> None + """Modify a value in the configuration. + """ + self._ensure_have_load_only() + + fname, parser = self._get_parser_to_modify() + + if parser is not None: + section, name = _disassemble_key(key) + + # Modify the parser and the configuration + if not parser.has_section(section): + parser.add_section(section) + parser.set(section, name, value) + + self._config[self.load_only][key] = value + self._mark_as_modified(fname, parser) + + def unset_value(self, key): + # type: (str) -> None + """Unset a value in the configuration. + """ + self._ensure_have_load_only() + + if key not in self._config[self.load_only]: + raise ConfigurationError("No such key - {}".format(key)) + + fname, parser = self._get_parser_to_modify() + + if parser is not None: + section, name = _disassemble_key(key) + + # Remove the key in the parser + modified_something = False + if parser.has_section(section): + # Returns whether the option was removed or not + modified_something = parser.remove_option(section, name) + + if modified_something: + # name removed from parser, section may now be empty + section_iter = iter(parser.items(section)) + try: + val = next(section_iter) + except StopIteration: + val = None + + if val is None: + parser.remove_section(section) + + self._mark_as_modified(fname, parser) + else: + raise ConfigurationError( + "Fatal Internal error [id=1]. Please report as a bug." + ) + + del self._config[self.load_only][key] + + def save(self): + # type: () -> None + """Save the current in-memory state. + """ + self._ensure_have_load_only() + + for fname, parser in self._modified_parsers: + logger.info("Writing to %s", fname) + + # Ensure directory exists. + ensure_dir(os.path.dirname(fname)) + + with open(fname, "w") as f: + parser.write(f) + + # + # Private routines + # + + def _ensure_have_load_only(self): + # type: () -> None + if self.load_only is None: + raise ConfigurationError("Needed a specific file to be modifying.") + logger.debug("Will be working with %s variant only", self.load_only) + + @property + def _dictionary(self): + # type: () -> Dict[str, Any] + """A dictionary representing the loaded configuration. + """ + # NOTE: Dictionaries are not populated if not loaded. So, conditionals + # are not needed here. + retval = {} + + for variant in self._override_order: + retval.update(self._config[variant]) + + return retval + + def _load_config_files(self): + # type: () -> None + """Loads configuration from configuration files + """ + config_files = dict(self._iter_config_files()) + if config_files[kinds.ENV][0:1] == [os.devnull]: + logger.debug( + "Skipping loading configuration files due to " + "environment's PIP_CONFIG_FILE being os.devnull" + ) + return + + for variant, files in config_files.items(): + for fname in files: + # If there's specific variant set in `load_only`, load only + # that variant, not the others. + if self.load_only is not None and variant != self.load_only: + logger.debug( + "Skipping file '%s' (variant: %s)", fname, variant + ) + continue + + parser = self._load_file(variant, fname) + + # Keeping track of the parsers used + self._parsers[variant].append((fname, parser)) + + def _load_file(self, variant, fname): + # type: (Kind, str) -> RawConfigParser + logger.debug("For variant '%s', will try loading '%s'", variant, fname) + parser = self._construct_parser(fname) + + for section in parser.sections(): + items = parser.items(section) + self._config[variant].update(self._normalized_keys(section, items)) + + return parser + + def _construct_parser(self, fname): + # type: (str) -> RawConfigParser + parser = configparser.RawConfigParser() + # If there is no such file, don't bother reading it but create the + # parser anyway, to hold the data. + # Doing this is useful when modifying and saving files, where we don't + # need to construct a parser. + if os.path.exists(fname): + try: + parser.read(fname) + except UnicodeDecodeError: + # See https://github.com/pypa/pip/issues/4963 + raise ConfigurationFileCouldNotBeLoaded( + reason="contains invalid {} characters".format( + locale.getpreferredencoding(False) + ), + fname=fname, + ) + except configparser.Error as error: + # See https://github.com/pypa/pip/issues/4893 + raise ConfigurationFileCouldNotBeLoaded(error=error) + return parser + + def _load_environment_vars(self): + # type: () -> None + """Loads configuration from environment variables + """ + self._config[kinds.ENV_VAR].update( + self._normalized_keys(":env:", self._get_environ_vars()) + ) + + def _normalized_keys(self, section, items): + # type: (str, Iterable[Tuple[str, Any]]) -> Dict[str, Any] + """Normalizes items to construct a dictionary with normalized keys. + + This routine is where the names become keys and are made the same + regardless of source - configuration files or environment. + """ + normalized = {} + for name, val in items: + key = section + "." + _normalize_name(name) + normalized[key] = val + return normalized + + def _get_environ_vars(self): + # type: () -> Iterable[Tuple[str, str]] + """Returns a generator with all environmental vars with prefix PIP_""" + for key, val in os.environ.items(): + should_be_yielded = ( + key.startswith("PIP_") and + key[4:].lower() not in self._ignore_env_names + ) + if should_be_yielded: + yield key[4:].lower(), val + + # XXX: This is patched in the tests. + def _iter_config_files(self): + # type: () -> Iterable[Tuple[Kind, List[str]]] + """Yields variant and configuration files associated with it. + + This should be treated like items of a dictionary. + """ + # SMELL: Move the conditions out of this function + + # environment variables have the lowest priority + config_file = os.environ.get('PIP_CONFIG_FILE', None) + if config_file is not None: + yield kinds.ENV, [config_file] + else: + yield kinds.ENV, [] + + config_files = get_configuration_files() + + # at the base we have any global configuration + yield kinds.GLOBAL, config_files[kinds.GLOBAL] + + # per-user configuration next + should_load_user_config = not self.isolated and not ( + config_file and os.path.exists(config_file) + ) + if should_load_user_config: + # The legacy config file is overridden by the new config file + yield kinds.USER, config_files[kinds.USER] + + # finally virtualenv configuration first trumping others + yield kinds.SITE, config_files[kinds.SITE] + + def _get_parser_to_modify(self): + # type: () -> Tuple[str, RawConfigParser] + # Determine which parser to modify + parsers = self._parsers[self.load_only] + if not parsers: + # This should not happen if everything works correctly. + raise ConfigurationError( + "Fatal Internal error [id=2]. Please report as a bug." + ) + + # Use the highest priority parser. + return parsers[-1] + + # XXX: This is patched in the tests. + def _mark_as_modified(self, fname, parser): + # type: (str, RawConfigParser) -> None + file_parser_tuple = (fname, parser) + if file_parser_tuple not in self._modified_parsers: + self._modified_parsers.append(file_parser_tuple) diff --git a/ubuntu/venv/pip/_internal/distributions/__init__.py b/ubuntu/venv/pip/_internal/distributions/__init__.py new file mode 100644 index 0000000..d5c1afc --- /dev/null +++ b/ubuntu/venv/pip/_internal/distributions/__init__.py @@ -0,0 +1,24 @@ +from pip._internal.distributions.sdist import SourceDistribution +from pip._internal.distributions.wheel import WheelDistribution +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pip._internal.distributions.base import AbstractDistribution + from pip._internal.req.req_install import InstallRequirement + + +def make_distribution_for_install_requirement(install_req): + # type: (InstallRequirement) -> AbstractDistribution + """Returns a Distribution for the given InstallRequirement + """ + # Editable requirements will always be source distributions. They use the + # legacy logic until we create a modern standard for them. + if install_req.editable: + return SourceDistribution(install_req) + + # If it's a wheel, it's a WheelDistribution + if install_req.is_wheel: + return WheelDistribution(install_req) + + # Otherwise, a SourceDistribution + return SourceDistribution(install_req) diff --git a/ubuntu/venv/pip/_internal/distributions/base.py b/ubuntu/venv/pip/_internal/distributions/base.py new file mode 100644 index 0000000..b836b98 --- /dev/null +++ b/ubuntu/venv/pip/_internal/distributions/base.py @@ -0,0 +1,45 @@ +import abc + +from pip._vendor.six import add_metaclass + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + + from pip._vendor.pkg_resources import Distribution + from pip._internal.req import InstallRequirement + from pip._internal.index.package_finder import PackageFinder + + +@add_metaclass(abc.ABCMeta) +class AbstractDistribution(object): + """A base class for handling installable artifacts. + + The requirements for anything installable are as follows: + + - we must be able to determine the requirement name + (or we can't correctly handle the non-upgrade case). + + - for packages with setup requirements, we must also be able + to determine their requirements without installing additional + packages (for the same reason as run-time dependencies) + + - we must be able to create a Distribution object exposing the + above metadata. + """ + + def __init__(self, req): + # type: (InstallRequirement) -> None + super(AbstractDistribution, self).__init__() + self.req = req + + @abc.abstractmethod + def get_pkg_resources_distribution(self): + # type: () -> Optional[Distribution] + raise NotImplementedError() + + @abc.abstractmethod + def prepare_distribution_metadata(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None + raise NotImplementedError() diff --git a/ubuntu/venv/pip/_internal/distributions/installed.py b/ubuntu/venv/pip/_internal/distributions/installed.py new file mode 100644 index 0000000..0d15bf4 --- /dev/null +++ b/ubuntu/venv/pip/_internal/distributions/installed.py @@ -0,0 +1,24 @@ +from pip._internal.distributions.base import AbstractDistribution +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + + from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder + + +class InstalledDistribution(AbstractDistribution): + """Represents an installed package. + + This does not need any preparation as the required information has already + been computed. + """ + + def get_pkg_resources_distribution(self): + # type: () -> Optional[Distribution] + return self.req.satisfied_by + + def prepare_distribution_metadata(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None + pass diff --git a/ubuntu/venv/pip/_internal/distributions/sdist.py b/ubuntu/venv/pip/_internal/distributions/sdist.py new file mode 100644 index 0000000..be3d7d9 --- /dev/null +++ b/ubuntu/venv/pip/_internal/distributions/sdist.py @@ -0,0 +1,104 @@ +import logging + +from pip._internal.build_env import BuildEnvironment +from pip._internal.distributions.base import AbstractDistribution +from pip._internal.exceptions import InstallationError +from pip._internal.utils.subprocess import runner_with_spinner_message +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Set, Tuple + + from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder + + +logger = logging.getLogger(__name__) + + +class SourceDistribution(AbstractDistribution): + """Represents a source distribution. + + The preparation step for these needs metadata for the packages to be + generated, either using PEP 517 or using the legacy `setup.py egg_info`. + """ + + def get_pkg_resources_distribution(self): + # type: () -> Distribution + return self.req.get_dist() + + def prepare_distribution_metadata(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None + # Load pyproject.toml, to determine whether PEP 517 is to be used + self.req.load_pyproject_toml() + + # Set up the build isolation, if this requirement should be isolated + should_isolate = self.req.use_pep517 and build_isolation + if should_isolate: + self._setup_isolation(finder) + + self.req.prepare_metadata() + + def _setup_isolation(self, finder): + # type: (PackageFinder) -> None + def _raise_conflicts(conflicting_with, conflicting_reqs): + # type: (str, Set[Tuple[str, str]]) -> None + format_string = ( + "Some build dependencies for {requirement} " + "conflict with {conflicting_with}: {description}." + ) + error_message = format_string.format( + requirement=self.req, + conflicting_with=conflicting_with, + description=', '.join( + '{} is incompatible with {}'.format(installed, wanted) + for installed, wanted in sorted(conflicting) + ) + ) + raise InstallationError(error_message) + + # Isolate in a BuildEnvironment and install the build-time + # requirements. + pyproject_requires = self.req.pyproject_requires + assert pyproject_requires is not None + + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, pyproject_requires, 'overlay', + "Installing build dependencies" + ) + conflicting, missing = self.req.build_env.check_requirements( + self.req.requirements_to_check + ) + if conflicting: + _raise_conflicts("PEP 517/518 supported requirements", + conflicting) + if missing: + logger.warning( + "Missing build requirements in pyproject.toml for %s.", + self.req, + ) + logger.warning( + "The project does not specify a build backend, and " + "pip cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) + ) + # Install any extra build dependencies that the backend requests. + # This must be done in a second pass, as the pyproject.toml + # dependencies must be installed before we can call the backend. + with self.req.build_env: + runner = runner_with_spinner_message( + "Getting requirements to build wheel" + ) + backend = self.req.pep517_backend + assert backend is not None + with backend.subprocess_runner(runner): + reqs = backend.get_requires_for_build_wheel() + + conflicting, missing = self.req.build_env.check_requirements(reqs) + if conflicting: + _raise_conflicts("the backend dependencies", conflicting) + self.req.build_env.install_requirements( + finder, missing, 'normal', + "Installing backend dependencies" + ) diff --git a/ubuntu/venv/pip/_internal/distributions/wheel.py b/ubuntu/venv/pip/_internal/distributions/wheel.py new file mode 100644 index 0000000..bf3482b --- /dev/null +++ b/ubuntu/venv/pip/_internal/distributions/wheel.py @@ -0,0 +1,36 @@ +from zipfile import ZipFile + +from pip._internal.distributions.base import AbstractDistribution +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel + +if MYPY_CHECK_RUNNING: + from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder + + +class WheelDistribution(AbstractDistribution): + """Represents a wheel distribution. + + This does not need any preparation as wheels can be directly unpacked. + """ + + def get_pkg_resources_distribution(self): + # type: () -> Distribution + """Loads the metadata from the wheel file into memory and returns a + Distribution that uses it, not relying on the wheel file or + requirement. + """ + # Set as part of preparation during download. + assert self.req.local_file_path + # Wheels are never unnamed. + assert self.req.name + + with ZipFile(self.req.local_file_path, allowZip64=True) as z: + return pkg_resources_distribution_for_wheel( + z, self.req.name, self.req.local_file_path + ) + + def prepare_distribution_metadata(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None + pass diff --git a/ubuntu/venv/pip/_internal/exceptions.py b/ubuntu/venv/pip/_internal/exceptions.py new file mode 100644 index 0000000..dddec78 --- /dev/null +++ b/ubuntu/venv/pip/_internal/exceptions.py @@ -0,0 +1,308 @@ +"""Exceptions used throughout package""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +from itertools import chain, groupby, repeat + +from pip._vendor.six import iteritems + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + from pip._vendor.pkg_resources import Distribution + from pip._internal.req.req_install import InstallRequirement + + +class PipError(Exception): + """Base pip exception""" + + +class ConfigurationError(PipError): + """General exception in configuration""" + + +class InstallationError(PipError): + """General exception during installation""" + + +class UninstallationError(PipError): + """General exception during uninstallation""" + + +class NoneMetadataError(PipError): + """ + Raised when accessing "METADATA" or "PKG-INFO" metadata for a + pip._vendor.pkg_resources.Distribution object and + `dist.has_metadata('METADATA')` returns True but + `dist.get_metadata('METADATA')` returns None (and similarly for + "PKG-INFO"). + """ + + def __init__(self, dist, metadata_name): + # type: (Distribution, str) -> None + """ + :param dist: A Distribution object. + :param metadata_name: The name of the metadata being accessed + (can be "METADATA" or "PKG-INFO"). + """ + self.dist = dist + self.metadata_name = metadata_name + + def __str__(self): + # type: () -> str + # Use `dist` in the error message because its stringification + # includes more information, like the version and location. + return ( + 'None {} metadata found for distribution: {}'.format( + self.metadata_name, self.dist, + ) + ) + + +class DistributionNotFound(InstallationError): + """Raised when a distribution cannot be found to satisfy a requirement""" + + +class RequirementsFileParseError(InstallationError): + """Raised when a general error occurs parsing a requirements file line.""" + + +class BestVersionAlreadyInstalled(PipError): + """Raised when the most up-to-date version of a package is already + installed.""" + + +class BadCommand(PipError): + """Raised when virtualenv or a command is not found""" + + +class CommandError(PipError): + """Raised when there is an error in command-line arguments""" + + +class PreviousBuildDirError(PipError): + """Raised when there's a previous conflicting build directory""" + + +class InvalidWheelFilename(InstallationError): + """Invalid wheel filename.""" + + +class UnsupportedWheel(InstallationError): + """Unsupported wheel.""" + + +class HashErrors(InstallationError): + """Multiple HashError instances rolled into one for reporting""" + + def __init__(self): + self.errors = [] + + def append(self, error): + self.errors.append(error) + + def __str__(self): + lines = [] + self.errors.sort(key=lambda e: e.order) + for cls, errors_of_cls in groupby(self.errors, lambda e: e.__class__): + lines.append(cls.head) + lines.extend(e.body() for e in errors_of_cls) + if lines: + return '\n'.join(lines) + + def __nonzero__(self): + return bool(self.errors) + + def __bool__(self): + return self.__nonzero__() + + +class HashError(InstallationError): + """ + A failure to verify a package against known-good hashes + + :cvar order: An int sorting hash exception classes by difficulty of + recovery (lower being harder), so the user doesn't bother fretting + about unpinned packages when he has deeper issues, like VCS + dependencies, to deal with. Also keeps error reports in a + deterministic order. + :cvar head: A section heading for display above potentially many + exceptions of this kind + :ivar req: The InstallRequirement that triggered this error. This is + pasted on after the exception is instantiated, because it's not + typically available earlier. + + """ + req = None # type: Optional[InstallRequirement] + head = '' + + def body(self): + """Return a summary of me for display under the heading. + + This default implementation simply prints a description of the + triggering requirement. + + :param req: The InstallRequirement that provoked this error, with + populate_link() having already been called + + """ + return ' %s' % self._requirement_name() + + def __str__(self): + return '%s\n%s' % (self.head, self.body()) + + def _requirement_name(self): + """Return a description of the requirement that triggered me. + + This default implementation returns long description of the req, with + line numbers + + """ + return str(self.req) if self.req else 'unknown package' + + +class VcsHashUnsupported(HashError): + """A hash was provided for a version-control-system-based requirement, but + we don't have a method for hashing those.""" + + order = 0 + head = ("Can't verify hashes for these requirements because we don't " + "have a way to hash version control repositories:") + + +class DirectoryUrlHashUnsupported(HashError): + """A hash was provided for a version-control-system-based requirement, but + we don't have a method for hashing those.""" + + order = 1 + head = ("Can't verify hashes for these file:// requirements because they " + "point to directories:") + + +class HashMissing(HashError): + """A hash was needed for a requirement but is absent.""" + + order = 2 + head = ('Hashes are required in --require-hashes mode, but they are ' + 'missing from some requirements. Here is a list of those ' + 'requirements along with the hashes their downloaded archives ' + 'actually had. Add lines like these to your requirements files to ' + 'prevent tampering. (If you did not enable --require-hashes ' + 'manually, note that it turns on automatically when any package ' + 'has a hash.)') + + def __init__(self, gotten_hash): + """ + :param gotten_hash: The hash of the (possibly malicious) archive we + just downloaded + """ + self.gotten_hash = gotten_hash + + def body(self): + # Dodge circular import. + from pip._internal.utils.hashes import FAVORITE_HASH + + package = None + if self.req: + # In the case of URL-based requirements, display the original URL + # seen in the requirements file rather than the package name, + # so the output can be directly copied into the requirements file. + package = (self.req.original_link if self.req.original_link + # In case someone feeds something downright stupid + # to InstallRequirement's constructor. + else getattr(self.req, 'req', None)) + return ' %s --hash=%s:%s' % (package or 'unknown package', + FAVORITE_HASH, + self.gotten_hash) + + +class HashUnpinned(HashError): + """A requirement had a hash specified but was not pinned to a specific + version.""" + + order = 3 + head = ('In --require-hashes mode, all requirements must have their ' + 'versions pinned with ==. These do not:') + + +class HashMismatch(HashError): + """ + Distribution file hash values don't match. + + :ivar package_name: The name of the package that triggered the hash + mismatch. Feel free to write to this after the exception is raise to + improve its error message. + + """ + order = 4 + head = ('THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS ' + 'FILE. If you have updated the package versions, please update ' + 'the hashes. Otherwise, examine the package contents carefully; ' + 'someone may have tampered with them.') + + def __init__(self, allowed, gots): + """ + :param allowed: A dict of algorithm names pointing to lists of allowed + hex digests + :param gots: A dict of algorithm names pointing to hashes we + actually got from the files under suspicion + """ + self.allowed = allowed + self.gots = gots + + def body(self): + return ' %s:\n%s' % (self._requirement_name(), + self._hash_comparison()) + + def _hash_comparison(self): + """ + Return a comparison of actual and expected hash values. + + Example:: + + Expected sha256 abcdeabcdeabcdeabcdeabcdeabcdeabcdeabcdeabcde + or 123451234512345123451234512345123451234512345 + Got bcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdefbcdef + + """ + def hash_then_or(hash_name): + # For now, all the decent hashes have 6-char names, so we can get + # away with hard-coding space literals. + return chain([hash_name], repeat(' or')) + + lines = [] + for hash_name, expecteds in iteritems(self.allowed): + prefix = hash_then_or(hash_name) + lines.extend((' Expected %s %s' % (next(prefix), e)) + for e in expecteds) + lines.append(' Got %s\n' % + self.gots[hash_name].hexdigest()) + return '\n'.join(lines) + + +class UnsupportedPythonVersion(InstallationError): + """Unsupported python version according to Requires-Python package + metadata.""" + + +class ConfigurationFileCouldNotBeLoaded(ConfigurationError): + """When there are errors while loading a configuration file + """ + + def __init__(self, reason="could not be loaded", fname=None, error=None): + super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) + self.reason = reason + self.fname = fname + self.error = error + + def __str__(self): + if self.fname is not None: + message_part = " in {}.".format(self.fname) + else: + assert self.error is not None + message_part = ".\n{}\n".format(self.error.message) + return "Configuration file {}{}".format(self.reason, message_part) diff --git a/ubuntu/venv/pip/_internal/index/__init__.py b/ubuntu/venv/pip/_internal/index/__init__.py new file mode 100644 index 0000000..7a17b7b --- /dev/null +++ b/ubuntu/venv/pip/_internal/index/__init__.py @@ -0,0 +1,2 @@ +"""Index interaction code +""" diff --git a/ubuntu/venv/pip/_internal/index/collector.py b/ubuntu/venv/pip/_internal/index/collector.py new file mode 100644 index 0000000..8330793 --- /dev/null +++ b/ubuntu/venv/pip/_internal/index/collector.py @@ -0,0 +1,544 @@ +""" +The main purpose of this module is to expose LinkCollector.collect_links(). +""" + +import cgi +import itertools +import logging +import mimetypes +import os +from collections import OrderedDict + +from pip._vendor import html5lib, requests +from pip._vendor.distlib.compat import unescape +from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.models.link import Link +from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pip._internal.utils.misc import redact_auth_from_url +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url, url_to_path +from pip._internal.vcs import is_url, vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Callable, Iterable, List, MutableMapping, Optional, Sequence, Tuple, + Union, + ) + import xml.etree.ElementTree + + from pip._vendor.requests import Response + + from pip._internal.models.search_scope import SearchScope + from pip._internal.network.session import PipSession + + HTMLElement = xml.etree.ElementTree.Element + ResponseHeaders = MutableMapping[str, str] + + +logger = logging.getLogger(__name__) + + +def _match_vcs_scheme(url): + # type: (str) -> Optional[str] + """Look for VCS schemes in the URL. + + Returns the matched VCS scheme, or None if there's no match. + """ + for scheme in vcs.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + return scheme + return None + + +def _is_url_like_archive(url): + # type: (str) -> bool + """Return whether the URL looks like an archive. + """ + filename = Link(url).filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + return True + return False + + +class _NotHTML(Exception): + def __init__(self, content_type, request_desc): + # type: (str, str) -> None + super(_NotHTML, self).__init__(content_type, request_desc) + self.content_type = content_type + self.request_desc = request_desc + + +def _ensure_html_header(response): + # type: (Response) -> None + """Check the Content-Type header to ensure the response contains HTML. + + Raises `_NotHTML` if the content type is not text/html. + """ + content_type = response.headers.get("Content-Type", "") + if not content_type.lower().startswith("text/html"): + raise _NotHTML(content_type, response.request.method) + + +class _NotHTTP(Exception): + pass + + +def _ensure_html_response(url, session): + # type: (str, PipSession) -> None + """Send a HEAD request to the URL, and ensure the response contains HTML. + + Raises `_NotHTTP` if the URL is not available for a HEAD request, or + `_NotHTML` if the content type is not text/html. + """ + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + raise _NotHTTP() + + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() + + _ensure_html_header(resp) + + +def _get_html_response(url, session): + # type: (str, PipSession) -> Response + """Access an HTML page with GET, and return the response. + + This consists of three parts: + + 1. If the URL looks suspiciously like an archive, send a HEAD first to + check the Content-Type is HTML, to avoid downloading a large file. + Raise `_NotHTTP` if the content type cannot be determined, or + `_NotHTML` if it is not HTML. + 2. Actually perform the request. Raise HTTP exceptions on network failures. + 3. Check the Content-Type header to make sure we got HTML, and raise + `_NotHTML` otherwise. + """ + if _is_url_like_archive(url): + _ensure_html_response(url, session=session) + + logger.debug('Getting page %s', redact_auth_from_url(url)) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() + + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + _ensure_html_header(resp) + + return resp + + +def _get_encoding_from_headers(headers): + # type: (ResponseHeaders) -> Optional[str] + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +def _determine_base_url(document, page_url): + # type: (HTMLElement, str) -> str + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _clean_link(url): + # type: (str) -> str + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + # Split the URL into parts according to the general structure + # `scheme://netloc/path;parameters?query#fragment`. Note that the + # `netloc` can be empty and the URI will then refer to a local + # filesystem path. + result = urllib_parse.urlparse(url) + # In both cases below we unquote prior to quoting to make sure + # nothing is double quoted. + if result.netloc == "": + # On Windows the path part might contain a drive letter which + # should not be quoted. On Linux where drive letters do not + # exist, the colon should be quoted. We rely on urllib.request + # to do the right thing here. + path = urllib_request.pathname2url( + urllib_request.url2pathname(result.path)) + else: + # In addition to the `/` character we protect `@` so that + # revision strings in VCS URLs are properly parsed. + path = urllib_parse.quote(urllib_parse.unquote(result.path), safe="/@") + return urllib_parse.urlunparse(result._replace(path=path)) + + +def _create_link_from_element( + anchor, # type: HTMLElement + page_url, # type: str + base_url, # type: str +): + # type: (...) -> Optional[Link] + """ + Convert an anchor element in a simple repository page to a Link. + """ + href = anchor.get("href") + if not href: + return None + + url = _clean_link(urllib_parse.urljoin(base_url, href)) + pyrequire = anchor.get('data-requires-python') + pyrequire = unescape(pyrequire) if pyrequire else None + + yanked_reason = anchor.get('data-yanked') + if yanked_reason: + # This is a unicode string in Python 2 (and 3). + yanked_reason = unescape(yanked_reason) + + link = Link( + url, + comes_from=page_url, + requires_python=pyrequire, + yanked_reason=yanked_reason, + ) + + return link + + +def parse_links(page): + # type: (HTMLPage) -> Iterable[Link] + """ + Parse an HTML document, and yield its anchor elements as Link objects. + """ + document = html5lib.parse( + page.content, + transport_encoding=page.encoding, + namespaceHTMLElements=False, + ) + + url = page.url + base_url = _determine_base_url(document, url) + for anchor in document.findall(".//a"): + link = _create_link_from_element( + anchor, + page_url=url, + base_url=base_url, + ) + if link is None: + continue + yield link + + +class HTMLPage(object): + """Represents one page, along with its URL""" + + def __init__( + self, + content, # type: bytes + encoding, # type: Optional[str] + url, # type: str + ): + # type: (...) -> None + """ + :param encoding: the encoding to decode the given content. + :param url: the URL from which the HTML was downloaded. + """ + self.content = content + self.encoding = encoding + self.url = url + + def __str__(self): + # type: () -> str + return redact_auth_from_url(self.url) + + +def _handle_get_page_fail( + link, # type: Link + reason, # type: Union[str, Exception] + meth=None # type: Optional[Callable[..., None]] +): + # type: (...) -> None + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _make_html_page(response): + # type: (Response) -> HTMLPage + encoding = _get_encoding_from_headers(response.headers) + return HTMLPage(response.content, encoding=encoding, url=response.url) + + +def _get_html_page(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" + ) + + url = link.url.split('#', 1)[0] + + # Check for VCS schemes that do not support lookup as web pages. + vcs_scheme = _match_vcs_scheme(url) + if vcs_scheme: + logger.debug('Cannot look at %s URL %s', vcs_scheme, link) + return None + + # Tack index.html onto file:// URLs that point to directories + scheme, _, path, _, _, _ = urllib_parse.urlparse(url) + if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) + + try: + resp = _get_html_response(url, session=session) + except _NotHTTP: + logger.debug( + 'Skipping page %s because it looks like an archive, and cannot ' + 'be checked by HEAD.', link, + ) + except _NotHTML as exc: + logger.debug( + 'Skipping page %s because the %s request got Content-Type: %s', + link, exc.request_desc, exc.content_type, + ) + except HTTPError as exc: + _handle_get_page_fail(link, exc) + except RetryError as exc: + _handle_get_page_fail(link, exc) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc) + except requests.Timeout: + _handle_get_page_fail(link, "timed out") + else: + return _make_html_page(resp) + return None + + +def _remove_duplicate_links(links): + # type: (Iterable[Link]) -> List[Link] + """ + Return a list of links, with duplicates removed and ordering preserved. + """ + # We preserve the ordering when removing duplicates because we can. + return list(OrderedDict.fromkeys(links)) + + +def group_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] + """ + Divide a list of locations into two groups: "files" (archives) and "urls." + + :return: A pair of lists (files, urls). + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate list + def sort_path(path): + # type: (str) -> None + url = path_to_url(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + + is_local_path = os.path.exists(url) + is_file_url = url.startswith('file:') + + if is_local_path or is_file_url: + if is_local_path: + path = url + else: + path = url_to_path(url) + if os.path.isdir(path): + if expand_dir: + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif is_file_url: + urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) + elif os.path.isfile(path): + sort_path(path) + else: + logger.warning( + "Url '%s' is ignored: it is neither a file " + "nor a directory.", url, + ) + elif is_url(url): + # Only add url with clear scheme + urls.append(url) + else: + logger.warning( + "Url '%s' is ignored. It is either a non-existing " + "path or lacks a specific scheme.", url, + ) + + return files, urls + + +class CollectedLinks(object): + + """ + Encapsulates the return value of a call to LinkCollector.collect_links(). + + The return value includes both URLs to project pages containing package + links, as well as individual package Link objects collected from other + sources. + + This info is stored separately as: + + (1) links from the configured file locations, + (2) links from the configured find_links, and + (3) urls to HTML project pages, as described by the PEP 503 simple + repository API. + """ + + def __init__( + self, + files, # type: List[Link] + find_links, # type: List[Link] + project_urls, # type: List[Link] + ): + # type: (...) -> None + """ + :param files: Links from file locations. + :param find_links: Links from find_links. + :param project_urls: URLs to HTML project pages, as described by + the PEP 503 simple repository API. + """ + self.files = files + self.find_links = find_links + self.project_urls = project_urls + + +class LinkCollector(object): + + """ + Responsible for collecting Link objects from all configured locations, + making network requests as needed. + + The class's main method is its collect_links() method. + """ + + def __init__( + self, + session, # type: PipSession + search_scope, # type: SearchScope + ): + # type: (...) -> None + self.search_scope = search_scope + self.session = session + + @property + def find_links(self): + # type: () -> List[str] + return self.search_scope.find_links + + def fetch_page(self, location): + # type: (Link) -> Optional[HTMLPage] + """ + Fetch an HTML page containing package links. + """ + return _get_html_page(location, session=self.session) + + def collect_links(self, project_name): + # type: (str) -> CollectedLinks + """Find all available links for the given project name. + + :return: All the Link objects (unfiltered), as a CollectedLinks object. + """ + search_scope = self.search_scope + index_locations = search_scope.get_index_urls_locations(project_name) + index_file_loc, index_url_loc = group_locations(index_locations) + fl_file_loc, fl_url_loc = group_locations( + self.find_links, expand_dir=True, + ) + + file_links = [ + Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) + ] + + # We trust every directly linked archive in find_links + find_link_links = [Link(url, '-f') for url in self.find_links] + + # We trust every url that the user has given us whether it was given + # via --index-url or --find-links. + # We want to filter out anything that does not have a secure origin. + url_locations = [ + link for link in itertools.chain( + (Link(url) for url in index_url_loc), + (Link(url) for url in fl_url_loc), + ) + if self.session.is_secure_origin(link) + ] + + url_locations = _remove_duplicate_links(url_locations) + lines = [ + '{} location(s) to search for versions of {}:'.format( + len(url_locations), project_name, + ), + ] + for link in url_locations: + lines.append('* {}'.format(link)) + logger.debug('\n'.join(lines)) + + return CollectedLinks( + files=file_links, + find_links=find_link_links, + project_urls=url_locations, + ) diff --git a/ubuntu/venv/pip/_internal/index/package_finder.py b/ubuntu/venv/pip/_internal/index/package_finder.py new file mode 100644 index 0000000..a74d78d --- /dev/null +++ b/ubuntu/venv/pip/_internal/index/package_finder.py @@ -0,0 +1,1013 @@ +"""Routines related to PyPI, indexes""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import re + +from pip._vendor.packaging import specifiers +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.exceptions import ( + BestVersionAlreadyInstalled, + DistributionNotFound, + InvalidWheelFilename, + UnsupportedWheel, +) +from pip._internal.index.collector import parse_links +from pip._internal.models.candidate import InstallationCandidate +from pip._internal.models.format_control import FormatControl +from pip._internal.models.link import Link +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.models.target_python import TargetPython +from pip._internal.models.wheel import Wheel +from pip._internal.utils.filetypes import WHEEL_EXTENSION +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import build_netloc +from pip._internal.utils.packaging import check_requires_python +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS +from pip._internal.utils.urls import url_to_path + +if MYPY_CHECK_RUNNING: + from typing import ( + FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union, + ) + + from pip._vendor.packaging.tags import Tag + from pip._vendor.packaging.version import _BaseVersion + + from pip._internal.index.collector import LinkCollector + from pip._internal.models.search_scope import SearchScope + from pip._internal.req import InstallRequirement + from pip._internal.utils.hashes import Hashes + + BuildTag = Union[Tuple[()], Tuple[int, str]] + CandidateSortingKey = ( + Tuple[int, int, int, _BaseVersion, BuildTag, Optional[int]] + ) + + +__all__ = ['FormatControl', 'BestCandidateResult', 'PackageFinder'] + + +logger = logging.getLogger(__name__) + + +def _check_link_requires_python( + link, # type: Link + version_info, # type: Tuple[int, int, int] + ignore_requires_python=False, # type: bool +): + # type: (...) -> bool + """ + Return whether the given Python version is compatible with a link's + "Requires-Python" value. + + :param version_info: A 3-tuple of ints representing the Python + major-minor-micro version to check. + :param ignore_requires_python: Whether to ignore the "Requires-Python" + value if the given Python version isn't compatible. + """ + try: + is_compatible = check_requires_python( + link.requires_python, version_info=version_info, + ) + except specifiers.InvalidSpecifier: + logger.debug( + "Ignoring invalid Requires-Python (%r) for link: %s", + link.requires_python, link, + ) + else: + if not is_compatible: + version = '.'.join(map(str, version_info)) + if not ignore_requires_python: + logger.debug( + 'Link requires a different Python (%s not in: %r): %s', + version, link.requires_python, link, + ) + return False + + logger.debug( + 'Ignoring failed Requires-Python check (%s not in: %r) ' + 'for link: %s', + version, link.requires_python, link, + ) + + return True + + +class LinkEvaluator(object): + + """ + Responsible for evaluating links for a particular project. + """ + + _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') + + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + def __init__( + self, + project_name, # type: str + canonical_name, # type: str + formats, # type: FrozenSet[str] + target_python, # type: TargetPython + allow_yanked, # type: bool + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> None + """ + :param project_name: The user supplied package name. + :param canonical_name: The canonical package name. + :param formats: The formats allowed for this package. Should be a set + with 'binary' or 'source' or both in it. + :param target_python: The target Python interpreter to use when + evaluating link compatibility. This is used, for example, to + check wheel compatibility, as well as when checking the Python + version, e.g. the Python version embedded in a link filename + (or egg fragment) and against an HTML link's optional PEP 503 + "data-requires-python" attribute. + :param allow_yanked: Whether files marked as yanked (in the sense + of PEP 592) are permitted to be candidates for install. + :param ignore_requires_python: Whether to ignore incompatible + PEP 503 "data-requires-python" values in HTML links. Defaults + to False. + """ + if ignore_requires_python is None: + ignore_requires_python = False + + self._allow_yanked = allow_yanked + self._canonical_name = canonical_name + self._ignore_requires_python = ignore_requires_python + self._formats = formats + self._target_python = target_python + + self.project_name = project_name + + def evaluate_link(self, link): + # type: (Link) -> Tuple[bool, Optional[Text]] + """ + Determine whether a link is a candidate for installation. + + :return: A tuple (is_candidate, result), where `result` is (1) a + version string if `is_candidate` is True, and (2) if + `is_candidate` is False, an optional string to log the reason + the link fails to qualify. + """ + version = None + if link.is_yanked and not self._allow_yanked: + reason = link.yanked_reason or '' + # Mark this as a unicode string to prevent "UnicodeEncodeError: + # 'ascii' codec can't encode character" in Python 2 when + # the reason contains non-ascii characters. + return (False, u'yanked for reason: {}'.format(reason)) + + if link.egg_fragment: + egg_info = link.egg_fragment + ext = link.ext + else: + egg_info, ext = link.splitext() + if not ext: + return (False, 'not a file') + if ext not in SUPPORTED_EXTENSIONS: + return (False, 'unsupported archive format: %s' % ext) + if "binary" not in self._formats and ext == WHEEL_EXTENSION: + reason = 'No binaries permitted for %s' % self.project_name + return (False, reason) + if "macosx10" in link.path and ext == '.zip': + return (False, 'macosx10 one') + if ext == WHEEL_EXTENSION: + try: + wheel = Wheel(link.filename) + except InvalidWheelFilename: + return (False, 'invalid wheel filename') + if canonicalize_name(wheel.name) != self._canonical_name: + reason = 'wrong project name (not %s)' % self.project_name + return (False, reason) + + supported_tags = self._target_python.get_tags() + if not wheel.supported(supported_tags): + # Include the wheel's tags in the reason string to + # simplify troubleshooting compatibility issues. + file_tags = wheel.get_formatted_file_tags() + reason = ( + "none of the wheel's tags match: {}".format( + ', '.join(file_tags) + ) + ) + return (False, reason) + + version = wheel.version + + # This should be up by the self.ok_binary check, but see issue 2700. + if "source" not in self._formats and ext != WHEEL_EXTENSION: + return (False, 'No sources permitted for %s' % self.project_name) + + if not version: + version = _extract_version_from_fragment( + egg_info, self._canonical_name, + ) + if not version: + return ( + False, 'Missing project version for %s' % self.project_name, + ) + + match = self._py_version_re.search(version) + if match: + version = version[:match.start()] + py_version = match.group(1) + if py_version != self._target_python.py_version: + return (False, 'Python version is incorrect') + + supports_python = _check_link_requires_python( + link, version_info=self._target_python.py_version_info, + ignore_requires_python=self._ignore_requires_python, + ) + if not supports_python: + # Return None for the reason text to suppress calling + # _log_skipped_link(). + return (False, None) + + logger.debug('Found link %s, version: %s', link, version) + + return (True, version) + + +def filter_unallowed_hashes( + candidates, # type: List[InstallationCandidate] + hashes, # type: Hashes + project_name, # type: str +): + # type: (...) -> List[InstallationCandidate] + """ + Filter out candidates whose hashes aren't allowed, and return a new + list of candidates. + + If at least one candidate has an allowed hash, then all candidates with + either an allowed hash or no hash specified are returned. Otherwise, + the given candidates are returned. + + Including the candidates with no hash specified when there is a match + allows a warning to be logged if there is a more preferred candidate + with no hash specified. Returning all candidates in the case of no + matches lets pip report the hash of the candidate that would otherwise + have been installed (e.g. permitting the user to more easily update + their requirements file with the desired hash). + """ + if not hashes: + logger.debug( + 'Given no hashes to check %s links for project %r: ' + 'discarding no candidates', + len(candidates), + project_name, + ) + # Make sure we're not returning back the given value. + return list(candidates) + + matches_or_no_digest = [] + # Collect the non-matches for logging purposes. + non_matches = [] + match_count = 0 + for candidate in candidates: + link = candidate.link + if not link.has_hash: + pass + elif link.is_hash_allowed(hashes=hashes): + match_count += 1 + else: + non_matches.append(candidate) + continue + + matches_or_no_digest.append(candidate) + + if match_count: + filtered = matches_or_no_digest + else: + # Make sure we're not returning back the given value. + filtered = list(candidates) + + if len(filtered) == len(candidates): + discard_message = 'discarding no candidates' + else: + discard_message = 'discarding {} non-matches:\n {}'.format( + len(non_matches), + '\n '.join(str(candidate.link) for candidate in non_matches) + ) + + logger.debug( + 'Checked %s links for project %r against %s hashes ' + '(%s matches, %s no digest): %s', + len(candidates), + project_name, + hashes.digest_count, + match_count, + len(matches_or_no_digest) - match_count, + discard_message + ) + + return filtered + + +class CandidatePreferences(object): + + """ + Encapsulates some of the preferences for filtering and sorting + InstallationCandidate objects. + """ + + def __init__( + self, + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + ): + # type: (...) -> None + """ + :param allow_all_prereleases: Whether to allow all pre-releases. + """ + self.allow_all_prereleases = allow_all_prereleases + self.prefer_binary = prefer_binary + + +class BestCandidateResult(object): + """A collection of candidates, returned by `PackageFinder.find_best_candidate`. + + This class is only intended to be instantiated by CandidateEvaluator's + `compute_best_candidate()` method. + """ + + def __init__( + self, + candidates, # type: List[InstallationCandidate] + applicable_candidates, # type: List[InstallationCandidate] + best_candidate, # type: Optional[InstallationCandidate] + ): + # type: (...) -> None + """ + :param candidates: A sequence of all available candidates found. + :param applicable_candidates: The applicable candidates. + :param best_candidate: The most preferred candidate found, or None + if no applicable candidates were found. + """ + assert set(applicable_candidates) <= set(candidates) + + if best_candidate is None: + assert not applicable_candidates + else: + assert best_candidate in applicable_candidates + + self._applicable_candidates = applicable_candidates + self._candidates = candidates + + self.best_candidate = best_candidate + + def iter_all(self): + # type: () -> Iterable[InstallationCandidate] + """Iterate through all candidates. + """ + return iter(self._candidates) + + def iter_applicable(self): + # type: () -> Iterable[InstallationCandidate] + """Iterate through the applicable candidates. + """ + return iter(self._applicable_candidates) + + +class CandidateEvaluator(object): + + """ + Responsible for filtering and sorting candidates for installation based + on what tags are valid. + """ + + @classmethod + def create( + cls, + project_name, # type: str + target_python=None, # type: Optional[TargetPython] + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> CandidateEvaluator + """Create a CandidateEvaluator object. + + :param target_python: The target Python interpreter to use when + checking compatibility. If None (the default), a TargetPython + object will be constructed from the running Python. + :param specifier: An optional object implementing `filter` + (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable + versions. + :param hashes: An optional collection of allowed hashes. + """ + if target_python is None: + target_python = TargetPython() + if specifier is None: + specifier = specifiers.SpecifierSet() + + supported_tags = target_python.get_tags() + + return cls( + project_name=project_name, + supported_tags=supported_tags, + specifier=specifier, + prefer_binary=prefer_binary, + allow_all_prereleases=allow_all_prereleases, + hashes=hashes, + ) + + def __init__( + self, + project_name, # type: str + supported_tags, # type: List[Tag] + specifier, # type: specifiers.BaseSpecifier + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> None + """ + :param supported_tags: The PEP 425 tags supported by the target + Python in order of preference (most preferred first). + """ + self._allow_all_prereleases = allow_all_prereleases + self._hashes = hashes + self._prefer_binary = prefer_binary + self._project_name = project_name + self._specifier = specifier + self._supported_tags = supported_tags + + def get_applicable_candidates( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> List[InstallationCandidate] + """ + Return the applicable candidates from a list of candidates. + """ + # Using None infers from the specifier instead. + allow_prereleases = self._allow_all_prereleases or None + specifier = self._specifier + versions = { + str(v) for v in specifier.filter( + # We turn the version object into a str here because otherwise + # when we're debundled but setuptools isn't, Python will see + # packaging.version.Version and + # pkg_resources._vendor.packaging.version.Version as different + # types. This way we'll use a str as a common data interchange + # format. If we stop using the pkg_resources provided specifier + # and start using our own, we can drop the cast to str(). + (str(c.version) for c in candidates), + prereleases=allow_prereleases, + ) + } + + # Again, converting version to str to deal with debundling. + applicable_candidates = [ + c for c in candidates if str(c.version) in versions + ] + + filtered_applicable_candidates = filter_unallowed_hashes( + candidates=applicable_candidates, + hashes=self._hashes, + project_name=self._project_name, + ) + + return sorted(filtered_applicable_candidates, key=self._sort_key) + + def _sort_key(self, candidate): + # type: (InstallationCandidate) -> CandidateSortingKey + """ + Function to pass as the `key` argument to a call to sorted() to sort + InstallationCandidates by preference. + + Returns a tuple such that tuples sorting as greater using Python's + default comparison operator are more preferred. + + The preference is as follows: + + First and foremost, candidates with allowed (matching) hashes are + always preferred over candidates without matching hashes. This is + because e.g. if the only candidate with an allowed hash is yanked, + we still want to use that candidate. + + Second, excepting hash considerations, candidates that have been + yanked (in the sense of PEP 592) are always less preferred than + candidates that haven't been yanked. Then: + + If not finding wheels, they are sorted by version only. + If finding wheels, then the sort order is by version, then: + 1. existing installs + 2. wheels ordered via Wheel.support_index_min(self._supported_tags) + 3. source archives + If prefer_binary was set, then all wheels are sorted above sources. + + Note: it was considered to embed this logic into the Link + comparison operators, but then different sdist links + with the same version, would have to be considered equal + """ + valid_tags = self._supported_tags + support_num = len(valid_tags) + build_tag = () # type: BuildTag + binary_preference = 0 + link = candidate.link + if link.is_wheel: + # can raise InvalidWheelFilename + wheel = Wheel(link.filename) + if not wheel.supported(valid_tags): + raise UnsupportedWheel( + "%s is not a supported wheel for this platform. It " + "can't be sorted." % wheel.filename + ) + if self._prefer_binary: + binary_preference = 1 + pri = -(wheel.support_index_min(valid_tags)) + if wheel.build_tag is not None: + match = re.match(r'^(\d+)(.*)$', wheel.build_tag) + build_tag_groups = match.groups() + build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) + else: # sdist + pri = -(support_num) + has_allowed_hash = int(link.is_hash_allowed(self._hashes)) + yank_value = -1 * int(link.is_yanked) # -1 for yanked. + return ( + has_allowed_hash, yank_value, binary_preference, candidate.version, + build_tag, pri, + ) + + def sort_best_candidate( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> Optional[InstallationCandidate] + """ + Return the best candidate per the instance's sort order, or None if + no candidate is acceptable. + """ + if not candidates: + return None + + best_candidate = max(candidates, key=self._sort_key) + + # Log a warning per PEP 592 if necessary before returning. + link = best_candidate.link + if link.is_yanked: + reason = link.yanked_reason or '' + msg = ( + # Mark this as a unicode string to prevent + # "UnicodeEncodeError: 'ascii' codec can't encode character" + # in Python 2 when the reason contains non-ascii characters. + u'The candidate selected for download or install is a ' + 'yanked version: {candidate}\n' + 'Reason for being yanked: {reason}' + ).format(candidate=best_candidate, reason=reason) + logger.warning(msg) + + return best_candidate + + def compute_best_candidate( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> BestCandidateResult + """ + Compute and return a `BestCandidateResult` instance. + """ + applicable_candidates = self.get_applicable_candidates(candidates) + + best_candidate = self.sort_best_candidate(applicable_candidates) + + return BestCandidateResult( + candidates, + applicable_candidates=applicable_candidates, + best_candidate=best_candidate, + ) + + +class PackageFinder(object): + """This finds packages. + + This is meant to match easy_install's technique for looking for + packages, by reading pages and looking for appropriate links. + """ + + def __init__( + self, + link_collector, # type: LinkCollector + target_python, # type: TargetPython + allow_yanked, # type: bool + format_control=None, # type: Optional[FormatControl] + candidate_prefs=None, # type: CandidatePreferences + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> None + """ + This constructor is primarily meant to be used by the create() class + method and from tests. + + :param format_control: A FormatControl object, used to control + the selection of source packages / binary packages when consulting + the index and links. + :param candidate_prefs: Options to use when creating a + CandidateEvaluator object. + """ + if candidate_prefs is None: + candidate_prefs = CandidatePreferences() + + format_control = format_control or FormatControl(set(), set()) + + self._allow_yanked = allow_yanked + self._candidate_prefs = candidate_prefs + self._ignore_requires_python = ignore_requires_python + self._link_collector = link_collector + self._target_python = target_python + + self.format_control = format_control + + # These are boring links that have already been logged somehow. + self._logged_links = set() # type: Set[Link] + + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + @classmethod + def create( + cls, + link_collector, # type: LinkCollector + selection_prefs, # type: SelectionPreferences + target_python=None, # type: Optional[TargetPython] + ): + # type: (...) -> PackageFinder + """Create a PackageFinder. + + :param selection_prefs: The candidate selection preferences, as a + SelectionPreferences object. + :param target_python: The target Python interpreter to use when + checking compatibility. If None (the default), a TargetPython + object will be constructed from the running Python. + """ + if target_python is None: + target_python = TargetPython() + + candidate_prefs = CandidatePreferences( + prefer_binary=selection_prefs.prefer_binary, + allow_all_prereleases=selection_prefs.allow_all_prereleases, + ) + + return cls( + candidate_prefs=candidate_prefs, + link_collector=link_collector, + target_python=target_python, + allow_yanked=selection_prefs.allow_yanked, + format_control=selection_prefs.format_control, + ignore_requires_python=selection_prefs.ignore_requires_python, + ) + + @property + def search_scope(self): + # type: () -> SearchScope + return self._link_collector.search_scope + + @search_scope.setter + def search_scope(self, search_scope): + # type: (SearchScope) -> None + self._link_collector.search_scope = search_scope + + @property + def find_links(self): + # type: () -> List[str] + return self._link_collector.find_links + + @property + def index_urls(self): + # type: () -> List[str] + return self.search_scope.index_urls + + @property + def trusted_hosts(self): + # type: () -> Iterable[str] + for host_port in self._link_collector.session.pip_trusted_origins: + yield build_netloc(*host_port) + + @property + def allow_all_prereleases(self): + # type: () -> bool + return self._candidate_prefs.allow_all_prereleases + + def set_allow_all_prereleases(self): + # type: () -> None + self._candidate_prefs.allow_all_prereleases = True + + def make_link_evaluator(self, project_name): + # type: (str) -> LinkEvaluator + canonical_name = canonicalize_name(project_name) + formats = self.format_control.get_allowed_formats(canonical_name) + + return LinkEvaluator( + project_name=project_name, + canonical_name=canonical_name, + formats=formats, + target_python=self._target_python, + allow_yanked=self._allow_yanked, + ignore_requires_python=self._ignore_requires_python, + ) + + def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] + """ + Returns elements of links in order, non-egg links first, egg links + second, while eliminating duplicates + """ + eggs, no_eggs = [], [] + seen = set() # type: Set[Link] + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + return no_eggs + eggs + + def _log_skipped_link(self, link, reason): + # type: (Link, Text) -> None + if link not in self._logged_links: + # Mark this as a unicode string to prevent "UnicodeEncodeError: + # 'ascii' codec can't encode character" in Python 2 when + # the reason contains non-ascii characters. + # Also, put the link at the end so the reason is more visible + # and because the link string is usually very long. + logger.debug(u'Skipping link: %s: %s', reason, link) + self._logged_links.add(link) + + def get_install_candidate(self, link_evaluator, link): + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] + """ + If the link is a candidate for install, convert it to an + InstallationCandidate and return it. Otherwise, return None. + """ + is_candidate, result = link_evaluator.evaluate_link(link) + if not is_candidate: + if result: + self._log_skipped_link(link, reason=result) + return None + + return InstallationCandidate( + name=link_evaluator.project_name, + link=link, + # Convert the Text result to str since InstallationCandidate + # accepts str. + version=str(result), + ) + + def evaluate_links(self, link_evaluator, links): + # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] + """ + Convert links that are candidates to InstallationCandidate objects. + """ + candidates = [] + for link in self._sort_links(links): + candidate = self.get_install_candidate(link_evaluator, link) + if candidate is not None: + candidates.append(candidate) + + return candidates + + def process_project_url(self, project_url, link_evaluator): + # type: (Link, LinkEvaluator) -> List[InstallationCandidate] + logger.debug( + 'Fetching project page and analyzing links: %s', project_url, + ) + html_page = self._link_collector.fetch_page(project_url) + if html_page is None: + return [] + + page_links = list(parse_links(html_page)) + + with indent_log(): + package_links = self.evaluate_links( + link_evaluator, + links=page_links, + ) + + return package_links + + def find_all_candidates(self, project_name): + # type: (str) -> List[InstallationCandidate] + """Find all available InstallationCandidate for project_name + + This checks index_urls and find_links. + All versions found are returned as an InstallationCandidate list. + + See LinkEvaluator.evaluate_link() for details on which files + are accepted. + """ + collected_links = self._link_collector.collect_links(project_name) + + link_evaluator = self.make_link_evaluator(project_name) + + find_links_versions = self.evaluate_links( + link_evaluator, + links=collected_links.find_links, + ) + + page_versions = [] + for project_url in collected_links.project_urls: + package_links = self.process_project_url( + project_url, link_evaluator=link_evaluator, + ) + page_versions.extend(package_links) + + file_versions = self.evaluate_links( + link_evaluator, + links=collected_links.files, + ) + if file_versions: + file_versions.sort(reverse=True) + logger.debug( + 'Local files found: %s', + ', '.join([ + url_to_path(candidate.link.url) + for candidate in file_versions + ]) + ) + + # This is an intentional priority ordering + return file_versions + find_links_versions + page_versions + + def make_candidate_evaluator( + self, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> CandidateEvaluator + """Create a CandidateEvaluator object to use. + """ + candidate_prefs = self._candidate_prefs + return CandidateEvaluator.create( + project_name=project_name, + target_python=self._target_python, + prefer_binary=candidate_prefs.prefer_binary, + allow_all_prereleases=candidate_prefs.allow_all_prereleases, + specifier=specifier, + hashes=hashes, + ) + + def find_best_candidate( + self, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> BestCandidateResult + """Find matches for the given project and specifier. + + :param specifier: An optional object implementing `filter` + (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable + versions. + + :return: A `BestCandidateResult` instance. + """ + candidates = self.find_all_candidates(project_name) + candidate_evaluator = self.make_candidate_evaluator( + project_name=project_name, + specifier=specifier, + hashes=hashes, + ) + return candidate_evaluator.compute_best_candidate(candidates) + + def find_requirement(self, req, upgrade): + # type: (InstallRequirement, bool) -> Optional[Link] + """Try to find a Link matching req + + Expects req, an InstallRequirement and upgrade, a boolean + Returns a Link if found, + Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise + """ + hashes = req.hashes(trust_internet=False) + best_candidate_result = self.find_best_candidate( + req.name, specifier=req.specifier, hashes=hashes, + ) + best_candidate = best_candidate_result.best_candidate + + installed_version = None # type: Optional[_BaseVersion] + if req.satisfied_by is not None: + installed_version = parse_version(req.satisfied_by.version) + + def _format_versions(cand_iter): + # type: (Iterable[InstallationCandidate]) -> str + # This repeated parse_version and str() conversion is needed to + # handle different vendoring sources from pip and pkg_resources. + # If we stop using the pkg_resources provided specifier and start + # using our own, we can drop the cast to str(). + return ", ".join(sorted( + {str(c.version) for c in cand_iter}, + key=parse_version, + )) or "none" + + if installed_version is None and best_candidate is None: + logger.critical( + 'Could not find a version that satisfies the requirement %s ' + '(from versions: %s)', + req, + _format_versions(best_candidate_result.iter_all()), + ) + + raise DistributionNotFound( + 'No matching distribution found for %s' % req + ) + + best_installed = False + if installed_version and ( + best_candidate is None or + best_candidate.version <= installed_version): + best_installed = True + + if not upgrade and installed_version is not None: + if best_installed: + logger.debug( + 'Existing installed version (%s) is most up-to-date and ' + 'satisfies requirement', + installed_version, + ) + else: + logger.debug( + 'Existing installed version (%s) satisfies requirement ' + '(most up-to-date version is %s)', + installed_version, + best_candidate.version, + ) + return None + + if best_installed: + # We have an existing version, and its the best version + logger.debug( + 'Installed version (%s) is most up-to-date (past versions: ' + '%s)', + installed_version, + _format_versions(best_candidate_result.iter_applicable()), + ) + raise BestVersionAlreadyInstalled + + logger.debug( + 'Using version %s (newest of versions: %s)', + best_candidate.version, + _format_versions(best_candidate_result.iter_applicable()), + ) + return best_candidate.link + + +def _find_name_version_sep(fragment, canonical_name): + # type: (str, str) -> int + """Find the separator's index based on the package's canonical name. + + :param fragment: A + filename "fragment" (stem) or + egg fragment. + :param canonical_name: The package's canonical name. + + This function is needed since the canonicalized name does not necessarily + have the same length as the egg info's name part. An example:: + + >>> fragment = 'foo__bar-1.0' + >>> canonical_name = 'foo-bar' + >>> _find_name_version_sep(fragment, canonical_name) + 8 + """ + # Project name and version must be separated by one single dash. Find all + # occurrences of dashes; if the string in front of it matches the canonical + # name, this is the one separating the name and version parts. + for i, c in enumerate(fragment): + if c != "-": + continue + if canonicalize_name(fragment[:i]) == canonical_name: + return i + raise ValueError("{} does not match {}".format(fragment, canonical_name)) + + +def _extract_version_from_fragment(fragment, canonical_name): + # type: (str, str) -> Optional[str] + """Parse the version string from a + filename + "fragment" (stem) or egg fragment. + + :param fragment: The string to parse. E.g. foo-2.1 + :param canonical_name: The canonicalized name of the package this + belongs to. + """ + try: + version_start = _find_name_version_sep(fragment, canonical_name) + 1 + except ValueError: + return None + version = fragment[version_start:] + if not version: + return None + return version diff --git a/ubuntu/venv/pip/_internal/legacy_resolve.py b/ubuntu/venv/pip/_internal/legacy_resolve.py new file mode 100644 index 0000000..ca26912 --- /dev/null +++ b/ubuntu/venv/pip/_internal/legacy_resolve.py @@ -0,0 +1,430 @@ +"""Dependency Resolution + +The dependency resolution in pip is performed as follows: + +for top-level requirements: + a. only one spec allowed per project, regardless of conflicts or not. + otherwise a "double requirement" exception is raised + b. they override sub-dependency requirements. +for sub-dependencies + a. "first found, wins" (where the order is breadth first) +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +import logging +import sys +from collections import defaultdict +from itertools import chain + +from pip._vendor.packaging import specifiers + +from pip._internal.exceptions import ( + BestVersionAlreadyInstalled, + DistributionNotFound, + HashError, + HashErrors, + UnsupportedPythonVersion, +) +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import dist_in_usersite, normalize_version_info +from pip._internal.utils.packaging import ( + check_requires_python, + get_requires_python, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Callable, DefaultDict, List, Optional, Set, Tuple + from pip._vendor import pkg_resources + + from pip._internal.distributions import AbstractDistribution + from pip._internal.index.package_finder import PackageFinder + from pip._internal.operations.prepare import RequirementPreparer + from pip._internal.req.req_install import InstallRequirement + from pip._internal.req.req_set import RequirementSet + + InstallRequirementProvider = Callable[ + [str, InstallRequirement], InstallRequirement + ] + DiscoveredDependencies = DefaultDict[str, List[InstallRequirement]] + +logger = logging.getLogger(__name__) + + +def _check_dist_requires_python( + dist, # type: pkg_resources.Distribution + version_info, # type: Tuple[int, int, int] + ignore_requires_python=False, # type: bool +): + # type: (...) -> None + """ + Check whether the given Python version is compatible with a distribution's + "Requires-Python" value. + + :param version_info: A 3-tuple of ints representing the Python + major-minor-micro version to check. + :param ignore_requires_python: Whether to ignore the "Requires-Python" + value if the given Python version isn't compatible. + + :raises UnsupportedPythonVersion: When the given Python version isn't + compatible. + """ + requires_python = get_requires_python(dist) + try: + is_compatible = check_requires_python( + requires_python, version_info=version_info, + ) + except specifiers.InvalidSpecifier as exc: + logger.warning( + "Package %r has an invalid Requires-Python: %s", + dist.project_name, exc, + ) + return + + if is_compatible: + return + + version = '.'.join(map(str, version_info)) + if ignore_requires_python: + logger.debug( + 'Ignoring failed Requires-Python check for package %r: ' + '%s not in %r', + dist.project_name, version, requires_python, + ) + return + + raise UnsupportedPythonVersion( + 'Package {!r} requires a different Python: {} not in {!r}'.format( + dist.project_name, version, requires_python, + )) + + +class Resolver(object): + """Resolves which packages need to be installed/uninstalled to perform \ + the requested operation without breaking the requirements of any package. + """ + + _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} + + def __init__( + self, + preparer, # type: RequirementPreparer + finder, # type: PackageFinder + make_install_req, # type: InstallRequirementProvider + use_user_site, # type: bool + ignore_dependencies, # type: bool + ignore_installed, # type: bool + ignore_requires_python, # type: bool + force_reinstall, # type: bool + upgrade_strategy, # type: str + py_version_info=None, # type: Optional[Tuple[int, ...]] + ): + # type: (...) -> None + super(Resolver, self).__init__() + assert upgrade_strategy in self._allowed_strategies + + if py_version_info is None: + py_version_info = sys.version_info[:3] + else: + py_version_info = normalize_version_info(py_version_info) + + self._py_version_info = py_version_info + + self.preparer = preparer + self.finder = finder + + self.upgrade_strategy = upgrade_strategy + self.force_reinstall = force_reinstall + self.ignore_dependencies = ignore_dependencies + self.ignore_installed = ignore_installed + self.ignore_requires_python = ignore_requires_python + self.use_user_site = use_user_site + self._make_install_req = make_install_req + + self._discovered_dependencies = \ + defaultdict(list) # type: DiscoveredDependencies + + def resolve(self, requirement_set): + # type: (RequirementSet) -> None + """Resolve what operations need to be done + + As a side-effect of this method, the packages (and their dependencies) + are downloaded, unpacked and prepared for installation. This + preparation is done by ``pip.operations.prepare``. + + Once PyPI has static dependency metadata available, it would be + possible to move the preparation to become a step separated from + dependency resolution. + """ + # If any top-level requirement has a hash specified, enter + # hash-checking mode, which requires hashes from all. + root_reqs = ( + requirement_set.unnamed_requirements + + list(requirement_set.requirements.values()) + ) + + # Actually prepare the files, and collect any exceptions. Most hash + # exceptions cannot be checked ahead of time, because + # req.populate_link() needs to be called before we can make decisions + # based on link type. + discovered_reqs = [] # type: List[InstallRequirement] + hash_errors = HashErrors() + for req in chain(root_reqs, discovered_reqs): + try: + discovered_reqs.extend(self._resolve_one(requirement_set, req)) + except HashError as exc: + exc.req = req + hash_errors.append(exc) + + if hash_errors: + raise hash_errors + + def _is_upgrade_allowed(self, req): + # type: (InstallRequirement) -> bool + if self.upgrade_strategy == "to-satisfy-only": + return False + elif self.upgrade_strategy == "eager": + return True + else: + assert self.upgrade_strategy == "only-if-needed" + return req.is_direct + + def _set_req_to_reinstall(self, req): + # type: (InstallRequirement) -> None + """ + Set a requirement to be installed. + """ + # Don't uninstall the conflict if doing a user install and the + # conflict is not a user install. + if not self.use_user_site or dist_in_usersite(req.satisfied_by): + req.should_reinstall = True + req.satisfied_by = None + + def _check_skip_installed(self, req_to_install): + # type: (InstallRequirement) -> Optional[str] + """Check if req_to_install should be skipped. + + This will check if the req is installed, and whether we should upgrade + or reinstall it, taking into account all the relevant user options. + + After calling this req_to_install will only have satisfied_by set to + None if the req_to_install is to be upgraded/reinstalled etc. Any + other value will be a dist recording the current thing installed that + satisfies the requirement. + + Note that for vcs urls and the like we can't assess skipping in this + routine - we simply identify that we need to pull the thing down, + then later on it is pulled down and introspected to assess upgrade/ + reinstalls etc. + + :return: A text reason for why it was skipped, or None. + """ + if self.ignore_installed: + return None + + req_to_install.check_if_exists(self.use_user_site) + if not req_to_install.satisfied_by: + return None + + if self.force_reinstall: + self._set_req_to_reinstall(req_to_install) + return None + + if not self._is_upgrade_allowed(req_to_install): + if self.upgrade_strategy == "only-if-needed": + return 'already satisfied, skipping upgrade' + return 'already satisfied' + + # Check for the possibility of an upgrade. For link-based + # requirements we have to pull the tree down and inspect to assess + # the version #, so it's handled way down. + if not req_to_install.link: + try: + self.finder.find_requirement(req_to_install, upgrade=True) + except BestVersionAlreadyInstalled: + # Then the best version is installed. + return 'already up-to-date' + except DistributionNotFound: + # No distribution found, so we squash the error. It will + # be raised later when we re-try later to do the install. + # Why don't we just raise here? + pass + + self._set_req_to_reinstall(req_to_install) + return None + + def _get_abstract_dist_for(self, req): + # type: (InstallRequirement) -> AbstractDistribution + """Takes a InstallRequirement and returns a single AbstractDist \ + representing a prepared variant of the same. + """ + if req.editable: + return self.preparer.prepare_editable_requirement(req) + + # satisfied_by is only evaluated by calling _check_skip_installed, + # so it must be None here. + assert req.satisfied_by is None + skip_reason = self._check_skip_installed(req) + + if req.satisfied_by: + return self.preparer.prepare_installed_requirement( + req, skip_reason + ) + + upgrade_allowed = self._is_upgrade_allowed(req) + + # We eagerly populate the link, since that's our "legacy" behavior. + require_hashes = self.preparer.require_hashes + req.populate_link(self.finder, upgrade_allowed, require_hashes) + abstract_dist = self.preparer.prepare_linked_requirement(req) + + # NOTE + # The following portion is for determining if a certain package is + # going to be re-installed/upgraded or not and reporting to the user. + # This should probably get cleaned up in a future refactor. + + # req.req is only avail after unpack for URL + # pkgs repeat check_if_exists to uninstall-on-upgrade + # (#14) + if not self.ignore_installed: + req.check_if_exists(self.use_user_site) + + if req.satisfied_by: + should_modify = ( + self.upgrade_strategy != "to-satisfy-only" or + self.force_reinstall or + self.ignore_installed or + req.link.scheme == 'file' + ) + if should_modify: + self._set_req_to_reinstall(req) + else: + logger.info( + 'Requirement already satisfied (use --upgrade to upgrade):' + ' %s', req, + ) + + return abstract_dist + + def _resolve_one( + self, + requirement_set, # type: RequirementSet + req_to_install, # type: InstallRequirement + ): + # type: (...) -> List[InstallRequirement] + """Prepare a single requirements file. + + :return: A list of additional InstallRequirements to also install. + """ + # Tell user what we are doing for this requirement: + # obtain (editable), skipping, processing (local url), collecting + # (remote url or package name) + if req_to_install.constraint or req_to_install.prepared: + return [] + + req_to_install.prepared = True + + # register tmp src for cleanup in case something goes wrong + requirement_set.reqs_to_cleanup.append(req_to_install) + + abstract_dist = self._get_abstract_dist_for(req_to_install) + + # Parse and return dependencies + dist = abstract_dist.get_pkg_resources_distribution() + # This will raise UnsupportedPythonVersion if the given Python + # version isn't compatible with the distribution's Requires-Python. + _check_dist_requires_python( + dist, version_info=self._py_version_info, + ignore_requires_python=self.ignore_requires_python, + ) + + more_reqs = [] # type: List[InstallRequirement] + + def add_req(subreq, extras_requested): + sub_install_req = self._make_install_req( + str(subreq), + req_to_install, + ) + parent_req_name = req_to_install.name + to_scan_again, add_to_parent = requirement_set.add_requirement( + sub_install_req, + parent_req_name=parent_req_name, + extras_requested=extras_requested, + ) + if parent_req_name and add_to_parent: + self._discovered_dependencies[parent_req_name].append( + add_to_parent + ) + more_reqs.extend(to_scan_again) + + with indent_log(): + # We add req_to_install before its dependencies, so that we + # can refer to it when adding dependencies. + if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here + # 'unnamed' requirements can only come from being directly + # provided by the user. + assert req_to_install.is_direct + requirement_set.add_requirement( + req_to_install, parent_req_name=None, + ) + + if not self.ignore_dependencies: + if req_to_install.extras: + logger.debug( + "Installing extra requirements: %r", + ','.join(req_to_install.extras), + ) + missing_requested = sorted( + set(req_to_install.extras) - set(dist.extras) + ) + for missing in missing_requested: + logger.warning( + '%s does not provide the extra \'%s\'', + dist, missing + ) + + available_requested = sorted( + set(dist.extras) & set(req_to_install.extras) + ) + for subreq in dist.requires(available_requested): + add_req(subreq, extras_requested=available_requested) + + if not req_to_install.editable and not req_to_install.satisfied_by: + # XXX: --no-install leads this to report 'Successfully + # downloaded' for only non-editable reqs, even though we took + # action on them. + requirement_set.successfully_downloaded.append(req_to_install) + + return more_reqs + + def get_installation_order(self, req_set): + # type: (RequirementSet) -> List[InstallRequirement] + """Create the installation order. + + The installation order is topological - requirements are installed + before the requiring thing. We break cycles at an arbitrary point, + and make no other guarantees. + """ + # The current implementation, which we may change at any point + # installs the user specified things in the order given, except when + # dependencies must come earlier to achieve topological order. + order = [] + ordered_reqs = set() # type: Set[InstallRequirement] + + def schedule(req): + if req.satisfied_by or req in ordered_reqs: + return + if req.constraint: + return + ordered_reqs.add(req) + for dep in self._discovered_dependencies[req.name]: + schedule(dep) + order.append(req) + + for install_req in req_set.requirements.values(): + schedule(install_req) + return order diff --git a/ubuntu/venv/pip/_internal/locations.py b/ubuntu/venv/pip/_internal/locations.py new file mode 100644 index 0000000..0c11553 --- /dev/null +++ b/ubuntu/venv/pip/_internal/locations.py @@ -0,0 +1,194 @@ +"""Locations where we look for configs, install stuff, etc""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import os +import os.path +import platform +import site +import sys +import sysconfig +from distutils import sysconfig as distutils_sysconfig +from distutils.command.install import SCHEME_KEYS # type: ignore +from distutils.command.install import install as distutils_install_command + +from pip._internal.models.scheme import Scheme +from pip._internal.utils import appdirs +from pip._internal.utils.compat import WINDOWS +from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast +from pip._internal.utils.virtualenv import running_under_virtualenv + +if MYPY_CHECK_RUNNING: + from typing import Dict, List, Optional, Union + + from distutils.cmd import Command as DistutilsCommand + + +# Application Directories +USER_CACHE_DIR = appdirs.user_cache_dir("pip") + + +def get_major_minor_version(): + # type: () -> str + """ + Return the major-minor version of the current Python as a string, e.g. + "3.7" or "3.10". + """ + return '{}.{}'.format(*sys.version_info) + + +def get_src_prefix(): + # type: () -> str + if running_under_virtualenv(): + src_prefix = os.path.join(sys.prefix, 'src') + else: + # FIXME: keep src in cwd for now (it is not a temporary folder) + try: + src_prefix = os.path.join(os.getcwd(), 'src') + except OSError: + # In case the current working directory has been renamed or deleted + sys.exit( + "The folder you are executing pip from can no longer be found." + ) + + # under macOS + virtualenv sys.prefix is not properly resolved + # it is something like /path/to/python/bin/.. + return os.path.abspath(src_prefix) + + +# FIXME doesn't account for venv linked to global site-packages + +site_packages = sysconfig.get_path("purelib") # type: Optional[str] + +# This is because of a bug in PyPy's sysconfig module, see +# https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths +# for more information. +if platform.python_implementation().lower() == "pypy": + site_packages = distutils_sysconfig.get_python_lib() +try: + # Use getusersitepackages if this is present, as it ensures that the + # value is initialised properly. + user_site = site.getusersitepackages() +except AttributeError: + user_site = site.USER_SITE + +if WINDOWS: + bin_py = os.path.join(sys.prefix, 'Scripts') + bin_user = os.path.join(user_site, 'Scripts') + # buildout uses 'bin' on Windows too? + if not os.path.exists(bin_py): + bin_py = os.path.join(sys.prefix, 'bin') + bin_user = os.path.join(user_site, 'bin') +else: + bin_py = os.path.join(sys.prefix, 'bin') + bin_user = os.path.join(user_site, 'bin') + + # Forcing to use /usr/local/bin for standard macOS framework installs + # Also log to ~/Library/Logs/ for use with the Console.app log viewer + if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': + bin_py = '/usr/local/bin' + + +def distutils_scheme( + dist_name, user=False, home=None, root=None, isolated=False, prefix=None +): + # type:(str, bool, str, str, bool, str) -> Dict[str, str] + """ + Return a distutils install scheme + """ + from distutils.dist import Distribution + + dist_args = {'name': dist_name} # type: Dict[str, Union[str, List[str]]] + if isolated: + dist_args["script_args"] = ["--no-user-cfg"] + + d = Distribution(dist_args) + d.parse_config_files() + obj = None # type: Optional[DistutilsCommand] + obj = d.get_command_obj('install', create=True) + assert obj is not None + i = cast(distutils_install_command, obj) + # NOTE: setting user or home has the side-effect of creating the home dir + # or user base for installations during finalize_options() + # ideally, we'd prefer a scheme class that has no side-effects. + assert not (user and prefix), "user={} prefix={}".format(user, prefix) + assert not (home and prefix), "home={} prefix={}".format(home, prefix) + i.user = user or i.user + if user or home: + i.prefix = "" + i.prefix = prefix or i.prefix + i.home = home or i.home + i.root = root or i.root + i.finalize_options() + + scheme = {} + for key in SCHEME_KEYS: + scheme[key] = getattr(i, 'install_' + key) + + # install_lib specified in setup.cfg should install *everything* + # into there (i.e. it takes precedence over both purelib and + # platlib). Note, i.install_lib is *always* set after + # finalize_options(); we only want to override here if the user + # has explicitly requested it hence going back to the config + if 'install_lib' in d.get_option_dict('install'): + scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) + + if running_under_virtualenv(): + scheme['headers'] = os.path.join( + sys.prefix, + 'include', + 'site', + 'python{}'.format(get_major_minor_version()), + dist_name, + ) + + if root is not None: + path_no_drive = os.path.splitdrive( + os.path.abspath(scheme["headers"]))[1] + scheme["headers"] = os.path.join( + root, + path_no_drive[1:], + ) + + return scheme + + +def get_scheme( + dist_name, # type: str + user=False, # type: bool + home=None, # type: Optional[str] + root=None, # type: Optional[str] + isolated=False, # type: bool + prefix=None, # type: Optional[str] +): + # type: (...) -> Scheme + """ + Get the "scheme" corresponding to the input parameters. The distutils + documentation provides the context for the available schemes: + https://docs.python.org/3/install/index.html#alternate-installation + + :param dist_name: the name of the package to retrieve the scheme for, used + in the headers scheme path + :param user: indicates to use the "user" scheme + :param home: indicates to use the "home" scheme and provides the base + directory for the same + :param root: root under which other directories are re-based + :param isolated: equivalent to --no-user-cfg, i.e. do not consider + ~/.pydistutils.cfg (posix) or ~/pydistutils.cfg (non-posix) for + scheme paths + :param prefix: indicates to use the "prefix" scheme and provides the + base directory for the same + """ + scheme = distutils_scheme( + dist_name, user, home, root, isolated, prefix + ) + return Scheme( + platlib=scheme["platlib"], + purelib=scheme["purelib"], + headers=scheme["headers"], + scripts=scheme["scripts"], + data=scheme["data"], + ) diff --git a/ubuntu/venv/pip/_internal/main.py b/ubuntu/venv/pip/_internal/main.py new file mode 100644 index 0000000..3208d5b --- /dev/null +++ b/ubuntu/venv/pip/_internal/main.py @@ -0,0 +1,16 @@ +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, List + + +def main(args=None): + # type: (Optional[List[str]]) -> int + """This is preserved for old console scripts that may still be referencing + it. + + For additional details, see https://github.com/pypa/pip/issues/7498. + """ + from pip._internal.utils.entrypoints import _wrapper + + return _wrapper(args) diff --git a/ubuntu/venv/pip/_internal/models/__init__.py b/ubuntu/venv/pip/_internal/models/__init__.py new file mode 100644 index 0000000..7855226 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/__init__.py @@ -0,0 +1,2 @@ +"""A package that contains models that represent entities. +""" diff --git a/ubuntu/venv/pip/_internal/models/candidate.py b/ubuntu/venv/pip/_internal/models/candidate.py new file mode 100644 index 0000000..1dc1a57 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/candidate.py @@ -0,0 +1,36 @@ +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.utils.models import KeyBasedCompareMixin +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pip._vendor.packaging.version import _BaseVersion + from pip._internal.models.link import Link + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, name, version, link): + # type: (str, str, Link) -> None + self.name = name + self.version = parse_version(version) # type: _BaseVersion + self.link = link + + super(InstallationCandidate, self).__init__( + key=(self.name, self.version, self.link), + defining_class=InstallationCandidate + ) + + def __repr__(self): + # type: () -> str + return "".format( + self.name, self.version, self.link, + ) + + def __str__(self): + # type: () -> str + return '{!r} candidate (version {} at {})'.format( + self.name, self.version, self.link, + ) diff --git a/ubuntu/venv/pip/_internal/models/format_control.py b/ubuntu/venv/pip/_internal/models/format_control.py new file mode 100644 index 0000000..2e13727 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/format_control.py @@ -0,0 +1,84 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal.exceptions import CommandError +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, FrozenSet + + +class FormatControl(object): + """Helper for managing formats from which a package can be installed. + """ + + def __init__(self, no_binary=None, only_binary=None): + # type: (Optional[Set[str]], Optional[Set[str]]) -> None + if no_binary is None: + no_binary = set() + if only_binary is None: + only_binary = set() + + self.no_binary = no_binary + self.only_binary = only_binary + + def __eq__(self, other): + # type: (object) -> bool + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + # type: (object) -> bool + return not self.__eq__(other) + + def __repr__(self): + # type: () -> str + return "{}({}, {})".format( + self.__class__.__name__, + self.no_binary, + self.only_binary + ) + + @staticmethod + def handle_mutual_excludes(value, target, other): + # type: (str, Optional[Set[str]], Optional[Set[str]]) -> None + if value.startswith('-'): + raise CommandError( + "--no-binary / --only-binary option requires 1 argument." + ) + new = value.split(',') + while ':all:' in new: + other.clear() + target.clear() + target.add(':all:') + del new[:new.index(':all:') + 1] + # Without a none, we want to discard everything as :all: covers it + if ':none:' not in new: + return + for name in new: + if name == ':none:': + target.clear() + continue + name = canonicalize_name(name) + other.discard(name) + target.add(name) + + def get_allowed_formats(self, canonical_name): + # type: (str) -> FrozenSet[str] + result = {"binary", "source"} + if canonical_name in self.only_binary: + result.discard('source') + elif canonical_name in self.no_binary: + result.discard('binary') + elif ':all:' in self.only_binary: + result.discard('source') + elif ':all:' in self.no_binary: + result.discard('binary') + return frozenset(result) + + def disallow_binaries(self): + # type: () -> None + self.handle_mutual_excludes( + ':all:', self.no_binary, self.only_binary, + ) diff --git a/ubuntu/venv/pip/_internal/models/index.py b/ubuntu/venv/pip/_internal/models/index.py new file mode 100644 index 0000000..ead1efb --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/index.py @@ -0,0 +1,31 @@ +from pip._vendor.six.moves.urllib import parse as urllib_parse + + +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + # type: (str, str) -> None + super(PackageIndex, self).__init__() + self.url = url + self.netloc = urllib_parse.urlsplit(url).netloc + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') + + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + + def _url_for_path(self, path): + # type: (str) -> str + return urllib_parse.urljoin(self.url, path) + + +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/ubuntu/venv/pip/_internal/models/link.py b/ubuntu/venv/pip/_internal/models/link.py new file mode 100644 index 0000000..34fbcbf --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/link.py @@ -0,0 +1,227 @@ +import os +import posixpath +import re + +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.utils.filetypes import WHEEL_EXTENSION +from pip._internal.utils.misc import ( + redact_auth_from_url, + split_auth_from_netloc, + splitext, +) +from pip._internal.utils.models import KeyBasedCompareMixin +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url, url_to_path + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Tuple, Union + from pip._internal.index.collector import HTMLPage + from pip._internal.utils.hashes import Hashes + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__( + self, + url, # type: str + comes_from=None, # type: Optional[Union[str, HTMLPage]] + requires_python=None, # type: Optional[str] + yanked_reason=None, # type: Optional[Text] + ): + # type: (...) -> None + """ + :param url: url of the resource pointed to (href of the link) + :param comes_from: instance of HTMLPage where the link was found, + or string. + :param requires_python: String containing the `Requires-Python` + metadata field, specified in PEP 345. This may be specified by + a data-requires-python attribute in the HTML link tag, as + described in PEP 503. + :param yanked_reason: the reason the file has been yanked, if the + file has been yanked, or None if the file hasn't been yanked. + This is the value of the "data-yanked" attribute, if present, in + a simple repository HTML link. If the file has been yanked but + no reason was provided, this should be the empty string. See + PEP 592 for more information and the specification. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self._parsed_url = urllib_parse.urlsplit(url) + # Store the url as a private attribute to prevent accidentally + # trying to set a new value. + self._url = url + + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + self.yanked_reason = yanked_reason + + super(Link, self).__init__(key=url, defining_class=Link) + + def __str__(self): + # type: () -> str + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (redact_auth_from_url(self._url), + self.comes_from, rp) + else: + return redact_auth_from_url(str(self._url)) + + def __repr__(self): + # type: () -> str + return '' % self + + @property + def url(self): + # type: () -> str + return self._url + + @property + def filename(self): + # type: () -> str + path = self.path.rstrip('/') + name = posixpath.basename(path) + if not name: + # Make sure we don't leak auth information if the netloc + # includes a username and password. + netloc, user_pass = split_auth_from_netloc(self.netloc) + return netloc + + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self._url) + return name + + @property + def file_path(self): + # type: () -> str + return url_to_path(self.url) + + @property + def scheme(self): + # type: () -> str + return self._parsed_url.scheme + + @property + def netloc(self): + # type: () -> str + """ + This can contain auth information. + """ + return self._parsed_url.netloc + + @property + def path(self): + # type: () -> str + return urllib_parse.unquote(self._parsed_url.path) + + def splitext(self): + # type: () -> Tuple[str, str] + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + # type: () -> str + return self.splitext()[1] + + @property + def url_without_fragment(self): + # type: () -> str + scheme, netloc, path, query, fragment = self._parsed_url + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + # type: () -> Optional[str] + match = self._egg_fragment_re.search(self._url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + # type: () -> Optional[str] + match = self._subdirectory_fragment_re.search(self._url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + # type: () -> Optional[str] + match = self._hash_re.search(self._url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + # type: () -> Optional[str] + match = self._hash_re.search(self._url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + # type: () -> str + return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_file(self): + # type: () -> bool + return self.scheme == 'file' + + def is_existing_dir(self): + # type: () -> bool + return self.is_file and os.path.isdir(self.file_path) + + @property + def is_wheel(self): + # type: () -> bool + return self.ext == WHEEL_EXTENSION + + @property + def is_vcs(self): + # type: () -> bool + from pip._internal.vcs import vcs + + return self.scheme in vcs.all_schemes + + @property + def is_yanked(self): + # type: () -> bool + return self.yanked_reason is not None + + @property + def has_hash(self): + # type: () -> bool + return self.hash_name is not None + + def is_hash_allowed(self, hashes): + # type: (Optional[Hashes]) -> bool + """ + Return True if the link has a hash and it is allowed. + """ + if hashes is None or not self.has_hash: + return False + # Assert non-None so mypy knows self.hash_name and self.hash are str. + assert self.hash_name is not None + assert self.hash is not None + + return hashes.is_hash_allowed(self.hash_name, hex_digest=self.hash) diff --git a/ubuntu/venv/pip/_internal/models/scheme.py b/ubuntu/venv/pip/_internal/models/scheme.py new file mode 100644 index 0000000..af07b40 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/scheme.py @@ -0,0 +1,25 @@ +""" +For types associated with installation schemes. + +For a general overview of available schemes and their context, see +https://docs.python.org/3/install/index.html#alternate-installation. +""" + + +class Scheme(object): + """A Scheme holds paths which are used as the base directories for + artifacts associated with a Python package. + """ + def __init__( + self, + platlib, # type: str + purelib, # type: str + headers, # type: str + scripts, # type: str + data, # type: str + ): + self.platlib = platlib + self.purelib = purelib + self.headers = headers + self.scripts = scripts + self.data = data diff --git a/ubuntu/venv/pip/_internal/models/search_scope.py b/ubuntu/venv/pip/_internal/models/search_scope.py new file mode 100644 index 0000000..138d1b6 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/search_scope.py @@ -0,0 +1,114 @@ +import itertools +import logging +import os +import posixpath + +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.models.index import PyPI +from pip._internal.utils.compat import has_tls +from pip._internal.utils.misc import normalize_path, redact_auth_from_url +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List + + +logger = logging.getLogger(__name__) + + +class SearchScope(object): + + """ + Encapsulates the locations that pip is configured to search. + """ + + @classmethod + def create( + cls, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> SearchScope + """ + Create a SearchScope object after normalizing the `find_links`. + """ + # Build find_links. If an argument starts with ~, it may be + # a local file relative to a home directory. So try normalizing + # it and if it exists, use the normalized version. + # This is deliberately conservative - it might be fine just to + # blindly normalize anything starting with a ~... + built_find_links = [] # type: List[str] + for link in find_links: + if link.startswith('~'): + new_link = normalize_path(link) + if os.path.exists(new_link): + link = new_link + built_find_links.append(link) + + # If we don't have TLS enabled, then WARN if anyplace we're looking + # relies on TLS. + if not has_tls(): + for link in itertools.chain(index_urls, built_find_links): + parsed = urllib_parse.urlparse(link) + if parsed.scheme == 'https': + logger.warning( + 'pip is configured with locations that require ' + 'TLS/SSL, however the ssl module in Python is not ' + 'available.' + ) + break + + return cls( + find_links=built_find_links, + index_urls=index_urls, + ) + + def __init__( + self, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> None + self.find_links = find_links + self.index_urls = index_urls + + def get_formatted_locations(self): + # type: () -> str + lines = [] + if self.index_urls and self.index_urls != [PyPI.simple_url]: + lines.append( + 'Looking in indexes: {}'.format(', '.join( + redact_auth_from_url(url) for url in self.index_urls)) + ) + if self.find_links: + lines.append( + 'Looking in links: {}'.format(', '.join( + redact_auth_from_url(url) for url in self.find_links)) + ) + return '\n'.join(lines) + + def get_index_urls_locations(self, project_name): + # type: (str) -> List[str] + """Returns the locations found via self.index_urls + + Checks the url_name on the main (first in the list) index and + use this url_name to produce all locations + """ + + def mkurl_pypi_url(url): + # type: (str) -> str + loc = posixpath.join( + url, + urllib_parse.quote(canonicalize_name(project_name))) + # For maximum compatibility with easy_install, ensure the path + # ends in a trailing slash. Although this isn't in the spec + # (and PyPI can handle it without the slash) some other index + # implementations might break if they relied on easy_install's + # behavior. + if not loc.endswith('/'): + loc = loc + '/' + return loc + + return [mkurl_pypi_url(url) for url in self.index_urls] diff --git a/ubuntu/venv/pip/_internal/models/selection_prefs.py b/ubuntu/venv/pip/_internal/models/selection_prefs.py new file mode 100644 index 0000000..f58fdce --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/selection_prefs.py @@ -0,0 +1,47 @@ +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + from pip._internal.models.format_control import FormatControl + + +class SelectionPreferences(object): + + """ + Encapsulates the candidate selection preferences for downloading + and installing files. + """ + + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + def __init__( + self, + allow_yanked, # type: bool + allow_all_prereleases=False, # type: bool + format_control=None, # type: Optional[FormatControl] + prefer_binary=False, # type: bool + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> None + """Create a SelectionPreferences object. + + :param allow_yanked: Whether files marked as yanked (in the sense + of PEP 592) are permitted to be candidates for install. + :param format_control: A FormatControl object or None. Used to control + the selection of source packages / binary packages when consulting + the index and links. + :param prefer_binary: Whether to prefer an old, but valid, binary + dist over a new source dist. + :param ignore_requires_python: Whether to ignore incompatible + "Requires-Python" values in links. Defaults to False. + """ + if ignore_requires_python is None: + ignore_requires_python = False + + self.allow_yanked = allow_yanked + self.allow_all_prereleases = allow_all_prereleases + self.format_control = format_control + self.prefer_binary = prefer_binary + self.ignore_requires_python = ignore_requires_python diff --git a/ubuntu/venv/pip/_internal/models/target_python.py b/ubuntu/venv/pip/_internal/models/target_python.py new file mode 100644 index 0000000..97ae85a --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/target_python.py @@ -0,0 +1,107 @@ +import sys + +from pip._internal.pep425tags import get_supported, version_info_to_nodot +from pip._internal.utils.misc import normalize_version_info +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Tuple + + from pip._vendor.packaging.tags import Tag + + +class TargetPython(object): + + """ + Encapsulates the properties of a Python interpreter one is targeting + for a package install, download, etc. + """ + + def __init__( + self, + platform=None, # type: Optional[str] + py_version_info=None, # type: Optional[Tuple[int, ...]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + ): + # type: (...) -> None + """ + :param platform: A string or None. If None, searches for packages + that are supported by the current system. Otherwise, will find + packages that can be built on the platform passed in. These + packages will only be downloaded for distribution: they will + not be built locally. + :param py_version_info: An optional tuple of ints representing the + Python version information to use (e.g. `sys.version_info[:3]`). + This can have length 1, 2, or 3 when provided. + :param abi: A string or None. This is passed to pep425tags.py's + get_supported() function as is. + :param implementation: A string or None. This is passed to + pep425tags.py's get_supported() function as is. + """ + # Store the given py_version_info for when we call get_supported(). + self._given_py_version_info = py_version_info + + if py_version_info is None: + py_version_info = sys.version_info[:3] + else: + py_version_info = normalize_version_info(py_version_info) + + py_version = '.'.join(map(str, py_version_info[:2])) + + self.abi = abi + self.implementation = implementation + self.platform = platform + self.py_version = py_version + self.py_version_info = py_version_info + + # This is used to cache the return value of get_tags(). + self._valid_tags = None # type: Optional[List[Tag]] + + def format_given(self): + # type: () -> str + """ + Format the given, non-None attributes for display. + """ + display_version = None + if self._given_py_version_info is not None: + display_version = '.'.join( + str(part) for part in self._given_py_version_info + ) + + key_values = [ + ('platform', self.platform), + ('version_info', display_version), + ('abi', self.abi), + ('implementation', self.implementation), + ] + return ' '.join( + '{}={!r}'.format(key, value) for key, value in key_values + if value is not None + ) + + def get_tags(self): + # type: () -> List[Tag] + """ + Return the supported PEP 425 tags to check wheel candidates against. + + The tags are returned in order of preference (most preferred first). + """ + if self._valid_tags is None: + # Pass versions=None if no py_version_info was given since + # versions=None uses special default logic. + py_version_info = self._given_py_version_info + if py_version_info is None: + version = None + else: + version = version_info_to_nodot(py_version_info) + + tags = get_supported( + version=version, + platform=self.platform, + abi=self.abi, + impl=self.implementation, + ) + self._valid_tags = tags + + return self._valid_tags diff --git a/ubuntu/venv/pip/_internal/models/wheel.py b/ubuntu/venv/pip/_internal/models/wheel.py new file mode 100644 index 0000000..f1e3f44 --- /dev/null +++ b/ubuntu/venv/pip/_internal/models/wheel.py @@ -0,0 +1,78 @@ +"""Represents a wheel file and provides access to the various parts of the +name that have meaning. +""" +import re + +from pip._vendor.packaging.tags import Tag + +from pip._internal.exceptions import InvalidWheelFilename +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List + + +class Wheel(object): + """A wheel file""" + + wheel_file_re = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl|\.dist-info)$""", + re.VERBOSE + ) + + def __init__(self, filename): + # type: (str) -> None + """ + :raises InvalidWheelFilename: when the filename is invalid for a wheel + """ + wheel_info = self.wheel_file_re.match(filename) + if not wheel_info: + raise InvalidWheelFilename( + "%s is not a valid wheel filename." % filename + ) + self.filename = filename + self.name = wheel_info.group('name').replace('_', '-') + # we'll assume "_" means "-" due to wheel naming scheme + # (https://github.com/pypa/pip/issues/1150) + self.version = wheel_info.group('ver').replace('_', '-') + self.build_tag = wheel_info.group('build') + self.pyversions = wheel_info.group('pyver').split('.') + self.abis = wheel_info.group('abi').split('.') + self.plats = wheel_info.group('plat').split('.') + + # All the tag combinations from this file + self.file_tags = { + Tag(x, y, z) for x in self.pyversions + for y in self.abis for z in self.plats + } + + def get_formatted_file_tags(self): + # type: () -> List[str] + """Return the wheel's tags as a sorted list of strings.""" + return sorted(str(tag) for tag in self.file_tags) + + def support_index_min(self, tags): + # type: (List[Tag]) -> int + """Return the lowest index that one of the wheel's file_tag combinations + achieves in the given list of supported tags. + + For example, if there are 8 supported tags and one of the file tags + is first in the list, then return 0. + + :param tags: the PEP 425 tags to check the wheel against, in order + with most preferred first. + + :raises ValueError: If none of the wheel's file tags match one of + the supported tags. + """ + return min(tags.index(tag) for tag in self.file_tags if tag in tags) + + def supported(self, tags): + # type: (List[Tag]) -> bool + """Return whether the wheel is compatible with one of the given tags. + + :param tags: the PEP 425 tags to check the wheel against. + """ + return not self.file_tags.isdisjoint(tags) diff --git a/ubuntu/venv/pip/_internal/network/__init__.py b/ubuntu/venv/pip/_internal/network/__init__.py new file mode 100644 index 0000000..b51bde9 --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/__init__.py @@ -0,0 +1,2 @@ +"""Contains purely network-related utilities. +""" diff --git a/ubuntu/venv/pip/_internal/network/auth.py b/ubuntu/venv/pip/_internal/network/auth.py new file mode 100644 index 0000000..1e1da54 --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/auth.py @@ -0,0 +1,298 @@ +"""Network Authentication Helpers + +Contains interface (MultiDomainBasicAuth) and associated glue code for +providing credentials in the context of network requests. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth +from pip._vendor.requests.utils import get_netrc_auth +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.utils.misc import ( + ask, + ask_input, + ask_password, + remove_auth_from_url, + split_auth_netloc_from_url, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Dict, Optional, Tuple + + from pip._internal.vcs.versioncontrol import AuthInfo + + Credentials = Tuple[str, str, str] + +logger = logging.getLogger(__name__) + +try: + import keyring # noqa +except ImportError: + keyring = None +except Exception as exc: + logger.warning( + "Keyring is skipped due to an exception: %s", str(exc), + ) + keyring = None + + +def get_keyring_auth(url, username): + """Return the tuple auth for a given url from keyring.""" + if not url or not keyring: + return None + + try: + try: + get_credential = keyring.get_credential + except AttributeError: + pass + else: + logger.debug("Getting credentials from keyring for %s", url) + cred = get_credential(url, username) + if cred is not None: + return cred.username, cred.password + return None + + if username: + logger.debug("Getting password from keyring for %s", url) + password = keyring.get_password(url, username) + if password: + return username, password + + except Exception as exc: + logger.warning( + "Keyring is skipped due to an exception: %s", str(exc), + ) + + +class MultiDomainBasicAuth(AuthBase): + + def __init__(self, prompting=True, index_urls=None): + # type: (bool, Optional[Values]) -> None + self.prompting = prompting + self.index_urls = index_urls + self.passwords = {} # type: Dict[str, AuthInfo] + # When the user is prompted to enter credentials and keyring is + # available, we will offer to save them. If the user accepts, + # this value is set to the credentials they entered. After the + # request authenticates, the caller should call + # ``save_credentials`` to save these. + self._credentials_to_save = None # type: Optional[Credentials] + + def _get_index_url(self, url): + """Return the original index URL matching the requested URL. + + Cached or dynamically generated credentials may work against + the original index URL rather than just the netloc. + + The provided url should have had its username and password + removed already. If the original index url had credentials then + they will be included in the return value. + + Returns None if no matching index was found, or if --no-index + was specified by the user. + """ + if not url or not self.index_urls: + return None + + for u in self.index_urls: + prefix = remove_auth_from_url(u).rstrip("/") + "/" + if url.startswith(prefix): + return u + + def _get_new_credentials(self, original_url, allow_netrc=True, + allow_keyring=True): + """Find and return credentials for the specified URL.""" + # Split the credentials and netloc from the url. + url, netloc, url_user_password = split_auth_netloc_from_url( + original_url, + ) + + # Start with the credentials embedded in the url + username, password = url_user_password + if username is not None and password is not None: + logger.debug("Found credentials in url for %s", netloc) + return url_user_password + + # Find a matching index url for this request + index_url = self._get_index_url(url) + if index_url: + # Split the credentials from the url. + index_info = split_auth_netloc_from_url(index_url) + if index_info: + index_url, _, index_url_user_password = index_info + logger.debug("Found index url %s", index_url) + + # If an index URL was found, try its embedded credentials + if index_url and index_url_user_password[0] is not None: + username, password = index_url_user_password + if username is not None and password is not None: + logger.debug("Found credentials in index url for %s", netloc) + return index_url_user_password + + # Get creds from netrc if we still don't have them + if allow_netrc: + netrc_auth = get_netrc_auth(original_url) + if netrc_auth: + logger.debug("Found credentials in netrc for %s", netloc) + return netrc_auth + + # If we don't have a password and keyring is available, use it. + if allow_keyring: + # The index url is more specific than the netloc, so try it first + kr_auth = ( + get_keyring_auth(index_url, username) or + get_keyring_auth(netloc, username) + ) + if kr_auth: + logger.debug("Found credentials in keyring for %s", netloc) + return kr_auth + + return username, password + + def _get_url_and_credentials(self, original_url): + """Return the credentials to use for the provided URL. + + If allowed, netrc and keyring may be used to obtain the + correct credentials. + + Returns (url_without_credentials, username, password). Note + that even if the original URL contains credentials, this + function may return a different username and password. + """ + url, netloc, _ = split_auth_netloc_from_url(original_url) + + # Use any stored credentials that we have for this netloc + username, password = self.passwords.get(netloc, (None, None)) + + if username is None and password is None: + # No stored credentials. Acquire new credentials without prompting + # the user. (e.g. from netrc, keyring, or the URL itself) + username, password = self._get_new_credentials(original_url) + + if username is not None or password is not None: + # Convert the username and password if they're None, so that + # this netloc will show up as "cached" in the conditional above. + # Further, HTTPBasicAuth doesn't accept None, so it makes sense to + # cache the value that is going to be used. + username = username or "" + password = password or "" + + # Store any acquired credentials. + self.passwords[netloc] = (username, password) + + assert ( + # Credentials were found + (username is not None and password is not None) or + # Credentials were not found + (username is None and password is None) + ), "Could not load credentials from url: {}".format(original_url) + + return url, username, password + + def __call__(self, req): + # Get credentials for this request + url, username, password = self._get_url_and_credentials(req.url) + + # Set the url of the request to the url without any credentials + req.url = url + + if username is not None and password is not None: + # Send the basic auth with this request + req = HTTPBasicAuth(username, password)(req) + + # Attach a hook to handle 401 responses + req.register_hook("response", self.handle_401) + + return req + + # Factored out to allow for easy patching in tests + def _prompt_for_password(self, netloc): + username = ask_input("User for %s: " % netloc) + if not username: + return None, None + auth = get_keyring_auth(netloc, username) + if auth: + return auth[0], auth[1], False + password = ask_password("Password: ") + return username, password, True + + # Factored out to allow for easy patching in tests + def _should_save_password_to_keyring(self): + if not keyring: + return False + return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y" + + def handle_401(self, resp, **kwargs): + # We only care about 401 responses, anything else we want to just + # pass through the actual response + if resp.status_code != 401: + return resp + + # We are not able to prompt the user so simply return the response + if not self.prompting: + return resp + + parsed = urllib_parse.urlparse(resp.url) + + # Prompt the user for a new username and password + username, password, save = self._prompt_for_password(parsed.netloc) + + # Store the new username and password to use for future requests + self._credentials_to_save = None + if username is not None and password is not None: + self.passwords[parsed.netloc] = (username, password) + + # Prompt to save the password to keyring + if save and self._should_save_password_to_keyring(): + self._credentials_to_save = (parsed.netloc, username, password) + + # Consume content and release the original connection to allow our new + # request to reuse the same one. + resp.content + resp.raw.release_conn() + + # Add our new username and password to the request + req = HTTPBasicAuth(username or "", password or "")(resp.request) + req.register_hook("response", self.warn_on_401) + + # On successful request, save the credentials that were used to + # keyring. (Note that if the user responded "no" above, this member + # is not set and nothing will be saved.) + if self._credentials_to_save: + req.register_hook("response", self.save_credentials) + + # Send our new request + new_resp = resp.connection.send(req, **kwargs) + new_resp.history.append(resp) + + return new_resp + + def warn_on_401(self, resp, **kwargs): + """Response callback to warn about incorrect credentials.""" + if resp.status_code == 401: + logger.warning( + '401 Error, Credentials not correct for %s', resp.request.url, + ) + + def save_credentials(self, resp, **kwargs): + """Response callback to save credentials on success.""" + assert keyring is not None, "should never reach here without keyring" + if not keyring: + return + + creds = self._credentials_to_save + self._credentials_to_save = None + if creds and resp.status_code < 400: + try: + logger.info('Saving credentials to keyring') + keyring.set_password(*creds) + except Exception: + logger.exception('Failed to save credentials') diff --git a/ubuntu/venv/pip/_internal/network/cache.py b/ubuntu/venv/pip/_internal/network/cache.py new file mode 100644 index 0000000..c9386e1 --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/cache.py @@ -0,0 +1,81 @@ +"""HTTP cache implementation. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import os +from contextlib import contextmanager + +from pip._vendor.cachecontrol.cache import BaseCache +from pip._vendor.cachecontrol.caches import FileCache +from pip._vendor.requests.models import Response + +from pip._internal.utils.filesystem import adjacent_tmp_file, replace +from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + + +def is_from_cache(response): + # type: (Response) -> bool + return getattr(response, "from_cache", False) + + +@contextmanager +def suppressed_cache_errors(): + """If we can't access the cache then we can just skip caching and process + requests as if caching wasn't enabled. + """ + try: + yield + except (OSError, IOError): + pass + + +class SafeFileCache(BaseCache): + """ + A file based cache which is safe to use even when the target directory may + not be accessible or writable. + """ + + def __init__(self, directory): + # type: (str) -> None + assert directory is not None, "Cache directory must not be None." + super(SafeFileCache, self).__init__() + self.directory = directory + + def _get_cache_path(self, name): + # type: (str) -> str + # From cachecontrol.caches.file_cache.FileCache._fn, brought into our + # class for backwards-compatibility and to avoid using a non-public + # method. + hashed = FileCache.encode(name) + parts = list(hashed[:5]) + [hashed] + return os.path.join(self.directory, *parts) + + def get(self, key): + # type: (str) -> Optional[bytes] + path = self._get_cache_path(key) + with suppressed_cache_errors(): + with open(path, 'rb') as f: + return f.read() + + def set(self, key, value): + # type: (str, bytes) -> None + path = self._get_cache_path(key) + with suppressed_cache_errors(): + ensure_dir(os.path.dirname(path)) + + with adjacent_tmp_file(path) as f: + f.write(value) + + replace(f.name, path) + + def delete(self, key): + # type: (str) -> None + path = self._get_cache_path(key) + with suppressed_cache_errors(): + os.remove(path) diff --git a/ubuntu/venv/pip/_internal/network/download.py b/ubuntu/venv/pip/_internal/network/download.py new file mode 100644 index 0000000..c90c4bf --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/download.py @@ -0,0 +1,200 @@ +"""Download files with progress indicators. +""" +import cgi +import logging +import mimetypes +import os + +from pip._vendor import requests +from pip._vendor.requests.models import CONTENT_CHUNK_SIZE + +from pip._internal.models.index import PyPI +from pip._internal.network.cache import is_from_cache +from pip._internal.network.utils import response_chunks +from pip._internal.utils.misc import ( + format_size, + redact_auth_from_url, + splitext, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import DownloadProgressProvider + +if MYPY_CHECK_RUNNING: + from typing import Iterable, Optional + + from pip._vendor.requests.models import Response + + from pip._internal.models.link import Link + from pip._internal.network.session import PipSession + +logger = logging.getLogger(__name__) + + +def _get_http_response_size(resp): + # type: (Response) -> Optional[int] + try: + return int(resp.headers['content-length']) + except (ValueError, KeyError, TypeError): + return None + + +def _prepare_download( + resp, # type: Response + link, # type: Link + progress_bar # type: str +): + # type: (...) -> Iterable[bytes] + total_length = _get_http_response_size(resp) + + if link.netloc == PyPI.file_storage_domain: + url = link.show_url + else: + url = link.url_without_fragment + + logged_url = redact_auth_from_url(url) + + if total_length: + logged_url = '{} ({})'.format(logged_url, format_size(total_length)) + + if is_from_cache(resp): + logger.info("Using cached %s", logged_url) + else: + logger.info("Downloading %s", logged_url) + + if logger.getEffectiveLevel() > logging.INFO: + show_progress = False + elif is_from_cache(resp): + show_progress = False + elif not total_length: + show_progress = True + elif total_length > (40 * 1000): + show_progress = True + else: + show_progress = False + + chunks = response_chunks(resp, CONTENT_CHUNK_SIZE) + + if not show_progress: + return chunks + + return DownloadProgressProvider( + progress_bar, max=total_length + )(chunks) + + +def sanitize_content_filename(filename): + # type: (str) -> str + """ + Sanitize the "filename" value from a Content-Disposition header. + """ + return os.path.basename(filename) + + +def parse_content_disposition(content_disposition, default_filename): + # type: (str, str) -> str + """ + Parse the "filename" value from a Content-Disposition header, and + return the default filename if the result is empty. + """ + _type, params = cgi.parse_header(content_disposition) + filename = params.get('filename') + if filename: + # We need to sanitize the filename to prevent directory traversal + # in case the filename contains ".." path parts. + filename = sanitize_content_filename(filename) + return filename or default_filename + + +def _get_http_response_filename(resp, link): + # type: (Response, Link) -> str + """Get an ideal filename from the given HTTP response, falling back to + the link filename if not provided. + """ + filename = link.filename # fallback + # Have a look at the Content-Disposition header for a better guess + content_disposition = resp.headers.get('content-disposition') + if content_disposition: + filename = parse_content_disposition(content_disposition, filename) + ext = splitext(filename)[1] # type: Optional[str] + if not ext: + ext = mimetypes.guess_extension( + resp.headers.get('content-type', '') + ) + if ext: + filename += ext + if not ext and link.url != resp.url: + ext = os.path.splitext(resp.url)[1] + if ext: + filename += ext + return filename + + +def _http_get_download(session, link): + # type: (PipSession, Link) -> Response + target_url = link.url.split('#', 1)[0] + resp = session.get( + target_url, + # We use Accept-Encoding: identity here because requests + # defaults to accepting compressed responses. This breaks in + # a variety of ways depending on how the server is configured. + # - Some servers will notice that the file isn't a compressible + # file and will leave the file alone and with an empty + # Content-Encoding + # - Some servers will notice that the file is already + # compressed and will leave the file alone and will add a + # Content-Encoding: gzip header + # - Some servers won't notice anything at all and will take + # a file that's already been compressed and compress it again + # and set the Content-Encoding: gzip header + # By setting this to request only the identity encoding We're + # hoping to eliminate the third case. Hopefully there does not + # exist a server which when given a file will notice it is + # already compressed and that you're not asking for a + # compressed file and will then decompress it before sending + # because if that's the case I don't think it'll ever be + # possible to make this work. + headers={"Accept-Encoding": "identity"}, + stream=True, + ) + resp.raise_for_status() + return resp + + +class Download(object): + def __init__( + self, + response, # type: Response + filename, # type: str + chunks, # type: Iterable[bytes] + ): + # type: (...) -> None + self.response = response + self.filename = filename + self.chunks = chunks + + +class Downloader(object): + def __init__( + self, + session, # type: PipSession + progress_bar, # type: str + ): + # type: (...) -> None + self._session = session + self._progress_bar = progress_bar + + def __call__(self, link): + # type: (Link) -> Download + try: + resp = _http_get_download(self._session, link) + except requests.HTTPError as e: + logger.critical( + "HTTP error %s while getting %s", e.response.status_code, link + ) + raise + + return Download( + resp, + _get_http_response_filename(resp, link), + _prepare_download(resp, link, self._progress_bar), + ) diff --git a/ubuntu/venv/pip/_internal/network/session.py b/ubuntu/venv/pip/_internal/network/session.py new file mode 100644 index 0000000..f5eb15e --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/session.py @@ -0,0 +1,405 @@ +"""PipSession and supporting code, containing all pip-specific +network request configuration and behavior. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import email.utils +import json +import logging +import mimetypes +import os +import platform +import sys +import warnings + +from pip._vendor import requests, six, urllib3 +from pip._vendor.cachecontrol import CacheControlAdapter +from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter +from pip._vendor.requests.models import Response +from pip._vendor.requests.structures import CaseInsensitiveDict +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.urllib3.exceptions import InsecureRequestWarning + +from pip import __version__ +from pip._internal.network.auth import MultiDomainBasicAuth +from pip._internal.network.cache import SafeFileCache +# Import ssl from compat so the initial import occurs in only one place. +from pip._internal.utils.compat import has_tls, ipaddress +from pip._internal.utils.glibc import libc_ver +from pip._internal.utils.misc import ( + build_url_from_netloc, + get_installed_version, + parse_netloc, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import url_to_path + +if MYPY_CHECK_RUNNING: + from typing import ( + Iterator, List, Optional, Tuple, Union, + ) + + from pip._internal.models.link import Link + + SecureOrigin = Tuple[str, str, Optional[Union[int, str]]] + + +logger = logging.getLogger(__name__) + + +# Ignore warning raised when using --trusted-host. +warnings.filterwarnings("ignore", category=InsecureRequestWarning) + + +SECURE_ORIGINS = [ + # protocol, hostname, port + # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC) + ("https", "*", "*"), + ("*", "localhost", "*"), + ("*", "127.0.0.0/8", "*"), + ("*", "::1/128", "*"), + ("file", "*", None), + # ssh is always secure. + ("ssh", "*", "*"), +] # type: List[SecureOrigin] + + +# These are environment variables present when running under various +# CI systems. For each variable, some CI systems that use the variable +# are indicated. The collection was chosen so that for each of a number +# of popular systems, at least one of the environment variables is used. +# This list is used to provide some indication of and lower bound for +# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive. +# For more background, see: https://github.com/pypa/pip/issues/5499 +CI_ENVIRONMENT_VARIABLES = ( + # Azure Pipelines + 'BUILD_BUILDID', + # Jenkins + 'BUILD_ID', + # AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI + 'CI', + # Explicit environment variable. + 'PIP_IS_CI', +) + + +def looks_like_ci(): + # type: () -> bool + """ + Return whether it looks like pip is running under CI. + """ + # We don't use the method of checking for a tty (e.g. using isatty()) + # because some CI systems mimic a tty (e.g. Travis CI). Thus that + # method doesn't provide definitive information in either direction. + return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES) + + +def user_agent(): + """ + Return a string representing the user agent. + """ + data = { + "installer": {"name": "pip", "version": __version__}, + "python": platform.python_version(), + "implementation": { + "name": platform.python_implementation(), + }, + } + + if data["implementation"]["name"] == 'CPython': + data["implementation"]["version"] = platform.python_version() + elif data["implementation"]["name"] == 'PyPy': + if sys.pypy_version_info.releaselevel == 'final': + pypy_version_info = sys.pypy_version_info[:3] + else: + pypy_version_info = sys.pypy_version_info + data["implementation"]["version"] = ".".join( + [str(x) for x in pypy_version_info] + ) + elif data["implementation"]["name"] == 'Jython': + # Complete Guess + data["implementation"]["version"] = platform.python_version() + elif data["implementation"]["name"] == 'IronPython': + # Complete Guess + data["implementation"]["version"] = platform.python_version() + + if sys.platform.startswith("linux"): + from pip._vendor import distro + distro_infos = dict(filter( + lambda x: x[1], + zip(["name", "version", "id"], distro.linux_distribution()), + )) + libc = dict(filter( + lambda x: x[1], + zip(["lib", "version"], libc_ver()), + )) + if libc: + distro_infos["libc"] = libc + if distro_infos: + data["distro"] = distro_infos + + if sys.platform.startswith("darwin") and platform.mac_ver()[0]: + data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} + + if platform.system(): + data.setdefault("system", {})["name"] = platform.system() + + if platform.release(): + data.setdefault("system", {})["release"] = platform.release() + + if platform.machine(): + data["cpu"] = platform.machine() + + if has_tls(): + import _ssl as ssl + data["openssl_version"] = ssl.OPENSSL_VERSION + + setuptools_version = get_installed_version("setuptools") + if setuptools_version is not None: + data["setuptools_version"] = setuptools_version + + # Use None rather than False so as not to give the impression that + # pip knows it is not being run under CI. Rather, it is a null or + # inconclusive result. Also, we include some value rather than no + # value to make it easier to know that the check has been run. + data["ci"] = True if looks_like_ci() else None + + user_data = os.environ.get("PIP_USER_AGENT_USER_DATA") + if user_data is not None: + data["user_data"] = user_data + + return "{data[installer][name]}/{data[installer][version]} {json}".format( + data=data, + json=json.dumps(data, separators=(",", ":"), sort_keys=True), + ) + + +class LocalFSAdapter(BaseAdapter): + + def send(self, request, stream=None, timeout=None, verify=None, cert=None, + proxies=None): + pathname = url_to_path(request.url) + + resp = Response() + resp.status_code = 200 + resp.url = request.url + + try: + stats = os.stat(pathname) + except OSError as exc: + resp.status_code = 404 + resp.raw = exc + else: + modified = email.utils.formatdate(stats.st_mtime, usegmt=True) + content_type = mimetypes.guess_type(pathname)[0] or "text/plain" + resp.headers = CaseInsensitiveDict({ + "Content-Type": content_type, + "Content-Length": stats.st_size, + "Last-Modified": modified, + }) + + resp.raw = open(pathname, "rb") + resp.close = resp.raw.close + + return resp + + def close(self): + pass + + +class InsecureHTTPAdapter(HTTPAdapter): + + def cert_verify(self, conn, url, verify, cert): + super(InsecureHTTPAdapter, self).cert_verify( + conn=conn, url=url, verify=False, cert=cert + ) + + +class PipSession(requests.Session): + + timeout = None # type: Optional[int] + + def __init__(self, *args, **kwargs): + """ + :param trusted_hosts: Domains not to emit warnings for when not using + HTTPS. + """ + retries = kwargs.pop("retries", 0) + cache = kwargs.pop("cache", None) + trusted_hosts = kwargs.pop("trusted_hosts", []) # type: List[str] + index_urls = kwargs.pop("index_urls", None) + + super(PipSession, self).__init__(*args, **kwargs) + + # Namespace the attribute with "pip_" just in case to prevent + # possible conflicts with the base class. + self.pip_trusted_origins = [] # type: List[Tuple[str, Optional[int]]] + + # Attach our User Agent to the request + self.headers["User-Agent"] = user_agent() + + # Attach our Authentication handler to the session + self.auth = MultiDomainBasicAuth(index_urls=index_urls) + + # Create our urllib3.Retry instance which will allow us to customize + # how we handle retries. + retries = urllib3.Retry( + # Set the total number of retries that a particular request can + # have. + total=retries, + + # A 503 error from PyPI typically means that the Fastly -> Origin + # connection got interrupted in some way. A 503 error in general + # is typically considered a transient error so we'll go ahead and + # retry it. + # A 500 may indicate transient error in Amazon S3 + # A 520 or 527 - may indicate transient error in CloudFlare + status_forcelist=[500, 503, 520, 527], + + # Add a small amount of back off between failed requests in + # order to prevent hammering the service. + backoff_factor=0.25, + ) + + # We want to _only_ cache responses on securely fetched origins. We do + # this because we can't validate the response of an insecurely fetched + # origin, and we don't want someone to be able to poison the cache and + # require manual eviction from the cache to fix it. + if cache: + secure_adapter = CacheControlAdapter( + cache=SafeFileCache(cache), + max_retries=retries, + ) + else: + secure_adapter = HTTPAdapter(max_retries=retries) + + # Our Insecure HTTPAdapter disables HTTPS validation. It does not + # support caching (see above) so we'll use it for all http:// URLs as + # well as any https:// host that we've marked as ignoring TLS errors + # for. + insecure_adapter = InsecureHTTPAdapter(max_retries=retries) + # Save this for later use in add_insecure_host(). + self._insecure_adapter = insecure_adapter + + self.mount("https://", secure_adapter) + self.mount("http://", insecure_adapter) + + # Enable file:// urls + self.mount("file://", LocalFSAdapter()) + + for host in trusted_hosts: + self.add_trusted_host(host, suppress_logging=True) + + def add_trusted_host(self, host, source=None, suppress_logging=False): + # type: (str, Optional[str], bool) -> None + """ + :param host: It is okay to provide a host that has previously been + added. + :param source: An optional source string, for logging where the host + string came from. + """ + if not suppress_logging: + msg = 'adding trusted host: {!r}'.format(host) + if source is not None: + msg += ' (from {})'.format(source) + logger.info(msg) + + host_port = parse_netloc(host) + if host_port not in self.pip_trusted_origins: + self.pip_trusted_origins.append(host_port) + + self.mount(build_url_from_netloc(host) + '/', self._insecure_adapter) + if not host_port[1]: + # Mount wildcard ports for the same host. + self.mount( + build_url_from_netloc(host) + ':', + self._insecure_adapter + ) + + def iter_secure_origins(self): + # type: () -> Iterator[SecureOrigin] + for secure_origin in SECURE_ORIGINS: + yield secure_origin + for host, port in self.pip_trusted_origins: + yield ('*', host, '*' if port is None else port) + + def is_secure_origin(self, location): + # type: (Link) -> bool + # Determine if this url used a secure transport mechanism + parsed = urllib_parse.urlparse(str(location)) + origin_protocol, origin_host, origin_port = ( + parsed.scheme, parsed.hostname, parsed.port, + ) + + # The protocol to use to see if the protocol matches. + # Don't count the repository type as part of the protocol: in + # cases such as "git+ssh", only use "ssh". (I.e., Only verify against + # the last scheme.) + origin_protocol = origin_protocol.rsplit('+', 1)[-1] + + # Determine if our origin is a secure origin by looking through our + # hardcoded list of secure origins, as well as any additional ones + # configured on this PackageFinder instance. + for secure_origin in self.iter_secure_origins(): + secure_protocol, secure_host, secure_port = secure_origin + if origin_protocol != secure_protocol and secure_protocol != "*": + continue + + try: + addr = ipaddress.ip_address( + None + if origin_host is None + else six.ensure_text(origin_host) + ) + network = ipaddress.ip_network( + six.ensure_text(secure_host) + ) + except ValueError: + # We don't have both a valid address or a valid network, so + # we'll check this origin against hostnames. + if ( + origin_host and + origin_host.lower() != secure_host.lower() and + secure_host != "*" + ): + continue + else: + # We have a valid address and network, so see if the address + # is contained within the network. + if addr not in network: + continue + + # Check to see if the port matches. + if ( + origin_port != secure_port and + secure_port != "*" and + secure_port is not None + ): + continue + + # If we've gotten here, then this origin matches the current + # secure origin and we should return True + return True + + # If we've gotten to this point, then the origin isn't secure and we + # will not accept it as a valid location to search. We will however + # log a warning that we are ignoring it. + logger.warning( + "The repository located at %s is not a trusted or secure host and " + "is being ignored. If this repository is available via HTTPS we " + "recommend you use HTTPS instead, otherwise you may silence " + "this warning and allow it anyway with '--trusted-host %s'.", + origin_host, + origin_host, + ) + + return False + + def request(self, method, url, *args, **kwargs): + # Allow setting a default timeout on a session + kwargs.setdefault("timeout", self.timeout) + + # Dispatch the actual request + return super(PipSession, self).request(method, url, *args, **kwargs) diff --git a/ubuntu/venv/pip/_internal/network/utils.py b/ubuntu/venv/pip/_internal/network/utils.py new file mode 100644 index 0000000..a19050b --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/utils.py @@ -0,0 +1,48 @@ +from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterator + + +def response_chunks(response, chunk_size=CONTENT_CHUNK_SIZE): + # type: (Response, int) -> Iterator[bytes] + """Given a requests Response, provide the data chunks. + """ + try: + # Special case for urllib3. + for chunk in response.raw.stream( + chunk_size, + # We use decode_content=False here because we don't + # want urllib3 to mess with the raw bytes we get + # from the server. If we decompress inside of + # urllib3 then we cannot verify the checksum + # because the checksum will be of the compressed + # file. This breakage will only occur if the + # server adds a Content-Encoding header, which + # depends on how the server was configured: + # - Some servers will notice that the file isn't a + # compressible file and will leave the file alone + # and with an empty Content-Encoding + # - Some servers will notice that the file is + # already compressed and will leave the file + # alone and will add a Content-Encoding: gzip + # header + # - Some servers won't notice anything at all and + # will take a file that's already been compressed + # and compress it again and set the + # Content-Encoding: gzip header + # + # By setting this not to decode automatically we + # hope to eliminate problems with the second case. + decode_content=False, + ): + yield chunk + except AttributeError: + # Standard file-like object. + while True: + chunk = response.raw.read(chunk_size) + if not chunk: + break + yield chunk diff --git a/ubuntu/venv/pip/_internal/network/xmlrpc.py b/ubuntu/venv/pip/_internal/network/xmlrpc.py new file mode 100644 index 0000000..121edd9 --- /dev/null +++ b/ubuntu/venv/pip/_internal/network/xmlrpc.py @@ -0,0 +1,44 @@ +"""xmlrpclib.Transport implementation +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pip._vendor import requests +# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import +from pip._vendor.six.moves import xmlrpc_client # type: ignore +from pip._vendor.six.moves.urllib import parse as urllib_parse + +logger = logging.getLogger(__name__) + + +class PipXmlrpcTransport(xmlrpc_client.Transport): + """Provide a `xmlrpclib.Transport` implementation via a `PipSession` + object. + """ + + def __init__(self, index_url, session, use_datetime=False): + xmlrpc_client.Transport.__init__(self, use_datetime) + index_parts = urllib_parse.urlparse(index_url) + self._scheme = index_parts.scheme + self._session = session + + def request(self, host, handler, request_body, verbose=False): + parts = (self._scheme, host, handler, None, None, None) + url = urllib_parse.urlunparse(parts) + try: + headers = {'Content-Type': 'text/xml'} + response = self._session.post(url, data=request_body, + headers=headers, stream=True) + response.raise_for_status() + self.verbose = verbose + return self.parse_response(response.raw) + except requests.HTTPError as exc: + logger.critical( + "HTTP error %s while getting %s", + exc.response.status_code, url, + ) + raise diff --git a/ubuntu/venv/pip/_internal/operations/__init__.py b/ubuntu/venv/pip/_internal/operations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/pip/_internal/operations/build/__init__.py b/ubuntu/venv/pip/_internal/operations/build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/pip/_internal/operations/build/metadata.py b/ubuntu/venv/pip/_internal/operations/build/metadata.py new file mode 100644 index 0000000..b13fbde --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/build/metadata.py @@ -0,0 +1,40 @@ +"""Metadata generation logic for source distributions. +""" + +import logging +import os + +from pip._internal.utils.subprocess import runner_with_spinner_message +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pip._internal.build_env import BuildEnvironment + from pip._vendor.pep517.wrappers import Pep517HookCaller + +logger = logging.getLogger(__name__) + + +def generate_metadata(build_env, backend): + # type: (BuildEnvironment, Pep517HookCaller) -> str + """Generate metadata using mechanisms described in PEP 517. + + Returns the generated metadata directory. + """ + metadata_tmpdir = TempDirectory( + kind="modern-metadata", globally_managed=True + ) + + metadata_dir = metadata_tmpdir.path + + with build_env: + # Note that Pep517HookCaller implements a fallback for + # prepare_metadata_for_build_wheel, so we don't have to + # consider the possibility that this hook doesn't exist. + runner = runner_with_spinner_message("Preparing wheel metadata") + with backend.subprocess_runner(runner): + distinfo_dir = backend.prepare_metadata_for_build_wheel( + metadata_dir + ) + + return os.path.join(metadata_dir, distinfo_dir) diff --git a/ubuntu/venv/pip/_internal/operations/build/metadata_legacy.py b/ubuntu/venv/pip/_internal/operations/build/metadata_legacy.py new file mode 100644 index 0000000..b6813f8 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/build/metadata_legacy.py @@ -0,0 +1,122 @@ +"""Metadata generation logic for legacy source distributions. +""" + +import logging +import os + +from pip._internal.exceptions import InstallationError +from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.setuptools_build import make_setuptools_egg_info_args +from pip._internal.utils.subprocess import call_subprocess +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + + from pip._internal.build_env import BuildEnvironment + +logger = logging.getLogger(__name__) + + +def _find_egg_info(source_directory, is_editable): + # type: (str, bool) -> str + """Find an .egg-info in `source_directory`, based on `is_editable`. + """ + + def looks_like_virtual_env(path): + # type: (str) -> bool + return ( + os.path.lexists(os.path.join(path, 'bin', 'python')) or + os.path.exists(os.path.join(path, 'Scripts', 'Python.exe')) + ) + + def locate_editable_egg_info(base): + # type: (str) -> List[str] + candidates = [] # type: List[str] + for root, dirs, files in os.walk(base): + for dir_ in vcs.dirnames: + if dir_ in dirs: + dirs.remove(dir_) + # Iterate over a copy of ``dirs``, since mutating + # a list while iterating over it can cause trouble. + # (See https://github.com/pypa/pip/pull/462.) + for dir_ in list(dirs): + if looks_like_virtual_env(os.path.join(root, dir_)): + dirs.remove(dir_) + # Also don't search through tests + elif dir_ == 'test' or dir_ == 'tests': + dirs.remove(dir_) + candidates.extend(os.path.join(root, dir_) for dir_ in dirs) + return [f for f in candidates if f.endswith('.egg-info')] + + def depth_of_directory(dir_): + # type: (str) -> int + return ( + dir_.count(os.path.sep) + + (os.path.altsep and dir_.count(os.path.altsep) or 0) + ) + + base = source_directory + if is_editable: + filenames = locate_editable_egg_info(base) + else: + base = os.path.join(base, 'pip-egg-info') + filenames = os.listdir(base) + + if not filenames: + raise InstallationError( + "Files/directories not found in {}".format(base) + ) + + # If we have more than one match, we pick the toplevel one. This + # can easily be the case if there is a dist folder which contains + # an extracted tarball for testing purposes. + if len(filenames) > 1: + filenames.sort(key=depth_of_directory) + + return os.path.join(base, filenames[0]) + + +def generate_metadata( + build_env, # type: BuildEnvironment + setup_py_path, # type: str + source_dir, # type: str + editable, # type: bool + isolated, # type: bool + details, # type: str +): + # type: (...) -> str + """Generate metadata using setup.py-based defacto mechanisms. + + Returns the generated metadata directory. + """ + logger.debug( + 'Running setup.py (path:%s) egg_info for package %s', + setup_py_path, details, + ) + + egg_info_dir = None # type: Optional[str] + # For non-editable installs, don't put the .egg-info files at the root, + # to avoid confusion due to the source code being considered an installed + # egg. + if not editable: + egg_info_dir = os.path.join(source_dir, 'pip-egg-info') + # setuptools complains if the target directory does not exist. + ensure_dir(egg_info_dir) + + args = make_setuptools_egg_info_args( + setup_py_path, + egg_info_dir=egg_info_dir, + no_user_config=isolated, + ) + + with build_env: + call_subprocess( + args, + cwd=source_dir, + command_desc='python setup.py egg_info', + ) + + # Return the .egg-info directory. + return _find_egg_info(source_dir, editable) diff --git a/ubuntu/venv/pip/_internal/operations/build/wheel.py b/ubuntu/venv/pip/_internal/operations/build/wheel.py new file mode 100644 index 0000000..1266ce0 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/build/wheel.py @@ -0,0 +1,46 @@ +import logging +import os + +from pip._internal.utils.subprocess import runner_with_spinner_message +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + from pip._vendor.pep517.wrappers import Pep517HookCaller + +logger = logging.getLogger(__name__) + + +def build_wheel_pep517( + name, # type: str + backend, # type: Pep517HookCaller + metadata_directory, # type: str + build_options, # type: List[str] + tempd, # type: str +): + # type: (...) -> Optional[str] + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + assert metadata_directory is not None + if build_options: + # PEP 517 does not support --build-options + logger.error('Cannot build wheel for %s using PEP 517 when ' + '--build-option is present' % (name,)) + return None + try: + logger.debug('Destination directory: %s', tempd) + + runner = runner_with_spinner_message( + 'Building wheel for {} (PEP 517)'.format(name) + ) + with backend.subprocess_runner(runner): + wheel_name = backend.build_wheel( + tempd, + metadata_directory=metadata_directory, + ) + except Exception: + logger.error('Failed building wheel for %s', name) + return None + return os.path.join(tempd, wheel_name) diff --git a/ubuntu/venv/pip/_internal/operations/build/wheel_legacy.py b/ubuntu/venv/pip/_internal/operations/build/wheel_legacy.py new file mode 100644 index 0000000..3ebd9fe --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/build/wheel_legacy.py @@ -0,0 +1,115 @@ +import logging +import os.path + +from pip._internal.utils.setuptools_build import ( + make_setuptools_bdist_wheel_args, +) +from pip._internal.utils.subprocess import ( + LOG_DIVIDER, + call_subprocess, + format_command_args, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import open_spinner + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Text + +logger = logging.getLogger(__name__) + + +def format_command_result( + command_args, # type: List[str] + command_output, # type: Text +): + # type: (...) -> str + """Format command information for logging.""" + command_desc = format_command_args(command_args) + text = 'Command arguments: {}\n'.format(command_desc) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + name, # type: str + command_args, # type: List[str] + command_output, # type: Text +): + # type: (...) -> Optional[str] + """Return the path to the wheel in the temporary build directory.""" + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(name) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(name, names) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + +def build_wheel_legacy( + name, # type: str + setup_py_path, # type: str + source_dir, # type: str + global_options, # type: List[str] + build_options, # type: List[str] + tempd, # type: str +): + # type: (...) -> Optional[str] + """Build one unpacked package using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + wheel_args = make_setuptools_bdist_wheel_args( + setup_py_path, + global_options=global_options, + build_options=build_options, + destination_dir=tempd, + ) + + spin_message = 'Building wheel for %s (setup.py)' % (name,) + with open_spinner(spin_message) as spinner: + logger.debug('Destination directory: %s', tempd) + + try: + output = call_subprocess( + wheel_args, + cwd=source_dir, + spinner=spinner, + ) + except Exception: + spinner.finish("error") + logger.error('Failed building wheel for %s', name) + return None + + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + name=name, + command_args=wheel_args, + command_output=output, + ) + return wheel_path diff --git a/ubuntu/venv/pip/_internal/operations/check.py b/ubuntu/venv/pip/_internal/operations/check.py new file mode 100644 index 0000000..b85a123 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/check.py @@ -0,0 +1,163 @@ +"""Validation of dependencies of packages +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +import logging +from collections import namedtuple + +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.pkg_resources import RequirementParseError + +from pip._internal.distributions import ( + make_distribution_for_install_requirement, +) +from pip._internal.utils.misc import get_installed_distributions +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +logger = logging.getLogger(__name__) + +if MYPY_CHECK_RUNNING: + from pip._internal.req.req_install import InstallRequirement + from typing import ( + Any, Callable, Dict, Optional, Set, Tuple, List + ) + + # Shorthands + PackageSet = Dict[str, 'PackageDetails'] + Missing = Tuple[str, Any] + Conflicting = Tuple[str, str, Any] + + MissingDict = Dict[str, List[Missing]] + ConflictingDict = Dict[str, List[Conflicting]] + CheckResult = Tuple[MissingDict, ConflictingDict] + +PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) + + +def create_package_set_from_installed(**kwargs): + # type: (**Any) -> Tuple[PackageSet, bool] + """Converts a list of distributions into a PackageSet. + """ + # Default to using all packages installed on the system + if kwargs == {}: + kwargs = {"local_only": False, "skip": ()} + + package_set = {} + problems = False + for dist in get_installed_distributions(**kwargs): + name = canonicalize_name(dist.project_name) + try: + package_set[name] = PackageDetails(dist.version, dist.requires()) + except RequirementParseError as e: + # Don't crash on broken metadata + logger.warning("Error parsing requirements for %s: %s", name, e) + problems = True + return package_set, problems + + +def check_package_set(package_set, should_ignore=None): + # type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult + """Check if a package set is consistent + + If should_ignore is passed, it should be a callable that takes a + package name and returns a boolean. + """ + if should_ignore is None: + def should_ignore(name): + return False + + missing = {} + conflicting = {} + + for package_name in package_set: + # Info about dependencies of package_name + missing_deps = set() # type: Set[Missing] + conflicting_deps = set() # type: Set[Conflicting] + + if should_ignore(package_name): + continue + + for req in package_set[package_name].requires: + name = canonicalize_name(req.project_name) # type: str + + # Check if it's missing + if name not in package_set: + missed = True + if req.marker is not None: + missed = req.marker.evaluate() + if missed: + missing_deps.add((name, req)) + continue + + # Check if there's a conflict + version = package_set[name].version # type: str + if not req.specifier.contains(version, prereleases=True): + conflicting_deps.add((name, version, req)) + + if missing_deps: + missing[package_name] = sorted(missing_deps, key=str) + if conflicting_deps: + conflicting[package_name] = sorted(conflicting_deps, key=str) + + return missing, conflicting + + +def check_install_conflicts(to_install): + # type: (List[InstallRequirement]) -> Tuple[PackageSet, CheckResult] + """For checking if the dependency graph would be consistent after \ + installing given requirements + """ + # Start from the current state + package_set, _ = create_package_set_from_installed() + # Install packages + would_be_installed = _simulate_installation_of(to_install, package_set) + + # Only warn about directly-dependent packages; create a whitelist of them + whitelist = _create_whitelist(would_be_installed, package_set) + + return ( + package_set, + check_package_set( + package_set, should_ignore=lambda name: name not in whitelist + ) + ) + + +def _simulate_installation_of(to_install, package_set): + # type: (List[InstallRequirement], PackageSet) -> Set[str] + """Computes the version of packages after installing to_install. + """ + + # Keep track of packages that were installed + installed = set() + + # Modify it as installing requirement_set would (assuming no errors) + for inst_req in to_install: + abstract_dist = make_distribution_for_install_requirement(inst_req) + dist = abstract_dist.get_pkg_resources_distribution() + + name = canonicalize_name(dist.key) + package_set[name] = PackageDetails(dist.version, dist.requires()) + + installed.add(name) + + return installed + + +def _create_whitelist(would_be_installed, package_set): + # type: (Set[str], PackageSet) -> Set[str] + packages_affected = set(would_be_installed) + + for package_name in package_set: + if package_name in packages_affected: + continue + + for req in package_set[package_name].requires: + if canonicalize_name(req.name) in packages_affected: + packages_affected.add(package_name) + break + + return packages_affected diff --git a/ubuntu/venv/pip/_internal/operations/freeze.py b/ubuntu/venv/pip/_internal/operations/freeze.py new file mode 100644 index 0000000..36a5c33 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/freeze.py @@ -0,0 +1,265 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import collections +import logging +import os +import re + +from pip._vendor import six +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.pkg_resources import RequirementParseError + +from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.req.constructors import ( + install_req_from_editable, + install_req_from_line, +) +from pip._internal.req.req_file import COMMENT_RE +from pip._internal.utils.misc import ( + dist_is_editable, + get_installed_distributions, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + ) + from pip._internal.cache import WheelCache + from pip._vendor.pkg_resources import ( + Distribution, Requirement + ) + + RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] + + +logger = logging.getLogger(__name__) + + +def freeze( + requirement=None, # type: Optional[List[str]] + find_links=None, # type: Optional[List[str]] + local_only=None, # type: Optional[bool] + user_only=None, # type: Optional[bool] + paths=None, # type: Optional[List[str]] + skip_regex=None, # type: Optional[str] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + exclude_editable=False, # type: bool + skip=() # type: Container[str] +): + # type: (...) -> Iterator[str] + find_links = find_links or [] + skip_match = None + + if skip_regex: + skip_match = re.compile(skip_regex).search + + for link in find_links: + yield '-f %s' % link + installations = {} # type: Dict[str, FrozenRequirement] + for dist in get_installed_distributions(local_only=local_only, + skip=(), + user_only=user_only, + paths=paths): + try: + req = FrozenRequirement.from_dist(dist) + except RequirementParseError as exc: + # We include dist rather than dist.project_name because the + # dist string includes more information, like the version and + # location. We also include the exception message to aid + # troubleshooting. + logger.warning( + 'Could not generate requirement for distribution %r: %s', + dist, exc + ) + continue + if exclude_editable and req.editable: + continue + installations[req.canonical_name] = req + + if requirement: + # the options that don't get turned into an InstallRequirement + # should only be emitted once, even if the same option is in multiple + # requirements files, so we need to keep track of what has been emitted + # so that we don't emit it again if it's seen again + emitted_options = set() # type: Set[str] + # keep track of which files a requirement is in so that we can + # give an accurate warning if a requirement appears multiple times. + req_files = collections.defaultdict(list) # type: Dict[str, List[str]] + for req_file_path in requirement: + with open(req_file_path) as req_file: + for line in req_file: + if (not line.strip() or + line.strip().startswith('#') or + (skip_match and skip_match(line)) or + line.startswith(( + '-r', '--requirement', + '-Z', '--always-unzip', + '-f', '--find-links', + '-i', '--index-url', + '--pre', + '--trusted-host', + '--process-dependency-links', + '--extra-index-url'))): + line = line.rstrip() + if line not in emitted_options: + emitted_options.add(line) + yield line + continue + + if line.startswith('-e') or line.startswith('--editable'): + if line.startswith('-e'): + line = line[2:].strip() + else: + line = line[len('--editable'):].strip().lstrip('=') + line_req = install_req_from_editable( + line, + isolated=isolated, + wheel_cache=wheel_cache, + ) + else: + line_req = install_req_from_line( + COMMENT_RE.sub('', line).strip(), + isolated=isolated, + wheel_cache=wheel_cache, + ) + + if not line_req.name: + logger.info( + "Skipping line in requirement file [%s] because " + "it's not clear what it would install: %s", + req_file_path, line.strip(), + ) + logger.info( + " (add #egg=PackageName to the URL to avoid" + " this warning)" + ) + else: + line_req_canonical_name = canonicalize_name( + line_req.name) + if line_req_canonical_name not in installations: + # either it's not installed, or it is installed + # but has been processed already + if not req_files[line_req.name]: + logger.warning( + "Requirement file [%s] contains %s, but " + "package %r is not installed", + req_file_path, + COMMENT_RE.sub('', line).strip(), + line_req.name + ) + else: + req_files[line_req.name].append(req_file_path) + else: + yield str(installations[ + line_req_canonical_name]).rstrip() + del installations[line_req_canonical_name] + req_files[line_req.name].append(req_file_path) + + # Warn about requirements that were included multiple times (in a + # single requirements file or in different requirements files). + for name, files in six.iteritems(req_files): + if len(files) > 1: + logger.warning("Requirement %s included multiple times [%s]", + name, ', '.join(sorted(set(files)))) + + yield( + '## The following requirements were added by ' + 'pip freeze:' + ) + for installation in sorted( + installations.values(), key=lambda x: x.name.lower()): + if installation.canonical_name not in skip: + yield str(installation).rstrip() + + +def get_requirement_info(dist): + # type: (Distribution) -> RequirementInfo + """ + Compute and return values (req, editable, comments) for use in + FrozenRequirement.from_dist(). + """ + if not dist_is_editable(dist): + return (None, False, []) + + location = os.path.normcase(os.path.abspath(dist.location)) + + from pip._internal.vcs import vcs, RemoteNotFoundError + vcs_backend = vcs.get_backend_for_dir(location) + + if vcs_backend is None: + req = dist.as_requirement() + logger.debug( + 'No VCS found for editable requirement "%s" in: %r', req, + location, + ) + comments = [ + '# Editable install with no version control ({})'.format(req) + ] + return (location, True, comments) + + try: + req = vcs_backend.get_src_requirement(location, dist.project_name) + except RemoteNotFoundError: + req = dist.as_requirement() + comments = [ + '# Editable {} install with no remote ({})'.format( + type(vcs_backend).__name__, req, + ) + ] + return (location, True, comments) + + except BadCommand: + logger.warning( + 'cannot determine version of editable source in %s ' + '(%s command not found in path)', + location, + vcs_backend.name, + ) + return (None, True, []) + + except InstallationError as exc: + logger.warning( + "Error when trying to get requirement for VCS system %s, " + "falling back to uneditable format", exc + ) + else: + if req is not None: + return (req, True, []) + + logger.warning( + 'Could not determine repository location of %s', location + ) + comments = ['## !! Could not determine repository location'] + + return (None, False, comments) + + +class FrozenRequirement(object): + def __init__(self, name, req, editable, comments=()): + # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None + self.name = name + self.canonical_name = canonicalize_name(name) + self.req = req + self.editable = editable + self.comments = comments + + @classmethod + def from_dist(cls, dist): + # type: (Distribution) -> FrozenRequirement + req, editable, comments = get_requirement_info(dist) + if req is None: + req = dist.as_requirement() + + return cls(dist.project_name, req, editable, comments=comments) + + def __str__(self): + req = self.req + if self.editable: + req = '-e %s' % req + return '\n'.join(list(self.comments) + [str(req)]) + '\n' diff --git a/ubuntu/venv/pip/_internal/operations/install/__init__.py b/ubuntu/venv/pip/_internal/operations/install/__init__.py new file mode 100644 index 0000000..24d6a5d --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/install/__init__.py @@ -0,0 +1,2 @@ +"""For modules related to installing packages. +""" diff --git a/ubuntu/venv/pip/_internal/operations/install/editable_legacy.py b/ubuntu/venv/pip/_internal/operations/install/editable_legacy.py new file mode 100644 index 0000000..a668a61 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/install/editable_legacy.py @@ -0,0 +1,52 @@ +"""Legacy editable installation process, i.e. `setup.py develop`. +""" +import logging + +from pip._internal.utils.logging import indent_log +from pip._internal.utils.setuptools_build import make_setuptools_develop_args +from pip._internal.utils.subprocess import call_subprocess +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Sequence + + from pip._internal.build_env import BuildEnvironment + + +logger = logging.getLogger(__name__) + + +def install_editable( + install_options, # type: List[str] + global_options, # type: Sequence[str] + prefix, # type: Optional[str] + home, # type: Optional[str] + use_user_site, # type: bool + name, # type: str + setup_py_path, # type: str + isolated, # type: bool + build_env, # type: BuildEnvironment + unpacked_source_directory, # type: str +): + # type: (...) -> None + """Install a package in editable mode. Most arguments are pass-through + to setuptools. + """ + logger.info('Running setup.py develop for %s', name) + + args = make_setuptools_develop_args( + setup_py_path, + global_options=global_options, + install_options=install_options, + no_user_config=isolated, + prefix=prefix, + home=home, + use_user_site=use_user_site, + ) + + with indent_log(): + with build_env: + call_subprocess( + args, + cwd=unpacked_source_directory, + ) diff --git a/ubuntu/venv/pip/_internal/operations/install/legacy.py b/ubuntu/venv/pip/_internal/operations/install/legacy.py new file mode 100644 index 0000000..2d4adc4 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/install/legacy.py @@ -0,0 +1,129 @@ +"""Legacy installation process, i.e. `setup.py install`. +""" + +import logging +import os +from distutils.util import change_root + +from pip._internal.utils.deprecation import deprecated +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.setuptools_build import make_setuptools_install_args +from pip._internal.utils.subprocess import runner_with_spinner_message +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Sequence + + from pip._internal.models.scheme import Scheme + from pip._internal.req.req_install import InstallRequirement + + +logger = logging.getLogger(__name__) + + +def install( + install_req, # type: InstallRequirement + install_options, # type: List[str] + global_options, # type: Sequence[str] + root, # type: Optional[str] + home, # type: Optional[str] + prefix, # type: Optional[str] + use_user_site, # type: bool + pycompile, # type: bool + scheme, # type: Scheme +): + # type: (...) -> None + # Extend the list of global and install options passed on to + # the setup.py call with the ones from the requirements file. + # Options specified in requirements file override those + # specified on the command line, since the last option given + # to setup.py is the one that is used. + global_options = list(global_options) + \ + install_req.options.get('global_options', []) + install_options = list(install_options) + \ + install_req.options.get('install_options', []) + + header_dir = scheme.headers + + with TempDirectory(kind="record") as temp_dir: + record_filename = os.path.join(temp_dir.path, 'install-record.txt') + install_args = make_setuptools_install_args( + install_req.setup_py_path, + global_options=global_options, + install_options=install_options, + record_filename=record_filename, + root=root, + prefix=prefix, + header_dir=header_dir, + home=home, + use_user_site=use_user_site, + no_user_config=install_req.isolated, + pycompile=pycompile, + ) + + runner = runner_with_spinner_message( + "Running setup.py install for {}".format(install_req.name) + ) + with indent_log(), install_req.build_env: + runner( + cmd=install_args, + cwd=install_req.unpacked_source_directory, + ) + + if not os.path.exists(record_filename): + logger.debug('Record file %s not found', record_filename) + return + install_req.install_succeeded = True + + # We intentionally do not use any encoding to read the file because + # setuptools writes the file using distutils.file_util.write_file, + # which does not specify an encoding. + with open(record_filename) as f: + record_lines = f.read().splitlines() + + def prepend_root(path): + # type: (str) -> str + if root is None or not os.path.isabs(path): + return path + else: + return change_root(root, path) + + for line in record_lines: + directory = os.path.dirname(line) + if directory.endswith('.egg-info'): + egg_info_dir = prepend_root(directory) + break + else: + deprecated( + reason=( + "{} did not indicate that it installed an " + ".egg-info directory. Only setup.py projects " + "generating .egg-info directories are supported." + ).format(install_req), + replacement=( + "for maintainers: updating the setup.py of {0}. " + "For users: contact the maintainers of {0} to let " + "them know to update their setup.py.".format( + install_req.name + ) + ), + gone_in="20.2", + issue=6998, + ) + # FIXME: put the record somewhere + return + new_lines = [] + for line in record_lines: + filename = line.strip() + if os.path.isdir(filename): + filename += os.path.sep + new_lines.append( + os.path.relpath(prepend_root(filename), egg_info_dir) + ) + new_lines.sort() + ensure_dir(egg_info_dir) + inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') + with open(inst_files_path, 'w') as f: + f.write('\n'.join(new_lines) + '\n') diff --git a/ubuntu/venv/pip/_internal/operations/install/wheel.py b/ubuntu/venv/pip/_internal/operations/install/wheel.py new file mode 100644 index 0000000..aac975c --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/install/wheel.py @@ -0,0 +1,615 @@ +"""Support for installing and building the "wheel" binary package format. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import collections +import compileall +import csv +import logging +import os.path +import re +import shutil +import stat +import sys +import warnings +from base64 import urlsafe_b64encode +from zipfile import ZipFile + +from pip._vendor import pkg_resources +from pip._vendor.distlib.scripts import ScriptMaker +from pip._vendor.distlib.util import get_export_entry +from pip._vendor.six import StringIO + +from pip._internal.exceptions import InstallationError +from pip._internal.locations import get_major_minor_version +from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.unpacking import unpack_file +from pip._internal.utils.wheel import parse_wheel + +if MYPY_CHECK_RUNNING: + from email.message import Message + from typing import ( + Dict, List, Optional, Sequence, Tuple, IO, Text, Any, + Iterable, Callable, Set, + ) + + from pip._internal.models.scheme import Scheme + + InstalledCSVRow = Tuple[str, ...] + + +logger = logging.getLogger(__name__) + + +def normpath(src, p): + # type: (str, str) -> str + return os.path.relpath(src, p).replace(os.path.sep, '/') + + +def rehash(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[str, str] + """Return (encoded_digest, length) for path using hashlib.sha256()""" + h, length = hash_file(path, blocksize) + digest = 'sha256=' + urlsafe_b64encode( + h.digest() + ).decode('latin1').rstrip('=') + # unicode/str python2 issues + return (digest, str(length)) # type: ignore + + +def open_for_csv(name, mode): + # type: (str, Text) -> IO[Any] + if sys.version_info[0] < 3: + nl = {} # type: Dict[str, Any] + bin = 'b' + else: + nl = {'newline': ''} # type: Dict[str, Any] + bin = '' + return open(name, mode + bin, **nl) + + +def fix_script(path): + # type: (str) -> Optional[bool] + """Replace #!python with #!/path/to/python + Return True if file was changed. + """ + # XXX RECORD hashes will need to be updated + if os.path.isfile(path): + with open(path, 'rb') as script: + firstline = script.readline() + if not firstline.startswith(b'#!python'): + return False + exename = sys.executable.encode(sys.getfilesystemencoding()) + firstline = b'#!' + exename + os.linesep.encode("ascii") + rest = script.read() + with open(path, 'wb') as script: + script.write(firstline) + script.write(rest) + return True + return None + + +def wheel_root_is_purelib(metadata): + # type: (Message) -> bool + return metadata.get("Root-Is-Purelib", "").lower() == "true" + + +def get_entrypoints(filename): + # type: (str) -> Tuple[Dict[str, str], Dict[str, str]] + if not os.path.exists(filename): + return {}, {} + + # This is done because you can pass a string to entry_points wrappers which + # means that they may or may not be valid INI files. The attempt here is to + # strip leading and trailing whitespace in order to make them valid INI + # files. + with open(filename) as fp: + data = StringIO() + for line in fp: + data.write(line.strip()) + data.write("\n") + data.seek(0) + + # get the entry points and then the script names + entry_points = pkg_resources.EntryPoint.parse_map(data) + console = entry_points.get('console_scripts', {}) + gui = entry_points.get('gui_scripts', {}) + + def _split_ep(s): + # type: (pkg_resources.EntryPoint) -> Tuple[str, str] + """get the string representation of EntryPoint, + remove space and split on '=' + """ + split_parts = str(s).replace(" ", "").split("=") + return split_parts[0], split_parts[1] + + # convert the EntryPoint objects into strings with module:function + console = dict(_split_ep(v) for v in console.values()) + gui = dict(_split_ep(v) for v in gui.values()) + return console, gui + + +def message_about_scripts_not_on_PATH(scripts): + # type: (Sequence[str]) -> Optional[str] + """Determine if any scripts are not on PATH and format a warning. + Returns a warning message if one or more scripts are not on PATH, + otherwise None. + """ + if not scripts: + return None + + # Group scripts by the path they were installed in + grouped_by_dir = collections.defaultdict(set) # type: Dict[str, Set[str]] + for destfile in scripts: + parent_dir = os.path.dirname(destfile) + script_name = os.path.basename(destfile) + grouped_by_dir[parent_dir].add(script_name) + + # We don't want to warn for directories that are on PATH. + not_warn_dirs = [ + os.path.normcase(i).rstrip(os.sep) for i in + os.environ.get("PATH", "").split(os.pathsep) + ] + # If an executable sits with sys.executable, we don't warn for it. + # This covers the case of venv invocations without activating the venv. + not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) + warn_for = { + parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() + if os.path.normcase(parent_dir) not in not_warn_dirs + } # type: Dict[str, Set[str]] + if not warn_for: + return None + + # Format a message + msg_lines = [] + for parent_dir, dir_scripts in warn_for.items(): + sorted_scripts = sorted(dir_scripts) # type: List[str] + if len(sorted_scripts) == 1: + start_text = "script {} is".format(sorted_scripts[0]) + else: + start_text = "scripts {} are".format( + ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1] + ) + + msg_lines.append( + "The {} installed in '{}' which is not on PATH." + .format(start_text, parent_dir) + ) + + last_line_fmt = ( + "Consider adding {} to PATH or, if you prefer " + "to suppress this warning, use --no-warn-script-location." + ) + if len(msg_lines) == 1: + msg_lines.append(last_line_fmt.format("this directory")) + else: + msg_lines.append(last_line_fmt.format("these directories")) + + # Add a note if any directory starts with ~ + warn_for_tilde = any( + i[0] == "~" for i in os.environ.get("PATH", "").split(os.pathsep) if i + ) + if warn_for_tilde: + tilde_warning_msg = ( + "NOTE: The current PATH contains path(s) starting with `~`, " + "which may not be expanded by all applications." + ) + msg_lines.append(tilde_warning_msg) + + # Returns the formatted multiline message + return "\n".join(msg_lines) + + +def sorted_outrows(outrows): + # type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow] + """Return the given rows of a RECORD file in sorted order. + + Each row is a 3-tuple (path, hash, size) and corresponds to a record of + a RECORD file (see PEP 376 and PEP 427 for details). For the rows + passed to this function, the size can be an integer as an int or string, + or the empty string. + """ + # Normally, there should only be one row per path, in which case the + # second and third elements don't come into play when sorting. + # However, in cases in the wild where a path might happen to occur twice, + # we don't want the sort operation to trigger an error (but still want + # determinism). Since the third element can be an int or string, we + # coerce each element to a string to avoid a TypeError in this case. + # For additional background, see-- + # https://github.com/pypa/pip/issues/5868 + return sorted(outrows, key=lambda row: tuple(str(x) for x in row)) + + +def get_csv_rows_for_installed( + old_csv_rows, # type: Iterable[List[str]] + installed, # type: Dict[str, str] + changed, # type: Set[str] + generated, # type: List[str] + lib_dir, # type: str +): + # type: (...) -> List[InstalledCSVRow] + """ + :param installed: A map from archive RECORD path to installation RECORD + path. + """ + installed_rows = [] # type: List[InstalledCSVRow] + for row in old_csv_rows: + if len(row) > 3: + logger.warning( + 'RECORD line has more than three elements: {}'.format(row) + ) + # Make a copy because we are mutating the row. + row = list(row) + old_path = row[0] + new_path = installed.pop(old_path, old_path) + row[0] = new_path + if new_path in changed: + digest, length = rehash(new_path) + row[1] = digest + row[2] = length + installed_rows.append(tuple(row)) + for f in generated: + digest, length = rehash(f) + installed_rows.append((normpath(f, lib_dir), digest, str(length))) + for f in installed: + installed_rows.append((installed[f], '', '')) + return installed_rows + + +class MissingCallableSuffix(Exception): + pass + + +def _raise_for_invalid_entrypoint(specification): + # type: (str) -> None + entry = get_export_entry(specification) + if entry is not None and entry.suffix is None: + raise MissingCallableSuffix(str(entry)) + + +class PipScriptMaker(ScriptMaker): + def make(self, specification, options=None): + # type: (str, Dict[str, Any]) -> List[str] + _raise_for_invalid_entrypoint(specification) + return super(PipScriptMaker, self).make(specification, options) + + +def install_unpacked_wheel( + name, # type: str + wheeldir, # type: str + wheel_zip, # type: ZipFile + scheme, # type: Scheme + req_description, # type: str + pycompile=True, # type: bool + warn_script_location=True # type: bool +): + # type: (...) -> None + """Install a wheel. + + :param name: Name of the project to install + :param wheeldir: Base directory of the unpacked wheel + :param wheel_zip: open ZipFile for wheel being installed + :param scheme: Distutils scheme dictating the install directories + :param req_description: String used in place of the requirement, for + logging + :param pycompile: Whether to byte-compile installed Python files + :param warn_script_location: Whether to check that scripts are installed + into a directory on PATH + :raises UnsupportedWheel: + * when the directory holds an unpacked wheel with incompatible + Wheel-Version + * when the .dist-info dir does not match the wheel + """ + # TODO: Investigate and break this up. + # TODO: Look into moving this into a dedicated class for representing an + # installation. + + source = wheeldir.rstrip(os.path.sep) + os.path.sep + + info_dir, metadata = parse_wheel(wheel_zip, name) + + if wheel_root_is_purelib(metadata): + lib_dir = scheme.purelib + else: + lib_dir = scheme.platlib + + subdirs = os.listdir(source) + data_dirs = [s for s in subdirs if s.endswith('.data')] + + # Record details of the files moved + # installed = files copied from the wheel to the destination + # changed = files changed while installing (scripts #! line typically) + # generated = files newly generated during the install (script wrappers) + installed = {} # type: Dict[str, str] + changed = set() + generated = [] # type: List[str] + + # Compile all of the pyc files that we're going to be installing + if pycompile: + with captured_stdout() as stdout: + with warnings.catch_warnings(): + warnings.filterwarnings('ignore') + compileall.compile_dir(source, force=True, quiet=True) + logger.debug(stdout.getvalue()) + + def record_installed(srcfile, destfile, modified=False): + # type: (str, str, bool) -> None + """Map archive RECORD paths to installation RECORD paths.""" + oldpath = normpath(srcfile, wheeldir) + newpath = normpath(destfile, lib_dir) + installed[oldpath] = newpath + if modified: + changed.add(destfile) + + def clobber( + source, # type: str + dest, # type: str + is_base, # type: bool + fixer=None, # type: Optional[Callable[[str], Any]] + filter=None # type: Optional[Callable[[str], bool]] + ): + # type: (...) -> None + ensure_dir(dest) # common for the 'include' path + + for dir, subdirs, files in os.walk(source): + basedir = dir[len(source):].lstrip(os.path.sep) + destdir = os.path.join(dest, basedir) + if is_base and basedir == '': + subdirs[:] = [s for s in subdirs if not s.endswith('.data')] + for f in files: + # Skip unwanted files + if filter and filter(f): + continue + srcfile = os.path.join(dir, f) + destfile = os.path.join(dest, basedir, f) + # directory creation is lazy and after the file filtering above + # to ensure we don't install empty dirs; empty dirs can't be + # uninstalled. + ensure_dir(destdir) + + # copyfile (called below) truncates the destination if it + # exists and then writes the new contents. This is fine in most + # cases, but can cause a segfault if pip has loaded a shared + # object (e.g. from pyopenssl through its vendored urllib3) + # Since the shared object is mmap'd an attempt to call a + # symbol in it will then cause a segfault. Unlinking the file + # allows writing of new contents while allowing the process to + # continue to use the old copy. + if os.path.exists(destfile): + os.unlink(destfile) + + # We use copyfile (not move, copy, or copy2) to be extra sure + # that we are not moving directories over (copyfile fails for + # directories) as well as to ensure that we are not copying + # over any metadata because we want more control over what + # metadata we actually copy over. + shutil.copyfile(srcfile, destfile) + + # Copy over the metadata for the file, currently this only + # includes the atime and mtime. + st = os.stat(srcfile) + if hasattr(os, "utime"): + os.utime(destfile, (st.st_atime, st.st_mtime)) + + # If our file is executable, then make our destination file + # executable. + if os.access(srcfile, os.X_OK): + st = os.stat(srcfile) + permissions = ( + st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) + os.chmod(destfile, permissions) + + changed = False + if fixer: + changed = fixer(destfile) + record_installed(srcfile, destfile, changed) + + clobber(source, lib_dir, True) + + dest_info_dir = os.path.join(lib_dir, info_dir) + + # Get the defined entry points + ep_file = os.path.join(dest_info_dir, 'entry_points.txt') + console, gui = get_entrypoints(ep_file) + + def is_entrypoint_wrapper(name): + # type: (str) -> bool + # EP, EP.exe and EP-script.py are scripts generated for + # entry point EP by setuptools + if name.lower().endswith('.exe'): + matchname = name[:-4] + elif name.lower().endswith('-script.py'): + matchname = name[:-10] + elif name.lower().endswith(".pya"): + matchname = name[:-4] + else: + matchname = name + # Ignore setuptools-generated scripts + return (matchname in console or matchname in gui) + + for datadir in data_dirs: + fixer = None + filter = None + for subdir in os.listdir(os.path.join(wheeldir, datadir)): + fixer = None + if subdir == 'scripts': + fixer = fix_script + filter = is_entrypoint_wrapper + source = os.path.join(wheeldir, datadir, subdir) + dest = getattr(scheme, subdir) + clobber(source, dest, False, fixer=fixer, filter=filter) + + maker = PipScriptMaker(None, scheme.scripts) + + # Ensure old scripts are overwritten. + # See https://github.com/pypa/pip/issues/1800 + maker.clobber = True + + # Ensure we don't generate any variants for scripts because this is almost + # never what somebody wants. + # See https://bitbucket.org/pypa/distlib/issue/35/ + maker.variants = {''} + + # This is required because otherwise distlib creates scripts that are not + # executable. + # See https://bitbucket.org/pypa/distlib/issue/32/ + maker.set_mode = True + + scripts_to_generate = [] + + # Special case pip and setuptools to generate versioned wrappers + # + # The issue is that some projects (specifically, pip and setuptools) use + # code in setup.py to create "versioned" entry points - pip2.7 on Python + # 2.7, pip3.3 on Python 3.3, etc. But these entry points are baked into + # the wheel metadata at build time, and so if the wheel is installed with + # a *different* version of Python the entry points will be wrong. The + # correct fix for this is to enhance the metadata to be able to describe + # such versioned entry points, but that won't happen till Metadata 2.0 is + # available. + # In the meantime, projects using versioned entry points will either have + # incorrect versioned entry points, or they will not be able to distribute + # "universal" wheels (i.e., they will need a wheel per Python version). + # + # Because setuptools and pip are bundled with _ensurepip and virtualenv, + # we need to use universal wheels. So, as a stopgap until Metadata 2.0, we + # override the versioned entry points in the wheel and generate the + # correct ones. This code is purely a short-term measure until Metadata 2.0 + # is available. + # + # To add the level of hack in this section of code, in order to support + # ensurepip this code will look for an ``ENSUREPIP_OPTIONS`` environment + # variable which will control which version scripts get installed. + # + # ENSUREPIP_OPTIONS=altinstall + # - Only pipX.Y and easy_install-X.Y will be generated and installed + # ENSUREPIP_OPTIONS=install + # - pipX.Y, pipX, easy_install-X.Y will be generated and installed. Note + # that this option is technically if ENSUREPIP_OPTIONS is set and is + # not altinstall + # DEFAULT + # - The default behavior is to install pip, pipX, pipX.Y, easy_install + # and easy_install-X.Y. + pip_script = console.pop('pip', None) + if pip_script: + if "ENSUREPIP_OPTIONS" not in os.environ: + scripts_to_generate.append('pip = ' + pip_script) + + if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": + scripts_to_generate.append( + 'pip%s = %s' % (sys.version_info[0], pip_script) + ) + + scripts_to_generate.append( + 'pip%s = %s' % (get_major_minor_version(), pip_script) + ) + # Delete any other versioned pip entry points + pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] + for k in pip_ep: + del console[k] + easy_install_script = console.pop('easy_install', None) + if easy_install_script: + if "ENSUREPIP_OPTIONS" not in os.environ: + scripts_to_generate.append( + 'easy_install = ' + easy_install_script + ) + + scripts_to_generate.append( + 'easy_install-%s = %s' % ( + get_major_minor_version(), easy_install_script + ) + ) + # Delete any other versioned easy_install entry points + easy_install_ep = [ + k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) + ] + for k in easy_install_ep: + del console[k] + + # Generate the console and GUI entry points specified in the wheel + scripts_to_generate.extend( + '%s = %s' % kv for kv in console.items() + ) + + gui_scripts_to_generate = [ + '%s = %s' % kv for kv in gui.items() + ] + + generated_console_scripts = [] # type: List[str] + + try: + generated_console_scripts = maker.make_multiple(scripts_to_generate) + generated.extend(generated_console_scripts) + + generated.extend( + maker.make_multiple(gui_scripts_to_generate, {'gui': True}) + ) + except MissingCallableSuffix as e: + entry = e.args[0] + raise InstallationError( + "Invalid script entry point: {} for req: {} - A callable " + "suffix is required. Cf https://packaging.python.org/" + "specifications/entry-points/#use-for-scripts for more " + "information.".format(entry, req_description) + ) + + if warn_script_location: + msg = message_about_scripts_not_on_PATH(generated_console_scripts) + if msg is not None: + logger.warning(msg) + + # Record pip as the installer + installer = os.path.join(dest_info_dir, 'INSTALLER') + temp_installer = os.path.join(dest_info_dir, 'INSTALLER.pip') + with open(temp_installer, 'wb') as installer_file: + installer_file.write(b'pip\n') + shutil.move(temp_installer, installer) + generated.append(installer) + + # Record details of all files installed + record = os.path.join(dest_info_dir, 'RECORD') + temp_record = os.path.join(dest_info_dir, 'RECORD.pip') + with open_for_csv(record, 'r') as record_in: + with open_for_csv(temp_record, 'w+') as record_out: + reader = csv.reader(record_in) + outrows = get_csv_rows_for_installed( + reader, installed=installed, changed=changed, + generated=generated, lib_dir=lib_dir, + ) + writer = csv.writer(record_out) + # Sort to simplify testing. + for row in sorted_outrows(outrows): + writer.writerow(row) + shutil.move(temp_record, record) + + +def install_wheel( + name, # type: str + wheel_path, # type: str + scheme, # type: Scheme + req_description, # type: str + pycompile=True, # type: bool + warn_script_location=True, # type: bool + _temp_dir_for_testing=None, # type: Optional[str] +): + # type: (...) -> None + with TempDirectory( + path=_temp_dir_for_testing, kind="unpacked-wheel" + ) as unpacked_dir, ZipFile(wheel_path, allowZip64=True) as z: + unpack_file(wheel_path, unpacked_dir.path) + install_unpacked_wheel( + name=name, + wheeldir=unpacked_dir.path, + wheel_zip=z, + scheme=scheme, + req_description=req_description, + pycompile=pycompile, + warn_script_location=warn_script_location, + ) diff --git a/ubuntu/venv/pip/_internal/operations/prepare.py b/ubuntu/venv/pip/_internal/operations/prepare.py new file mode 100644 index 0000000..0b61f20 --- /dev/null +++ b/ubuntu/venv/pip/_internal/operations/prepare.py @@ -0,0 +1,591 @@ +"""Prepares a distribution for installation +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import logging +import mimetypes +import os +import shutil +import sys + +from pip._vendor import requests +from pip._vendor.six import PY2 + +from pip._internal.distributions import ( + make_distribution_for_install_requirement, +) +from pip._internal.distributions.installed import InstalledDistribution +from pip._internal.exceptions import ( + DirectoryUrlHashUnsupported, + HashMismatch, + HashUnpinned, + InstallationError, + PreviousBuildDirError, + VcsHashUnsupported, +) +from pip._internal.utils.filesystem import copy2_fixed +from pip._internal.utils.hashes import MissingHashes +from pip._internal.utils.logging import indent_log +from pip._internal.utils.marker_files import write_delete_marker_file +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + path_to_display, + rmtree, +) +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.unpacking import unpack_file +from pip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Callable, List, Optional, Tuple, + ) + + from mypy_extensions import TypedDict + + from pip._internal.distributions import AbstractDistribution + from pip._internal.index.package_finder import PackageFinder + from pip._internal.models.link import Link + from pip._internal.network.download import Downloader + from pip._internal.req.req_install import InstallRequirement + from pip._internal.req.req_tracker import RequirementTracker + from pip._internal.utils.hashes import Hashes + + if PY2: + CopytreeKwargs = TypedDict( + 'CopytreeKwargs', + { + 'ignore': Callable[[str, List[str]], List[str]], + 'symlinks': bool, + }, + total=False, + ) + else: + CopytreeKwargs = TypedDict( + 'CopytreeKwargs', + { + 'copy_function': Callable[[str, str], None], + 'ignore': Callable[[str, List[str]], List[str]], + 'ignore_dangling_symlinks': bool, + 'symlinks': bool, + }, + total=False, + ) + +logger = logging.getLogger(__name__) + + +def _get_prepared_distribution( + req, # type: InstallRequirement + req_tracker, # type: RequirementTracker + finder, # type: PackageFinder + build_isolation # type: bool +): + # type: (...) -> AbstractDistribution + """Prepare a distribution for installation. + """ + abstract_dist = make_distribution_for_install_requirement(req) + with req_tracker.track(req): + abstract_dist.prepare_distribution_metadata(finder, build_isolation) + return abstract_dist + + +def unpack_vcs_link(link, location): + # type: (Link, str) -> None + vcs_backend = vcs.get_backend_for_scheme(link.scheme) + assert vcs_backend is not None + vcs_backend.unpack(location, url=hide_url(link.url)) + + +def _copy_file(filename, location, link): + # type: (str, str, Link) -> None + copy = True + download_location = os.path.join(location, link.filename) + if os.path.exists(download_location): + response = ask_path_exists( + 'The file {} exists. (i)gnore, (w)ipe, (b)ackup, (a)abort'.format( + display_path(download_location) + ), + ('i', 'w', 'b', 'a'), + ) + if response == 'i': + copy = False + elif response == 'w': + logger.warning('Deleting %s', display_path(download_location)) + os.remove(download_location) + elif response == 'b': + dest_file = backup_dir(download_location) + logger.warning( + 'Backing up %s to %s', + display_path(download_location), + display_path(dest_file), + ) + shutil.move(download_location, dest_file) + elif response == 'a': + sys.exit(-1) + if copy: + shutil.copy(filename, download_location) + logger.info('Saved %s', display_path(download_location)) + + +def unpack_http_url( + link, # type: Link + location, # type: str + downloader, # type: Downloader + download_dir=None, # type: Optional[str] + hashes=None, # type: Optional[Hashes] +): + # type: (...) -> str + temp_dir = TempDirectory(kind="unpack", globally_managed=True) + # If a download dir is specified, is the file already downloaded there? + already_downloaded_path = None + if download_dir: + already_downloaded_path = _check_download_dir( + link, download_dir, hashes + ) + + if already_downloaded_path: + from_path = already_downloaded_path + content_type = mimetypes.guess_type(from_path)[0] + else: + # let's download to a tmp dir + from_path, content_type = _download_http_url( + link, downloader, temp_dir.path, hashes + ) + + # unpack the archive to the build dir location. even when only + # downloading archives, they have to be unpacked to parse dependencies + unpack_file(from_path, location, content_type) + + return from_path + + +def _copy2_ignoring_special_files(src, dest): + # type: (str, str) -> None + """Copying special files is not supported, but as a convenience to users + we skip errors copying them. This supports tools that may create e.g. + socket files in the project source directory. + """ + try: + copy2_fixed(src, dest) + except shutil.SpecialFileError as e: + # SpecialFileError may be raised due to either the source or + # destination. If the destination was the cause then we would actually + # care, but since the destination directory is deleted prior to + # copy we ignore all of them assuming it is caused by the source. + logger.warning( + "Ignoring special file error '%s' encountered copying %s to %s.", + str(e), + path_to_display(src), + path_to_display(dest), + ) + + +def _copy_source_tree(source, target): + # type: (str, str) -> None + def ignore(d, names): + # type: (str, List[str]) -> List[str] + # Pulling in those directories can potentially be very slow, + # exclude the following directories if they appear in the top + # level dir (and only it). + # See discussion at https://github.com/pypa/pip/pull/6770 + return ['.tox', '.nox'] if d == source else [] + + kwargs = dict(ignore=ignore, symlinks=True) # type: CopytreeKwargs + + if not PY2: + # Python 2 does not support copy_function, so we only ignore + # errors on special file copy in Python 3. + kwargs['copy_function'] = _copy2_ignoring_special_files + + shutil.copytree(source, target, **kwargs) + + +def unpack_file_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + hashes=None # type: Optional[Hashes] +): + # type: (...) -> Optional[str] + """Unpack link into location. + """ + link_path = link.file_path + # If it's a url to a local directory + if link.is_existing_dir(): + if os.path.isdir(location): + rmtree(location) + _copy_source_tree(link_path, location) + return None + + # If a download dir is specified, is the file already there and valid? + already_downloaded_path = None + if download_dir: + already_downloaded_path = _check_download_dir( + link, download_dir, hashes + ) + + if already_downloaded_path: + from_path = already_downloaded_path + else: + from_path = link_path + + # If --require-hashes is off, `hashes` is either empty, the + # link's embedded hash, or MissingHashes; it is required to + # match. If --require-hashes is on, we are satisfied by any + # hash in `hashes` matching: a URL-based or an option-based + # one; no internet-sourced hash will be in `hashes`. + if hashes: + hashes.check_against_path(from_path) + + content_type = mimetypes.guess_type(from_path)[0] + + # unpack the archive to the build dir location. even when only downloading + # archives, they have to be unpacked to parse dependencies + unpack_file(from_path, location, content_type) + + return from_path + + +def unpack_url( + link, # type: Link + location, # type: str + downloader, # type: Downloader + download_dir=None, # type: Optional[str] + hashes=None, # type: Optional[Hashes] +): + # type: (...) -> Optional[str] + """Unpack link into location, downloading if required. + + :param hashes: A Hashes object, one of whose embedded hashes must match, + or HashMismatch will be raised. If the Hashes is empty, no matches are + required, and unhashable types of requirements (like VCS ones, which + would ordinarily raise HashUnsupported) are allowed. + """ + # non-editable vcs urls + if link.is_vcs: + unpack_vcs_link(link, location) + return None + + # file urls + elif link.is_file: + return unpack_file_url(link, location, download_dir, hashes=hashes) + + # http urls + else: + return unpack_http_url( + link, + location, + downloader, + download_dir, + hashes=hashes, + ) + + +def _download_http_url( + link, # type: Link + downloader, # type: Downloader + temp_dir, # type: str + hashes, # type: Optional[Hashes] +): + # type: (...) -> Tuple[str, str] + """Download link url into temp_dir using provided session""" + download = downloader(link) + + file_path = os.path.join(temp_dir, download.filename) + with open(file_path, 'wb') as content_file: + for chunk in download.chunks: + content_file.write(chunk) + + if hashes: + hashes.check_against_path(file_path) + + return file_path, download.response.headers.get('content-type', '') + + +def _check_download_dir(link, download_dir, hashes): + # type: (Link, str, Optional[Hashes]) -> Optional[str] + """ Check download_dir for previously downloaded file with correct hash + If a correct file is found return its path else None + """ + download_path = os.path.join(download_dir, link.filename) + + if not os.path.exists(download_path): + return None + + # If already downloaded, does its hash match? + logger.info('File was already downloaded %s', download_path) + if hashes: + try: + hashes.check_against_path(download_path) + except HashMismatch: + logger.warning( + 'Previously-downloaded file %s has bad hash. ' + 'Re-downloading.', + download_path + ) + os.unlink(download_path) + return None + return download_path + + +class RequirementPreparer(object): + """Prepares a Requirement + """ + + def __init__( + self, + build_dir, # type: str + download_dir, # type: Optional[str] + src_dir, # type: str + wheel_download_dir, # type: Optional[str] + build_isolation, # type: bool + req_tracker, # type: RequirementTracker + downloader, # type: Downloader + finder, # type: PackageFinder + require_hashes, # type: bool + use_user_site, # type: bool + ): + # type: (...) -> None + super(RequirementPreparer, self).__init__() + + self.src_dir = src_dir + self.build_dir = build_dir + self.req_tracker = req_tracker + self.downloader = downloader + self.finder = finder + + # Where still-packed archives should be written to. If None, they are + # not saved, and are deleted immediately after unpacking. + self.download_dir = download_dir + + # Where still-packed .whl files should be written to. If None, they are + # written to the download_dir parameter. Separate to download_dir to + # permit only keeping wheel archives for pip wheel. + self.wheel_download_dir = wheel_download_dir + + # NOTE + # download_dir and wheel_download_dir overlap semantically and may + # be combined if we're willing to have non-wheel archives present in + # the wheelhouse output by 'pip wheel'. + + # Is build isolation allowed? + self.build_isolation = build_isolation + + # Should hash-checking be required? + self.require_hashes = require_hashes + + # Should install in user site-packages? + self.use_user_site = use_user_site + + @property + def _download_should_save(self): + # type: () -> bool + if not self.download_dir: + return False + + if os.path.exists(self.download_dir): + return True + + logger.critical('Could not find download directory') + raise InstallationError( + "Could not find or access download directory '{}'" + .format(self.download_dir)) + + def prepare_linked_requirement( + self, + req, # type: InstallRequirement + ): + # type: (...) -> AbstractDistribution + """Prepare a requirement that would be obtained from req.link + """ + assert req.link + link = req.link + + # TODO: Breakup into smaller functions + if link.scheme == 'file': + path = link.file_path + logger.info('Processing %s', display_path(path)) + else: + logger.info('Collecting %s', req.req or req) + + with indent_log(): + # @@ if filesystem packages are not marked + # editable in a req, a non deterministic error + # occurs when the script attempts to unpack the + # build directory + # Since source_dir is only set for editable requirements. + assert req.source_dir is None + req.ensure_has_source_dir(self.build_dir) + # If a checkout exists, it's unwise to keep going. version + # inconsistencies are logged later, but do not fail the + # installation. + # FIXME: this won't upgrade when there's an existing + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): + raise PreviousBuildDirError( + "pip can't proceed with requirements '{}' due to a" + " pre-existing build directory ({}). This is " + "likely due to a previous installation that failed" + ". pip is being responsible and not assuming it " + "can delete this. Please delete it and try again." + .format(req, req.source_dir) + ) + + # Now that we have the real link, we can tell what kind of + # requirements we have and raise some more informative errors + # than otherwise. (For example, we can raise VcsHashUnsupported + # for a VCS URL rather than HashMissing.) + if self.require_hashes: + # We could check these first 2 conditions inside + # unpack_url and save repetition of conditions, but then + # we would report less-useful error messages for + # unhashable requirements, complaining that there's no + # hash provided. + if link.is_vcs: + raise VcsHashUnsupported() + elif link.is_existing_dir(): + raise DirectoryUrlHashUnsupported() + if not req.original_link and not req.is_pinned: + # Unpinned packages are asking for trouble when a new + # version is uploaded. This isn't a security check, but + # it saves users a surprising hash mismatch in the + # future. + # + # file:/// URLs aren't pinnable, so don't complain + # about them not being pinned. + raise HashUnpinned() + + hashes = req.hashes(trust_internet=not self.require_hashes) + if self.require_hashes and not hashes: + # Known-good hashes are missing for this requirement, so + # shim it with a facade object that will provoke hash + # computation and then raise a HashMissing exception + # showing the user what the hash should be. + hashes = MissingHashes() + + download_dir = self.download_dir + if link.is_wheel and self.wheel_download_dir: + # when doing 'pip wheel` we download wheels to a + # dedicated dir. + download_dir = self.wheel_download_dir + + try: + local_path = unpack_url( + link, req.source_dir, self.downloader, download_dir, + hashes=hashes, + ) + except requests.HTTPError as exc: + logger.critical( + 'Could not install requirement %s because of error %s', + req, + exc, + ) + raise InstallationError( + 'Could not install requirement {} because of HTTP ' + 'error {} for URL {}'.format(req, exc, link) + ) + + # For use in later processing, preserve the file path on the + # requirement. + if local_path: + req.local_file_path = local_path + + if link.is_wheel: + if download_dir: + # When downloading, we only unpack wheels to get + # metadata. + autodelete_unpacked = True + else: + # When installing a wheel, we use the unpacked + # wheel. + autodelete_unpacked = False + else: + # We always delete unpacked sdists after pip runs. + autodelete_unpacked = True + if autodelete_unpacked: + write_delete_marker_file(req.source_dir) + + abstract_dist = _get_prepared_distribution( + req, self.req_tracker, self.finder, self.build_isolation, + ) + + if download_dir: + if link.is_existing_dir(): + logger.info('Link is a directory, ignoring download_dir') + elif local_path and not os.path.exists( + os.path.join(download_dir, link.filename) + ): + _copy_file(local_path, download_dir, link) + + if self._download_should_save: + # Make a .zip of the source_dir we already created. + if link.is_vcs: + req.archive(self.download_dir) + return abstract_dist + + def prepare_editable_requirement( + self, + req, # type: InstallRequirement + ): + # type: (...) -> AbstractDistribution + """Prepare an editable requirement + """ + assert req.editable, "cannot prepare a non-editable req as editable" + + logger.info('Obtaining %s', req) + + with indent_log(): + if self.require_hashes: + raise InstallationError( + 'The editable requirement {} cannot be installed when ' + 'requiring hashes, because there is no single file to ' + 'hash.'.format(req) + ) + req.ensure_has_source_dir(self.src_dir) + req.update_editable(not self._download_should_save) + + abstract_dist = _get_prepared_distribution( + req, self.req_tracker, self.finder, self.build_isolation, + ) + + if self._download_should_save: + req.archive(self.download_dir) + req.check_if_exists(self.use_user_site) + + return abstract_dist + + def prepare_installed_requirement( + self, + req, # type: InstallRequirement + skip_reason # type: str + ): + # type: (...) -> AbstractDistribution + """Prepare an already-installed requirement + """ + assert req.satisfied_by, "req should have been satisfied but isn't" + assert skip_reason is not None, ( + "did not get skip reason skipped but req.satisfied_by " + "is set to {}".format(req.satisfied_by) + ) + logger.info( + 'Requirement %s: %s (%s)', + skip_reason, req, req.satisfied_by.version + ) + with indent_log(): + if self.require_hashes: + logger.debug( + 'Since it is already installed, we are trusting this ' + 'package without checking its hash. To ensure a ' + 'completely repeatable environment, install into an ' + 'empty virtualenv.' + ) + abstract_dist = InstalledDistribution(req) + + return abstract_dist diff --git a/ubuntu/venv/pip/_internal/pep425tags.py b/ubuntu/venv/pip/_internal/pep425tags.py new file mode 100644 index 0000000..a2386ee --- /dev/null +++ b/ubuntu/venv/pip/_internal/pep425tags.py @@ -0,0 +1,167 @@ +"""Generate and work with PEP 425 Compatibility Tags.""" +from __future__ import absolute_import + +import logging +import re + +from pip._vendor.packaging.tags import ( + Tag, + compatible_tags, + cpython_tags, + generic_tags, + interpreter_name, + interpreter_version, + mac_platforms, +) + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Tuple + + from pip._vendor.packaging.tags import PythonVersion + +logger = logging.getLogger(__name__) + +_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') + + +def version_info_to_nodot(version_info): + # type: (Tuple[int, ...]) -> str + # Only use up to the first two numbers. + return ''.join(map(str, version_info[:2])) + + +def _mac_platforms(arch): + # type: (str) -> List[str] + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + mac_version = (int(major), int(minor)) + arches = [ + # Since we have always only checked that the platform starts + # with "macosx", for backwards-compatibility we extract the + # actual prefix provided by the user in case they provided + # something like "macosxcustom_". It may be good to remove + # this as undocumented or deprecate it in the future. + '{}_{}'.format(name, arch[len('macosx_'):]) + for arch in mac_platforms(mac_version, actual_arch) + ] + else: + # arch pattern didn't match (?!) + arches = [arch] + return arches + + +def _custom_manylinux_platforms(arch): + # type: (str) -> List[str] + arches = [arch] + arch_prefix, arch_sep, arch_suffix = arch.partition('_') + if arch_prefix == 'manylinux2014': + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {'i686', 'x86_64'}: + arches.append('manylinux2010' + arch_sep + arch_suffix) + arches.append('manylinux1' + arch_sep + arch_suffix) + elif arch_prefix == 'manylinux2010': + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches.append('manylinux1' + arch_sep + arch_suffix) + return arches + + +def _get_custom_platforms(arch): + # type: (str) -> List[str] + arch_prefix, arch_sep, arch_suffix = arch.partition('_') + if arch.startswith('macosx'): + arches = _mac_platforms(arch) + elif arch_prefix in ['manylinux2014', 'manylinux2010']: + arches = _custom_manylinux_platforms(arch) + else: + arches = [arch] + return arches + + +def _get_python_version(version): + # type: (str) -> PythonVersion + if len(version) > 1: + return int(version[0]), int(version[1:]) + else: + return (int(version[0]),) + + +def _get_custom_interpreter(implementation=None, version=None): + # type: (Optional[str], Optional[str]) -> str + if implementation is None: + implementation = interpreter_name() + if version is None: + version = interpreter_version() + return "{}{}".format(implementation, version) + + +def get_supported( + version=None, # type: Optional[str] + platform=None, # type: Optional[str] + impl=None, # type: Optional[str] + abi=None # type: Optional[str] +): + # type: (...) -> List[Tag] + """Return a list of supported tags for each version specified in + `versions`. + + :param version: a string version, of the form "33" or "32", + or None. The version will be assumed to support our ABI. + :param platform: specify the exact platform you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abi: specify the exact abi you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] # type: List[Tag] + + python_version = None # type: Optional[PythonVersion] + if version is not None: + python_version = _get_python_version(version) + + interpreter = _get_custom_interpreter(impl, version) + + abis = None # type: Optional[List[str]] + if abi is not None: + abis = [abi] + + platforms = None # type: Optional[List[str]] + if platform is not None: + platforms = _get_custom_platforms(platform) + + is_cpython = (impl or interpreter_name()) == "cp" + if is_cpython: + supported.extend( + cpython_tags( + python_version=python_version, + abis=abis, + platforms=platforms, + ) + ) + else: + supported.extend( + generic_tags( + interpreter=interpreter, + abis=abis, + platforms=platforms, + ) + ) + supported.extend( + compatible_tags( + python_version=python_version, + interpreter=interpreter, + platforms=platforms, + ) + ) + + return supported diff --git a/ubuntu/venv/pip/_internal/pyproject.py b/ubuntu/venv/pip/_internal/pyproject.py new file mode 100644 index 0000000..6b4faf7 --- /dev/null +++ b/ubuntu/venv/pip/_internal/pyproject.py @@ -0,0 +1,196 @@ +from __future__ import absolute_import + +import io +import os +import sys +from collections import namedtuple + +from pip._vendor import six, toml +from pip._vendor.packaging.requirements import InvalidRequirement, Requirement + +from pip._internal.exceptions import InstallationError +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Optional, List + + +def _is_list_of_str(obj): + # type: (Any) -> bool + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) + + +def make_pyproject_path(unpacked_source_directory): + # type: (str) -> str + path = os.path.join(unpacked_source_directory, 'pyproject.toml') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(path, six.text_type): + path = path.encode(sys.getfilesystemencoding()) + + return path + + +BuildSystemDetails = namedtuple('BuildSystemDetails', [ + 'requires', 'backend', 'check', 'backend_path' +]) + + +def load_pyproject_toml( + use_pep517, # type: Optional[bool] + pyproject_toml, # type: str + setup_py, # type: str + req_name # type: str +): + # type: (...) -> Optional[BuildSystemDetails] + """Load the pyproject.toml file. + + Parameters: + use_pep517 - Has the user requested PEP 517 processing? None + means the user hasn't explicitly specified. + pyproject_toml - Location of the project's pyproject.toml file + setup_py - Location of the project's setup.py file + req_name - The name of the requirement we're processing (for + error reporting) + + Returns: + None if we should use the legacy code path, otherwise a tuple + ( + requirements from pyproject.toml, + name of PEP 517 backend, + requirements we should check are installed after setting + up the build environment + directory paths to import the backend from (backend-path), + relative to the project root. + ) + """ + has_pyproject = os.path.isfile(pyproject_toml) + has_setup = os.path.isfile(setup_py) + + if has_pyproject: + with io.open(pyproject_toml, encoding="utf-8") as f: + pp_toml = toml.load(f) + build_system = pp_toml.get("build-system") + else: + build_system = None + + # The following cases must use PEP 517 + # We check for use_pep517 being non-None and falsey because that means + # the user explicitly requested --no-use-pep517. The value 0 as + # opposed to False can occur when the value is provided via an + # environment variable or config file option (due to the quirk of + # strtobool() returning an integer in pip's configuration code). + if has_pyproject and not has_setup: + if use_pep517 is not None and not use_pep517: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project does not have a setup.py" + ) + use_pep517 = True + elif build_system and "build-backend" in build_system: + if use_pep517 is not None and not use_pep517: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project specifies a build backend of {} " + "in pyproject.toml".format( + build_system["build-backend"] + ) + ) + use_pep517 = True + + # If we haven't worked out whether to use PEP 517 yet, + # and the user hasn't explicitly stated a preference, + # we do so if the project has a pyproject.toml file. + elif use_pep517 is None: + use_pep517 = has_pyproject + + # At this point, we know whether we're going to use PEP 517. + assert use_pep517 is not None + + # If we're using the legacy code path, there is nothing further + # for us to do here. + if not use_pep517: + return None + + if build_system is None: + # Either the user has a pyproject.toml with no build-system + # section, or the user has no pyproject.toml, but has opted in + # explicitly via --use-pep517. + # In the absence of any explicit backend specification, we + # assume the setuptools backend that most closely emulates the + # traditional direct setup.py execution, and require wheel and + # a version of setuptools that supports that backend. + + build_system = { + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", + } + + # If we're using PEP 517, we have build system information (either + # from pyproject.toml, or defaulted by the code above). + # Note that at this point, we do not know if the user has actually + # specified a backend, though. + assert build_system is not None + + # Ensure that the build-system section in pyproject.toml conforms + # to PEP 518. + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + + # Specifying the build-system table but not the requires key is invalid + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=req_name, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) + ) + + # Error out if requires is not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=req_name, + reason="'build-system.requires' is not a list of strings.", + )) + + # Each requirement must be valid as per PEP 508 + for requirement in requires: + try: + Requirement(requirement) + except InvalidRequirement: + raise InstallationError( + error_template.format( + package=req_name, + reason=( + "'build-system.requires' contains an invalid " + "requirement: {!r}".format(requirement) + ), + ) + ) + + backend = build_system.get("build-backend") + backend_path = build_system.get("backend-path", []) + check = [] # type: List[str] + if backend is None: + # If the user didn't specify a backend, we assume they want to use + # the setuptools backend. But we can't be sure they have included + # a version of setuptools which supplies the backend, or wheel + # (which is needed by the backend) in their requirements. So we + # make a note to check that those requirements are present once + # we have set up the environment. + # This is quite a lot of work to check for a very specific case. But + # the problem is, that case is potentially quite common - projects that + # adopted PEP 518 early for the ability to specify requirements to + # execute setup.py, but never considered needing to mention the build + # tools themselves. The original PEP 518 code had a similar check (but + # implemented in a different way). + backend = "setuptools.build_meta:__legacy__" + check = ["setuptools>=40.8.0", "wheel"] + + return BuildSystemDetails(requires, backend, check, backend_path) diff --git a/ubuntu/venv/pip/_internal/req/__init__.py b/ubuntu/venv/pip/_internal/req/__init__.py new file mode 100644 index 0000000..d2d027a --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/__init__.py @@ -0,0 +1,92 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging + +from pip._internal.utils.logging import indent_log +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +from .req_file import parse_requirements +from .req_install import InstallRequirement +from .req_set import RequirementSet + +if MYPY_CHECK_RUNNING: + from typing import Any, List, Sequence + +__all__ = [ + "RequirementSet", "InstallRequirement", + "parse_requirements", "install_given_reqs", +] + +logger = logging.getLogger(__name__) + + +class InstallationResult(object): + def __init__(self, name): + # type: (str) -> None + self.name = name + + def __repr__(self): + # type: () -> str + return "InstallationResult(name={!r})".format(self.name) + + +def install_given_reqs( + to_install, # type: List[InstallRequirement] + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + *args, # type: Any + **kwargs # type: Any +): + # type: (...) -> List[InstallationResult] + """ + Install everything in the given list. + + (to be called after having downloaded and unpacked the packages) + """ + + if to_install: + logger.info( + 'Installing collected packages: %s', + ', '.join([req.name for req in to_install]), + ) + + installed = [] + + with indent_log(): + for requirement in to_install: + if requirement.should_reinstall: + logger.info('Attempting uninstall: %s', requirement.name) + with indent_log(): + uninstalled_pathset = requirement.uninstall( + auto_confirm=True + ) + try: + requirement.install( + install_options, + global_options, + *args, + **kwargs + ) + except Exception: + should_rollback = ( + requirement.should_reinstall and + not requirement.install_succeeded + ) + # if install did not succeed, rollback previous uninstall + if should_rollback: + uninstalled_pathset.rollback() + raise + else: + should_commit = ( + requirement.should_reinstall and + requirement.install_succeeded + ) + if should_commit: + uninstalled_pathset.commit() + + installed.append(InstallationResult(requirement.name)) + + return installed diff --git a/ubuntu/venv/pip/_internal/req/constructors.py b/ubuntu/venv/pip/_internal/req/constructors.py new file mode 100644 index 0000000..1f3cd8a --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/constructors.py @@ -0,0 +1,436 @@ +"""Backing implementation for InstallRequirement's various constructors + +The idea here is that these formed a major chunk of InstallRequirement's size +so, moving them and support code dedicated to them outside of that class +helps creates for better understandability for the rest of the code. + +These are meant to be used elsewhere within pip to create instances of +InstallRequirement. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import logging +import os +import re + +from pip._vendor.packaging.markers import Marker +from pip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pip._vendor.packaging.specifiers import Specifier +from pip._vendor.pkg_resources import RequirementParseError, parse_requirements + +from pip._internal.exceptions import InstallationError +from pip._internal.models.index import PyPI, TestPyPI +from pip._internal.models.link import Link +from pip._internal.models.wheel import Wheel +from pip._internal.pyproject import make_pyproject_path +from pip._internal.req.req_install import InstallRequirement +from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pip._internal.utils.misc import is_installable_dir, splitext +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs import is_url, vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Optional, Set, Tuple, Union, + ) + from pip._internal.cache import WheelCache + + +__all__ = [ + "install_req_from_editable", "install_req_from_line", + "parse_editable" +] + +logger = logging.getLogger(__name__) +operators = Specifier._operators.keys() + + +def is_archive_file(name): + # type: (str) -> bool + """Return True if `name` is a considered as an archive file.""" + ext = splitext(name)[1].lower() + if ext in ARCHIVE_EXTENSIONS: + return True + return False + + +def _strip_extras(path): + # type: (str) -> Tuple[str, Optional[str]] + m = re.match(r'^(.+)(\[[^\]]+\])$', path) + extras = None + if m: + path_no_extras = m.group(1) + extras = m.group(2) + else: + path_no_extras = path + + return path_no_extras, extras + + +def convert_extras(extras): + # type: (Optional[str]) -> Set[str] + if not extras: + return set() + return Requirement("placeholder" + extras.lower()).extras + + +def parse_editable(editable_req): + # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + - editable options + Accepted requirements: + svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + .[some_extra] + """ + + url = editable_req + + # If a file path is specified with extras, strip off the extras. + url_no_extras, extras = _strip_extras(url) + + if os.path.isdir(url_no_extras): + if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): + msg = ( + 'File "setup.py" not found. Directory cannot be installed ' + 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) + ) + pyproject_path = make_pyproject_path(url_no_extras) + if os.path.isfile(pyproject_path): + msg += ( + '\n(A "pyproject.toml" file was found, but editable ' + 'mode currently requires a setup.py based build.)' + ) + raise InstallationError(msg) + + # Treating it as code that has already been checked out + url_no_extras = path_to_url(url_no_extras) + + if url_no_extras.lower().startswith('file:'): + package_name = Link(url_no_extras).egg_fragment + if extras: + return ( + package_name, + url_no_extras, + Requirement("placeholder" + extras.lower()).extras, + ) + else: + return package_name, url_no_extras, None + + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + break + + if '+' not in url: + raise InstallationError( + '{} is not a valid editable requirement. ' + 'It should either be a path to a local project or a VCS URL ' + '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req) + ) + + vc_type = url.split('+', 1)[0].lower() + + if not vcs.get_backend(vc_type): + error_message = 'For --editable=%s only ' % editable_req + \ + ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ + ' is currently supported' + raise InstallationError(error_message) + + package_name = Link(url).egg_fragment + if not package_name: + raise InstallationError( + "Could not detect requirement name for '%s', please specify one " + "with #egg=your_package_name" % editable_req + ) + return package_name, url, None + + +def deduce_helpful_msg(req): + # type: (str) -> str + """Returns helpful msg in case requirements file does not exist, + or cannot be parsed. + + :params req: Requirements file path + """ + msg = "" + if os.path.exists(req): + msg = " It does exist." + # Try to parse and check if it is a requirements file. + try: + with open(req, 'r') as fp: + # parse first line only + next(parse_requirements(fp.read())) + msg += " The argument you provided " + \ + "(%s) appears to be a" % (req) + \ + " requirements file. If that is the" + \ + " case, use the '-r' flag to install" + \ + " the packages specified within it." + except RequirementParseError: + logger.debug("Cannot parse '%s' as requirements \ + file" % (req), exc_info=True) + else: + msg += " File '%s' does not exist." % (req) + return msg + + +class RequirementParts(object): + def __init__( + self, + requirement, # type: Optional[Requirement] + link, # type: Optional[Link] + markers, # type: Optional[Marker] + extras, # type: Set[str] + ): + self.requirement = requirement + self.link = link + self.markers = markers + self.extras = extras + + +def parse_req_from_editable(editable_req): + # type: (str) -> RequirementParts + name, url, extras_override = parse_editable(editable_req) + + if name is not None: + try: + req = Requirement(name) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % name) + else: + req = None + + link = Link(url) + + return RequirementParts(req, link, None, extras_override) + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, # type: str + comes_from=None, # type: Optional[str] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool +): + # type: (...) -> InstallRequirement + + parts = parse_req_from_editable(editable_req) + + source_dir = parts.link.file_path if parts.link.scheme == 'file' else None + + return InstallRequirement( + parts.requirement, comes_from, source_dir=source_dir, + editable=True, + link=parts.link, + constraint=constraint, + use_pep517=use_pep517, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + extras=parts.extras, + ) + + +def _looks_like_path(name): + # type: (str) -> bool + """Checks whether the string "looks like" a path on the filesystem. + + This does not check whether the target actually exists, only judge from the + appearance. + + Returns true if any of the following conditions is true: + * a path separator is found (either os.path.sep or os.path.altsep); + * a dot is found (which represents the current directory). + """ + if os.path.sep in name: + return True + if os.path.altsep is not None and os.path.altsep in name: + return True + if name.startswith("."): + return True + return False + + +def _get_url_from_path(path, name): + # type: (str, str) -> str + """ + First, it checks whether a provided path is an installable directory + (e.g. it has a setup.py). If it is, returns the path. + + If false, check if the path is an archive file (such as a .whl). + The function checks if the path is a file. If false, if the path has + an @, it will treat it as a PEP 440 URL requirement and return the path. + """ + if _looks_like_path(name) and os.path.isdir(path): + if is_installable_dir(path): + return path_to_url(path) + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + if not is_archive_file(path): + return None + if os.path.isfile(path): + return path_to_url(path) + urlreq_parts = name.split('@', 1) + if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): + # If the path contains '@' and the part before it does not look + # like a path, try to treat it as a PEP 440 URL req instead. + return None + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + return path_to_url(path) + + +def parse_req_from_line(name, line_source): + # type: (str, Optional[str]) -> RequirementParts + if is_url(name): + marker_sep = '; ' + else: + marker_sep = ';' + if marker_sep in name: + name, markers_as_string = name.split(marker_sep, 1) + markers_as_string = markers_as_string.strip() + if not markers_as_string: + markers = None + else: + markers = Marker(markers_as_string) + else: + markers = None + name = name.strip() + req_as_string = None + path = os.path.normpath(os.path.abspath(name)) + link = None + extras_as_string = None + + if is_url(name): + link = Link(name) + else: + p, extras_as_string = _strip_extras(path) + url = _get_url_from_path(p, name) + if url is not None: + link = Link(url) + + # it's a local file, dir, or url + if link: + # Handle relative file URLs + if link.scheme == 'file' and re.search(r'\.\./', link.url): + link = Link( + path_to_url(os.path.normpath(os.path.abspath(link.path)))) + # wheel file + if link.is_wheel: + wheel = Wheel(link.filename) # can raise InvalidWheelFilename + req_as_string = "%s==%s" % (wheel.name, wheel.version) + else: + # set the req to the egg fragment. when it's not there, this + # will become an 'unnamed' requirement + req_as_string = link.egg_fragment + + # a requirement specifier + else: + req_as_string = name + + extras = convert_extras(extras_as_string) + + def with_source(text): + # type: (str) -> str + if not line_source: + return text + return '{} (from {})'.format(text, line_source) + + if req_as_string is not None: + try: + req = Requirement(req_as_string) + except InvalidRequirement: + if os.path.sep in req_as_string: + add_msg = "It looks like a path." + add_msg += deduce_helpful_msg(req_as_string) + elif ('=' in req_as_string and + not any(op in req_as_string for op in operators)): + add_msg = "= is not a valid operator. Did you mean == ?" + else: + add_msg = '' + msg = with_source( + 'Invalid requirement: {!r}'.format(req_as_string) + ) + if add_msg: + msg += '\nHint: {}'.format(add_msg) + raise InstallationError(msg) + else: + req = None + + return RequirementParts(req, link, markers, extras) + + +def install_req_from_line( + name, # type: str + comes_from=None, # type: Optional[Union[str, InstallRequirement]] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + line_source=None, # type: Optional[str] +): + # type: (...) -> InstallRequirement + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + + :param line_source: An optional string describing where the line is from, + for logging purposes in case of an error. + """ + parts = parse_req_from_line(name, line_source) + + return InstallRequirement( + parts.requirement, comes_from, link=parts.link, markers=parts.markers, + use_pep517=use_pep517, isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + constraint=constraint, + extras=parts.extras, + ) + + +def install_req_from_req_string( + req_string, # type: str + comes_from=None, # type: Optional[InstallRequirement] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> InstallRequirement + try: + req = Requirement(req_string) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % req_string) + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if (req.url and comes_from and comes_from.link and + comes_from.link.netloc in domains_not_allowed): + # Explicitly disallow pypi packages that depend on external urls + raise InstallationError( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) + ) + + return InstallRequirement( + req, comes_from, isolated=isolated, wheel_cache=wheel_cache, + use_pep517=use_pep517 + ) diff --git a/ubuntu/venv/pip/_internal/req/req_file.py b/ubuntu/venv/pip/_internal/req/req_file.py new file mode 100644 index 0000000..8c78104 --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/req_file.py @@ -0,0 +1,546 @@ +""" +Requirements file parsing +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import optparse +import os +import re +import shlex +import sys + +from pip._vendor.six.moves import filterfalse +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.cli import cmdoptions +from pip._internal.exceptions import ( + InstallationError, + RequirementsFileParseError, +) +from pip._internal.models.search_scope import SearchScope +from pip._internal.req.constructors import ( + install_req_from_editable, + install_req_from_line, +) +from pip._internal.utils.encoding import auto_decode +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import ( + Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple, + ) + + from pip._internal.req import InstallRequirement + from pip._internal.cache import WheelCache + from pip._internal.index.package_finder import PackageFinder + from pip._internal.network.session import PipSession + + ReqFileLines = Iterator[Tuple[int, Text]] + + LineParser = Callable[[Text], Tuple[str, Values]] + + +__all__ = ['parse_requirements'] + +SCHEME_RE = re.compile(r'^(http|https|file):', re.I) +COMMENT_RE = re.compile(r'(^|\s+)#.*$') + +# Matches environment variable-style values in '${MY_VARIABLE_1}' with the +# variable name consisting of only uppercase letters, digits or the '_' +# (underscore). This follows the POSIX standard defined in IEEE Std 1003.1, +# 2013 Edition. +ENV_VAR_RE = re.compile(r'(?P\$\{(?P[A-Z0-9_]+)\})') + +SUPPORTED_OPTIONS = [ + cmdoptions.index_url, + cmdoptions.extra_index_url, + cmdoptions.no_index, + cmdoptions.constraints, + cmdoptions.requirements, + cmdoptions.editable, + cmdoptions.find_links, + cmdoptions.no_binary, + cmdoptions.only_binary, + cmdoptions.require_hashes, + cmdoptions.pre, + cmdoptions.trusted_host, + cmdoptions.always_unzip, # Deprecated +] # type: List[Callable[..., optparse.Option]] + +# options to be passed to requirements +SUPPORTED_OPTIONS_REQ = [ + cmdoptions.install_options, + cmdoptions.global_options, + cmdoptions.hash, +] # type: List[Callable[..., optparse.Option]] + +# the 'dest' string values +SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] + + +class ParsedLine(object): + def __init__( + self, + filename, # type: str + lineno, # type: int + comes_from, # type: str + args, # type: str + opts, # type: Values + constraint, # type: bool + ): + # type: (...) -> None + self.filename = filename + self.lineno = lineno + self.comes_from = comes_from + self.args = args + self.opts = opts + self.constraint = constraint + + +def parse_requirements( + filename, # type: str + session, # type: PipSession + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + constraint=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> Iterator[InstallRequirement] + """Parse a requirements file and yield InstallRequirement instances. + + :param filename: Path or url of requirements file. + :param session: PipSession instance. + :param finder: Instance of pip.index.PackageFinder. + :param comes_from: Origin description of requirements. + :param options: cli options. + :param constraint: If true, parsing a constraint file rather than + requirements file. + :param wheel_cache: Instance of pip.wheel.WheelCache + :param use_pep517: Value of the --use-pep517 option. + """ + skip_requirements_regex = ( + options.skip_requirements_regex if options else None + ) + line_parser = get_line_parser(finder) + parser = RequirementsFileParser( + session, line_parser, comes_from, skip_requirements_regex + ) + + for parsed_line in parser.parse(filename, constraint): + req = handle_line( + parsed_line, finder, options, session, wheel_cache, use_pep517 + ) + if req is not None: + yield req + + +def preprocess(content, skip_requirements_regex): + # type: (Text, Optional[str]) -> ReqFileLines + """Split, filter, and join lines, and return a line iterator + + :param content: the content of the requirements file + :param options: cli options + """ + lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines + lines_enum = join_lines(lines_enum) + lines_enum = ignore_comments(lines_enum) + if skip_requirements_regex: + lines_enum = skip_regex(lines_enum, skip_requirements_regex) + lines_enum = expand_env_variables(lines_enum) + return lines_enum + + +def handle_line( + line, # type: ParsedLine + finder=None, # type: Optional[PackageFinder] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None, # type: Optional[bool] +): + # type: (...) -> Optional[InstallRequirement] + """Handle a single parsed requirements line; This can result in + creating/yielding requirements, or updating the finder. + + For lines that contain requirements, the only options that have an effect + are from SUPPORTED_OPTIONS_REQ, and they are scoped to the + requirement. Other options from SUPPORTED_OPTIONS may be present, but are + ignored. + + For lines that do not contain requirements, the only options that have an + effect are from SUPPORTED_OPTIONS. Options from SUPPORTED_OPTIONS_REQ may + be present, but are ignored. These lines may contain multiple options + (although our docs imply only one is supported), and all our parsed and + affect the finder. + """ + + # preserve for the nested code path + line_comes_from = '%s %s (line %s)' % ( + '-c' if line.constraint else '-r', line.filename, line.lineno, + ) + + # return a line requirement + if line.args: + isolated = options.isolated_mode if options else False + if options: + cmdoptions.check_install_build_global(options, line.opts) + # get the options that apply to requirements + req_options = {} + for dest in SUPPORTED_OPTIONS_REQ_DEST: + if dest in line.opts.__dict__ and line.opts.__dict__[dest]: + req_options[dest] = line.opts.__dict__[dest] + line_source = 'line {} of {}'.format(line.lineno, line.filename) + return install_req_from_line( + line.args, + comes_from=line_comes_from, + use_pep517=use_pep517, + isolated=isolated, + options=req_options, + wheel_cache=wheel_cache, + constraint=line.constraint, + line_source=line_source, + ) + + # return an editable requirement + elif line.opts.editables: + isolated = options.isolated_mode if options else False + return install_req_from_editable( + line.opts.editables[0], comes_from=line_comes_from, + use_pep517=use_pep517, + constraint=line.constraint, isolated=isolated, + wheel_cache=wheel_cache + ) + + # percolate hash-checking option upward + elif line.opts.require_hashes: + options.require_hashes = line.opts.require_hashes + + # set finder options + elif finder: + find_links = finder.find_links + index_urls = finder.index_urls + if line.opts.index_url: + index_urls = [line.opts.index_url] + if line.opts.no_index is True: + index_urls = [] + if line.opts.extra_index_urls: + index_urls.extend(line.opts.extra_index_urls) + if line.opts.find_links: + # FIXME: it would be nice to keep track of the source + # of the find_links: support a find-links local path + # relative to a requirements file. + value = line.opts.find_links[0] + req_dir = os.path.dirname(os.path.abspath(line.filename)) + relative_to_reqs_file = os.path.join(req_dir, value) + if os.path.exists(relative_to_reqs_file): + value = relative_to_reqs_file + find_links.append(value) + + search_scope = SearchScope( + find_links=find_links, + index_urls=index_urls, + ) + finder.search_scope = search_scope + + if line.opts.pre: + finder.set_allow_all_prereleases() + + if session: + for host in line.opts.trusted_hosts or []: + source = 'line {} of {}'.format(line.lineno, line.filename) + session.add_trusted_host(host, source=source) + + return None + + +class RequirementsFileParser(object): + def __init__( + self, + session, # type: PipSession + line_parser, # type: LineParser + comes_from, # type: str + skip_requirements_regex, # type: Optional[str] + ): + # type: (...) -> None + self._session = session + self._line_parser = line_parser + self._comes_from = comes_from + self._skip_requirements_regex = skip_requirements_regex + + def parse(self, filename, constraint): + # type: (str, bool) -> Iterator[ParsedLine] + """Parse a given file, yielding parsed lines. + """ + for line in self._parse_and_recurse(filename, constraint): + yield line + + def _parse_and_recurse(self, filename, constraint): + # type: (str, bool) -> Iterator[ParsedLine] + for line in self._parse_file(filename, constraint): + if ( + not line.args and + not line.opts.editables and + (line.opts.requirements or line.opts.constraints) + ): + # parse a nested requirements file + if line.opts.requirements: + req_path = line.opts.requirements[0] + nested_constraint = False + else: + req_path = line.opts.constraints[0] + nested_constraint = True + + # original file is over http + if SCHEME_RE.search(filename): + # do a url join so relative paths work + req_path = urllib_parse.urljoin(filename, req_path) + # original file and nested file are paths + elif not SCHEME_RE.search(req_path): + # do a join so relative paths work + req_path = os.path.join( + os.path.dirname(filename), req_path, + ) + + for inner_line in self._parse_and_recurse( + req_path, nested_constraint, + ): + yield inner_line + else: + yield line + + def _parse_file(self, filename, constraint): + # type: (str, bool) -> Iterator[ParsedLine] + _, content = get_file_content( + filename, self._session, comes_from=self._comes_from + ) + + lines_enum = preprocess(content, self._skip_requirements_regex) + + for line_number, line in lines_enum: + try: + args_str, opts = self._line_parser(line) + except OptionParsingError as e: + # add offending line + msg = 'Invalid requirement: %s\n%s' % (line, e.msg) + raise RequirementsFileParseError(msg) + + yield ParsedLine( + filename, + line_number, + self._comes_from, + args_str, + opts, + constraint, + ) + + +def get_line_parser(finder): + # type: (Optional[PackageFinder]) -> LineParser + def parse_line(line): + # type: (Text) -> Tuple[str, Values] + # Build new parser for each line since it accumulates appendable + # options. + parser = build_parser() + defaults = parser.get_default_values() + defaults.index_url = None + if finder: + defaults.format_control = finder.format_control + + args_str, options_str = break_args_options(line) + # Prior to 2.7.3, shlex cannot deal with unicode entries + if sys.version_info < (2, 7, 3): + # https://github.com/python/mypy/issues/1174 + options_str = options_str.encode('utf8') # type: ignore + + # https://github.com/python/mypy/issues/1174 + opts, _ = parser.parse_args( + shlex.split(options_str), defaults) # type: ignore + + return args_str, opts + + return parse_line + + +def break_args_options(line): + # type: (Text) -> Tuple[str, Text] + """Break up the line into an args and options string. We only want to shlex + (and then optparse) the options, not the args. args can contain markers + which are corrupted by shlex. + """ + tokens = line.split(' ') + args = [] + options = tokens[:] + for token in tokens: + if token.startswith('-') or token.startswith('--'): + break + else: + args.append(token) + options.pop(0) + return ' '.join(args), ' '.join(options) # type: ignore + + +class OptionParsingError(Exception): + def __init__(self, msg): + # type: (str) -> None + self.msg = msg + + +def build_parser(): + # type: () -> optparse.OptionParser + """ + Return a parser for parsing requirement lines + """ + parser = optparse.OptionParser(add_help_option=False) + + option_factories = SUPPORTED_OPTIONS + SUPPORTED_OPTIONS_REQ + for option_factory in option_factories: + option = option_factory() + parser.add_option(option) + + # By default optparse sys.exits on parsing errors. We want to wrap + # that in our own exception. + def parser_exit(self, msg): + # type: (Any, str) -> NoReturn + raise OptionParsingError(msg) + # NOTE: mypy disallows assigning to a method + # https://github.com/python/mypy/issues/2427 + parser.exit = parser_exit # type: ignore + + return parser + + +def join_lines(lines_enum): + # type: (ReqFileLines) -> ReqFileLines + """Joins a line ending in '\' with the previous line (except when following + comments). The joined line takes on the index of the first line. + """ + primary_line_number = None + new_line = [] # type: List[Text] + for line_number, line in lines_enum: + if not line.endswith('\\') or COMMENT_RE.match(line): + if COMMENT_RE.match(line): + # this ensures comments are always matched later + line = ' ' + line + if new_line: + new_line.append(line) + yield primary_line_number, ''.join(new_line) + new_line = [] + else: + yield line_number, line + else: + if not new_line: + primary_line_number = line_number + new_line.append(line.strip('\\')) + + # last line contains \ + if new_line: + yield primary_line_number, ''.join(new_line) + + # TODO: handle space after '\'. + + +def ignore_comments(lines_enum): + # type: (ReqFileLines) -> ReqFileLines + """ + Strips comments and filter empty lines. + """ + for line_number, line in lines_enum: + line = COMMENT_RE.sub('', line) + line = line.strip() + if line: + yield line_number, line + + +def skip_regex(lines_enum, pattern): + # type: (ReqFileLines, str) -> ReqFileLines + """ + Skip lines that match the provided pattern + + Note: the regex pattern is only built once + """ + matcher = re.compile(pattern) + lines_enum = filterfalse(lambda e: matcher.search(e[1]), lines_enum) + return lines_enum + + +def expand_env_variables(lines_enum): + # type: (ReqFileLines) -> ReqFileLines + """Replace all environment variables that can be retrieved via `os.getenv`. + + The only allowed format for environment variables defined in the + requirement file is `${MY_VARIABLE_1}` to ensure two things: + + 1. Strings that contain a `$` aren't accidentally (partially) expanded. + 2. Ensure consistency across platforms for requirement files. + + These points are the result of a discussion on the `github pull + request #3514 `_. + + Valid characters in variable names follow the `POSIX standard + `_ and are limited + to uppercase letter, digits and the `_` (underscore). + """ + for line_number, line in lines_enum: + for env_var, var_name in ENV_VAR_RE.findall(line): + value = os.getenv(var_name) + if not value: + continue + + line = line.replace(env_var, value) + + yield line_number, line + + +def get_file_content(url, session, comes_from=None): + # type: (str, PipSession, Optional[str]) -> Tuple[str, Text] + """Gets the content of a file; it may be a filename, file: URL, or + http: URL. Returns (location, content). Content is unicode. + Respects # -*- coding: declarations on the retrieved files. + + :param url: File path or url. + :param session: PipSession instance. + :param comes_from: Origin description of requirements. + """ + scheme = get_url_scheme(url) + + if scheme in ['http', 'https']: + # FIXME: catch some errors + resp = session.get(url) + resp.raise_for_status() + return resp.url, resp.text + + elif scheme == 'file': + if comes_from and comes_from.startswith('http'): + raise InstallationError( + 'Requirements file %s references URL %s, which is local' + % (comes_from, url)) + + path = url.split(':', 1)[1] + path = path.replace('\\', '/') + match = _url_slash_drive_re.match(path) + if match: + path = match.group(1) + ':' + path.split('|', 1)[1] + path = urllib_parse.unquote(path) + if path.startswith('/'): + path = '/' + path.lstrip('/') + url = path + + try: + with open(url, 'rb') as f: + content = auto_decode(f.read()) + except IOError as exc: + raise InstallationError( + 'Could not open requirements file: %s' % str(exc) + ) + return url, content + + +_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) diff --git a/ubuntu/venv/pip/_internal/req/req_install.py b/ubuntu/venv/pip/_internal/req/req_install.py new file mode 100644 index 0000000..22ac24b --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/req_install.py @@ -0,0 +1,830 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import os +import shutil +import sys +import zipfile + +from pip._vendor import pkg_resources, six +from pip._vendor.packaging.requirements import Requirement +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import parse as parse_version +from pip._vendor.pep517.wrappers import Pep517HookCaller + +from pip._internal import pep425tags +from pip._internal.build_env import NoOpBuildEnvironment +from pip._internal.exceptions import InstallationError +from pip._internal.locations import get_scheme +from pip._internal.models.link import Link +from pip._internal.operations.build.metadata import generate_metadata +from pip._internal.operations.build.metadata_legacy import \ + generate_metadata as generate_metadata_legacy +from pip._internal.operations.install.editable_legacy import \ + install_editable as install_editable_legacy +from pip._internal.operations.install.legacy import install as install_legacy +from pip._internal.operations.install.wheel import install_wheel +from pip._internal.pyproject import load_pyproject_toml, make_pyproject_path +from pip._internal.req.req_uninstall import UninstallPathSet +from pip._internal.utils.deprecation import deprecated +from pip._internal.utils.hashes import Hashes +from pip._internal.utils.logging import indent_log +from pip._internal.utils.marker_files import ( + PIP_DELETE_MARKER_FILENAME, + has_delete_marker_file, + write_delete_marker_file, +) +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + dist_in_site_packages, + dist_in_usersite, + get_installed_version, + hide_url, + redact_auth_from_url, + rmtree, +) +from pip._internal.utils.packaging import get_metadata +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.virtualenv import running_under_virtualenv +from pip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Optional, Sequence, Union, + ) + from pip._internal.build_env import BuildEnvironment + from pip._internal.cache import WheelCache + from pip._internal.index.package_finder import PackageFinder + from pip._vendor.pkg_resources import Distribution + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.packaging.markers import Marker + + +logger = logging.getLogger(__name__) + + +def _get_dist(metadata_directory): + # type: (str) -> Distribution + """Return a pkg_resources.Distribution for the provided + metadata directory. + """ + dist_dir = metadata_directory.rstrip(os.sep) + + # Determine the correct Distribution object type. + if dist_dir.endswith(".egg-info"): + dist_cls = pkg_resources.Distribution + else: + assert dist_dir.endswith(".dist-info") + dist_cls = pkg_resources.DistInfoDistribution + + # Build a PathMetadata object, from path to metadata. :wink: + base_dir, dist_dir_name = os.path.split(dist_dir) + dist_name = os.path.splitext(dist_dir_name)[0] + metadata = pkg_resources.PathMetadata(base_dir, dist_dir) + + return dist_cls( + base_dir, + project_name=dist_name, + metadata=metadata, + ) + + +class InstallRequirement(object): + """ + Represents something that may be installed later on, may have information + about where to fetch the relevant requirement and also contains logic for + installing the said requirement. + """ + + def __init__( + self, + req, # type: Optional[Requirement] + comes_from, # type: Optional[Union[str, InstallRequirement]] + source_dir=None, # type: Optional[str] + editable=False, # type: bool + link=None, # type: Optional[Link] + markers=None, # type: Optional[Marker] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + extras=() # type: Iterable[str] + ): + # type: (...) -> None + assert req is None or isinstance(req, Requirement), req + self.req = req + self.comes_from = comes_from + self.constraint = constraint + if source_dir is None: + self.source_dir = None # type: Optional[str] + else: + self.source_dir = os.path.normpath(os.path.abspath(source_dir)) + self.editable = editable + + self._wheel_cache = wheel_cache + if link is None and req and req.url: + # PEP 508 URL requirement + link = Link(req.url) + self.link = self.original_link = link + # Path to any downloaded or already-existing package. + self.local_file_path = None # type: Optional[str] + if self.link and self.link.is_file: + self.local_file_path = self.link.file_path + + if extras: + self.extras = extras + elif req: + self.extras = { + pkg_resources.safe_extra(extra) for extra in req.extras + } + else: + self.extras = set() + if markers is None and req: + markers = req.marker + self.markers = markers + + # This holds the pkg_resources.Distribution object if this requirement + # is already available: + self.satisfied_by = None # type: Optional[Distribution] + # Whether the installation process should try to uninstall an existing + # distribution before installing this requirement. + self.should_reinstall = False + # Temporary build location + self._temp_build_dir = None # type: Optional[TempDirectory] + # Set to True after successful installation + self.install_succeeded = None # type: Optional[bool] + self.options = options if options else {} + # Set to True after successful preparation of this requirement + self.prepared = False + self.is_direct = False + + self.isolated = isolated + self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment + + # For PEP 517, the directory where we request the project metadata + # gets stored. We need this to pass to build_wheel, so the backend + # can ensure that the wheel matches the metadata (see the PEP for + # details). + self.metadata_directory = None # type: Optional[str] + + # The static build requirements (from pyproject.toml) + self.pyproject_requires = None # type: Optional[List[str]] + + # Build requirements that we will check are available + self.requirements_to_check = [] # type: List[str] + + # The PEP 517 backend we should use to build the project + self.pep517_backend = None # type: Optional[Pep517HookCaller] + + # Are we using PEP 517 for this requirement? + # After pyproject.toml has been loaded, the only valid values are True + # and False. Before loading, None is valid (meaning "use the default"). + # Setting an explicit value before loading pyproject.toml is supported, + # but after loading this flag should be treated as read only. + self.use_pep517 = use_pep517 + + def __str__(self): + # type: () -> str + if self.req: + s = str(self.req) + if self.link: + s += ' from %s' % redact_auth_from_url(self.link.url) + elif self.link: + s = redact_auth_from_url(self.link.url) + else: + s = '' + if self.satisfied_by is not None: + s += ' in %s' % display_path(self.satisfied_by.location) + if self.comes_from: + if isinstance(self.comes_from, six.string_types): + comes_from = self.comes_from # type: Optional[str] + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += ' (from %s)' % comes_from + return s + + def __repr__(self): + # type: () -> str + return '<%s object: %s editable=%r>' % ( + self.__class__.__name__, str(self), self.editable) + + def format_debug(self): + # type: () -> str + """An un-tested helper for getting state, for debugging. + """ + attributes = vars(self) + names = sorted(attributes) + + state = ( + "{}={!r}".format(attr, attributes[attr]) for attr in sorted(names) + ) + return '<{name} object: {{{state}}}>'.format( + name=self.__class__.__name__, + state=", ".join(state), + ) + + def populate_link(self, finder, upgrade, require_hashes): + # type: (PackageFinder, bool, bool) -> None + """Ensure that if a link can be found for this, that it is found. + + Note that self.link may still be None - if Upgrade is False and the + requirement is already installed. + + If require_hashes is True, don't use the wheel cache, because cached + wheels, always built locally, have different hashes than the files + downloaded from the index server and thus throw false hash mismatches. + Furthermore, cached wheels at present have undeterministic contents due + to file modification times. + """ + if self.link is None: + self.link = finder.find_requirement(self, upgrade) + if self._wheel_cache is not None and not require_hashes: + old_link = self.link + supported_tags = pep425tags.get_supported() + self.link = self._wheel_cache.get( + link=self.link, + package_name=self.name, + supported_tags=supported_tags, + ) + if old_link != self.link: + logger.debug('Using cached wheel link: %s', self.link) + + # Things that are valid for all kinds of requirements? + @property + def name(self): + # type: () -> Optional[str] + if self.req is None: + return None + return six.ensure_str(pkg_resources.safe_name(self.req.name)) + + @property + def specifier(self): + # type: () -> SpecifierSet + return self.req.specifier + + @property + def is_pinned(self): + # type: () -> bool + """Return whether I am pinned to an exact version. + + For example, some-package==1.2 is pinned; some-package>1.2 is not. + """ + specifiers = self.specifier + return (len(specifiers) == 1 and + next(iter(specifiers)).operator in {'==', '==='}) + + @property + def installed_version(self): + # type: () -> Optional[str] + return get_installed_version(self.name) + + def match_markers(self, extras_requested=None): + # type: (Optional[Iterable[str]]) -> bool + if not extras_requested: + # Provide an extra to safely evaluate the markers + # without matching any extra + extras_requested = ('',) + if self.markers is not None: + return any( + self.markers.evaluate({'extra': extra}) + for extra in extras_requested) + else: + return True + + @property + def has_hash_options(self): + # type: () -> bool + """Return whether any known-good hashes are specified as options. + + These activate --require-hashes mode; hashes specified as part of a + URL do not. + + """ + return bool(self.options.get('hashes', {})) + + def hashes(self, trust_internet=True): + # type: (bool) -> Hashes + """Return a hash-comparer that considers my option- and URL-based + hashes to be known-good. + + Hashes in URLs--ones embedded in the requirements file, not ones + downloaded from an index server--are almost peers with ones from + flags. They satisfy --require-hashes (whether it was implicitly or + explicitly activated) but do not activate it. md5 and sha224 are not + allowed in flags, which should nudge people toward good algos. We + always OR all hashes together, even ones from URLs. + + :param trust_internet: Whether to trust URL-based (#md5=...) hashes + downloaded from the internet, as by populate_link() + + """ + good_hashes = self.options.get('hashes', {}).copy() + link = self.link if trust_internet else self.original_link + if link and link.hash: + good_hashes.setdefault(link.hash_name, []).append(link.hash) + return Hashes(good_hashes) + + def from_path(self): + # type: () -> Optional[str] + """Format a nice indicator to show where this "comes from" + """ + if self.req is None: + return None + s = str(self.req) + if self.comes_from: + if isinstance(self.comes_from, six.string_types): + comes_from = self.comes_from + else: + comes_from = self.comes_from.from_path() + if comes_from: + s += '->' + comes_from + return s + + def ensure_build_location(self, build_dir): + # type: (str) -> str + assert build_dir is not None + if self._temp_build_dir is not None: + assert self._temp_build_dir.path + return self._temp_build_dir.path + if self.req is None: + # Some systems have /tmp as a symlink which confuses custom + # builds (such as numpy). Thus, we ensure that the real path + # is returned. + self._temp_build_dir = TempDirectory(kind="req-build") + + return self._temp_build_dir.path + if self.editable: + name = self.name.lower() + else: + name = self.name + # FIXME: Is there a better place to create the build_dir? (hg and bzr + # need this) + if not os.path.exists(build_dir): + logger.debug('Creating directory %s', build_dir) + os.makedirs(build_dir) + write_delete_marker_file(build_dir) + return os.path.join(build_dir, name) + + def _set_requirement(self): + # type: () -> None + """Set requirement after generating metadata. + """ + assert self.req is None + assert self.metadata is not None + assert self.source_dir is not None + + # Construct a Requirement object from the generated metadata + if isinstance(parse_version(self.metadata["Version"]), Version): + op = "==" + else: + op = "===" + + self.req = Requirement( + "".join([ + self.metadata["Name"], + op, + self.metadata["Version"], + ]) + ) + + def warn_on_mismatching_name(self): + # type: () -> None + metadata_name = canonicalize_name(self.metadata["Name"]) + if canonicalize_name(self.req.name) == metadata_name: + # Everything is fine. + return + + # If we're here, there's a mismatch. Log a warning about it. + logger.warning( + 'Generating metadata for package %s ' + 'produced metadata for project name %s. Fix your ' + '#egg=%s fragments.', + self.name, metadata_name, self.name + ) + self.req = Requirement(metadata_name) + + def remove_temporary_source(self): + # type: () -> None + """Remove the source files from this requirement, if they are marked + for deletion""" + if self.source_dir and has_delete_marker_file(self.source_dir): + logger.debug('Removing source in %s', self.source_dir) + rmtree(self.source_dir) + self.source_dir = None + if self._temp_build_dir: + self._temp_build_dir.cleanup() + self._temp_build_dir = None + self.build_env.cleanup() + + def check_if_exists(self, use_user_site): + # type: (bool) -> None + """Find an installed distribution that satisfies or conflicts + with this requirement, and set self.satisfied_by or + self.should_reinstall appropriately. + """ + if self.req is None: + return + # get_distribution() will resolve the entire list of requirements + # anyway, and we've already determined that we need the requirement + # in question, so strip the marker so that we don't try to + # evaluate it. + no_marker = Requirement(str(self.req)) + no_marker.marker = None + try: + self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) + except pkg_resources.DistributionNotFound: + return + except pkg_resources.VersionConflict: + existing_dist = pkg_resources.get_distribution( + self.req.name + ) + if use_user_site: + if dist_in_usersite(existing_dist): + self.should_reinstall = True + elif (running_under_virtualenv() and + dist_in_site_packages(existing_dist)): + raise InstallationError( + "Will not install to the user site because it will " + "lack sys.path precedence to %s in %s" % + (existing_dist.project_name, existing_dist.location) + ) + else: + self.should_reinstall = True + else: + if self.editable and self.satisfied_by: + self.should_reinstall = True + # when installing editables, nothing pre-existing should ever + # satisfy + self.satisfied_by = None + + # Things valid for wheels + @property + def is_wheel(self): + # type: () -> bool + if not self.link: + return False + return self.link.is_wheel + + # Things valid for sdists + @property + def unpacked_source_directory(self): + # type: () -> str + return os.path.join( + self.source_dir, + self.link and self.link.subdirectory_fragment or '') + + @property + def setup_py_path(self): + # type: () -> str + assert self.source_dir, "No source dir for %s" % self + setup_py = os.path.join(self.unpacked_source_directory, 'setup.py') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(setup_py, six.text_type): + setup_py = setup_py.encode(sys.getfilesystemencoding()) + + return setup_py + + @property + def pyproject_toml_path(self): + # type: () -> str + assert self.source_dir, "No source dir for %s" % self + return make_pyproject_path(self.unpacked_source_directory) + + def load_pyproject_toml(self): + # type: () -> None + """Load the pyproject.toml file. + + After calling this routine, all of the attributes related to PEP 517 + processing for this requirement have been set. In particular, the + use_pep517 attribute can be used to determine whether we should + follow the PEP 517 or legacy (setup.py) code path. + """ + pyproject_toml_data = load_pyproject_toml( + self.use_pep517, + self.pyproject_toml_path, + self.setup_py_path, + str(self) + ) + + if pyproject_toml_data is None: + self.use_pep517 = False + return + + self.use_pep517 = True + requires, backend, check, backend_path = pyproject_toml_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller( + self.unpacked_source_directory, backend, backend_path=backend_path, + ) + + def _generate_metadata(self): + # type: () -> str + """Invokes metadata generator functions, with the required arguments. + """ + if not self.use_pep517: + assert self.unpacked_source_directory + + return generate_metadata_legacy( + build_env=self.build_env, + setup_py_path=self.setup_py_path, + source_dir=self.unpacked_source_directory, + editable=self.editable, + isolated=self.isolated, + details=self.name or "from {}".format(self.link) + ) + + assert self.pep517_backend is not None + + return generate_metadata( + build_env=self.build_env, + backend=self.pep517_backend, + ) + + def prepare_metadata(self): + # type: () -> None + """Ensure that project metadata is available. + + Under PEP 517, call the backend hook to prepare the metadata. + Under legacy processing, call setup.py egg-info. + """ + assert self.source_dir + + with indent_log(): + self.metadata_directory = self._generate_metadata() + + # Act on the newly generated metadata, based on the name and version. + if not self.name: + self._set_requirement() + else: + self.warn_on_mismatching_name() + + self.assert_source_matches_version() + + @property + def metadata(self): + # type: () -> Any + if not hasattr(self, '_metadata'): + self._metadata = get_metadata(self.get_dist()) + + return self._metadata + + def get_dist(self): + # type: () -> Distribution + return _get_dist(self.metadata_directory) + + def assert_source_matches_version(self): + # type: () -> None + assert self.source_dir + version = self.metadata['version'] + if self.req.specifier and version not in self.req.specifier: + logger.warning( + 'Requested %s, but installing version %s', + self, + version, + ) + else: + logger.debug( + 'Source in %s has version %s, which satisfies requirement %s', + display_path(self.source_dir), + version, + self, + ) + + # For both source distributions and editables + def ensure_has_source_dir(self, parent_dir): + # type: (str) -> None + """Ensure that a source_dir is set. + + This will create a temporary build dir if the name of the requirement + isn't known yet. + + :param parent_dir: The ideal pip parent_dir for the source_dir. + Generally src_dir for editables and build_dir for sdists. + :return: self.source_dir + """ + if self.source_dir is None: + self.source_dir = self.ensure_build_location(parent_dir) + + # For editable installations + def update_editable(self, obtain=True): + # type: (bool) -> None + if not self.link: + logger.debug( + "Cannot update repository at %s; repository location is " + "unknown", + self.source_dir, + ) + return + assert self.editable + assert self.source_dir + if self.link.scheme == 'file': + # Static paths don't get updated + return + assert '+' in self.link.url, "bad url: %r" % self.link.url + vc_type, url = self.link.url.split('+', 1) + vcs_backend = vcs.get_backend(vc_type) + if vcs_backend: + if not self.link.is_vcs: + reason = ( + "This form of VCS requirement is being deprecated: {}." + ).format( + self.link.url + ) + replacement = None + if self.link.url.startswith("git+git@"): + replacement = ( + "git+https://git@example.com/..., " + "git+ssh://git@example.com/..., " + "or the insecure git+git://git@example.com/..." + ) + deprecated(reason, replacement, gone_in="21.0", issue=7554) + hidden_url = hide_url(self.link.url) + if obtain: + vcs_backend.obtain(self.source_dir, url=hidden_url) + else: + vcs_backend.export(self.source_dir, url=hidden_url) + else: + assert 0, ( + 'Unexpected version control type (in %s): %s' + % (self.link, vc_type)) + + # Top-level Actions + def uninstall(self, auto_confirm=False, verbose=False): + # type: (bool, bool) -> Optional[UninstallPathSet] + """ + Uninstall the distribution currently satisfying this requirement. + + Prompts before removing or modifying files unless + ``auto_confirm`` is True. + + Refuses to delete or modify files outside of ``sys.prefix`` - + thus uninstallation within a virtual environment can only + modify that virtual environment, even if the virtualenv is + linked to global site-packages. + + """ + assert self.req + try: + dist = pkg_resources.get_distribution(self.req.name) + except pkg_resources.DistributionNotFound: + logger.warning("Skipping %s as it is not installed.", self.name) + return None + else: + logger.info('Found existing installation: %s', dist) + + uninstalled_pathset = UninstallPathSet.from_dist(dist) + uninstalled_pathset.remove(auto_confirm, verbose) + return uninstalled_pathset + + def _get_archive_name(self, path, parentdir, rootdir): + # type: (str, str, str) -> str + + def _clean_zip_name(name, prefix): + # type: (str, str) -> str + assert name.startswith(prefix + os.path.sep), ( + "name %r doesn't start with prefix %r" % (name, prefix) + ) + name = name[len(prefix) + 1:] + name = name.replace(os.path.sep, '/') + return name + + path = os.path.join(parentdir, path) + name = _clean_zip_name(path, rootdir) + return self.name + '/' + name + + def archive(self, build_dir): + # type: (str) -> None + """Saves archive to provided build_dir. + + Used for saving downloaded VCS requirements as part of `pip download`. + """ + assert self.source_dir + + create_archive = True + archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) + archive_path = os.path.join(build_dir, archive_name) + + if os.path.exists(archive_path): + response = ask_path_exists( + 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' % + display_path(archive_path), ('i', 'w', 'b', 'a')) + if response == 'i': + create_archive = False + elif response == 'w': + logger.warning('Deleting %s', display_path(archive_path)) + os.remove(archive_path) + elif response == 'b': + dest_file = backup_dir(archive_path) + logger.warning( + 'Backing up %s to %s', + display_path(archive_path), + display_path(dest_file), + ) + shutil.move(archive_path, dest_file) + elif response == 'a': + sys.exit(-1) + + if not create_archive: + return + + zip_output = zipfile.ZipFile( + archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True, + ) + with zip_output: + dir = os.path.normcase( + os.path.abspath(self.unpacked_source_directory) + ) + for dirpath, dirnames, filenames in os.walk(dir): + if 'pip-egg-info' in dirnames: + dirnames.remove('pip-egg-info') + for dirname in dirnames: + dir_arcname = self._get_archive_name( + dirname, parentdir=dirpath, rootdir=dir, + ) + zipdir = zipfile.ZipInfo(dir_arcname + '/') + zipdir.external_attr = 0x1ED << 16 # 0o755 + zip_output.writestr(zipdir, '') + for filename in filenames: + if filename == PIP_DELETE_MARKER_FILENAME: + continue + file_arcname = self._get_archive_name( + filename, parentdir=dirpath, rootdir=dir, + ) + filename = os.path.join(dirpath, filename) + zip_output.write(filename, file_arcname) + + logger.info('Saved %s', display_path(archive_path)) + + def install( + self, + install_options, # type: List[str] + global_options=None, # type: Optional[Sequence[str]] + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None + scheme = get_scheme( + self.name, + user=use_user_site, + home=home, + root=root, + isolated=self.isolated, + prefix=prefix, + ) + + global_options = global_options if global_options is not None else [] + if self.editable: + install_editable_legacy( + install_options, + global_options, + prefix=prefix, + home=home, + use_user_site=use_user_site, + name=self.name, + setup_py_path=self.setup_py_path, + isolated=self.isolated, + build_env=self.build_env, + unpacked_source_directory=self.unpacked_source_directory, + ) + self.install_succeeded = True + return + + if self.is_wheel: + assert self.local_file_path + install_wheel( + self.name, + self.local_file_path, + scheme=scheme, + req_description=str(self.req), + pycompile=pycompile, + warn_script_location=warn_script_location, + ) + self.install_succeeded = True + return + + install_legacy( + self, + install_options=install_options, + global_options=global_options, + root=root, + home=home, + prefix=prefix, + use_user_site=use_user_site, + pycompile=pycompile, + scheme=scheme, + ) diff --git a/ubuntu/venv/pip/_internal/req/req_set.py b/ubuntu/venv/pip/_internal/req/req_set.py new file mode 100644 index 0000000..087ac59 --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/req_set.py @@ -0,0 +1,209 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +from collections import OrderedDict + +from pip._vendor.packaging.utils import canonicalize_name + +from pip._internal import pep425tags +from pip._internal.exceptions import InstallationError +from pip._internal.models.wheel import Wheel +from pip._internal.utils.logging import indent_log +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Dict, Iterable, List, Optional, Tuple + from pip._internal.req.req_install import InstallRequirement + + +logger = logging.getLogger(__name__) + + +class RequirementSet(object): + + def __init__(self, check_supported_wheels=True): + # type: (bool) -> None + """Create a RequirementSet. + """ + + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 + self.check_supported_wheels = check_supported_wheels + + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] + + def __str__(self): + # type: () -> str + requirements = sorted( + (req for req in self.requirements.values() if not req.comes_from), + key=lambda req: canonicalize_name(req.name), + ) + return ' '.join(str(req.req) for req in requirements) + + def __repr__(self): + # type: () -> str + requirements = sorted( + self.requirements.values(), + key=lambda req: canonicalize_name(req.name), + ) + + format_string = '<{classname} object; {count} requirement(s): {reqs}>' + return format_string.format( + classname=self.__class__.__name__, + count=len(requirements), + reqs=', '.join(str(req.req) for req in requirements), + ) + + def add_unnamed_requirement(self, install_req): + # type: (InstallRequirement) -> None + assert not install_req.name + self.unnamed_requirements.append(install_req) + + def add_named_requirement(self, install_req): + # type: (InstallRequirement) -> None + assert install_req.name + + project_name = canonicalize_name(install_req.name) + self.requirements[project_name] = install_req + + def add_requirement( + self, + install_req, # type: InstallRequirement + parent_req_name=None, # type: Optional[str] + extras_requested=None # type: Optional[Iterable[str]] + ): + # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 + """Add install_req as a requirement to install. + + :param parent_req_name: The name of the requirement that needed this + added. The name is used because when multiple unnamed requirements + resolve to the same name, we could otherwise end up with dependency + links that point outside the Requirements set. parent_req must + already be added. Note that None implies that this is a user + supplied requirement, vs an inferred one. + :param extras_requested: an iterable of extras used to evaluate the + environment markers. + :return: Additional requirements to scan. That is either [] if + the requirement is not applicable, or [install_req] if the + requirement is applicable and has just been added. + """ + # If the markers do not match, ignore this requirement. + if not install_req.match_markers(extras_requested): + logger.info( + "Ignoring %s: markers '%s' don't match your environment", + install_req.name, install_req.markers, + ) + return [], None + + # If the wheel is not supported, raise an error. + # Should check this after filtering out based on environment markers to + # allow specifying different wheels based on the environment/OS, in a + # single requirements file. + if install_req.link and install_req.link.is_wheel: + wheel = Wheel(install_req.link.filename) + tags = pep425tags.get_supported() + if (self.check_supported_wheels and not wheel.supported(tags)): + raise InstallationError( + "%s is not a supported wheel on this platform." % + wheel.filename + ) + + # This next bit is really a sanity check. + assert install_req.is_direct == (parent_req_name is None), ( + "a direct req shouldn't have a parent and also, " + "a non direct req should have a parent" + ) + + # Unnamed requirements are scanned again and the requirement won't be + # added as a dependency until after scanning. + if not install_req.name: + self.add_unnamed_requirement(install_req) + return [install_req], None + + try: + existing_req = self.get_requirement(install_req.name) + except KeyError: + existing_req = None + + has_conflicting_requirement = ( + parent_req_name is None and + existing_req and + not existing_req.constraint and + existing_req.extras == install_req.extras and + existing_req.req.specifier != install_req.req.specifier + ) + if has_conflicting_requirement: + raise InstallationError( + "Double requirement given: %s (already in %s, name=%r)" + % (install_req, existing_req, install_req.name) + ) + + # When no existing requirement exists, add the requirement as a + # dependency and it will be scanned again after. + if not existing_req: + self.add_named_requirement(install_req) + # We'd want to rescan this requirement later + return [install_req], install_req + + # Assume there's no need to scan, and that we've already + # encountered this for scanning. + if install_req.constraint or not existing_req.constraint: + return [], existing_req + + does_not_satisfy_constraint = ( + install_req.link and + not ( + existing_req.link and + install_req.link.path == existing_req.link.path + ) + ) + if does_not_satisfy_constraint: + self.reqs_to_cleanup.append(install_req) + raise InstallationError( + "Could not satisfy constraints for '%s': " + "installation from path or url cannot be " + "constrained to a version" % install_req.name, + ) + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + existing_req.extras = tuple(sorted( + set(existing_req.extras) | set(install_req.extras) + )) + logger.debug( + "Setting %s extras to: %s", + existing_req, existing_req.extras, + ) + # Return the existing requirement for addition to the parent and + # scanning again. + return [existing_req], existing_req + + def has_requirement(self, name): + # type: (str) -> bool + project_name = canonicalize_name(name) + + return ( + project_name in self.requirements and + not self.requirements[project_name].constraint + ) + + def get_requirement(self, name): + # type: (str) -> InstallRequirement + project_name = canonicalize_name(name) + + if project_name in self.requirements: + return self.requirements[project_name] + + raise KeyError("No project with the name %r" % name) + + def cleanup_files(self): + # type: () -> None + """Clean up files, remove builds.""" + logger.debug('Cleaning up...') + with indent_log(): + for req in self.reqs_to_cleanup: + req.remove_temporary_source() diff --git a/ubuntu/venv/pip/_internal/req/req_tracker.py b/ubuntu/venv/pip/_internal/req/req_tracker.py new file mode 100644 index 0000000..84e0c04 --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/req_tracker.py @@ -0,0 +1,150 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import contextlib +import errno +import hashlib +import logging +import os + +from pip._vendor import contextlib2 + +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from types import TracebackType + from typing import Dict, Iterator, Optional, Set, Type, Union + from pip._internal.req.req_install import InstallRequirement + from pip._internal.models.link import Link + +logger = logging.getLogger(__name__) + + +@contextlib.contextmanager +def update_env_context_manager(**changes): + # type: (str) -> Iterator[None] + target = os.environ + + # Save values from the target and change them. + non_existent_marker = object() + saved_values = {} # type: Dict[str, Union[object, str]] + for name, new_value in changes.items(): + try: + saved_values[name] = target[name] + except KeyError: + saved_values[name] = non_existent_marker + target[name] = new_value + + try: + yield + finally: + # Restore original values in the target. + for name, original_value in saved_values.items(): + if original_value is non_existent_marker: + del target[name] + else: + assert isinstance(original_value, str) # for mypy + target[name] = original_value + + +@contextlib.contextmanager +def get_requirement_tracker(): + # type: () -> Iterator[RequirementTracker] + root = os.environ.get('PIP_REQ_TRACKER') + with contextlib2.ExitStack() as ctx: + if root is None: + root = ctx.enter_context( + TempDirectory(kind='req-tracker') + ).path + ctx.enter_context(update_env_context_manager(PIP_REQ_TRACKER=root)) + logger.debug("Initialized build tracking at %s", root) + + with RequirementTracker(root) as tracker: + yield tracker + + +class RequirementTracker(object): + + def __init__(self, root): + # type: (str) -> None + self._root = root + self._entries = set() # type: Set[InstallRequirement] + logger.debug("Created build tracker: %s", self._root) + + def __enter__(self): + # type: () -> RequirementTracker + logger.debug("Entered build tracker: %s", self._root) + return self + + def __exit__( + self, + exc_type, # type: Optional[Type[BaseException]] + exc_val, # type: Optional[BaseException] + exc_tb # type: Optional[TracebackType] + ): + # type: (...) -> None + self.cleanup() + + def _entry_path(self, link): + # type: (Link) -> str + hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + return os.path.join(self._root, hashed) + + def add(self, req): + # type: (InstallRequirement) -> None + """Add an InstallRequirement to build tracking. + """ + + # Get the file to write information about this requirement. + entry_path = self._entry_path(req.link) + + # Try reading from the file. If it exists and can be read from, a build + # is already in progress, so a LookupError is raised. + try: + with open(entry_path) as fp: + contents = fp.read() + except IOError as e: + # if the error is anything other than "file does not exist", raise. + if e.errno != errno.ENOENT: + raise + else: + message = '%s is already being built: %s' % (req.link, contents) + raise LookupError(message) + + # If we're here, req should really not be building already. + assert req not in self._entries + + # Start tracking this requirement. + with open(entry_path, 'w') as fp: + fp.write(str(req)) + self._entries.add(req) + + logger.debug('Added %s to build tracker %r', req, self._root) + + def remove(self, req): + # type: (InstallRequirement) -> None + """Remove an InstallRequirement from build tracking. + """ + + # Delete the created file and the corresponding entries. + os.unlink(self._entry_path(req.link)) + self._entries.remove(req) + + logger.debug('Removed %s from build tracker %r', req, self._root) + + def cleanup(self): + # type: () -> None + for req in set(self._entries): + self.remove(req) + + logger.debug("Removed build tracker: %r", self._root) + + @contextlib.contextmanager + def track(self, req): + # type: (InstallRequirement) -> Iterator[None] + self.add(req) + yield + self.remove(req) diff --git a/ubuntu/venv/pip/_internal/req/req_uninstall.py b/ubuntu/venv/pip/_internal/req/req_uninstall.py new file mode 100644 index 0000000..5971b13 --- /dev/null +++ b/ubuntu/venv/pip/_internal/req/req_uninstall.py @@ -0,0 +1,644 @@ +from __future__ import absolute_import + +import csv +import functools +import logging +import os +import sys +import sysconfig + +from pip._vendor import pkg_resources + +from pip._internal.exceptions import UninstallationError +from pip._internal.locations import bin_py, bin_user +from pip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import ( + FakeFile, + ask, + dist_in_usersite, + dist_is_local, + egg_link_path, + is_local, + normalize_path, + renames, + rmtree, +) +from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, + ) + from pip._vendor.pkg_resources import Distribution + +logger = logging.getLogger(__name__) + + +def _script_names(dist, script_name, is_gui): + # type: (Distribution, str, bool) -> List[str] + """Create the fully qualified name of the files created by + {console,gui}_scripts for the given ``dist``. + Returns the list of file names + """ + if dist_in_usersite(dist): + bin_dir = bin_user + else: + bin_dir = bin_py + exe_name = os.path.join(bin_dir, script_name) + paths_to_remove = [exe_name] + if WINDOWS: + paths_to_remove.append(exe_name + '.exe') + paths_to_remove.append(exe_name + '.exe.manifest') + if is_gui: + paths_to_remove.append(exe_name + '-script.pyw') + else: + paths_to_remove.append(exe_name + '-script.py') + return paths_to_remove + + +def _unique(fn): + # type: (Callable[..., Iterator[Any]]) -> Callable[..., Iterator[Any]] + @functools.wraps(fn) + def unique(*args, **kw): + # type: (Any, Any) -> Iterator[Any] + seen = set() # type: Set[Any] + for item in fn(*args, **kw): + if item not in seen: + seen.add(item) + yield item + return unique + + +@_unique +def uninstallation_paths(dist): + # type: (Distribution) -> Iterator[str] + """ + Yield all the uninstallation paths for dist based on RECORD-without-.py[co] + + Yield paths to all the files in RECORD. For each .py file in RECORD, add + the .pyc and .pyo in the same directory. + + UninstallPathSet.add() takes care of the __pycache__ .py[co]. + """ + r = csv.reader(FakeFile(dist.get_metadata_lines('RECORD'))) + for row in r: + path = os.path.join(dist.location, row[0]) + yield path + if path.endswith('.py'): + dn, fn = os.path.split(path) + base = fn[:-3] + path = os.path.join(dn, base + '.pyc') + yield path + path = os.path.join(dn, base + '.pyo') + yield path + + +def compact(paths): + # type: (Iterable[str]) -> Set[str] + """Compact a path set to contain the minimal number of paths + necessary to contain all paths in the set. If /a/path/ and + /a/path/to/a/file.txt are both in the set, leave only the + shorter path.""" + + sep = os.path.sep + short_paths = set() # type: Set[str] + for path in sorted(paths, key=len): + should_skip = any( + path.startswith(shortpath.rstrip("*")) and + path[len(shortpath.rstrip("*").rstrip(sep))] == sep + for shortpath in short_paths + ) + if not should_skip: + short_paths.add(path) + return short_paths + + +def compress_for_rename(paths): + # type: (Iterable[str]) -> Set[str] + """Returns a set containing the paths that need to be renamed. + + This set may include directories when the original sequence of paths + included every file on disk. + """ + case_map = dict((os.path.normcase(p), p) for p in paths) + remaining = set(case_map) + unchecked = sorted(set(os.path.split(p)[0] + for p in case_map.values()), key=len) + wildcards = set() # type: Set[str] + + def norm_join(*a): + # type: (str) -> str + return os.path.normcase(os.path.join(*a)) + + for root in unchecked: + if any(os.path.normcase(root).startswith(w) + for w in wildcards): + # This directory has already been handled. + continue + + all_files = set() # type: Set[str] + all_subdirs = set() # type: Set[str] + for dirname, subdirs, files in os.walk(root): + all_subdirs.update(norm_join(root, dirname, d) + for d in subdirs) + all_files.update(norm_join(root, dirname, f) + for f in files) + # If all the files we found are in our remaining set of files to + # remove, then remove them from the latter set and add a wildcard + # for the directory. + if not (all_files - remaining): + remaining.difference_update(all_files) + wildcards.add(root + os.sep) + + return set(map(case_map.__getitem__, remaining)) | wildcards + + +def compress_for_output_listing(paths): + # type: (Iterable[str]) -> Tuple[Set[str], Set[str]] + """Returns a tuple of 2 sets of which paths to display to user + + The first set contains paths that would be deleted. Files of a package + are not added and the top-level directory of the package has a '*' added + at the end - to signify that all it's contents are removed. + + The second set contains files that would have been skipped in the above + folders. + """ + + will_remove = set(paths) + will_skip = set() + + # Determine folders and files + folders = set() + files = set() + for path in will_remove: + if path.endswith(".pyc"): + continue + if path.endswith("__init__.py") or ".dist-info" in path: + folders.add(os.path.dirname(path)) + files.add(path) + + # probably this one https://github.com/python/mypy/issues/390 + _normcased_files = set(map(os.path.normcase, files)) # type: ignore + + folders = compact(folders) + + # This walks the tree using os.walk to not miss extra folders + # that might get added. + for folder in folders: + for dirpath, _, dirfiles in os.walk(folder): + for fname in dirfiles: + if fname.endswith(".pyc"): + continue + + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): + # We are skipping this file. Add it to the set. + will_skip.add(file_) + + will_remove = files | { + os.path.join(folder, "*") for folder in folders + } + + return will_remove, will_skip + + +class StashedUninstallPathSet(object): + """A set of file rename operations to stash files while + tentatively uninstalling them.""" + def __init__(self): + # type: () -> None + # Mapping from source file root to [Adjacent]TempDirectory + # for files under that directory. + self._save_dirs = {} # type: Dict[str, TempDirectory] + # (old path, new path) tuples for each move that may need + # to be undone. + self._moves = [] # type: List[Tuple[str, str]] + + def _get_directory_stash(self, path): + # type: (str) -> str + """Stashes a directory. + + Directories are stashed adjacent to their original location if + possible, or else moved/copied into the user's temp dir.""" + + try: + save_dir = AdjacentTempDirectory(path) # type: TempDirectory + except OSError: + save_dir = TempDirectory(kind="uninstall") + self._save_dirs[os.path.normcase(path)] = save_dir + + return save_dir.path + + def _get_file_stash(self, path): + # type: (str) -> str + """Stashes a file. + + If no root has been provided, one will be created for the directory + in the user's temp directory.""" + path = os.path.normcase(path) + head, old_head = os.path.dirname(path), None + save_dir = None + + while head != old_head: + try: + save_dir = self._save_dirs[head] + break + except KeyError: + pass + head, old_head = os.path.dirname(head), head + else: + # Did not find any suitable root + head = os.path.dirname(path) + save_dir = TempDirectory(kind='uninstall') + self._save_dirs[head] = save_dir + + relpath = os.path.relpath(path, head) + if relpath and relpath != os.path.curdir: + return os.path.join(save_dir.path, relpath) + return save_dir.path + + def stash(self, path): + # type: (str) -> str + """Stashes the directory or file and returns its new location. + Handle symlinks as files to avoid modifying the symlink targets. + """ + path_is_dir = os.path.isdir(path) and not os.path.islink(path) + if path_is_dir: + new_path = self._get_directory_stash(path) + else: + new_path = self._get_file_stash(path) + + self._moves.append((path, new_path)) + if (path_is_dir and os.path.isdir(new_path)): + # If we're moving a directory, we need to + # remove the destination first or else it will be + # moved to inside the existing directory. + # We just created new_path ourselves, so it will + # be removable. + os.rmdir(new_path) + renames(path, new_path) + return new_path + + def commit(self): + # type: () -> None + """Commits the uninstall by removing stashed files.""" + for _, save_dir in self._save_dirs.items(): + save_dir.cleanup() + self._moves = [] + self._save_dirs = {} + + def rollback(self): + # type: () -> None + """Undoes the uninstall by moving stashed files back.""" + for p in self._moves: + logger.info("Moving to %s\n from %s", *p) + + for new_path, path in self._moves: + try: + logger.debug('Replacing %s from %s', new_path, path) + if os.path.isfile(new_path) or os.path.islink(new_path): + os.unlink(new_path) + elif os.path.isdir(new_path): + rmtree(new_path) + renames(path, new_path) + except OSError as ex: + logger.error("Failed to restore %s", new_path) + logger.debug("Exception: %s", ex) + + self.commit() + + @property + def can_rollback(self): + # type: () -> bool + return bool(self._moves) + + +class UninstallPathSet(object): + """A set of file paths to be removed in the uninstallation of a + requirement.""" + def __init__(self, dist): + # type: (Distribution) -> None + self.paths = set() # type: Set[str] + self._refuse = set() # type: Set[str] + self.pth = {} # type: Dict[str, UninstallPthEntries] + self.dist = dist + self._moved_paths = StashedUninstallPathSet() + + def _permitted(self, path): + # type: (str) -> bool + """ + Return True if the given path is one we are permitted to + remove/modify, False otherwise. + + """ + return is_local(path) + + def add(self, path): + # type: (str) -> None + head, tail = os.path.split(path) + + # we normalize the head to resolve parent directory symlinks, but not + # the tail, since we only want to uninstall symlinks, not their targets + path = os.path.join(normalize_path(head), os.path.normcase(tail)) + + if not os.path.exists(path): + return + if self._permitted(path): + self.paths.add(path) + else: + self._refuse.add(path) + + # __pycache__ files can show up after 'installed-files.txt' is created, + # due to imports + if os.path.splitext(path)[1] == '.py' and uses_pycache: + self.add(cache_from_source(path)) + + def add_pth(self, pth_file, entry): + # type: (str, str) -> None + pth_file = normalize_path(pth_file) + if self._permitted(pth_file): + if pth_file not in self.pth: + self.pth[pth_file] = UninstallPthEntries(pth_file) + self.pth[pth_file].add(entry) + else: + self._refuse.add(pth_file) + + def remove(self, auto_confirm=False, verbose=False): + # type: (bool, bool) -> None + """Remove paths in ``self.paths`` with confirmation (unless + ``auto_confirm`` is True).""" + + if not self.paths: + logger.info( + "Can't uninstall '%s'. No files were found to uninstall.", + self.dist.project_name, + ) + return + + dist_name_version = ( + self.dist.project_name + "-" + self.dist.version + ) + logger.info('Uninstalling %s:', dist_name_version) + + with indent_log(): + if auto_confirm or self._allowed_to_proceed(verbose): + moved = self._moved_paths + + for_rename = compress_for_rename(self.paths) + + for path in sorted(compact(for_rename)): + moved.stash(path) + logger.debug('Removing file or directory %s', path) + + for pth in self.pth.values(): + pth.remove() + + logger.info('Successfully uninstalled %s', dist_name_version) + + def _allowed_to_proceed(self, verbose): + # type: (bool) -> bool + """Display which files would be deleted and prompt for confirmation + """ + + def _display(msg, paths): + # type: (str, Iterable[str]) -> None + if not paths: + return + + logger.info(msg) + with indent_log(): + for path in sorted(compact(paths)): + logger.info(path) + + if not verbose: + will_remove, will_skip = compress_for_output_listing(self.paths) + else: + # In verbose mode, display all the files that are going to be + # deleted. + will_remove = set(self.paths) + will_skip = set() + + _display('Would remove:', will_remove) + _display('Would not remove (might be manually added):', will_skip) + _display('Would not remove (outside of prefix):', self._refuse) + if verbose: + _display('Will actually move:', compress_for_rename(self.paths)) + + return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' + + def rollback(self): + # type: () -> None + """Rollback the changes previously made by remove().""" + if not self._moved_paths.can_rollback: + logger.error( + "Can't roll back %s; was not uninstalled", + self.dist.project_name, + ) + return + logger.info('Rolling back uninstall of %s', self.dist.project_name) + self._moved_paths.rollback() + for pth in self.pth.values(): + pth.rollback() + + def commit(self): + # type: () -> None + """Remove temporary save dir: rollback will no longer be possible.""" + self._moved_paths.commit() + + @classmethod + def from_dist(cls, dist): + # type: (Distribution) -> UninstallPathSet + dist_path = normalize_path(dist.location) + if not dist_is_local(dist): + logger.info( + "Not uninstalling %s at %s, outside environment %s", + dist.key, + dist_path, + sys.prefix, + ) + return cls(dist) + + if dist_path in {p for p in {sysconfig.get_path("stdlib"), + sysconfig.get_path("platstdlib")} + if p}: + logger.info( + "Not uninstalling %s at %s, as it is in the standard library.", + dist.key, + dist_path, + ) + return cls(dist) + + paths_to_remove = cls(dist) + develop_egg_link = egg_link_path(dist) + develop_egg_link_egg_info = '{}.egg-info'.format( + pkg_resources.to_filename(dist.project_name)) + egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) + # Special case for distutils installed package + distutils_egg_info = getattr(dist._provider, 'path', None) + + # Uninstall cases order do matter as in the case of 2 installs of the + # same package, pip needs to uninstall the currently detected version + if (egg_info_exists and dist.egg_info.endswith('.egg-info') and + not dist.egg_info.endswith(develop_egg_link_egg_info)): + # if dist.egg_info.endswith(develop_egg_link_egg_info), we + # are in fact in the develop_egg_link case + paths_to_remove.add(dist.egg_info) + if dist.has_metadata('installed-files.txt'): + for installed_file in dist.get_metadata( + 'installed-files.txt').splitlines(): + path = os.path.normpath( + os.path.join(dist.egg_info, installed_file) + ) + paths_to_remove.add(path) + # FIXME: need a test for this elif block + # occurs with --single-version-externally-managed/--record outside + # of pip + elif dist.has_metadata('top_level.txt'): + if dist.has_metadata('namespace_packages.txt'): + namespaces = dist.get_metadata('namespace_packages.txt') + else: + namespaces = [] + for top_level_pkg in [ + p for p + in dist.get_metadata('top_level.txt').splitlines() + if p and p not in namespaces]: + path = os.path.join(dist.location, top_level_pkg) + paths_to_remove.add(path) + paths_to_remove.add(path + '.py') + paths_to_remove.add(path + '.pyc') + paths_to_remove.add(path + '.pyo') + + elif distutils_egg_info: + raise UninstallationError( + "Cannot uninstall {!r}. It is a distutils installed project " + "and thus we cannot accurately determine which files belong " + "to it which would lead to only a partial uninstall.".format( + dist.project_name, + ) + ) + + elif dist.location.endswith('.egg'): + # package installed by easy_install + # We cannot match on dist.egg_name because it can slightly vary + # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg + paths_to_remove.add(dist.location) + easy_install_egg = os.path.split(dist.location)[1] + easy_install_pth = os.path.join(os.path.dirname(dist.location), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) + + elif egg_info_exists and dist.egg_info.endswith('.dist-info'): + for path in uninstallation_paths(dist): + paths_to_remove.add(path) + + elif develop_egg_link: + # develop egg + with open(develop_egg_link, 'r') as fh: + link_pointer = os.path.normcase(fh.readline().strip()) + assert (link_pointer == dist.location), ( + 'Egg-link %s does not match installed location of %s ' + '(at %s)' % (link_pointer, dist.project_name, dist.location) + ) + paths_to_remove.add(develop_egg_link) + easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), + 'easy-install.pth') + paths_to_remove.add_pth(easy_install_pth, dist.location) + + else: + logger.debug( + 'Not sure how to uninstall: %s - Check: %s', + dist, dist.location, + ) + + # find distutils scripts= scripts + if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): + for script in dist.metadata_listdir('scripts'): + if dist_in_usersite(dist): + bin_dir = bin_user + else: + bin_dir = bin_py + paths_to_remove.add(os.path.join(bin_dir, script)) + if WINDOWS: + paths_to_remove.add(os.path.join(bin_dir, script) + '.bat') + + # find console_scripts + _scripts_to_remove = [] + console_scripts = dist.get_entry_map(group='console_scripts') + for name in console_scripts.keys(): + _scripts_to_remove.extend(_script_names(dist, name, False)) + # find gui_scripts + gui_scripts = dist.get_entry_map(group='gui_scripts') + for name in gui_scripts.keys(): + _scripts_to_remove.extend(_script_names(dist, name, True)) + + for s in _scripts_to_remove: + paths_to_remove.add(s) + + return paths_to_remove + + +class UninstallPthEntries(object): + def __init__(self, pth_file): + # type: (str) -> None + if not os.path.isfile(pth_file): + raise UninstallationError( + "Cannot remove entries from nonexistent file %s" % pth_file + ) + self.file = pth_file + self.entries = set() # type: Set[str] + self._saved_lines = None # type: Optional[List[bytes]] + + def add(self, entry): + # type: (str) -> None + entry = os.path.normcase(entry) + # On Windows, os.path.normcase converts the entry to use + # backslashes. This is correct for entries that describe absolute + # paths outside of site-packages, but all the others use forward + # slashes. + # os.path.splitdrive is used instead of os.path.isabs because isabs + # treats non-absolute paths with drive letter markings like c:foo\bar + # as absolute paths. It also does not recognize UNC paths if they don't + # have more than "\\sever\share". Valid examples: "\\server\share\" or + # "\\server\share\folder". Python 2.7.8+ support UNC in splitdrive. + if WINDOWS and not os.path.splitdrive(entry)[0]: + entry = entry.replace('\\', '/') + self.entries.add(entry) + + def remove(self): + # type: () -> None + logger.debug('Removing pth entries from %s:', self.file) + with open(self.file, 'rb') as fh: + # windows uses '\r\n' with py3k, but uses '\n' with py2.x + lines = fh.readlines() + self._saved_lines = lines + if any(b'\r\n' in line for line in lines): + endline = '\r\n' + else: + endline = '\n' + # handle missing trailing newline + if lines and not lines[-1].endswith(endline.encode("utf-8")): + lines[-1] = lines[-1] + endline.encode("utf-8") + for entry in self.entries: + try: + logger.debug('Removing entry: %s', entry) + lines.remove((entry + endline).encode("utf-8")) + except ValueError: + pass + with open(self.file, 'wb') as fh: + fh.writelines(lines) + + def rollback(self): + # type: () -> bool + if self._saved_lines is None: + logger.error( + 'Cannot roll back changes to %s, none were made', self.file + ) + return False + logger.debug('Rolling %s back to previous state', self.file) + with open(self.file, 'wb') as fh: + fh.writelines(self._saved_lines) + return True diff --git a/ubuntu/venv/pip/_internal/self_outdated_check.py b/ubuntu/venv/pip/_internal/self_outdated_check.py new file mode 100644 index 0000000..8fc3c59 --- /dev/null +++ b/ubuntu/venv/pip/_internal/self_outdated_check.py @@ -0,0 +1,242 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import datetime +import hashlib +import json +import logging +import os.path +import sys + +from pip._vendor import pkg_resources +from pip._vendor.packaging import version as packaging_version +from pip._vendor.six import ensure_binary + +from pip._internal.index.collector import LinkCollector +from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.search_scope import SearchScope +from pip._internal.models.selection_prefs import SelectionPreferences +from pip._internal.utils.filesystem import ( + adjacent_tmp_file, + check_path_owner, + replace, +) +from pip._internal.utils.misc import ( + ensure_dir, + get_installed_version, + redact_auth_from_url, +) +from pip._internal.utils.packaging import get_installer +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + import optparse + from optparse import Values + from typing import Any, Dict, Text, Union + + from pip._internal.network.session import PipSession + + +SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" + + +logger = logging.getLogger(__name__) + + +def make_link_collector( + session, # type: PipSession + options, # type: Values + suppress_no_index=False, # type: bool +): + # type: (...) -> LinkCollector + """ + :param session: The Session to use to make requests. + :param suppress_no_index: Whether to ignore the --no-index option + when constructing the SearchScope object. + """ + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index and not suppress_no_index: + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_auth_from_url(url) for url in index_urls), + ) + index_urls = [] + + # Make sure find_links is a list before passing to create(). + find_links = options.find_links or [] + + search_scope = SearchScope.create( + find_links=find_links, index_urls=index_urls, + ) + + link_collector = LinkCollector(session=session, search_scope=search_scope) + + return link_collector + + +def _get_statefile_name(key): + # type: (Union[str, Text]) -> str + key_bytes = ensure_binary(key) + name = hashlib.sha224(key_bytes).hexdigest() + return name + + +class SelfCheckState(object): + def __init__(self, cache_dir): + # type: (str) -> None + self.state = {} # type: Dict[str, Any] + self.statefile_path = None + + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join( + cache_dir, "selfcheck", _get_statefile_name(self.key) + ) + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile) + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass + + @property + def key(self): + return sys.prefix + + def save(self, pypi_version, current_time): + # type: (str, datetime.datetime) -> None + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + + # Check to make sure that we own the directory + if not check_path_owner(os.path.dirname(self.statefile_path)): + return + + # Now that we've ensured the directory is owned by this user, we'll go + # ahead and make sure that all our directories are created. + ensure_dir(os.path.dirname(self.statefile_path)) + + state = { + # Include the key so it's easy to tell which pip wrote the + # file. + "key": self.key, + "last_check": current_time.strftime(SELFCHECK_DATE_FMT), + "pypi_version": pypi_version, + } + + text = json.dumps(state, sort_keys=True, separators=(",", ":")) + + with adjacent_tmp_file(self.statefile_path) as f: + f.write(ensure_binary(text)) + + try: + # Since we have a prefix-specific state file, we can just + # overwrite whatever is there, no need to check. + replace(f.name, self.statefile_path) + except OSError: + # Best effort. + pass + + +def was_installed_by_pip(pkg): + # type: (str) -> bool + """Checks whether pkg was installed by pip + + This is used not to display the upgrade message when pip is in fact + installed by system package manager, such as dnf on Fedora. + """ + try: + dist = pkg_resources.get_distribution(pkg) + return "pip" == get_installer(dist) + except pkg_resources.DistributionNotFound: + return False + + +def pip_self_version_check(session, options): + # type: (PipSession, optparse.Values) -> None + """Check for an update for pip. + + Limit the frequency of checks to once per week. State is stored either in + the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix + of the pip script path. + """ + installed_version = get_installed_version("pip") + if not installed_version: + return + + pip_version = packaging_version.parse(installed_version) + pypi_version = None + + try: + state = SelfCheckState(cache_dir=options.cache_dir) + + current_time = datetime.datetime.utcnow() + # Determine if we need to refresh the state + if "last_check" in state.state and "pypi_version" in state.state: + last_check = datetime.datetime.strptime( + state.state["last_check"], + SELFCHECK_DATE_FMT + ) + if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60: + pypi_version = state.state["pypi_version"] + + # Refresh the version if we need to or just see if we need to warn + if pypi_version is None: + # Lets use PackageFinder to see what the latest pip version is + link_collector = make_link_collector( + session, + options=options, + suppress_no_index=True, + ) + + # Pass allow_yanked=False so we don't suggest upgrading to a + # yanked version. + selection_prefs = SelectionPreferences( + allow_yanked=False, + allow_all_prereleases=False, # Explicitly set to False + ) + + finder = PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + ) + best_candidate = finder.find_best_candidate("pip").best_candidate + if best_candidate is None: + return + pypi_version = str(best_candidate.version) + + # save that we've performed a check + state.save(pypi_version, current_time) + + remote_version = packaging_version.parse(pypi_version) + + local_version_is_older = ( + pip_version < remote_version and + pip_version.base_version != remote_version.base_version and + was_installed_by_pip('pip') + ) + + # Determine if our pypi_version is older + if not local_version_is_older: + return + + # We cannot tell how the current pip is available in the current + # command context, so be pragmatic here and suggest the command + # that's always available. This does not accommodate spaces in + # `sys.executable`. + pip_cmd = "{} -m pip".format(sys.executable) + logger.warning( + "You are using pip version %s; however, version %s is " + "available.\nYou should consider upgrading via the " + "'%s install --upgrade pip' command.", + pip_version, pypi_version, pip_cmd + ) + except Exception: + logger.debug( + "There was an error checking the latest version of pip", + exc_info=True, + ) diff --git a/ubuntu/venv/pip/_internal/utils/__init__.py b/ubuntu/venv/pip/_internal/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/pip/_internal/utils/appdirs.py b/ubuntu/venv/pip/_internal/utils/appdirs.py new file mode 100644 index 0000000..93d17b5 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/appdirs.py @@ -0,0 +1,44 @@ +""" +This code wraps the vendored appdirs module to so the return values are +compatible for the current pip code base. + +The intention is to rewrite current usages gradually, keeping the tests pass, +and eventually drop this after all usages are changed. +""" + +from __future__ import absolute_import + +import os + +from pip._vendor import appdirs as _appdirs + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List + + +def user_cache_dir(appname): + # type: (str) -> str + return _appdirs.user_cache_dir(appname, appauthor=False) + + +def user_config_dir(appname, roaming=True): + # type: (str, bool) -> str + return _appdirs.user_config_dir(appname, appauthor=False, roaming=roaming) + + +def user_data_dir(appname, roaming=False): + # type: (str, bool) -> str + return _appdirs.user_data_dir(appname, appauthor=False, roaming=roaming) + + +# for the discussion regarding site_config_dir locations +# see +def site_config_dirs(appname): + # type: (str) -> List[str] + dirval = _appdirs.site_config_dir(appname, appauthor=False, multipath=True) + if _appdirs.system not in ["win32", "darwin"]: + # always look in /etc directly as well + return dirval.split(os.pathsep) + ['/etc'] + return [dirval] diff --git a/ubuntu/venv/pip/_internal/utils/compat.py b/ubuntu/venv/pip/_internal/utils/compat.py new file mode 100644 index 0000000..6efa52a --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/compat.py @@ -0,0 +1,269 @@ +"""Stuff that differs in different Python versions and platform +distributions.""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import, division + +import codecs +import locale +import logging +import os +import shutil +import sys + +from pip._vendor.six import PY2, text_type + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Tuple, Union + +try: + import ipaddress +except ImportError: + try: + from pip._vendor import ipaddress # type: ignore + except ImportError: + import ipaddr as ipaddress # type: ignore + ipaddress.ip_address = ipaddress.IPAddress # type: ignore + ipaddress.ip_network = ipaddress.IPNetwork # type: ignore + + +__all__ = [ + "ipaddress", "uses_pycache", "console_to_str", + "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", +] + + +logger = logging.getLogger(__name__) + +if PY2: + import imp + + try: + cache_from_source = imp.cache_from_source # type: ignore + except AttributeError: + # does not use __pycache__ + cache_from_source = None + + uses_pycache = cache_from_source is not None +else: + uses_pycache = True + from importlib.util import cache_from_source + + +if PY2: + # In Python 2.7, backslashreplace exists + # but does not support use for decoding. + # We implement our own replace handler for this + # situation, so that we can consistently use + # backslash replacement for all versions. + def backslashreplace_decode_fn(err): + raw_bytes = (err.object[i] for i in range(err.start, err.end)) + # Python 2 gave us characters - convert to numeric bytes + raw_bytes = (ord(b) for b in raw_bytes) + return u"".join(u"\\x%x" % c for c in raw_bytes), err.end + codecs.register_error( + "backslashreplace_decode", + backslashreplace_decode_fn, + ) + backslashreplace_decode = "backslashreplace_decode" +else: + backslashreplace_decode = "backslashreplace" + + +def has_tls(): + # type: () -> bool + try: + import _ssl # noqa: F401 # ignore unused + return True + except ImportError: + pass + + from pip._vendor.urllib3.util import IS_PYOPENSSL + return IS_PYOPENSSL + + +def str_to_display(data, desc=None): + # type: (Union[bytes, Text], Optional[str]) -> Text + """ + For display or logging purposes, convert a bytes object (or text) to + text (e.g. unicode in Python 2) safe for output. + + :param desc: An optional phrase describing the input data, for use in + the log message if a warning is logged. Defaults to "Bytes object". + + This function should never error out and so can take a best effort + approach. It is okay to be lossy if needed since the return value is + just for display. + + We assume the data is in the locale preferred encoding. If it won't + decode properly, we warn the user but decode as best we can. + + We also ensure that the output can be safely written to standard output + without encoding errors. + """ + if isinstance(data, text_type): + return data + + # Otherwise, data is a bytes object (str in Python 2). + # First, get the encoding we assume. This is the preferred + # encoding for the locale, unless that is not found, or + # it is ASCII, in which case assume UTF-8 + encoding = locale.getpreferredencoding() + if (not encoding) or codecs.lookup(encoding).name == "ascii": + encoding = "utf-8" + + # Now try to decode the data - if we fail, warn the user and + # decode with replacement. + try: + decoded_data = data.decode(encoding) + except UnicodeDecodeError: + if desc is None: + desc = 'Bytes object' + msg_format = '{} does not appear to be encoded as %s'.format(desc) + logger.warning(msg_format, encoding) + decoded_data = data.decode(encoding, errors=backslashreplace_decode) + + # Make sure we can print the output, by encoding it to the output + # encoding with replacement of unencodable characters, and then + # decoding again. + # We use stderr's encoding because it's less likely to be + # redirected and if we don't find an encoding we skip this + # step (on the assumption that output is wrapped by something + # that won't fail). + # The double getattr is to deal with the possibility that we're + # being called in a situation where sys.__stderr__ doesn't exist, + # or doesn't have an encoding attribute. Neither of these cases + # should occur in normal pip use, but there's no harm in checking + # in case people use pip in (unsupported) unusual situations. + output_encoding = getattr(getattr(sys, "__stderr__", None), + "encoding", None) + + if output_encoding: + output_encoded = decoded_data.encode( + output_encoding, + errors="backslashreplace" + ) + decoded_data = output_encoded.decode(output_encoding) + + return decoded_data + + +def console_to_str(data): + # type: (bytes) -> Text + """Return a string, safe for output, of subprocess output. + """ + return str_to_display(data, desc='Subprocess output') + + +def get_path_uid(path): + # type: (str) -> int + """ + Return path's uid. + + Does not follow symlinks: + https://github.com/pypa/pip/pull/935#discussion_r5307003 + + Placed this function in compat due to differences on AIX and + Jython, that should eventually go away. + + :raises OSError: When path is a symlink or can't be read. + """ + if hasattr(os, 'O_NOFOLLOW'): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + file_uid = os.fstat(fd).st_uid + os.close(fd) + else: # AIX and Jython + # WARNING: time of check vulnerability, but best we can do w/o NOFOLLOW + if not os.path.islink(path): + # older versions of Jython don't have `os.fstat` + file_uid = os.stat(path).st_uid + else: + # raise OSError for parity with os.O_NOFOLLOW above + raise OSError( + "%s is a symlink; Will not return uid for symlinks" % path + ) + return file_uid + + +def expanduser(path): + # type: (str) -> str + """ + Expand ~ and ~user constructions. + + Includes a workaround for https://bugs.python.org/issue14768 + """ + expanded = os.path.expanduser(path) + if path.startswith('~/') and expanded.startswith('//'): + expanded = expanded[1:] + return expanded + + +# packages in the stdlib that may have installation metadata, but should not be +# considered 'installed'. this theoretically could be determined based on +# dist.location (py27:`sysconfig.get_paths()['stdlib']`, +# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may +# make this ineffective, so hard-coding +stdlib_pkgs = {"python", "wsgiref", "argparse"} + + +# windows detection, covers cpython and ironpython +WINDOWS = (sys.platform.startswith("win") or + (sys.platform == 'cli' and os.name == 'nt')) + + +def samefile(file1, file2): + # type: (str, str) -> bool + """Provide an alternative for os.path.samefile on Windows/Python2""" + if hasattr(os.path, 'samefile'): + return os.path.samefile(file1, file2) + else: + path1 = os.path.normcase(os.path.abspath(file1)) + path2 = os.path.normcase(os.path.abspath(file2)) + return path1 == path2 + + +if hasattr(shutil, 'get_terminal_size'): + def get_terminal_size(): + # type: () -> Tuple[int, int] + """ + Returns a tuple (x, y) representing the width(x) and the height(y) + in characters of the terminal window. + """ + return tuple(shutil.get_terminal_size()) # type: ignore +else: + def get_terminal_size(): + # type: () -> Tuple[int, int] + """ + Returns a tuple (x, y) representing the width(x) and the height(y) + in characters of the terminal window. + """ + def ioctl_GWINSZ(fd): + try: + import fcntl + import termios + import struct + cr = struct.unpack_from( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') + ) + except Exception: + return None + if cr == (0, 0): + return None + return cr + cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + if not cr: + if sys.platform != "win32": + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except Exception: + pass + if not cr: + cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + return int(cr[1]), int(cr[0]) diff --git a/ubuntu/venv/pip/_internal/utils/deprecation.py b/ubuntu/venv/pip/_internal/utils/deprecation.py new file mode 100644 index 0000000..2f20cfd --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/deprecation.py @@ -0,0 +1,104 @@ +""" +A module that implements tooling to enable easy warnings about deprecations. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import warnings + +from pip._vendor.packaging.version import parse + +from pip import __version__ as current_version +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Optional + + +DEPRECATION_MSG_PREFIX = "DEPRECATION: " + + +class PipDeprecationWarning(Warning): + pass + + +_original_showwarning = None # type: Any + + +# Warnings <-> Logging Integration +def _showwarning(message, category, filename, lineno, file=None, line=None): + if file is not None: + if _original_showwarning is not None: + _original_showwarning( + message, category, filename, lineno, file, line, + ) + elif issubclass(category, PipDeprecationWarning): + # We use a specially named logger which will handle all of the + # deprecation messages for pip. + logger = logging.getLogger("pip._internal.deprecations") + logger.warning(message) + else: + _original_showwarning( + message, category, filename, lineno, file, line, + ) + + +def install_warning_logger(): + # type: () -> None + # Enable our Deprecation Warnings + warnings.simplefilter("default", PipDeprecationWarning, append=True) + + global _original_showwarning + + if _original_showwarning is None: + _original_showwarning = warnings.showwarning + warnings.showwarning = _showwarning + + +def deprecated(reason, replacement, gone_in, issue=None): + # type: (str, Optional[str], Optional[str], Optional[int]) -> None + """Helper to deprecate existing functionality. + + reason: + Textual reason shown to the user about why this functionality has + been deprecated. + replacement: + Textual suggestion shown to the user about what alternative + functionality they can use. + gone_in: + The version of pip does this functionality should get removed in. + Raises errors if pip's current version is greater than or equal to + this. + issue: + Issue number on the tracker that would serve as a useful place for + users to find related discussion and provide feedback. + + Always pass replacement, gone_in and issue as keyword arguments for clarity + at the call site. + """ + + # Construct a nice message. + # This is eagerly formatted as we want it to get logged as if someone + # typed this entire message out. + sentences = [ + (reason, DEPRECATION_MSG_PREFIX + "{}"), + (gone_in, "pip {} will remove support for this functionality."), + (replacement, "A possible replacement is {}."), + (issue, ( + "You can find discussion regarding this at " + "https://github.com/pypa/pip/issues/{}." + )), + ] + message = " ".join( + template.format(val) for val, template in sentences if val is not None + ) + + # Raise as an error if it has to be removed. + if gone_in is not None and parse(current_version) >= parse(gone_in): + raise PipDeprecationWarning(message) + + warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) diff --git a/ubuntu/venv/pip/_internal/utils/distutils_args.py b/ubuntu/venv/pip/_internal/utils/distutils_args.py new file mode 100644 index 0000000..e38e402 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/distutils_args.py @@ -0,0 +1,48 @@ +from distutils.errors import DistutilsArgError +from distutils.fancy_getopt import FancyGetopt + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Dict, List + + +_options = [ + ("exec-prefix=", None, ""), + ("home=", None, ""), + ("install-base=", None, ""), + ("install-data=", None, ""), + ("install-headers=", None, ""), + ("install-lib=", None, ""), + ("install-platlib=", None, ""), + ("install-purelib=", None, ""), + ("install-scripts=", None, ""), + ("prefix=", None, ""), + ("root=", None, ""), + ("user", None, ""), +] + + +# typeshed doesn't permit Tuple[str, None, str], see python/typeshed#3469. +_distutils_getopt = FancyGetopt(_options) # type: ignore + + +def parse_distutils_args(args): + # type: (List[str]) -> Dict[str, str] + """Parse provided arguments, returning an object that has the + matched arguments. + + Any unknown arguments are ignored. + """ + result = {} + for arg in args: + try: + _, match = _distutils_getopt.getopt(args=[arg]) + except DistutilsArgError: + # We don't care about any other options, which here may be + # considered unrecognized since our option list is not + # exhaustive. + pass + else: + result.update(match.__dict__) + return result diff --git a/ubuntu/venv/pip/_internal/utils/encoding.py b/ubuntu/venv/pip/_internal/utils/encoding.py new file mode 100644 index 0000000..ab4d4b9 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/encoding.py @@ -0,0 +1,42 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import codecs +import locale +import re +import sys + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Tuple, Text + +BOMS = [ + (codecs.BOM_UTF8, 'utf-8'), + (codecs.BOM_UTF16, 'utf-16'), + (codecs.BOM_UTF16_BE, 'utf-16-be'), + (codecs.BOM_UTF16_LE, 'utf-16-le'), + (codecs.BOM_UTF32, 'utf-32'), + (codecs.BOM_UTF32_BE, 'utf-32-be'), + (codecs.BOM_UTF32_LE, 'utf-32-le'), +] # type: List[Tuple[bytes, Text]] + +ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') + + +def auto_decode(data): + # type: (bytes) -> Text + """Check a bytes string for a BOM to correctly detect the encoding + + Fallback to locale.getpreferredencoding(False) like open() on Python3""" + for bom, encoding in BOMS: + if data.startswith(bom): + return data[len(bom):].decode(encoding) + # Lets check the first two lines as in PEP263 + for line in data.split(b'\n')[:2]: + if line[0:1] == b'#' and ENCODING_RE.search(line): + encoding = ENCODING_RE.search(line).groups()[0].decode('ascii') + return data.decode(encoding) + return data.decode( + locale.getpreferredencoding(False) or sys.getdefaultencoding(), + ) diff --git a/ubuntu/venv/pip/_internal/utils/entrypoints.py b/ubuntu/venv/pip/_internal/utils/entrypoints.py new file mode 100644 index 0000000..befd01c --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/entrypoints.py @@ -0,0 +1,31 @@ +import sys + +from pip._internal.cli.main import main +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, List + + +def _wrapper(args=None): + # type: (Optional[List[str]]) -> int + """Central wrapper for all old entrypoints. + + Historically pip has had several entrypoints defined. Because of issues + arising from PATH, sys.path, multiple Pythons, their interactions, and most + of them having a pip installed, users suffer every time an entrypoint gets + moved. + + To alleviate this pain, and provide a mechanism for warning users and + directing them to an appropriate place for help, we now define all of + our old entrypoints as wrappers for the current one. + """ + sys.stderr.write( + "WARNING: pip is being invoked by an old script wrapper. This will " + "fail in a future version of pip.\n" + "Please see https://github.com/pypa/pip/issues/5599 for advice on " + "fixing the underlying issue.\n" + "To avoid this problem you can invoke Python with '-m pip' instead of " + "running pip directly.\n" + ) + return main(args) diff --git a/ubuntu/venv/pip/_internal/utils/filesystem.py b/ubuntu/venv/pip/_internal/utils/filesystem.py new file mode 100644 index 0000000..6f1537e --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/filesystem.py @@ -0,0 +1,171 @@ +import errno +import os +import os.path +import random +import shutil +import stat +import sys +from contextlib import contextmanager +from tempfile import NamedTemporaryFile + +# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import. +from pip._vendor.retrying import retry # type: ignore +from pip._vendor.six import PY2 + +from pip._internal.utils.compat import get_path_uid +from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast + +if MYPY_CHECK_RUNNING: + from typing import BinaryIO, Iterator + + class NamedTemporaryFileResult(BinaryIO): + @property + def file(self): + # type: () -> BinaryIO + pass + + +def check_path_owner(path): + # type: (str) -> bool + # If we don't have a way to check the effective uid of this process, then + # we'll just assume that we own the directory. + if sys.platform == "win32" or not hasattr(os, "geteuid"): + return True + + assert os.path.isabs(path) + + previous = None + while path != previous: + if os.path.lexists(path): + # Check if path is writable by current user. + if os.geteuid() == 0: + # Special handling for root user in order to handle properly + # cases where users use sudo without -H flag. + try: + path_uid = get_path_uid(path) + except OSError: + return False + return path_uid == 0 + else: + return os.access(path, os.W_OK) + else: + previous, path = path, os.path.dirname(path) + return False # assume we don't own the path + + +def copy2_fixed(src, dest): + # type: (str, str) -> None + """Wrap shutil.copy2() but map errors copying socket files to + SpecialFileError as expected. + + See also https://bugs.python.org/issue37700. + """ + try: + shutil.copy2(src, dest) + except (OSError, IOError): + for f in [src, dest]: + try: + is_socket_file = is_socket(f) + except OSError: + # An error has already occurred. Another error here is not + # a problem and we can ignore it. + pass + else: + if is_socket_file: + raise shutil.SpecialFileError("`%s` is a socket" % f) + + raise + + +def is_socket(path): + # type: (str) -> bool + return stat.S_ISSOCK(os.lstat(path).st_mode) + + +@contextmanager +def adjacent_tmp_file(path): + # type: (str) -> Iterator[NamedTemporaryFileResult] + """Given a path to a file, open a temp file next to it securely and ensure + it is written to disk after the context reaches its end. + """ + with NamedTemporaryFile( + delete=False, + dir=os.path.dirname(path), + prefix=os.path.basename(path), + suffix='.tmp', + ) as f: + result = cast('NamedTemporaryFileResult', f) + try: + yield result + finally: + result.file.flush() + os.fsync(result.file.fileno()) + + +_replace_retry = retry(stop_max_delay=1000, wait_fixed=250) + +if PY2: + @_replace_retry + def replace(src, dest): + # type: (str, str) -> None + try: + os.rename(src, dest) + except OSError: + os.remove(dest) + os.rename(src, dest) + +else: + replace = _replace_retry(os.replace) + + +# test_writable_dir and _test_writable_dir_win are copied from Flit, +# with the author's agreement to also place them under pip's license. +def test_writable_dir(path): + # type: (str) -> bool + """Check if a directory is writable. + + Uses os.access() on POSIX, tries creating files on Windows. + """ + # If the directory doesn't exist, find the closest parent that does. + while not os.path.isdir(path): + parent = os.path.dirname(path) + if parent == path: + break # Should never get here, but infinite loops are bad + path = parent + + if os.name == 'posix': + return os.access(path, os.W_OK) + + return _test_writable_dir_win(path) + + +def _test_writable_dir_win(path): + # type: (str) -> bool + # os.access doesn't work on Windows: http://bugs.python.org/issue2528 + # and we can't use tempfile: http://bugs.python.org/issue22107 + basename = 'accesstest_deleteme_fishfingers_custard_' + alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789' + for i in range(10): + name = basename + ''.join(random.choice(alphabet) for _ in range(6)) + file = os.path.join(path, name) + try: + fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL) + except OSError as e: + if e.errno == errno.EEXIST: + continue + if e.errno == errno.EPERM: + # This could be because there's a directory with the same name. + # But it's highly unlikely there's a directory called that, + # so we'll assume it's because the parent dir is not writable. + return False + raise + else: + os.close(fd) + os.unlink(file) + return True + + # This should never be reached + raise EnvironmentError( + 'Unexpected condition testing for writable directory' + ) diff --git a/ubuntu/venv/pip/_internal/utils/filetypes.py b/ubuntu/venv/pip/_internal/utils/filetypes.py new file mode 100644 index 0000000..daa0ca7 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/filetypes.py @@ -0,0 +1,16 @@ +"""Filetype information. +""" +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple + +WHEEL_EXTENSION = '.whl' +BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') # type: Tuple[str, ...] +XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', + '.tar.lz', '.tar.lzma') # type: Tuple[str, ...] +ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) # type: Tuple[str, ...] +TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') # type: Tuple[str, ...] +ARCHIVE_EXTENSIONS = ( + ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS +) diff --git a/ubuntu/venv/pip/_internal/utils/glibc.py b/ubuntu/venv/pip/_internal/utils/glibc.py new file mode 100644 index 0000000..3610424 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/glibc.py @@ -0,0 +1,98 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import os +import sys + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + + +def glibc_version_string(): + # type: () -> Optional[str] + "Returns glibc version string, or None if not using glibc." + return glibc_version_string_confstr() or glibc_version_string_ctypes() + + +def glibc_version_string_confstr(): + # type: () -> Optional[str] + "Primary implementation of glibc_version_string using os.confstr." + # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely + # to be broken or missing. This strategy is used in the standard library + # platform module: + # https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183 + if sys.platform == "win32": + return None + try: + # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() + except (AttributeError, OSError, ValueError): + # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... + return None + return version + + +def glibc_version_string_ctypes(): + # type: () -> Optional[str] + "Fallback implementation of glibc_version_string using ctypes." + + try: + import ctypes + except ImportError: + return None + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +# platform.libc_ver regularly returns completely nonsensical glibc +# versions. E.g. on my computer, platform says: +# +# ~$ python2.7 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.7') +# ~$ python3.5 -c 'import platform; print(platform.libc_ver())' +# ('glibc', '2.9') +# +# But the truth is: +# +# ~$ ldd --version +# ldd (Debian GLIBC 2.22-11) 2.22 +# +# This is unfortunate, because it means that the linehaul data on libc +# versions that was generated by pip 8.1.2 and earlier is useless and +# misleading. Solution: instead of using platform, use our code that actually +# works. +def libc_ver(): + # type: () -> Tuple[str, str] + """Try to determine the glibc version + + Returns a tuple of strings (lib, version) which default to empty strings + in case the lookup fails. + """ + glibc_version = glibc_version_string() + if glibc_version is None: + return ("", "") + else: + return ("glibc", glibc_version) diff --git a/ubuntu/venv/pip/_internal/utils/hashes.py b/ubuntu/venv/pip/_internal/utils/hashes.py new file mode 100644 index 0000000..4c41551 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/hashes.py @@ -0,0 +1,131 @@ +from __future__ import absolute_import + +import hashlib + +from pip._vendor.six import iteritems, iterkeys, itervalues + +from pip._internal.exceptions import ( + HashMismatch, + HashMissing, + InstallationError, +) +from pip._internal.utils.misc import read_chunks +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Dict, List, BinaryIO, NoReturn, Iterator + ) + from pip._vendor.six import PY3 + if PY3: + from hashlib import _Hash + else: + from hashlib import _hash as _Hash + + +# The recommended hash algo of the moment. Change this whenever the state of +# the art changes; it won't hurt backward compatibility. +FAVORITE_HASH = 'sha256' + + +# Names of hashlib algorithms allowed by the --hash option and ``pip hash`` +# Currently, those are the ones at least as collision-resistant as sha256. +STRONG_HASHES = ['sha256', 'sha384', 'sha512'] + + +class Hashes(object): + """A wrapper that builds multiple hashes at once and checks them against + known-good values + + """ + def __init__(self, hashes=None): + # type: (Dict[str, List[str]]) -> None + """ + :param hashes: A dict of algorithm names pointing to lists of allowed + hex digests + """ + self._allowed = {} if hashes is None else hashes + + @property + def digest_count(self): + # type: () -> int + return sum(len(digests) for digests in self._allowed.values()) + + def is_hash_allowed( + self, + hash_name, # type: str + hex_digest, # type: str + ): + # type: (...) -> bool + """Return whether the given hex digest is allowed.""" + return hex_digest in self._allowed.get(hash_name, []) + + def check_against_chunks(self, chunks): + # type: (Iterator[bytes]) -> None + """Check good hashes against ones built from iterable of chunks of + data. + + Raise HashMismatch if none match. + + """ + gots = {} + for hash_name in iterkeys(self._allowed): + try: + gots[hash_name] = hashlib.new(hash_name) + except (ValueError, TypeError): + raise InstallationError('Unknown hash name: %s' % hash_name) + + for chunk in chunks: + for hash in itervalues(gots): + hash.update(chunk) + + for hash_name, got in iteritems(gots): + if got.hexdigest() in self._allowed[hash_name]: + return + self._raise(gots) + + def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn + raise HashMismatch(self._allowed, gots) + + def check_against_file(self, file): + # type: (BinaryIO) -> None + """Check good hashes against a file-like object + + Raise HashMismatch if none match. + + """ + return self.check_against_chunks(read_chunks(file)) + + def check_against_path(self, path): + # type: (str) -> None + with open(path, 'rb') as file: + return self.check_against_file(file) + + def __nonzero__(self): + # type: () -> bool + """Return whether I know any known-good hashes.""" + return bool(self._allowed) + + def __bool__(self): + # type: () -> bool + return self.__nonzero__() + + +class MissingHashes(Hashes): + """A workalike for Hashes used when we're missing a hash for a requirement + + It computes the actual hash of the requirement and raises a HashMissing + exception showing it to the user. + + """ + def __init__(self): + # type: () -> None + """Don't offer the ``hashes`` kwarg.""" + # Pass our favorite hash in to generate a "gotten hash". With the + # empty list, it will never match, so an error will always raise. + super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) + + def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn + raise HashMissing(gots[FAVORITE_HASH].hexdigest()) diff --git a/ubuntu/venv/pip/_internal/utils/inject_securetransport.py b/ubuntu/venv/pip/_internal/utils/inject_securetransport.py new file mode 100644 index 0000000..5b93b1d --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/inject_securetransport.py @@ -0,0 +1,36 @@ +"""A helper module that injects SecureTransport, on import. + +The import should be done as early as possible, to ensure all requests and +sessions (or whatever) are created after injecting SecureTransport. + +Note that we only do the injection on macOS, when the linked OpenSSL is too +old to handle TLSv1.2. +""" + +import sys + + +def inject_securetransport(): + # type: () -> None + # Only relevant on macOS + if sys.platform != "darwin": + return + + try: + import ssl + except ImportError: + return + + # Checks for OpenSSL 1.0.1 + if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100f: + return + + try: + from pip._vendor.urllib3.contrib import securetransport + except (ImportError, OSError): + return + + securetransport.inject_into_urllib3() + + +inject_securetransport() diff --git a/ubuntu/venv/pip/_internal/utils/logging.py b/ubuntu/venv/pip/_internal/utils/logging.py new file mode 100644 index 0000000..7767111 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/logging.py @@ -0,0 +1,398 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import contextlib +import errno +import logging +import logging.handlers +import os +import sys +from logging import Filter, getLogger + +from pip._vendor.six import PY2 + +from pip._internal.utils.compat import WINDOWS +from pip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX +from pip._internal.utils.misc import ensure_dir + +try: + import threading +except ImportError: + import dummy_threading as threading # type: ignore + + +try: + # Use "import as" and set colorama in the else clause to avoid mypy + # errors and get the following correct revealed type for colorama: + # `Union[_importlib_modulespec.ModuleType, None]` + # Otherwise, we get an error like the following in the except block: + # > Incompatible types in assignment (expression has type "None", + # variable has type Module) + # TODO: eliminate the need to use "import as" once mypy addresses some + # of its issues with conditional imports. Here is an umbrella issue: + # https://github.com/python/mypy/issues/1297 + from pip._vendor import colorama as _colorama +# Lots of different errors can come from this, including SystemError and +# ImportError. +except Exception: + colorama = None +else: + # Import Fore explicitly rather than accessing below as colorama.Fore + # to avoid the following error running mypy: + # > Module has no attribute "Fore" + # TODO: eliminate the need to import Fore once mypy addresses some of its + # issues with conditional imports. This particular case could be an + # instance of the following issue (but also see the umbrella issue above): + # https://github.com/python/mypy/issues/3500 + from pip._vendor.colorama import Fore + + colorama = _colorama + + +_log_state = threading.local() +_log_state.indentation = 0 +subprocess_logger = getLogger('pip.subprocessor') + + +class BrokenStdoutLoggingError(Exception): + """ + Raised if BrokenPipeError occurs for the stdout stream while logging. + """ + pass + + +# BrokenPipeError does not exist in Python 2 and, in addition, manifests +# differently in Windows and non-Windows. +if WINDOWS: + # In Windows, a broken pipe can show up as EINVAL rather than EPIPE: + # https://bugs.python.org/issue19612 + # https://bugs.python.org/issue30418 + if PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and + exc.errno in (errno.EINVAL, errno.EPIPE)) + else: + # In Windows, a broken pipe IOError became OSError in Python 3. + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return ((exc_class is BrokenPipeError) or # noqa: F821 + (exc_class is OSError and + exc.errno in (errno.EINVAL, errno.EPIPE))) +elif PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and exc.errno == errno.EPIPE) +else: + # Then we are in the non-Windows Python 3 case. + def _is_broken_pipe_error(exc_class, exc): + """ + Return whether an exception is a broken pipe error. + + Args: + exc_class: an exception class. + exc: an exception instance. + """ + return (exc_class is BrokenPipeError) # noqa: F821 + + +@contextlib.contextmanager +def indent_log(num=2): + """ + A context manager which will cause the log output to be indented for any + log messages emitted inside it. + """ + _log_state.indentation += num + try: + yield + finally: + _log_state.indentation -= num + + +def get_indentation(): + return getattr(_log_state, 'indentation', 0) + + +class IndentingFormatter(logging.Formatter): + + def __init__(self, *args, **kwargs): + """ + A logging.Formatter that obeys the indent_log() context manager. + + :param add_timestamp: A bool indicating output lines should be prefixed + with their record's timestamp. + """ + self.add_timestamp = kwargs.pop("add_timestamp", False) + super(IndentingFormatter, self).__init__(*args, **kwargs) + + def get_message_start(self, formatted, levelno): + """ + Return the start of the formatted log message (not counting the + prefix to add to each line). + """ + if levelno < logging.WARNING: + return '' + if formatted.startswith(DEPRECATION_MSG_PREFIX): + # Then the message already has a prefix. We don't want it to + # look like "WARNING: DEPRECATION: ...." + return '' + if levelno < logging.ERROR: + return 'WARNING: ' + + return 'ERROR: ' + + def format(self, record): + """ + Calls the standard formatter, but will indent all of the log message + lines by our current indentation level. + """ + formatted = super(IndentingFormatter, self).format(record) + message_start = self.get_message_start(formatted, record.levelno) + formatted = message_start + formatted + + prefix = '' + if self.add_timestamp: + # TODO: Use Formatter.default_time_format after dropping PY2. + t = self.formatTime(record, "%Y-%m-%dT%H:%M:%S") + prefix = '%s,%03d ' % (t, record.msecs) + prefix += " " * get_indentation() + formatted = "".join([ + prefix + line + for line in formatted.splitlines(True) + ]) + return formatted + + +def _color_wrap(*colors): + def wrapped(inp): + return "".join(list(colors) + [inp, colorama.Style.RESET_ALL]) + return wrapped + + +class ColorizedStreamHandler(logging.StreamHandler): + + # Don't build up a list of colors if we don't have colorama + if colorama: + COLORS = [ + # This needs to be in order from highest logging level to lowest. + (logging.ERROR, _color_wrap(Fore.RED)), + (logging.WARNING, _color_wrap(Fore.YELLOW)), + ] + else: + COLORS = [] + + def __init__(self, stream=None, no_color=None): + logging.StreamHandler.__init__(self, stream) + self._no_color = no_color + + if WINDOWS and colorama: + self.stream = colorama.AnsiToWin32(self.stream) + + def _using_stdout(self): + """ + Return whether the handler is using sys.stdout. + """ + if WINDOWS and colorama: + # Then self.stream is an AnsiToWin32 object. + return self.stream.wrapped is sys.stdout + + return self.stream is sys.stdout + + def should_color(self): + # Don't colorize things if we do not have colorama or if told not to + if not colorama or self._no_color: + return False + + real_stream = ( + self.stream if not isinstance(self.stream, colorama.AnsiToWin32) + else self.stream.wrapped + ) + + # If the stream is a tty we should color it + if hasattr(real_stream, "isatty") and real_stream.isatty(): + return True + + # If we have an ANSI term we should color it + if os.environ.get("TERM") == "ANSI": + return True + + # If anything else we should not color it + return False + + def format(self, record): + msg = logging.StreamHandler.format(self, record) + + if self.should_color(): + for level, color in self.COLORS: + if record.levelno >= level: + msg = color(msg) + break + + return msg + + # The logging module says handleError() can be customized. + def handleError(self, record): + exc_class, exc = sys.exc_info()[:2] + # If a broken pipe occurred while calling write() or flush() on the + # stdout stream in logging's Handler.emit(), then raise our special + # exception so we can handle it in main() instead of logging the + # broken pipe error and continuing. + if (exc_class and self._using_stdout() and + _is_broken_pipe_error(exc_class, exc)): + raise BrokenStdoutLoggingError() + + return super(ColorizedStreamHandler, self).handleError(record) + + +class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): + + def _open(self): + ensure_dir(os.path.dirname(self.baseFilename)) + return logging.handlers.RotatingFileHandler._open(self) + + +class MaxLevelFilter(Filter): + + def __init__(self, level): + self.level = level + + def filter(self, record): + return record.levelno < self.level + + +class ExcludeLoggerFilter(Filter): + + """ + A logging Filter that excludes records from a logger (or its children). + """ + + def filter(self, record): + # The base Filter class allows only records from a logger (or its + # children). + return not super(ExcludeLoggerFilter, self).filter(record) + + +def setup_logging(verbosity, no_color, user_log_file): + """Configures and sets up all of the logging + + Returns the requested logging level, as its integer value. + """ + + # Determine the level to be logging at. + if verbosity >= 1: + level = "DEBUG" + elif verbosity == -1: + level = "WARNING" + elif verbosity == -2: + level = "ERROR" + elif verbosity <= -3: + level = "CRITICAL" + else: + level = "INFO" + + level_number = getattr(logging, level) + + # The "root" logger should match the "console" level *unless* we also need + # to log to a user log file. + include_user_log = user_log_file is not None + if include_user_log: + additional_log_file = user_log_file + root_level = "DEBUG" + else: + additional_log_file = "/dev/null" + root_level = level + + # Disable any logging besides WARNING unless we have DEBUG level logging + # enabled for vendored libraries. + vendored_log_level = "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" + + # Shorthands for clarity + log_streams = { + "stdout": "ext://sys.stdout", + "stderr": "ext://sys.stderr", + } + handler_classes = { + "stream": "pip._internal.utils.logging.ColorizedStreamHandler", + "file": "pip._internal.utils.logging.BetterRotatingFileHandler", + } + handlers = ["console", "console_errors", "console_subprocess"] + ( + ["user_log"] if include_user_log else [] + ) + + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": False, + "filters": { + "exclude_warnings": { + "()": "pip._internal.utils.logging.MaxLevelFilter", + "level": logging.WARNING, + }, + "restrict_to_subprocess": { + "()": "logging.Filter", + "name": subprocess_logger.name, + }, + "exclude_subprocess": { + "()": "pip._internal.utils.logging.ExcludeLoggerFilter", + "name": subprocess_logger.name, + }, + }, + "formatters": { + "indent": { + "()": IndentingFormatter, + "format": "%(message)s", + }, + "indent_with_timestamp": { + "()": IndentingFormatter, + "format": "%(message)s", + "add_timestamp": True, + }, + }, + "handlers": { + "console": { + "level": level, + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stdout"], + "filters": ["exclude_subprocess", "exclude_warnings"], + "formatter": "indent", + }, + "console_errors": { + "level": "WARNING", + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stderr"], + "filters": ["exclude_subprocess"], + "formatter": "indent", + }, + # A handler responsible for logging to the console messages + # from the "subprocessor" logger. + "console_subprocess": { + "level": level, + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stderr"], + "filters": ["restrict_to_subprocess"], + "formatter": "indent", + }, + "user_log": { + "level": "DEBUG", + "class": handler_classes["file"], + "filename": additional_log_file, + "delay": True, + "formatter": "indent_with_timestamp", + }, + }, + "root": { + "level": root_level, + "handlers": handlers, + }, + "loggers": { + "pip._vendor": { + "level": vendored_log_level + } + }, + }) + + return level_number diff --git a/ubuntu/venv/pip/_internal/utils/marker_files.py b/ubuntu/venv/pip/_internal/utils/marker_files.py new file mode 100644 index 0000000..42ea814 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/marker_files.py @@ -0,0 +1,25 @@ +import os.path + +DELETE_MARKER_MESSAGE = '''\ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). +''' +PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' + + +def has_delete_marker_file(directory): + # type: (str) -> bool + return os.path.exists(os.path.join(directory, PIP_DELETE_MARKER_FILENAME)) + + +def write_delete_marker_file(directory): + # type: (str) -> None + """ + Write the pip delete marker file into this directory. + """ + filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME) + with open(filepath, 'w') as marker_fp: + marker_fp.write(DELETE_MARKER_MESSAGE) diff --git a/ubuntu/venv/pip/_internal/utils/misc.py b/ubuntu/venv/pip/_internal/utils/misc.py new file mode 100644 index 0000000..554af0b --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/misc.py @@ -0,0 +1,904 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import contextlib +import errno +import getpass +import hashlib +import io +import logging +import os +import posixpath +import shutil +import stat +import sys +from collections import deque + +from pip._vendor import pkg_resources +# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import. +from pip._vendor.retrying import retry # type: ignore +from pip._vendor.six import PY2, text_type +from pip._vendor.six.moves import input +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote + +from pip import __version__ +from pip._internal.exceptions import CommandError +from pip._internal.locations import ( + get_major_minor_version, + site_packages, + user_site, +) +from pip._internal.utils.compat import ( + WINDOWS, + expanduser, + stdlib_pkgs, + str_to_display, +) +from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast +from pip._internal.utils.virtualenv import ( + running_under_virtualenv, + virtualenv_no_global, +) + +if PY2: + from io import BytesIO as StringIO +else: + from io import StringIO + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, AnyStr, Container, Iterable, List, Optional, Text, + Tuple, Union, + ) + from pip._vendor.pkg_resources import Distribution + + VersionInfo = Tuple[int, int, int] + + +__all__ = ['rmtree', 'display_path', 'backup_dir', + 'ask', 'splitext', + 'format_size', 'is_installable_dir', + 'normalize_path', + 'renames', 'get_prog', + 'captured_stdout', 'ensure_dir', + 'get_installed_version', 'remove_auth_from_url'] + + +logger = logging.getLogger(__name__) + + +def get_pip_version(): + # type: () -> str + pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..") + pip_pkg_dir = os.path.abspath(pip_pkg_dir) + + return ( + 'pip {} from {} (python {})'.format( + __version__, pip_pkg_dir, get_major_minor_version(), + ) + ) + + +def normalize_version_info(py_version_info): + # type: (Tuple[int, ...]) -> Tuple[int, int, int] + """ + Convert a tuple of ints representing a Python version to one of length + three. + + :param py_version_info: a tuple of ints representing a Python version, + or None to specify no version. The tuple can have any length. + + :return: a tuple of length three if `py_version_info` is non-None. + Otherwise, return `py_version_info` unchanged (i.e. None). + """ + if len(py_version_info) < 3: + py_version_info += (3 - len(py_version_info)) * (0,) + elif len(py_version_info) > 3: + py_version_info = py_version_info[:3] + + return cast('VersionInfo', py_version_info) + + +def ensure_dir(path): + # type: (AnyStr) -> None + """os.path.makedirs without EEXIST.""" + try: + os.makedirs(path) + except OSError as e: + # Windows can raise spurious ENOTEMPTY errors. See #6426. + if e.errno != errno.EEXIST and e.errno != errno.ENOTEMPTY: + raise + + +def get_prog(): + # type: () -> str + try: + prog = os.path.basename(sys.argv[0]) + if prog in ('__main__.py', '-c'): + return "%s -m pip" % sys.executable + else: + return prog + except (AttributeError, TypeError, IndexError): + pass + return 'pip' + + +# Retry every half second for up to 3 seconds +@retry(stop_max_delay=3000, wait_fixed=500) +def rmtree(dir, ignore_errors=False): + # type: (str, bool) -> None + shutil.rmtree(dir, ignore_errors=ignore_errors, + onerror=rmtree_errorhandler) + + +def rmtree_errorhandler(func, path, exc_info): + """On Windows, the files in .svn are read-only, so when rmtree() tries to + remove them, an exception is thrown. We catch that here, remove the + read-only attribute, and hopefully continue without problems.""" + try: + has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE) + except (IOError, OSError): + # it's equivalent to os.path.exists + return + + if has_attr_readonly: + # convert to read/write + os.chmod(path, stat.S_IWRITE) + # use the original function to repeat the operation + func(path) + return + else: + raise + + +def path_to_display(path): + # type: (Optional[Union[str, Text]]) -> Optional[Text] + """ + Convert a bytes (or text) path to text (unicode in Python 2) for display + and logging purposes. + + This function should never error out. Also, this function is mainly needed + for Python 2 since in Python 3 str paths are already text. + """ + if path is None: + return None + if isinstance(path, text_type): + return path + # Otherwise, path is a bytes object (str in Python 2). + try: + display_path = path.decode(sys.getfilesystemencoding(), 'strict') + except UnicodeDecodeError: + # Include the full bytes to make troubleshooting easier, even though + # it may not be very human readable. + if PY2: + # Convert the bytes to a readable str representation using + # repr(), and then convert the str to unicode. + # Also, we add the prefix "b" to the repr() return value both + # to make the Python 2 output look like the Python 3 output, and + # to signal to the user that this is a bytes representation. + display_path = str_to_display('b{!r}'.format(path)) + else: + # Silence the "F821 undefined name 'ascii'" flake8 error since + # in Python 3 ascii() is a built-in. + display_path = ascii(path) # noqa: F821 + + return display_path + + +def display_path(path): + # type: (Union[str, Text]) -> str + """Gives the display value for a given path, making it relative to cwd + if possible.""" + path = os.path.normcase(os.path.abspath(path)) + if sys.version_info[0] == 2: + path = path.decode(sys.getfilesystemencoding(), 'replace') + path = path.encode(sys.getdefaultencoding(), 'replace') + if path.startswith(os.getcwd() + os.path.sep): + path = '.' + path[len(os.getcwd()):] + return path + + +def backup_dir(dir, ext='.bak'): + # type: (str, str) -> str + """Figure out the name of a directory to back up the given dir to + (adding .bak, .bak2, etc)""" + n = 1 + extension = ext + while os.path.exists(dir + extension): + n += 1 + extension = ext + str(n) + return dir + extension + + +def ask_path_exists(message, options): + # type: (str, Iterable[str]) -> str + for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): + if action in options: + return action + return ask(message, options) + + +def _check_no_input(message): + # type: (str) -> None + """Raise an error if no input is allowed.""" + if os.environ.get('PIP_NO_INPUT'): + raise Exception( + 'No input was expected ($PIP_NO_INPUT set); question: %s' % + message + ) + + +def ask(message, options): + # type: (str, Iterable[str]) -> str + """Ask the message interactively, with the given possible responses""" + while 1: + _check_no_input(message) + response = input(message) + response = response.strip().lower() + if response not in options: + print( + 'Your response (%r) was not one of the expected responses: ' + '%s' % (response, ', '.join(options)) + ) + else: + return response + + +def ask_input(message): + # type: (str) -> str + """Ask for input interactively.""" + _check_no_input(message) + return input(message) + + +def ask_password(message): + # type: (str) -> str + """Ask for a password interactively.""" + _check_no_input(message) + return getpass.getpass(message) + + +def format_size(bytes): + # type: (float) -> str + if bytes > 1000 * 1000: + return '%.1f MB' % (bytes / 1000.0 / 1000) + elif bytes > 10 * 1000: + return '%i kB' % (bytes / 1000) + elif bytes > 1000: + return '%.1f kB' % (bytes / 1000.0) + else: + return '%i bytes' % bytes + + +def is_installable_dir(path): + # type: (str) -> bool + """Is path is a directory containing setup.py or pyproject.toml? + """ + if not os.path.isdir(path): + return False + setup_py = os.path.join(path, 'setup.py') + if os.path.isfile(setup_py): + return True + pyproject_toml = os.path.join(path, 'pyproject.toml') + if os.path.isfile(pyproject_toml): + return True + return False + + +def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): + """Yield pieces of data from a file-like object until EOF.""" + while True: + chunk = file.read(size) + if not chunk: + break + yield chunk + + +def normalize_path(path, resolve_symlinks=True): + # type: (str, bool) -> str + """ + Convert a path to its canonical, case-normalized, absolute version. + + """ + path = expanduser(path) + if resolve_symlinks: + path = os.path.realpath(path) + else: + path = os.path.abspath(path) + return os.path.normcase(path) + + +def splitext(path): + # type: (str) -> Tuple[str, str] + """Like os.path.splitext, but take off .tar too""" + base, ext = posixpath.splitext(path) + if base.lower().endswith('.tar'): + ext = base[-4:] + ext + base = base[:-4] + return base, ext + + +def renames(old, new): + # type: (str, str) -> None + """Like os.renames(), but handles renaming across devices.""" + # Implementation borrowed from os.renames(). + head, tail = os.path.split(new) + if head and tail and not os.path.exists(head): + os.makedirs(head) + + shutil.move(old, new) + + head, tail = os.path.split(old) + if head and tail: + try: + os.removedirs(head) + except OSError: + pass + + +def is_local(path): + # type: (str) -> bool + """ + Return True if this is a path pip is allowed to modify. + + If we're in a virtualenv, sys.prefix points to the virtualenv's + prefix; only sys.prefix is considered local. + + If we're not in a virtualenv, in general we can modify anything. + However, if the OS vendor has configured distutils to install + somewhere other than sys.prefix (which could be a subdirectory of + sys.prefix, e.g. /usr/local), we consider sys.prefix itself nonlocal + and the domain of the OS vendor. (In other words, everything _other + than_ sys.prefix is considered local.) + + Caution: this function assumes the head of path has been normalized + with normalize_path. + """ + + path = normalize_path(path) + prefix = normalize_path(sys.prefix) + + if running_under_virtualenv(): + return path.startswith(normalize_path(sys.prefix)) + else: + from pip._internal.locations import distutils_scheme + if path.startswith(prefix): + for local_path in distutils_scheme("").values(): + if path.startswith(normalize_path(local_path)): + return True + return False + else: + return True + + +def dist_is_local(dist): + # type: (Distribution) -> bool + """ + Return True if given Distribution object is installed somewhere pip + is allowed to modify. + + """ + return is_local(dist_location(dist)) + + +def dist_in_usersite(dist): + # type: (Distribution) -> bool + """ + Return True if given Distribution is installed in user site. + """ + return dist_location(dist).startswith(normalize_path(user_site)) + + +def dist_in_site_packages(dist): + # type: (Distribution) -> bool + """ + Return True if given Distribution is installed in + sysconfig.get_python_lib(). + """ + return dist_location(dist).startswith(normalize_path(site_packages)) + + +def dist_is_editable(dist): + # type: (Distribution) -> bool + """ + Return True if given Distribution is an editable install. + """ + for path_item in sys.path: + egg_link = os.path.join(path_item, dist.project_name + '.egg-link') + if os.path.isfile(egg_link): + return True + return False + + +def get_installed_distributions( + local_only=True, # type: bool + skip=stdlib_pkgs, # type: Container[str] + include_editables=True, # type: bool + editables_only=False, # type: bool + user_only=False, # type: bool + paths=None # type: Optional[List[str]] +): + # type: (...) -> List[Distribution] + """ + Return a list of installed Distribution objects. + + If ``local_only`` is True (default), only return installations + local to the current virtualenv, if in a virtualenv. + + ``skip`` argument is an iterable of lower-case project names to + ignore; defaults to stdlib_pkgs + + If ``include_editables`` is False, don't report editables. + + If ``editables_only`` is True , only report editables. + + If ``user_only`` is True , only report installations in the user + site directory. + + If ``paths`` is set, only report the distributions present at the + specified list of locations. + """ + if paths: + working_set = pkg_resources.WorkingSet(paths) + else: + working_set = pkg_resources.working_set + + if local_only: + local_test = dist_is_local + else: + def local_test(d): + return True + + if include_editables: + def editable_test(d): + return True + else: + def editable_test(d): + return not dist_is_editable(d) + + if editables_only: + def editables_only_test(d): + return dist_is_editable(d) + else: + def editables_only_test(d): + return True + + if user_only: + user_test = dist_in_usersite + else: + def user_test(d): + return True + + return [d for d in working_set + if local_test(d) and + d.key not in skip and + editable_test(d) and + editables_only_test(d) and + user_test(d) + ] + + +def egg_link_path(dist): + # type: (Distribution) -> Optional[str] + """ + Return the path for the .egg-link file if it exists, otherwise, None. + + There's 3 scenarios: + 1) not in a virtualenv + try to find in site.USER_SITE, then site_packages + 2) in a no-global virtualenv + try to find in site_packages + 3) in a yes-global virtualenv + try to find in site_packages, then site.USER_SITE + (don't look in global location) + + For #1 and #3, there could be odd cases, where there's an egg-link in 2 + locations. + + This method will just return the first one found. + """ + sites = [] + if running_under_virtualenv(): + sites.append(site_packages) + if not virtualenv_no_global() and user_site: + sites.append(user_site) + else: + if user_site: + sites.append(user_site) + sites.append(site_packages) + + for site in sites: + egglink = os.path.join(site, dist.project_name) + '.egg-link' + if os.path.isfile(egglink): + return egglink + return None + + +def dist_location(dist): + # type: (Distribution) -> str + """ + Get the site-packages location of this distribution. Generally + this is dist.location, except in the case of develop-installed + packages, where dist.location is the source code location, and we + want to know where the egg-link file is. + + The returned location is normalized (in particular, with symlinks removed). + """ + egg_link = egg_link_path(dist) + if egg_link: + return normalize_path(egg_link) + return normalize_path(dist.location) + + +def write_output(msg, *args): + # type: (str, str) -> None + logger.info(msg, *args) + + +class FakeFile(object): + """Wrap a list of lines in an object with readline() to make + ConfigParser happy.""" + def __init__(self, lines): + self._gen = (l for l in lines) + + def readline(self): + try: + try: + return next(self._gen) + except NameError: + return self._gen.next() + except StopIteration: + return '' + + def __iter__(self): + return self._gen + + +class StreamWrapper(StringIO): + + @classmethod + def from_stream(cls, orig_stream): + cls.orig_stream = orig_stream + return cls() + + # compileall.compile_dir() needs stdout.encoding to print to stdout + @property + def encoding(self): + return self.orig_stream.encoding + + +@contextlib.contextmanager +def captured_output(stream_name): + """Return a context manager used by captured_stdout/stdin/stderr + that temporarily replaces the sys stream *stream_name* with a StringIO. + + Taken from Lib/support/__init__.py in the CPython repo. + """ + orig_stdout = getattr(sys, stream_name) + setattr(sys, stream_name, StreamWrapper.from_stream(orig_stdout)) + try: + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stdout) + + +def captured_stdout(): + """Capture the output of sys.stdout: + + with captured_stdout() as stdout: + print('hello') + self.assertEqual(stdout.getvalue(), 'hello\n') + + Taken from Lib/support/__init__.py in the CPython repo. + """ + return captured_output('stdout') + + +def captured_stderr(): + """ + See captured_stdout(). + """ + return captured_output('stderr') + + +class cached_property(object): + """A property that is only computed once per instance and then replaces + itself with an ordinary attribute. Deleting the attribute resets the + property. + + Source: https://github.com/bottlepy/bottle/blob/0.11.5/bottle.py#L175 + """ + + def __init__(self, func): + self.__doc__ = getattr(func, '__doc__') + self.func = func + + def __get__(self, obj, cls): + if obj is None: + # We're being accessed from the class itself, not from an object + return self + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + +def get_installed_version(dist_name, working_set=None): + """Get the installed version of dist_name avoiding pkg_resources cache""" + # Create a requirement that we'll look for inside of setuptools. + req = pkg_resources.Requirement.parse(dist_name) + + if working_set is None: + # We want to avoid having this cached, so we need to construct a new + # working set each time. + working_set = pkg_resources.WorkingSet() + + # Get the installed distribution from our working set + dist = working_set.find(req) + + # Check to see if we got an installed distribution or not, if we did + # we want to return it's version. + return dist.version if dist else None + + +def consume(iterator): + """Consume an iterable at C speed.""" + deque(iterator, maxlen=0) + + +# Simulates an enum +def enum(*sequential, **named): + enums = dict(zip(sequential, range(len(sequential))), **named) + reverse = {value: key for key, value in enums.items()} + enums['reverse_mapping'] = reverse + return type('Enum', (), enums) + + +def build_netloc(host, port): + # type: (str, Optional[int]) -> str + """ + Build a netloc from a host-port pair + """ + if port is None: + return host + if ':' in host: + # Only wrap host with square brackets when it is IPv6 + host = '[{}]'.format(host) + return '{}:{}'.format(host, port) + + +def build_url_from_netloc(netloc, scheme='https'): + # type: (str, str) -> str + """ + Build a full URL from a netloc. + """ + if netloc.count(':') >= 2 and '@' not in netloc and '[' not in netloc: + # It must be a bare IPv6 address, so wrap it with brackets. + netloc = '[{}]'.format(netloc) + return '{}://{}'.format(scheme, netloc) + + +def parse_netloc(netloc): + # type: (str) -> Tuple[str, Optional[int]] + """ + Return the host-port pair from a netloc. + """ + url = build_url_from_netloc(netloc) + parsed = urllib_parse.urlparse(url) + return parsed.hostname, parsed.port + + +def split_auth_from_netloc(netloc): + """ + Parse out and remove the auth information from a netloc. + + Returns: (netloc, (username, password)). + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (which can be checked using + # the password attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (which again can be checked + # using the password attribute of the return value) + user_pass = auth.split(':', 1) + else: + user_pass = auth, None + + user_pass = tuple( + None if x is None else urllib_unquote(x) for x in user_pass + ) + + return netloc, user_pass + + +def redact_netloc(netloc): + # type: (str) -> str + """ + Replace the sensitive data in a netloc with "****", if it exists. + + For example: + - "user:pass@example.com" returns "user:****@example.com" + - "accesstoken@example.com" returns "****@example.com" + """ + netloc, (user, password) = split_auth_from_netloc(netloc) + if user is None: + return netloc + if password is None: + user = '****' + password = '' + else: + user = urllib_parse.quote(user) + password = ':****' + return '{user}{password}@{netloc}'.format(user=user, + password=password, + netloc=netloc) + + +def _transform_url(url, transform_netloc): + """Transform and replace netloc in a url. + + transform_netloc is a function taking the netloc and returning a + tuple. The first element of this tuple is the new netloc. The + entire tuple is returned. + + Returns a tuple containing the transformed url as item 0 and the + original tuple returned by transform_netloc as item 1. + """ + purl = urllib_parse.urlsplit(url) + netloc_tuple = transform_netloc(purl.netloc) + # stripped url + url_pieces = ( + purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment + ) + surl = urllib_parse.urlunsplit(url_pieces) + return surl, netloc_tuple + + +def _get_netloc(netloc): + return split_auth_from_netloc(netloc) + + +def _redact_netloc(netloc): + return (redact_netloc(netloc),) + + +def split_auth_netloc_from_url(url): + # type: (str) -> Tuple[str, str, Tuple[str, str]] + """ + Parse a url into separate netloc, auth, and url with no auth. + + Returns: (url_without_auth, netloc, (username, password)) + """ + url_without_auth, (netloc, auth) = _transform_url(url, _get_netloc) + return url_without_auth, netloc, auth + + +def remove_auth_from_url(url): + # type: (str) -> str + """Return a copy of url with 'username:password@' removed.""" + # username/pass params are passed to subversion through flags + # and are not recognized in the url. + return _transform_url(url, _get_netloc)[0] + + +def redact_auth_from_url(url): + # type: (str) -> str + """Replace the password in a given url with ****.""" + return _transform_url(url, _redact_netloc)[0] + + +class HiddenText(object): + def __init__( + self, + secret, # type: str + redacted, # type: str + ): + # type: (...) -> None + self.secret = secret + self.redacted = redacted + + def __repr__(self): + # type: (...) -> str + return ''.format(str(self)) + + def __str__(self): + # type: (...) -> str + return self.redacted + + # This is useful for testing. + def __eq__(self, other): + # type: (Any) -> bool + if type(self) != type(other): + return False + + # The string being used for redaction doesn't also have to match, + # just the raw, original string. + return (self.secret == other.secret) + + # We need to provide an explicit __ne__ implementation for Python 2. + # TODO: remove this when we drop PY2 support. + def __ne__(self, other): + # type: (Any) -> bool + return not self == other + + +def hide_value(value): + # type: (str) -> HiddenText + return HiddenText(value, redacted='****') + + +def hide_url(url): + # type: (str) -> HiddenText + redacted = redact_auth_from_url(url) + return HiddenText(url, redacted=redacted) + + +def protect_pip_from_modification_on_windows(modifying_pip): + # type: (bool) -> None + """Protection of pip.exe from modification on Windows + + On Windows, any operation modifying pip should be run as: + python -m pip ... + """ + pip_names = [ + "pip.exe", + "pip{}.exe".format(sys.version_info[0]), + "pip{}.{}.exe".format(*sys.version_info[:2]) + ] + + # See https://github.com/pypa/pip/issues/1299 for more discussion + should_show_use_python_msg = ( + modifying_pip and + WINDOWS and + os.path.basename(sys.argv[0]) in pip_names + ) + + if should_show_use_python_msg: + new_command = [ + sys.executable, "-m", "pip" + ] + sys.argv[1:] + raise CommandError( + 'To modify pip, please run the following command:\n{}' + .format(" ".join(new_command)) + ) + + +def is_console_interactive(): + # type: () -> bool + """Is this console interactive? + """ + return sys.stdin is not None and sys.stdin.isatty() + + +def hash_file(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[Any, int] + """Return (hash, length) for path using hashlib.sha256() + """ + + h = hashlib.sha256() + length = 0 + with open(path, 'rb') as f: + for block in read_chunks(f, size=blocksize): + length += len(block) + h.update(block) + return h, length + + +def is_wheel_installed(): + """ + Return whether the wheel package is installed. + """ + try: + import wheel # noqa: F401 + except ImportError: + return False + + return True diff --git a/ubuntu/venv/pip/_internal/utils/models.py b/ubuntu/venv/pip/_internal/utils/models.py new file mode 100644 index 0000000..29e1441 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/models.py @@ -0,0 +1,42 @@ +"""Utilities for defining models +""" +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparison capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/ubuntu/venv/pip/_internal/utils/packaging.py b/ubuntu/venv/pip/_internal/utils/packaging.py new file mode 100644 index 0000000..68aa86e --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/packaging.py @@ -0,0 +1,94 @@ +from __future__ import absolute_import + +import logging +from email.parser import FeedParser + +from pip._vendor import pkg_resources +from pip._vendor.packaging import specifiers, version + +from pip._internal.exceptions import NoneMetadataError +from pip._internal.utils.misc import display_path +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from email.message import Message + from pip._vendor.pkg_resources import Distribution + + +logger = logging.getLogger(__name__) + + +def check_requires_python(requires_python, version_info): + # type: (Optional[str], Tuple[int, ...]) -> bool + """ + Check if the given Python version matches a "Requires-Python" specifier. + + :param version_info: A 3-tuple of ints representing a Python + major-minor-micro version to check (e.g. `sys.version_info[:3]`). + + :return: `True` if the given Python version satisfies the requirement. + Otherwise, return `False`. + + :raises InvalidSpecifier: If `requires_python` has an invalid format. + """ + if requires_python is None: + # The package provides no information + return True + requires_python_specifier = specifiers.SpecifierSet(requires_python) + + python_version = version.parse('.'.join(map(str, version_info))) + return python_version in requires_python_specifier + + +def get_metadata(dist): + # type: (Distribution) -> Message + """ + :raises NoneMetadataError: if the distribution reports `has_metadata()` + True but `get_metadata()` returns None. + """ + metadata_name = 'METADATA' + if (isinstance(dist, pkg_resources.DistInfoDistribution) and + dist.has_metadata(metadata_name)): + metadata = dist.get_metadata(metadata_name) + elif dist.has_metadata('PKG-INFO'): + metadata_name = 'PKG-INFO' + metadata = dist.get_metadata(metadata_name) + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' + + if metadata is None: + raise NoneMetadataError(dist, metadata_name) + + feed_parser = FeedParser() + # The following line errors out if with a "NoneType" TypeError if + # passed metadata=None. + feed_parser.feed(metadata) + return feed_parser.close() + + +def get_requires_python(dist): + # type: (pkg_resources.Distribution) -> Optional[str] + """ + Return the "Requires-Python" metadata for a distribution, or None + if not present. + """ + pkg_info_dict = get_metadata(dist) + requires_python = pkg_info_dict.get('Requires-Python') + + if requires_python is not None: + # Convert to a str to satisfy the type checker, since requires_python + # can be a Header object. + requires_python = str(requires_python) + + return requires_python + + +def get_installer(dist): + # type: (Distribution) -> str + if dist.has_metadata('INSTALLER'): + for line in dist.get_metadata_lines('INSTALLER'): + if line.strip(): + return line.strip() + return '' diff --git a/ubuntu/venv/pip/_internal/utils/pkg_resources.py b/ubuntu/venv/pip/_internal/utils/pkg_resources.py new file mode 100644 index 0000000..0bc129a --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/pkg_resources.py @@ -0,0 +1,44 @@ +from pip._vendor.pkg_resources import yield_lines +from pip._vendor.six import ensure_str + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Dict, Iterable, List + + +class DictMetadata(object): + """IMetadataProvider that reads metadata files from a dictionary. + """ + def __init__(self, metadata): + # type: (Dict[str, bytes]) -> None + self._metadata = metadata + + def has_metadata(self, name): + # type: (str) -> bool + return name in self._metadata + + def get_metadata(self, name): + # type: (str) -> str + try: + return ensure_str(self._metadata[name]) + except UnicodeDecodeError as e: + # Mirrors handling done in pkg_resources.NullProvider. + e.reason += " in {} file".format(name) + raise + + def get_metadata_lines(self, name): + # type: (str) -> Iterable[str] + return yield_lines(self.get_metadata(name)) + + def metadata_isdir(self, name): + # type: (str) -> bool + return False + + def metadata_listdir(self, name): + # type: (str) -> List[str] + return [] + + def run_script(self, script_name, namespace): + # type: (str, str) -> None + pass diff --git a/ubuntu/venv/pip/_internal/utils/setuptools_build.py b/ubuntu/venv/pip/_internal/utils/setuptools_build.py new file mode 100644 index 0000000..4147a65 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/setuptools_build.py @@ -0,0 +1,181 @@ +import sys + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Sequence + +# Shim to wrap setup.py invocation with setuptools +# +# We set sys.argv[0] to the path to the underlying setup.py file so +# setuptools / distutils don't take the path to the setup.py to be "-c" when +# invoking via the shim. This avoids e.g. the following manifest_maker +# warning: "warning: manifest_maker: standard file '-c' not found". +_SETUPTOOLS_SHIM = ( + "import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) + + +def make_setuptools_shim_args( + setup_py_path, # type: str + global_options=None, # type: Sequence[str] + no_user_config=False, # type: bool + unbuffered_output=False # type: bool +): + # type: (...) -> List[str] + """ + Get setuptools command arguments with shim wrapped setup file invocation. + + :param setup_py_path: The path to setup.py to be wrapped. + :param global_options: Additional global options. + :param no_user_config: If True, disables personal user configuration. + :param unbuffered_output: If True, adds the unbuffered switch to the + argument list. + """ + args = [sys.executable] + if unbuffered_output: + args += ["-u"] + args += ["-c", _SETUPTOOLS_SHIM.format(setup_py_path)] + if global_options: + args += global_options + if no_user_config: + args += ["--no-user-cfg"] + return args + + +def make_setuptools_bdist_wheel_args( + setup_py_path, # type: str + global_options, # type: Sequence[str] + build_options, # type: Sequence[str] + destination_dir, # type: str +): + # type: (...) -> List[str] + # NOTE: Eventually, we'd want to also -S to the flags here, when we're + # isolating. Currently, it breaks Python in virtualenvs, because it + # relies on site.py to find parts of the standard library outside the + # virtualenv. + args = make_setuptools_shim_args( + setup_py_path, + global_options=global_options, + unbuffered_output=True + ) + args += ["bdist_wheel", "-d", destination_dir] + args += build_options + return args + + +def make_setuptools_clean_args( + setup_py_path, # type: str + global_options, # type: Sequence[str] +): + # type: (...) -> List[str] + args = make_setuptools_shim_args( + setup_py_path, + global_options=global_options, + unbuffered_output=True + ) + args += ["clean", "--all"] + return args + + +def make_setuptools_develop_args( + setup_py_path, # type: str + global_options, # type: Sequence[str] + install_options, # type: Sequence[str] + no_user_config, # type: bool + prefix, # type: Optional[str] + home, # type: Optional[str] + use_user_site, # type: bool +): + # type: (...) -> List[str] + assert not (use_user_site and prefix) + + args = make_setuptools_shim_args( + setup_py_path, + global_options=global_options, + no_user_config=no_user_config, + ) + + args += ["develop", "--no-deps"] + + args += install_options + + if prefix: + args += ["--prefix", prefix] + if home is not None: + args += ["--home", home] + + if use_user_site: + args += ["--user", "--prefix="] + + return args + + +def make_setuptools_egg_info_args( + setup_py_path, # type: str + egg_info_dir, # type: Optional[str] + no_user_config, # type: bool +): + # type: (...) -> List[str] + args = make_setuptools_shim_args(setup_py_path) + if no_user_config: + args += ["--no-user-cfg"] + + args += ["egg_info"] + + if egg_info_dir: + args += ["--egg-base", egg_info_dir] + + return args + + +def make_setuptools_install_args( + setup_py_path, # type: str + global_options, # type: Sequence[str] + install_options, # type: Sequence[str] + record_filename, # type: str + root, # type: Optional[str] + prefix, # type: Optional[str] + header_dir, # type: Optional[str] + home, # type: Optional[str] + use_user_site, # type: bool + no_user_config, # type: bool + pycompile # type: bool +): + # type: (...) -> List[str] + assert not (use_user_site and prefix) + assert not (use_user_site and root) + + args = make_setuptools_shim_args( + setup_py_path, + global_options=global_options, + no_user_config=no_user_config, + unbuffered_output=True + ) + args += ["install", "--record", record_filename] + args += ["--single-version-externally-managed"] + + if root is not None: + args += ["--root", root] + if prefix is not None: + args += ["--prefix", prefix] + if home is not None: + args += ["--home", home] + if use_user_site: + args += ["--user", "--prefix="] + + if pycompile: + args += ["--compile"] + else: + args += ["--no-compile"] + + if header_dir: + args += ["--install-headers", header_dir] + + args += install_options + + return args diff --git a/ubuntu/venv/pip/_internal/utils/subprocess.py b/ubuntu/venv/pip/_internal/utils/subprocess.py new file mode 100644 index 0000000..ea0176d --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/subprocess.py @@ -0,0 +1,278 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import os +import subprocess + +from pip._vendor.six.moves import shlex_quote + +from pip._internal.exceptions import InstallationError +from pip._internal.utils.compat import console_to_str, str_to_display +from pip._internal.utils.logging import subprocess_logger +from pip._internal.utils.misc import HiddenText, path_to_display +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.ui import open_spinner + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Iterable, List, Mapping, Optional, Text, Union, + ) + from pip._internal.utils.ui import SpinnerInterface + + CommandArgs = List[Union[str, HiddenText]] + + +LOG_DIVIDER = '----------------------------------------' + + +def make_command(*args): + # type: (Union[str, HiddenText, CommandArgs]) -> CommandArgs + """ + Create a CommandArgs object. + """ + command_args = [] # type: CommandArgs + for arg in args: + # Check for list instead of CommandArgs since CommandArgs is + # only known during type-checking. + if isinstance(arg, list): + command_args.extend(arg) + else: + # Otherwise, arg is str or HiddenText. + command_args.append(arg) + + return command_args + + +def format_command_args(args): + # type: (Union[List[str], CommandArgs]) -> str + """ + Format command arguments for display. + """ + # For HiddenText arguments, display the redacted form by calling str(). + # Also, we don't apply str() to arguments that aren't HiddenText since + # this can trigger a UnicodeDecodeError in Python 2 if the argument + # has type unicode and includes a non-ascii character. (The type + # checker doesn't ensure the annotations are correct in all cases.) + return ' '.join( + shlex_quote(str(arg)) if isinstance(arg, HiddenText) + else shlex_quote(arg) for arg in args + ) + + +def reveal_command_args(args): + # type: (Union[List[str], CommandArgs]) -> List[str] + """ + Return the arguments in their raw, unredacted form. + """ + return [ + arg.secret if isinstance(arg, HiddenText) else arg for arg in args + ] + + +def make_subprocess_output_error( + cmd_args, # type: Union[List[str], CommandArgs] + cwd, # type: Optional[str] + lines, # type: List[Text] + exit_status, # type: int +): + # type: (...) -> Text + """ + Create and return the error message to use to log a subprocess error + with command output. + + :param lines: A list of lines, each ending with a newline. + """ + command = format_command_args(cmd_args) + # Convert `command` and `cwd` to text (unicode in Python 2) so we can use + # them as arguments in the unicode format string below. This avoids + # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2 + # if either contains a non-ascii character. + command_display = str_to_display(command, desc='command bytes') + cwd_display = path_to_display(cwd) + + # We know the joined output value ends in a newline. + output = ''.join(lines) + msg = ( + # Use a unicode string to avoid "UnicodeEncodeError: 'ascii' + # codec can't encode character ..." in Python 2 when a format + # argument (e.g. `output`) has a non-ascii character. + u'Command errored out with exit status {exit_status}:\n' + ' command: {command_display}\n' + ' cwd: {cwd_display}\n' + 'Complete output ({line_count} lines):\n{output}{divider}' + ).format( + exit_status=exit_status, + command_display=command_display, + cwd_display=cwd_display, + line_count=len(lines), + output=output, + divider=LOG_DIVIDER, + ) + return msg + + +def call_subprocess( + cmd, # type: Union[List[str], CommandArgs] + show_stdout=False, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + unset_environ=None, # type: Optional[Iterable[str]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True # type: Optional[bool] +): + # type: (...) -> Text + """ + Args: + show_stdout: if true, use INFO to log the subprocess's stderr and + stdout streams. Otherwise, use DEBUG. Defaults to False. + extra_ok_returncodes: an iterable of integer return codes that are + acceptable, in addition to 0. Defaults to None, which means []. + unset_environ: an iterable of environment variable names to unset + prior to calling subprocess.Popen(). + log_failed_cmd: if false, failed commands are not logged, only raised. + """ + if extra_ok_returncodes is None: + extra_ok_returncodes = [] + if unset_environ is None: + unset_environ = [] + # Most places in pip use show_stdout=False. What this means is-- + # + # - We connect the child's output (combined stderr and stdout) to a + # single pipe, which we read. + # - We log this output to stderr at DEBUG level as it is received. + # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't + # requested), then we show a spinner so the user can still see the + # subprocess is in progress. + # - If the subprocess exits with an error, we log the output to stderr + # at ERROR level if it hasn't already been displayed to the console + # (e.g. if --verbose logging wasn't enabled). This way we don't log + # the output to the console twice. + # + # If show_stdout=True, then the above is still done, but with DEBUG + # replaced by INFO. + if show_stdout: + # Then log the subprocess output at INFO level. + log_subprocess = subprocess_logger.info + used_level = logging.INFO + else: + # Then log the subprocess output using DEBUG. This also ensures + # it will be logged to the log file (aka user_log), if enabled. + log_subprocess = subprocess_logger.debug + used_level = logging.DEBUG + + # Whether the subprocess will be visible in the console. + showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level + + # Only use the spinner if we're not showing the subprocess output + # and we have a spinner. + use_spinner = not showing_subprocess and spinner is not None + + if command_desc is None: + command_desc = format_command_args(cmd) + + log_subprocess("Running command %s", command_desc) + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + for name in unset_environ: + env.pop(name, None) + try: + proc = subprocess.Popen( + # Convert HiddenText objects to the underlying str. + reveal_command_args(cmd), + stderr=subprocess.STDOUT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, cwd=cwd, env=env, + ) + proc.stdin.close() + except Exception as exc: + if log_failed_cmd: + subprocess_logger.critical( + "Error %s while executing command %s", exc, command_desc, + ) + raise + all_output = [] + while True: + # The "line" value is a unicode string in Python 2. + line = console_to_str(proc.stdout.readline()) + if not line: + break + line = line.rstrip() + all_output.append(line + '\n') + + # Show the line immediately. + log_subprocess(line) + # Update the spinner. + if use_spinner: + spinner.spin() + try: + proc.wait() + finally: + if proc.stdout: + proc.stdout.close() + proc_had_error = ( + proc.returncode and proc.returncode not in extra_ok_returncodes + ) + if use_spinner: + if proc_had_error: + spinner.finish("error") + else: + spinner.finish("done") + if proc_had_error: + if on_returncode == 'raise': + if not showing_subprocess and log_failed_cmd: + # Then the subprocess streams haven't been logged to the + # console yet. + msg = make_subprocess_output_error( + cmd_args=cmd, + cwd=cwd, + lines=all_output, + exit_status=proc.returncode, + ) + subprocess_logger.error(msg) + exc_msg = ( + 'Command errored out with exit status {}: {} ' + 'Check the logs for full command output.' + ).format(proc.returncode, command_desc) + raise InstallationError(exc_msg) + elif on_returncode == 'warn': + subprocess_logger.warning( + 'Command "%s" had error code %s in %s', + command_desc, proc.returncode, cwd, + ) + elif on_returncode == 'ignore': + pass + else: + raise ValueError('Invalid value: on_returncode=%s' % + repr(on_returncode)) + return ''.join(all_output) + + +def runner_with_spinner_message(message): + # type: (str) -> Callable[..., None] + """Provide a subprocess_runner that shows a spinner message. + + Intended for use with for pep517's Pep517HookCaller. Thus, the runner has + an API that matches what's expected by Pep517HookCaller.subprocess_runner. + """ + + def runner( + cmd, # type: List[str] + cwd=None, # type: Optional[str] + extra_environ=None # type: Optional[Mapping[str, Any]] + ): + # type: (...) -> None + with open_spinner(message) as spinner: + call_subprocess( + cmd, + cwd=cwd, + extra_environ=extra_environ, + spinner=spinner, + ) + + return runner diff --git a/ubuntu/venv/pip/_internal/utils/temp_dir.py b/ubuntu/venv/pip/_internal/utils/temp_dir.py new file mode 100644 index 0000000..65e41bc --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/temp_dir.py @@ -0,0 +1,250 @@ +from __future__ import absolute_import + +import errno +import itertools +import logging +import os.path +import tempfile +from contextlib import contextmanager + +from pip._vendor.contextlib2 import ExitStack + +from pip._internal.utils.misc import rmtree +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Dict, Iterator, Optional, TypeVar + + _T = TypeVar('_T', bound='TempDirectory') + + +logger = logging.getLogger(__name__) + + +_tempdir_manager = None # type: Optional[ExitStack] + + +@contextmanager +def global_tempdir_manager(): + # type: () -> Iterator[None] + global _tempdir_manager + with ExitStack() as stack: + old_tempdir_manager, _tempdir_manager = _tempdir_manager, stack + try: + yield + finally: + _tempdir_manager = old_tempdir_manager + + +class TempDirectoryTypeRegistry(object): + """Manages temp directory behavior + """ + + def __init__(self): + # type: () -> None + self._should_delete = {} # type: Dict[str, bool] + + def set_delete(self, kind, value): + # type: (str, bool) -> None + """Indicate whether a TempDirectory of the given kind should be + auto-deleted. + """ + self._should_delete[kind] = value + + def get_delete(self, kind): + # type: (str) -> bool + """Get configured auto-delete flag for a given TempDirectory type, + default True. + """ + return self._should_delete.get(kind, True) + + +_tempdir_registry = None # type: Optional[TempDirectoryTypeRegistry] + + +@contextmanager +def tempdir_registry(): + # type: () -> Iterator[TempDirectoryTypeRegistry] + """Provides a scoped global tempdir registry that can be used to dictate + whether directories should be deleted. + """ + global _tempdir_registry + old_tempdir_registry = _tempdir_registry + _tempdir_registry = TempDirectoryTypeRegistry() + try: + yield _tempdir_registry + finally: + _tempdir_registry = old_tempdir_registry + + +class TempDirectory(object): + """Helper class that owns and cleans up a temporary directory. + + This class can be used as a context manager or as an OO representation of a + temporary directory. + + Attributes: + path + Location to the created temporary directory + delete + Whether the directory should be deleted when exiting + (when used as a contextmanager) + + Methods: + cleanup() + Deletes the temporary directory + + When used as a context manager, if the delete attribute is True, on + exiting the context the temporary directory is deleted. + """ + + def __init__( + self, + path=None, # type: Optional[str] + delete=None, # type: Optional[bool] + kind="temp", # type: str + globally_managed=False, # type: bool + ): + super(TempDirectory, self).__init__() + + # If we were given an explicit directory, resolve delete option now. + # Otherwise we wait until cleanup and see what tempdir_registry says. + if path is not None and delete is None: + delete = False + + if path is None: + path = self._create(kind) + + self._path = path + self._deleted = False + self.delete = delete + self.kind = kind + + if globally_managed: + assert _tempdir_manager is not None + _tempdir_manager.enter_context(self) + + @property + def path(self): + # type: () -> str + assert not self._deleted, ( + "Attempted to access deleted path: {}".format(self._path) + ) + return self._path + + def __repr__(self): + # type: () -> str + return "<{} {!r}>".format(self.__class__.__name__, self.path) + + def __enter__(self): + # type: (_T) -> _T + return self + + def __exit__(self, exc, value, tb): + # type: (Any, Any, Any) -> None + if self.delete is not None: + delete = self.delete + elif _tempdir_registry: + delete = _tempdir_registry.get_delete(self.kind) + else: + delete = True + + if delete: + self.cleanup() + + def _create(self, kind): + # type: (str) -> str + """Create a temporary directory and store its path in self.path + """ + # We realpath here because some systems have their default tmpdir + # symlinked to another directory. This tends to confuse build + # scripts, so we canonicalize the path by traversing potential + # symlinks here. + path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(kind)) + ) + logger.debug("Created temporary directory: {}".format(path)) + return path + + def cleanup(self): + # type: () -> None + """Remove the temporary directory created and reset state + """ + self._deleted = True + if os.path.exists(self._path): + rmtree(self._path) + + +class AdjacentTempDirectory(TempDirectory): + """Helper class that creates a temporary directory adjacent to a real one. + + Attributes: + original + The original directory to create a temp directory for. + path + After calling create() or entering, contains the full + path to the temporary directory. + delete + Whether the directory should be deleted when exiting + (when used as a contextmanager) + + """ + # The characters that may be used to name the temp directory + # We always prepend a ~ and then rotate through these until + # a usable name is found. + # pkg_resources raises a different error for .dist-info folder + # with leading '-' and invalid metadata + LEADING_CHARS = "-~.=%0123456789" + + def __init__(self, original, delete=None): + # type: (str, Optional[bool]) -> None + self.original = original.rstrip('/\\') + super(AdjacentTempDirectory, self).__init__(delete=delete) + + @classmethod + def _generate_names(cls, name): + # type: (str) -> Iterator[str] + """Generates a series of temporary names. + + The algorithm replaces the leading characters in the name + with ones that are valid filesystem characters, but are not + valid package names (for both Python and pip definitions of + package). + """ + for i in range(1, len(name)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i - 1): + new_name = '~' + ''.join(candidate) + name[i:] + if new_name != name: + yield new_name + + # If we make it this far, we will have to make a longer name + for i in range(len(cls.LEADING_CHARS)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i): + new_name = '~' + ''.join(candidate) + name + if new_name != name: + yield new_name + + def _create(self, kind): + # type: (str) -> str + root, name = os.path.split(self.original) + for candidate in self._generate_names(name): + path = os.path.join(root, candidate) + try: + os.mkdir(path) + except OSError as ex: + # Continue if the name exists already + if ex.errno != errno.EEXIST: + raise + else: + path = os.path.realpath(path) + break + else: + # Final fallback on the default behavior. + path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(kind)) + ) + + logger.debug("Created temporary directory: {}".format(path)) + return path diff --git a/ubuntu/venv/pip/_internal/utils/typing.py b/ubuntu/venv/pip/_internal/utils/typing.py new file mode 100644 index 0000000..8505a29 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/typing.py @@ -0,0 +1,38 @@ +"""For neatly implementing static typing in pip. + +`mypy` - the static type analysis tool we use - uses the `typing` module, which +provides core functionality fundamental to mypy's functioning. + +Generally, `typing` would be imported at runtime and used in that fashion - +it acts as a no-op at runtime and does not have any run-time overhead by +design. + +As it turns out, `typing` is not vendorable - it uses separate sources for +Python 2/Python 3. Thus, this codebase can not expect it to be present. +To work around this, mypy allows the typing import to be behind a False-y +optional to prevent it from running at runtime and type-comments can be used +to remove the need for the types to be accessible directly during runtime. + +This module provides the False-y guard in a nicely named fashion so that a +curious maintainer can reach here to read this. + +In pip, all static-typing related imports should be guarded as follows: + + from pip._internal.utils.typing import MYPY_CHECK_RUNNING + + if MYPY_CHECK_RUNNING: + from typing import ... + +Ref: https://github.com/python/mypy/issues/3216 +""" + +MYPY_CHECK_RUNNING = False + + +if MYPY_CHECK_RUNNING: + from typing import cast +else: + # typing's cast() is needed at runtime, but we don't want to import typing. + # Thus, we use a dummy no-op version, which we tell mypy to ignore. + def cast(type_, value): # type: ignore + return value diff --git a/ubuntu/venv/pip/_internal/utils/ui.py b/ubuntu/venv/pip/_internal/utils/ui.py new file mode 100644 index 0000000..87782aa --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/ui.py @@ -0,0 +1,428 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import, division + +import contextlib +import itertools +import logging +import sys +import time +from signal import SIGINT, default_int_handler, signal + +from pip._vendor import six +from pip._vendor.progress import HIDE_CURSOR, SHOW_CURSOR +from pip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar +from pip._vendor.progress.spinner import Spinner + +from pip._internal.utils.compat import WINDOWS +from pip._internal.utils.logging import get_indentation +from pip._internal.utils.misc import format_size +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Iterator, IO + +try: + from pip._vendor import colorama +# Lots of different errors can come from this, including SystemError and +# ImportError. +except Exception: + colorama = None + +logger = logging.getLogger(__name__) + + +def _select_progress_class(preferred, fallback): + encoding = getattr(preferred.file, "encoding", None) + + # If we don't know what encoding this file is in, then we'll just assume + # that it doesn't support unicode and use the ASCII bar. + if not encoding: + return fallback + + # Collect all of the possible characters we want to use with the preferred + # bar. + characters = [ + getattr(preferred, "empty_fill", six.text_type()), + getattr(preferred, "fill", six.text_type()), + ] + characters += list(getattr(preferred, "phases", [])) + + # Try to decode the characters we're using for the bar using the encoding + # of the given file, if this works then we'll assume that we can use the + # fancier bar and if not we'll fall back to the plaintext bar. + try: + six.text_type().join(characters).encode(encoding) + except UnicodeEncodeError: + return fallback + else: + return preferred + + +_BaseBar = _select_progress_class(IncrementalBar, Bar) # type: Any + + +class InterruptibleMixin(object): + """ + Helper to ensure that self.finish() gets called on keyboard interrupt. + + This allows downloads to be interrupted without leaving temporary state + (like hidden cursors) behind. + + This class is similar to the progress library's existing SigIntMixin + helper, but as of version 1.2, that helper has the following problems: + + 1. It calls sys.exit(). + 2. It discards the existing SIGINT handler completely. + 3. It leaves its own handler in place even after an uninterrupted finish, + which will have unexpected delayed effects if the user triggers an + unrelated keyboard interrupt some time after a progress-displaying + download has already completed, for example. + """ + + def __init__(self, *args, **kwargs): + """ + Save the original SIGINT handler for later. + """ + super(InterruptibleMixin, self).__init__(*args, **kwargs) + + self.original_handler = signal(SIGINT, self.handle_sigint) + + # If signal() returns None, the previous handler was not installed from + # Python, and we cannot restore it. This probably should not happen, + # but if it does, we must restore something sensible instead, at least. + # The least bad option should be Python's default SIGINT handler, which + # just raises KeyboardInterrupt. + if self.original_handler is None: + self.original_handler = default_int_handler + + def finish(self): + """ + Restore the original SIGINT handler after finishing. + + This should happen regardless of whether the progress display finishes + normally, or gets interrupted. + """ + super(InterruptibleMixin, self).finish() + signal(SIGINT, self.original_handler) + + def handle_sigint(self, signum, frame): + """ + Call self.finish() before delegating to the original SIGINT handler. + + This handler should only be in place while the progress display is + active. + """ + self.finish() + self.original_handler(signum, frame) + + +class SilentBar(Bar): + + def update(self): + pass + + +class BlueEmojiBar(IncrementalBar): + + suffix = "%(percent)d%%" + bar_prefix = " " + bar_suffix = " " + phases = (u"\U0001F539", u"\U0001F537", u"\U0001F535") # type: Any + + +class DownloadProgressMixin(object): + + def __init__(self, *args, **kwargs): + super(DownloadProgressMixin, self).__init__(*args, **kwargs) + self.message = (" " * (get_indentation() + 2)) + self.message + + @property + def downloaded(self): + return format_size(self.index) + + @property + def download_speed(self): + # Avoid zero division errors... + if self.avg == 0.0: + return "..." + return format_size(1 / self.avg) + "/s" + + @property + def pretty_eta(self): + if self.eta: + return "eta %s" % self.eta_td + return "" + + def iter(self, it): + for x in it: + yield x + self.next(len(x)) + self.finish() + + +class WindowsMixin(object): + + def __init__(self, *args, **kwargs): + # The Windows terminal does not support the hide/show cursor ANSI codes + # even with colorama. So we'll ensure that hide_cursor is False on + # Windows. + # This call needs to go before the super() call, so that hide_cursor + # is set in time. The base progress bar class writes the "hide cursor" + # code to the terminal in its init, so if we don't set this soon + # enough, we get a "hide" with no corresponding "show"... + if WINDOWS and self.hide_cursor: + self.hide_cursor = False + + super(WindowsMixin, self).__init__(*args, **kwargs) + + # Check if we are running on Windows and we have the colorama module, + # if we do then wrap our file with it. + if WINDOWS and colorama: + self.file = colorama.AnsiToWin32(self.file) + # The progress code expects to be able to call self.file.isatty() + # but the colorama.AnsiToWin32() object doesn't have that, so we'll + # add it. + self.file.isatty = lambda: self.file.wrapped.isatty() + # The progress code expects to be able to call self.file.flush() + # but the colorama.AnsiToWin32() object doesn't have that, so we'll + # add it. + self.file.flush = lambda: self.file.wrapped.flush() + + +class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, + DownloadProgressMixin): + + file = sys.stdout + message = "%(percent)d%%" + suffix = "%(downloaded)s %(download_speed)s %(pretty_eta)s" + +# NOTE: The "type: ignore" comments on the following classes are there to +# work around https://github.com/python/typing/issues/241 + + +class DefaultDownloadProgressBar(BaseDownloadProgressBar, + _BaseBar): + pass + + +class DownloadSilentBar(BaseDownloadProgressBar, SilentBar): # type: ignore + pass + + +class DownloadBar(BaseDownloadProgressBar, # type: ignore + Bar): + pass + + +class DownloadFillingCirclesBar(BaseDownloadProgressBar, # type: ignore + FillingCirclesBar): + pass + + +class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar, # type: ignore + BlueEmojiBar): + pass + + +class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin, + DownloadProgressMixin, Spinner): + + file = sys.stdout + suffix = "%(downloaded)s %(download_speed)s" + + def next_phase(self): + if not hasattr(self, "_phaser"): + self._phaser = itertools.cycle(self.phases) + return next(self._phaser) + + def update(self): + message = self.message % self + phase = self.next_phase() + suffix = self.suffix % self + line = ''.join([ + message, + " " if message else "", + phase, + " " if suffix else "", + suffix, + ]) + + self.writeln(line) + + +BAR_TYPES = { + "off": (DownloadSilentBar, DownloadSilentBar), + "on": (DefaultDownloadProgressBar, DownloadProgressSpinner), + "ascii": (DownloadBar, DownloadProgressSpinner), + "pretty": (DownloadFillingCirclesBar, DownloadProgressSpinner), + "emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner) +} + + +def DownloadProgressProvider(progress_bar, max=None): + if max is None or max == 0: + return BAR_TYPES[progress_bar][1]().iter + else: + return BAR_TYPES[progress_bar][0](max=max).iter + + +################################################################ +# Generic "something is happening" spinners +# +# We don't even try using progress.spinner.Spinner here because it's actually +# simpler to reimplement from scratch than to coerce their code into doing +# what we need. +################################################################ + +@contextlib.contextmanager +def hidden_cursor(file): + # type: (IO[Any]) -> Iterator[None] + # The Windows terminal does not support the hide/show cursor ANSI codes, + # even via colorama. So don't even try. + if WINDOWS: + yield + # We don't want to clutter the output with control characters if we're + # writing to a file, or if the user is running with --quiet. + # See https://github.com/pypa/pip/issues/3418 + elif not file.isatty() or logger.getEffectiveLevel() > logging.INFO: + yield + else: + file.write(HIDE_CURSOR) + try: + yield + finally: + file.write(SHOW_CURSOR) + + +class RateLimiter(object): + def __init__(self, min_update_interval_seconds): + # type: (float) -> None + self._min_update_interval_seconds = min_update_interval_seconds + self._last_update = 0 # type: float + + def ready(self): + # type: () -> bool + now = time.time() + delta = now - self._last_update + return delta >= self._min_update_interval_seconds + + def reset(self): + # type: () -> None + self._last_update = time.time() + + +class SpinnerInterface(object): + def spin(self): + # type: () -> None + raise NotImplementedError() + + def finish(self, final_status): + # type: (str) -> None + raise NotImplementedError() + + +class InteractiveSpinner(SpinnerInterface): + def __init__(self, message, file=None, spin_chars="-\\|/", + # Empirically, 8 updates/second looks nice + min_update_interval_seconds=0.125): + self._message = message + if file is None: + file = sys.stdout + self._file = file + self._rate_limiter = RateLimiter(min_update_interval_seconds) + self._finished = False + + self._spin_cycle = itertools.cycle(spin_chars) + + self._file.write(" " * get_indentation() + self._message + " ... ") + self._width = 0 + + def _write(self, status): + assert not self._finished + # Erase what we wrote before by backspacing to the beginning, writing + # spaces to overwrite the old text, and then backspacing again + backup = "\b" * self._width + self._file.write(backup + " " * self._width + backup) + # Now we have a blank slate to add our status + self._file.write(status) + self._width = len(status) + self._file.flush() + self._rate_limiter.reset() + + def spin(self): + # type: () -> None + if self._finished: + return + if not self._rate_limiter.ready(): + return + self._write(next(self._spin_cycle)) + + def finish(self, final_status): + # type: (str) -> None + if self._finished: + return + self._write(final_status) + self._file.write("\n") + self._file.flush() + self._finished = True + + +# Used for dumb terminals, non-interactive installs (no tty), etc. +# We still print updates occasionally (once every 60 seconds by default) to +# act as a keep-alive for systems like Travis-CI that take lack-of-output as +# an indication that a task has frozen. +class NonInteractiveSpinner(SpinnerInterface): + def __init__(self, message, min_update_interval_seconds=60): + # type: (str, float) -> None + self._message = message + self._finished = False + self._rate_limiter = RateLimiter(min_update_interval_seconds) + self._update("started") + + def _update(self, status): + assert not self._finished + self._rate_limiter.reset() + logger.info("%s: %s", self._message, status) + + def spin(self): + # type: () -> None + if self._finished: + return + if not self._rate_limiter.ready(): + return + self._update("still running...") + + def finish(self, final_status): + # type: (str) -> None + if self._finished: + return + self._update("finished with status '%s'" % (final_status,)) + self._finished = True + + +@contextlib.contextmanager +def open_spinner(message): + # type: (str) -> Iterator[SpinnerInterface] + # Interactive spinner goes directly to sys.stdout rather than being routed + # through the logging system, but it acts like it has level INFO, + # i.e. it's only displayed if we're at level INFO or better. + # Non-interactive spinner goes through the logging system, so it is always + # in sync with logging configuration. + if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: + spinner = InteractiveSpinner(message) # type: SpinnerInterface + else: + spinner = NonInteractiveSpinner(message) + try: + with hidden_cursor(sys.stdout): + yield spinner + except KeyboardInterrupt: + spinner.finish("canceled") + raise + except Exception: + spinner.finish("error") + raise + else: + spinner.finish("done") diff --git a/ubuntu/venv/pip/_internal/utils/unpacking.py b/ubuntu/venv/pip/_internal/utils/unpacking.py new file mode 100644 index 0000000..7252dc2 --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/unpacking.py @@ -0,0 +1,272 @@ +"""Utilities related archives. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os +import shutil +import stat +import tarfile +import zipfile + +from pip._internal.exceptions import InstallationError +from pip._internal.utils.filetypes import ( + BZ2_EXTENSIONS, + TAR_EXTENSIONS, + XZ_EXTENSIONS, + ZIP_EXTENSIONS, +) +from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterable, List, Optional, Text, Union + + +logger = logging.getLogger(__name__) + + +SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS + +try: + import bz2 # noqa + SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS +except ImportError: + logger.debug('bz2 module is not available') + +try: + # Only for Python 3.3+ + import lzma # noqa + SUPPORTED_EXTENSIONS += XZ_EXTENSIONS +except ImportError: + logger.debug('lzma module is not available') + + +def current_umask(): + """Get the current umask which involves having to set it temporarily.""" + mask = os.umask(0) + os.umask(mask) + return mask + + +def split_leading_dir(path): + # type: (Union[str, Text]) -> List[Union[str, Text]] + path = path.lstrip('/').lstrip('\\') + if ( + '/' in path and ( + ('\\' in path and path.find('/') < path.find('\\')) or + '\\' not in path + ) + ): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return [path, ''] + + +def has_leading_dir(paths): + # type: (Iterable[Union[str, Text]]) -> bool + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def is_within_directory(directory, target): + # type: ((Union[str, Text]), (Union[str, Text])) -> bool + """ + Return true if the absolute path of target is within the directory + """ + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + return prefix == abs_directory + + +def unzip_file(filename, location, flatten=True): + # type: (str, str, bool) -> None + """ + Unzip the file (with path `filename`) to the destination `location`. All + files are written based on system defaults and umask (i.e. permissions are + not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp, allowZip64=True) + leading = has_leading_dir(zip.namelist()) and flatten + for info in zip.infolist(): + name = info.filename + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not is_within_directory(location, fn): + message = ( + 'The zip file ({}) has a file ({}) trying to install ' + 'outside target directory ({})' + ) + raise InstallationError(message.format(filename, fn, location)) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + ensure_dir(fn) + else: + ensure_dir(dir) + # Don't use read() to avoid allocating an arbitrarily large + # chunk of memory for the file's content + fp = zip.open(name) + try: + with open(fn, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + finally: + fp.close() + mode = info.external_attr >> 16 + # if mode and regular file and any execute permissions for + # user/group/world? + if mode and stat.S_ISREG(mode) and mode & 0o111: + # make dest file have execute for user/group/world + # (chmod +x) no-op on windows per python docs + os.chmod(fn, (0o777 - current_umask() | 0o111)) + finally: + zipfp.close() + + +def untar_file(filename, location): + # type: (str, str) -> None + """ + Untar the file (with path `filename`) to the destination `location`. + All files are written based on system defaults and umask (i.e. permissions + are not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif filename.lower().endswith(BZ2_EXTENSIONS): + mode = 'r:bz2' + elif filename.lower().endswith(XZ_EXTENSIONS): + mode = 'r:xz' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + logger.warning( + 'Cannot determine compression type for file %s', filename, + ) + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + leading = has_leading_dir([ + member.name for member in tar.getmembers() + ]) + for member in tar.getmembers(): + fn = member.name + if leading: + # https://github.com/python/mypy/issues/1174 + fn = split_leading_dir(fn)[1] # type: ignore + path = os.path.join(location, fn) + if not is_within_directory(location, path): + message = ( + 'The tar file ({}) has a file ({}) trying to install ' + 'outside target directory ({})' + ) + raise InstallationError( + message.format(filename, path, location) + ) + if member.isdir(): + ensure_dir(path) + elif member.issym(): + try: + # https://github.com/python/typeshed/issues/2673 + tar._extract_member(member, path) # type: ignore + except Exception as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError) as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + ensure_dir(os.path.dirname(path)) + with open(path, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + fp.close() + # Update the timestamp (useful for cython compiled files) + # https://github.com/python/typeshed/issues/2673 + tar.utime(member, path) # type: ignore + # member have any execute permissions for user/group/world? + if member.mode & 0o111: + # make dest file have execute for user/group/world + # no-op on windows per python docs + os.chmod(path, (0o777 - current_umask() | 0o111)) + finally: + tar.close() + + +def unpack_file( + filename, # type: str + location, # type: str + content_type=None, # type: Optional[str] +): + # type: (...) -> None + filename = os.path.realpath(filename) + if ( + content_type == 'application/zip' or + filename.lower().endswith(ZIP_EXTENSIONS) or + zipfile.is_zipfile(filename) + ): + unzip_file( + filename, + location, + flatten=not filename.endswith('.whl') + ) + elif ( + content_type == 'application/x-gzip' or + tarfile.is_tarfile(filename) or + filename.lower().endswith( + TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS + ) + ): + untar_file(filename, location) + else: + # FIXME: handle? + # FIXME: magic signatures? + logger.critical( + 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' + 'cannot detect archive format', + filename, location, content_type, + ) + raise InstallationError( + 'Cannot determine archive format of {}'.format(location) + ) diff --git a/ubuntu/venv/pip/_internal/utils/urls.py b/ubuntu/venv/pip/_internal/utils/urls.py new file mode 100644 index 0000000..9ad40fe --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/urls.py @@ -0,0 +1,54 @@ +import os +import sys + +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Union + + +def get_url_scheme(url): + # type: (Union[str, Text]) -> Optional[Text] + if ':' not in url: + return None + return url.split(':', 1)[0].lower() + + +def path_to_url(path): + # type: (Union[str, Text]) -> str + """ + Convert a path to a file: URL. The path will be made absolute and have + quoted path parts. + """ + path = os.path.normpath(os.path.abspath(path)) + url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) + return url + + +def url_to_path(url): + # type: (str) -> str + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + + _, netloc, path, _, _ = urllib_parse.urlsplit(url) + + if not netloc or netloc == 'localhost': + # According to RFC 8089, same as empty authority. + netloc = '' + elif sys.platform == 'win32': + # If we have a UNC path, prepend UNC share notation. + netloc = '\\\\' + netloc + else: + raise ValueError( + 'non-local file URIs are not supported on this platform: %r' + % url + ) + + path = urllib_request.url2pathname(netloc + path) + return path diff --git a/ubuntu/venv/pip/_internal/utils/virtualenv.py b/ubuntu/venv/pip/_internal/utils/virtualenv.py new file mode 100644 index 0000000..d81e6ac --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/virtualenv.py @@ -0,0 +1,115 @@ +from __future__ import absolute_import + +import logging +import os +import re +import site +import sys + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional + +logger = logging.getLogger(__name__) +_INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile( + r"include-system-site-packages\s*=\s*(?Ptrue|false)" +) + + +def _running_under_venv(): + # type: () -> bool + """Checks if sys.base_prefix and sys.prefix match. + + This handles PEP 405 compliant virtual environments. + """ + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + +def _running_under_regular_virtualenv(): + # type: () -> bool + """Checks if sys.real_prefix is set. + + This handles virtual environments created with pypa's virtualenv. + """ + # pypa/virtualenv case + return hasattr(sys, 'real_prefix') + + +def running_under_virtualenv(): + # type: () -> bool + """Return True if we're running inside a virtualenv, False otherwise. + """ + return _running_under_venv() or _running_under_regular_virtualenv() + + +def _get_pyvenv_cfg_lines(): + # type: () -> Optional[List[str]] + """Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines + + Returns None, if it could not read/access the file. + """ + pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg') + try: + with open(pyvenv_cfg_file) as f: + return f.read().splitlines() # avoids trailing newlines + except IOError: + return None + + +def _no_global_under_venv(): + # type: () -> bool + """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion + + PEP 405 specifies that when system site-packages are not supposed to be + visible from a virtual environment, `pyvenv.cfg` must contain the following + line: + + include-system-site-packages = false + + Additionally, log a warning if accessing the file fails. + """ + cfg_lines = _get_pyvenv_cfg_lines() + if cfg_lines is None: + # We're not in a "sane" venv, so assume there is no system + # site-packages access (since that's PEP 405's default state). + logger.warning( + "Could not access 'pyvenv.cfg' despite a virtual environment " + "being active. Assuming global site-packages is not accessible " + "in this environment." + ) + return True + + for line in cfg_lines: + match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line) + if match is not None and match.group('value') == 'false': + return True + return False + + +def _no_global_under_regular_virtualenv(): + # type: () -> bool + """Check if "no-global-site-packages.txt" exists beside site.py + + This mirrors logic in pypa/virtualenv for determining whether system + site-packages are visible in the virtual environment. + """ + site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) + no_global_site_packages_file = os.path.join( + site_mod_dir, 'no-global-site-packages.txt', + ) + return os.path.exists(no_global_site_packages_file) + + +def virtualenv_no_global(): + # type: () -> bool + """Returns a boolean, whether running in venv with no system site-packages. + """ + + if _running_under_regular_virtualenv(): + return _no_global_under_regular_virtualenv() + + if _running_under_venv(): + return _no_global_under_venv() + + return False diff --git a/ubuntu/venv/pip/_internal/utils/wheel.py b/ubuntu/venv/pip/_internal/utils/wheel.py new file mode 100644 index 0000000..837e0af --- /dev/null +++ b/ubuntu/venv/pip/_internal/utils/wheel.py @@ -0,0 +1,225 @@ +"""Support functions for working with wheel files. +""" + +from __future__ import absolute_import + +import logging +from email.parser import Parser +from zipfile import ZipFile + +from pip._vendor.packaging.utils import canonicalize_name +from pip._vendor.pkg_resources import DistInfoDistribution +from pip._vendor.six import PY2, ensure_str + +from pip._internal.exceptions import UnsupportedWheel +from pip._internal.utils.pkg_resources import DictMetadata +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from email.message import Message + from typing import Dict, Tuple + + from pip._vendor.pkg_resources import Distribution + +if PY2: + from zipfile import BadZipfile as BadZipFile +else: + from zipfile import BadZipFile + + +VERSION_COMPATIBLE = (1, 0) + + +logger = logging.getLogger(__name__) + + +class WheelMetadata(DictMetadata): + """Metadata provider that maps metadata decoding exceptions to our + internal exception type. + """ + def __init__(self, metadata, wheel_name): + # type: (Dict[str, bytes], str) -> None + super(WheelMetadata, self).__init__(metadata) + self._wheel_name = wheel_name + + def get_metadata(self, name): + # type: (str) -> str + try: + return super(WheelMetadata, self).get_metadata(name) + except UnicodeDecodeError as e: + # Augment the default error with the origin of the file. + raise UnsupportedWheel( + "Error decoding metadata for {}: {}".format( + self._wheel_name, e + ) + ) + + +def pkg_resources_distribution_for_wheel(wheel_zip, name, location): + # type: (ZipFile, str, str) -> Distribution + """Get a pkg_resources distribution given a wheel. + + :raises UnsupportedWheel: on any errors + """ + info_dir, _ = parse_wheel(wheel_zip, name) + + metadata_files = [ + p for p in wheel_zip.namelist() if p.startswith("{}/".format(info_dir)) + ] + + metadata_text = {} # type: Dict[str, bytes] + for path in metadata_files: + # If a flag is set, namelist entries may be unicode in Python 2. + # We coerce them to native str type to match the types used in the rest + # of the code. This cannot fail because unicode can always be encoded + # with UTF-8. + full_path = ensure_str(path) + _, metadata_name = full_path.split("/", 1) + + try: + metadata_text[metadata_name] = read_wheel_metadata_file( + wheel_zip, full_path + ) + except UnsupportedWheel as e: + raise UnsupportedWheel( + "{} has an invalid wheel, {}".format(name, str(e)) + ) + + metadata = WheelMetadata(metadata_text, location) + + return DistInfoDistribution( + location=location, metadata=metadata, project_name=name + ) + + +def parse_wheel(wheel_zip, name): + # type: (ZipFile, str) -> Tuple[str, Message] + """Extract information from the provided wheel, ensuring it meets basic + standards. + + Returns the name of the .dist-info directory and the parsed WHEEL metadata. + """ + try: + info_dir = wheel_dist_info_dir(wheel_zip, name) + metadata = wheel_metadata(wheel_zip, info_dir) + version = wheel_version(metadata) + except UnsupportedWheel as e: + raise UnsupportedWheel( + "{} has an invalid wheel, {}".format(name, str(e)) + ) + + check_compatibility(version, name) + + return info_dir, metadata + + +def wheel_dist_info_dir(source, name): + # type: (ZipFile, str) -> str + """Returns the name of the contained .dist-info directory. + + Raises AssertionError or UnsupportedWheel if not found, >1 found, or + it doesn't match the provided name. + """ + # Zip file path separators must be / + subdirs = list(set(p.split("/")[0] for p in source.namelist())) + + info_dirs = [s for s in subdirs if s.endswith('.dist-info')] + + if not info_dirs: + raise UnsupportedWheel(".dist-info directory not found") + + if len(info_dirs) > 1: + raise UnsupportedWheel( + "multiple .dist-info directories found: {}".format( + ", ".join(info_dirs) + ) + ) + + info_dir = info_dirs[0] + + info_dir_name = canonicalize_name(info_dir) + canonical_name = canonicalize_name(name) + if not info_dir_name.startswith(canonical_name): + raise UnsupportedWheel( + ".dist-info directory {!r} does not start with {!r}".format( + info_dir, canonical_name + ) + ) + + # Zip file paths can be unicode or str depending on the zip entry flags, + # so normalize it. + return ensure_str(info_dir) + + +def read_wheel_metadata_file(source, path): + # type: (ZipFile, str) -> bytes + try: + return source.read(path) + # BadZipFile for general corruption, KeyError for missing entry, + # and RuntimeError for password-protected files + except (BadZipFile, KeyError, RuntimeError) as e: + raise UnsupportedWheel( + "could not read {!r} file: {!r}".format(path, e) + ) + + +def wheel_metadata(source, dist_info_dir): + # type: (ZipFile, str) -> Message + """Return the WHEEL metadata of an extracted wheel, if possible. + Otherwise, raise UnsupportedWheel. + """ + path = "{}/WHEEL".format(dist_info_dir) + # Zip file path separators must be / + wheel_contents = read_wheel_metadata_file(source, path) + + try: + wheel_text = ensure_str(wheel_contents) + except UnicodeDecodeError as e: + raise UnsupportedWheel("error decoding {!r}: {!r}".format(path, e)) + + # FeedParser (used by Parser) does not raise any exceptions. The returned + # message may have .defects populated, but for backwards-compatibility we + # currently ignore them. + return Parser().parsestr(wheel_text) + + +def wheel_version(wheel_data): + # type: (Message) -> Tuple[int, ...] + """Given WHEEL metadata, return the parsed Wheel-Version. + Otherwise, raise UnsupportedWheel. + """ + version_text = wheel_data["Wheel-Version"] + if version_text is None: + raise UnsupportedWheel("WHEEL is missing Wheel-Version") + + version = version_text.strip() + + try: + return tuple(map(int, version.split('.'))) + except ValueError: + raise UnsupportedWheel("invalid Wheel-Version: {!r}".format(version)) + + +def check_compatibility(version, name): + # type: (Tuple[int, ...], str) -> None + """Raises errors or warns if called with an incompatible Wheel-Version. + + Pip should refuse to install a Wheel-Version that's a major series + ahead of what it's compatible with (e.g 2.0 > 1.1); and warn when + installing a version only minor version ahead (e.g 1.2 > 1.1). + + version: a 2-tuple representing a Wheel-Version (Major, Minor) + name: name of wheel or package to raise exception about + + :raises UnsupportedWheel: when an incompatible Wheel-Version is given + """ + if version[0] > VERSION_COMPATIBLE[0]: + raise UnsupportedWheel( + "%s's Wheel-Version (%s) is not compatible with this version " + "of pip" % (name, '.'.join(map(str, version))) + ) + elif version > VERSION_COMPATIBLE: + logger.warning( + 'Installing from a newer Wheel-Version (%s)', + '.'.join(map(str, version)), + ) diff --git a/ubuntu/venv/pip/_internal/vcs/__init__.py b/ubuntu/venv/pip/_internal/vcs/__init__.py new file mode 100644 index 0000000..2a4eb13 --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/__init__.py @@ -0,0 +1,15 @@ +# Expose a limited set of classes and functions so callers outside of +# the vcs package don't need to import deeper than `pip._internal.vcs`. +# (The test directory and imports protected by MYPY_CHECK_RUNNING may +# still need to import from a vcs sub-package.) +# Import all vcs modules to register each VCS in the VcsSupport object. +import pip._internal.vcs.bazaar +import pip._internal.vcs.git +import pip._internal.vcs.mercurial +import pip._internal.vcs.subversion # noqa: F401 +from pip._internal.vcs.versioncontrol import ( # noqa: F401 + RemoteNotFoundError, + is_url, + make_vcs_requirement_url, + vcs, +) diff --git a/ubuntu/venv/pip/_internal/vcs/bazaar.py b/ubuntu/venv/pip/_internal/vcs/bazaar.py new file mode 100644 index 0000000..347c06f --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/bazaar.py @@ -0,0 +1,120 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.utils.misc import display_path, rmtree +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs.versioncontrol import VersionControl, vcs + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +logger = logging.getLogger(__name__) + + +class Bazaar(VersionControl): + name = 'bzr' + dirname = '.bzr' + repo_name = 'branch' + schemes = ( + 'bzr', 'bzr+http', 'bzr+https', 'bzr+ssh', 'bzr+sftp', 'bzr+ftp', + 'bzr+lp', + ) + + def __init__(self, *args, **kwargs): + super(Bazaar, self).__init__(*args, **kwargs) + # This is only needed for python <2.7.5 + # Register lp but do not expose as a scheme to support bzr+lp. + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(['lp']) + + @staticmethod + def get_base_rev_args(rev): + return ['-r', rev] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the Bazaar repository at the url to the destination location + """ + # Remove the location to make sure Bazaar can export it correctly + if os.path.exists(location): + rmtree(location) + + url, rev_options = self.get_url_rev_options(url) + self.run_command( + make_command('export', location, url, rev_options.to_args()), + show_stdout=False, + ) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = ( + make_command('branch', '-q', rev_options.to_args(), url, dest) + ) + self.run_command(cmd_args) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command(make_command('switch', url), cwd=dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command('pull', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it + url, rev, user_pass = super(Bazaar, cls).get_url_rev_and_auth(url) + if url.startswith('ssh://'): + url = 'bzr+' + url + return url, rev, user_pass + + @classmethod + def get_remote_url(cls, location): + urls = cls.run_command(['info'], show_stdout=False, cwd=location) + for line in urls.splitlines(): + line = line.strip() + for x in ('checkout of branch: ', + 'parent branch: '): + if line.startswith(x): + repo = line.split(x)[1] + if cls._is_local_repository(repo): + return path_to_url(repo) + return repo + return None + + @classmethod + def get_revision(cls, location): + revision = cls.run_command( + ['revno'], show_stdout=False, cwd=location, + ) + return revision.splitlines()[-1] + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + +vcs.register(Bazaar) diff --git a/ubuntu/venv/pip/_internal/vcs/git.py b/ubuntu/venv/pip/_internal/vcs/git.py new file mode 100644 index 0000000..d706064 --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/git.py @@ -0,0 +1,395 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os.path +import re + +from pip._vendor.packaging.version import parse as parse_version +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.misc import display_path, hide_url +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs.versioncontrol import ( + RemoteNotFoundError, + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +urlsplit = urllib_parse.urlsplit +urlunsplit = urllib_parse.urlunsplit + + +logger = logging.getLogger(__name__) + + +HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') + + +def looks_like_hash(sha): + return bool(HASH_REGEX.match(sha)) + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ( + 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', + ) + # Prevent the user's environment variables from interfering with pip: + # https://github.com/pypa/pip/issues/1130 + unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') + default_arg_rev = 'HEAD' + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def is_immutable_rev_checkout(self, url, dest): + # type: (str, str) -> bool + _, rev_options = self.get_url_rev_options(hide_url(url)) + if not rev_options.rev: + return False + if not self.is_commit_id_equal(dest, rev_options.rev): + # the current commit is different from rev, + # which means rev was something else than a commit hash + return False + # return False in the rare case rev is both a commit hash + # and a tag or a branch; we don't want to cache in that case + # because that branch/tag could point to something else in the future + is_tag_or_branch = bool( + self.get_revision_sha(dest, rev_options.rev)[0] + ) + return not is_tag_or_branch + + def get_git_version(self): + VERSION_PFX = 'git version ' + version = self.run_command(['version'], show_stdout=False) + if version.startswith(VERSION_PFX): + version = version[len(VERSION_PFX):].split()[0] + else: + version = '' + # get first 3 positions of the git version because + # on windows it is x.y.z.windows.t, and this parses as + # LegacyVersion which always smaller than a Version. + version = '.'.join(version.split('.')[:3]) + return parse_version(version) + + @classmethod + def get_current_branch(cls, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = cls.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() + + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] + + return None + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Git repository at the url to the destination location""" + if not location.endswith('/'): + location = location + '/' + + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + self.run_command( + ['checkout-index', '-a', '-f', '--prefix', location], + show_stdout=False, cwd=temp_dir.path + ) + + @classmethod + def get_revision_sha(cls, dest, rev): + """ + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. + + Args: + dest: the repository directory. + rev: the revision name. + """ + # Pass rev to pre-filter the list. + output = cls.run_command(['show-ref', rev], cwd=dest, + show_stdout=False, on_returncode='ignore') + refs = {} + # NOTE: We do not use splitlines here since that would split on other + # unicode separators, which can be maliciously used to install a + # different revision. + for line in output.strip().split("\n"): + line = line.rstrip("\r") + if not line: + continue + try: + sha, ref = line.split(" ", maxsplit=2) + except ValueError: + # Include the offending line to simplify troubleshooting if + # this error ever occurs. + raise ValueError('unexpected show-ref line: {!r}'.format(line)) + + refs[ref] = sha + + branch_ref = 'refs/remotes/origin/{}'.format(rev) + tag_ref = 'refs/tags/{}'.format(rev) + + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) + + sha = refs.get(tag_ref) + + return (sha, False) + + @classmethod + def resolve_revision(cls, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> RevOptions + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. + + Args: + rev_options: a RevOptions object. + """ + rev = rev_options.arg_rev + # The arg_rev property's implementation for Git ensures that the + # rev return value is always non-None. + assert rev is not None + + sha, is_branch = cls.get_revision_sha(dest, rev) + + if sha is not None: + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options + + # Do not show a warning for the common case of something that has + # the form of a Git commit hash. + if not looks_like_hash(rev): + logger.warning( + "Did not find branch or tag '%s', assuming revision or ref.", + rev, + ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + cls.run_command( + make_command('fetch', '-q', url, rev_options.to_args()), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = cls.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + + return rev_options + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the current commit hash equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + if not name: + # Then avoid an unnecessary subprocess call. + return False + + return cls.get_revision(dest) == name + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) + self.run_command(make_command('clone', '-q', url, dest)) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = make_command( + 'checkout', '-q', rev_options.to_args(), + ) + self.run_command(cmd_args, cwd=dest) + elif self.get_current_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command( + make_command('config', 'remote.origin.url', url), + cwd=dest, + ) + cmd_args = make_command('checkout', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + self.update_submodules(dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + # First fetch changes from the default remote + if self.get_git_version() >= parse_version('1.9.0'): + # fetch tags in addition to everything else + self.run_command(['fetch', '-q', '--tags'], cwd=dest) + else: + self.run_command(['fetch', '-q'], cwd=dest) + # Then reset to wanted revision (maybe even origin/master) + rev_options = self.resolve_revision(dest, url, rev_options) + cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + #: update submodules + self.update_submodules(dest) + + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( + ['config', '--get-regexp', r'remote\..*\.url'], + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + + for remote in remotes: + if remote.startswith('remote.origin.url '): + found_remote = remote + break + url = found_remote.split(' ')[1] + return url.strip() + + @classmethod + def get_revision(cls, location, rev=None): + if rev is None: + rev = 'HEAD' + current_rev = cls.run_command( + ['rev-parse', rev], show_stdout=False, cwd=location, + ) + return current_rev.strip() + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + git_dir = cls.run_command( + ['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() + if not os.path.isabs(git_dir): + git_dir = os.path.join(location, git_dir) + repo_root = os.path.abspath(os.path.join(git_dir, '..')) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes don't + work with a ssh:// scheme (e.g. GitHub). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + # Works around an apparent Git bug + # (see https://article.gmane.org/gmane.comp.version-control.git/146500) + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = ( + initial_slashes + + urllib_request.url2pathname(path) + .replace('\\', '/').lstrip('/') + ) + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+') + 1 + url = scheme[:after_plus] + urlunsplit( + (scheme[after_plus:], netloc, newpath, query, fragment), + ) + + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + url = url.replace('ssh://', '') + else: + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + + return url, rev, user_pass + + @classmethod + def update_submodules(cls, location): + if not os.path.exists(os.path.join(location, '.gitmodules')): + return + cls.run_command( + ['submodule', 'update', '--init', '--recursive', '-q'], + cwd=location, + ) + + @classmethod + def controls_location(cls, location): + if super(Git, cls).controls_location(location): + return True + try: + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + log_failed_cmd=False) + return not r + except BadCommand: + logger.debug("could not determine if %s is under git control " + "because git is not available", location) + return False + + +vcs.register(Git) diff --git a/ubuntu/venv/pip/_internal/vcs/mercurial.py b/ubuntu/venv/pip/_internal/vcs/mercurial.py new file mode 100644 index 0000000..d9b58cf --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/mercurial.py @@ -0,0 +1,155 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._vendor.six.moves import configparser + +from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs.versioncontrol import ( + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import RevOptions + + +logger = logging.getLogger(__name__) + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ( + 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', + ) + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Hg repository at the url to the destination location""" + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + + self.run_command( + ['archive', location], show_stdout=False, cwd=temp_dir.path + ) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) + self.run_command( + make_command('update', '-q', rev_options.to_args()), + cwd=dest, + ) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = configparser.RawConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url.secret) + with open(repo_config, 'w') as config_file: + config.write(config_file) + except (OSError, configparser.NoSectionError) as exc: + logger.warning( + 'Could not switch Mercurial repository to %s: %s', url, exc, + ) + else: + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command(['pull', '-q'], cwd=dest) + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( + ['showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if cls._is_local_repository(url): + url = path_to_url(url) + return url.strip() + + @classmethod + def get_revision(cls, location): + """ + Return the repository-local changeset revision number, as an integer. + """ + current_revision = cls.run_command( + ['parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + @classmethod + def get_requirement_revision(cls, location): + """ + Return the changeset identification hash, as a 40-character + hexadecimal string + """ + current_rev_hash = cls.run_command( + ['parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + repo_root = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() + if not os.path.isabs(repo_root): + repo_root = os.path.abspath(os.path.join(location, repo_root)) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + return True + except (BadCommand, InstallationError): + return False + + +vcs.register(Mercurial) diff --git a/ubuntu/venv/pip/_internal/vcs/subversion.py b/ubuntu/venv/pip/_internal/vcs/subversion.py new file mode 100644 index 0000000..6c76d1a --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/subversion.py @@ -0,0 +1,333 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os +import re + +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import ( + display_path, + is_console_interactive, + rmtree, + split_auth_from_netloc, +) +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs.versioncontrol import VersionControl, vcs + +_svn_xml_url_re = re.compile('url="([^"]+)"') +_svn_rev_re = re.compile(r'committed-rev="(\d+)"') +_svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') +_svn_info_xml_url_re = re.compile(r'(.*)') + + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.subprocess import CommandArgs + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +logger = logging.getLogger(__name__) + + +class Subversion(VersionControl): + name = 'svn' + dirname = '.svn' + repo_name = 'checkout' + schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn') + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + return True + + @staticmethod + def get_base_rev_args(rev): + return ['-r', rev] + + @classmethod + def get_revision(cls, location): + """ + Return the maximum revision for all files under a given location + """ + # Note: taken from setuptools.command.egg_info + revision = 0 + + for base, dirs, files in os.walk(location): + if cls.dirname not in dirs: + dirs[:] = [] + continue # no sense walking uncontrolled subdirs + dirs.remove(cls.dirname) + entries_fn = os.path.join(base, cls.dirname, 'entries') + if not os.path.exists(entries_fn): + # FIXME: should we warn? + continue + + dirurl, localrev = cls._get_svn_url_rev(base) + + if base == location: + base = dirurl + '/' # save the root url + elif not dirurl or not dirurl.startswith(base): + dirs[:] = [] + continue # not part of the same svn tree, skip it + revision = max(revision, localrev) + return revision + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + """ + This override allows the auth information to be passed to svn via the + --username and --password options instead of via the URL. + """ + if scheme == 'ssh': + # The --username and --password options can't be used for + # svn+ssh URLs, so keep the auth information in the URL. + return super(Subversion, cls).get_netloc_and_auth(netloc, scheme) + + return split_auth_from_netloc(netloc) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it + url, rev, user_pass = super(Subversion, cls).get_url_rev_and_auth(url) + if url.startswith('ssh://'): + url = 'svn+' + url + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + extra_args = [] # type: CommandArgs + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] + + return extra_args + + @classmethod + def get_remote_url(cls, location): + # In cases where the source is in a subdirectory, not alongside + # setup.py we have to look up in the location until we find a real + # setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + return cls._get_svn_url_rev(location)[0] + + @classmethod + def _get_svn_url_rev(cls, location): + from pip._internal.exceptions import InstallationError + + entries_path = os.path.join(location, cls.dirname, 'entries') + if os.path.exists(entries_path): + with open(entries_path) as f: + data = f.read() + else: # subversion >= 1.7 does not have the 'entries' file + data = '' + + if (data.startswith('8') or + data.startswith('9') or + data.startswith('10')): + data = list(map(str.splitlines, data.split('\n\x0c\n'))) + del data[0][0] # get rid of the '8' + url = data[0][3] + revs = [int(d[9]) for d in data if len(d) > 9 and d[9]] + [0] + elif data.startswith('= 1.7 + # Note that using get_remote_call_options is not necessary here + # because `svn info` is being run against a local directory. + # We don't need to worry about making sure interactive mode + # is being used to prompt for passwords, because passwords + # are only potentially needed for remote server requests. + xml = cls.run_command( + ['info', '--xml', location], + show_stdout=False, + ) + url = _svn_info_xml_url_re.search(xml).group(1) + revs = [ + int(m.group(1)) for m in _svn_info_xml_rev_re.finditer(xml) + ] + except InstallationError: + url, revs = None, [] + + if revs: + rev = max(revs) + else: + rev = 0 + + return url, rev + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + def __init__(self, use_interactive=None): + # type: (bool) -> None + if use_interactive is None: + use_interactive = is_console_interactive() + self.use_interactive = use_interactive + + # This member is used to cache the fetched version of the current + # ``svn`` client. + # Special value definitions: + # None: Not evaluated yet. + # Empty tuple: Could not parse version. + self._vcs_version = None # type: Optional[Tuple[int, ...]] + + super(Subversion, self).__init__() + + def call_vcs_version(self): + # type: () -> Tuple[int, ...] + """Query the version of the currently installed Subversion client. + + :return: A tuple containing the parts of the version information or + ``()`` if the version returned from ``svn`` could not be parsed. + :raises: BadCommand: If ``svn`` is not installed. + """ + # Example versions: + # svn, version 1.10.3 (r1842928) + # compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0 + # svn, version 1.7.14 (r1542130) + # compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu + version_prefix = 'svn, version ' + version = self.run_command(['--version'], show_stdout=False) + if not version.startswith(version_prefix): + return () + + version = version[len(version_prefix):].split()[0] + version_list = version.split('.') + try: + parsed_version = tuple(map(int, version_list)) + except ValueError: + return () + + return parsed_version + + def get_vcs_version(self): + # type: () -> Tuple[int, ...] + """Return the version of the currently installed Subversion client. + + If the version of the Subversion client has already been queried, + a cached value will be used. + + :return: A tuple containing the parts of the version information or + ``()`` if the version returned from ``svn`` could not be parsed. + :raises: BadCommand: If ``svn`` is not installed. + """ + if self._vcs_version is not None: + # Use cached version, if available. + # If parsing the version failed previously (empty tuple), + # do not attempt to parse it again. + return self._vcs_version + + vcs_version = self.call_vcs_version() + self._vcs_version = vcs_version + return vcs_version + + def get_remote_call_options(self): + # type: () -> CommandArgs + """Return options to be used on calls to Subversion that contact the server. + + These options are applicable for the following ``svn`` subcommands used + in this class. + + - checkout + - export + - switch + - update + + :return: A list of command line arguments to pass to ``svn``. + """ + if not self.use_interactive: + # --non-interactive switch is available since Subversion 0.14.4. + # Subversion < 1.8 runs in interactive mode by default. + return ['--non-interactive'] + + svn_version = self.get_vcs_version() + # By default, Subversion >= 1.8 runs in non-interactive mode if + # stdin is not a TTY. Since that is how pip invokes SVN, in + # call_subprocess(), pip must pass --force-interactive to ensure + # the user can be prompted for a password, if required. + # SVN added the --force-interactive option in SVN 1.8. Since + # e.g. RHEL/CentOS 7, which is supported until 2024, ships with + # SVN 1.7, pip should continue to support SVN 1.7. Therefore, pip + # can't safely add the option if the SVN version is < 1.8 (or unknown). + if svn_version >= (1, 8): + return ['--force-interactive'] + + return [] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the svn repository at the url to the destination location""" + url, rev_options = self.get_url_rev_options(url) + + logger.info('Exporting svn repository %s to %s', url, location) + with indent_log(): + if os.path.exists(location): + # Subversion doesn't like to check out over an existing + # directory --force fixes this, but was only added in svn 1.5 + rmtree(location) + cmd_args = make_command( + 'export', self.get_remote_call_options(), + rev_options.to_args(), url, location, + ) + self.run_command(cmd_args, show_stdout=False) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = make_command( + 'checkout', '-q', self.get_remote_call_options(), + rev_options.to_args(), url, dest, + ) + self.run_command(cmd_args) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command( + 'switch', self.get_remote_call_options(), rev_options.to_args(), + url, dest, + ) + self.run_command(cmd_args) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command( + 'update', self.get_remote_call_options(), rev_options.to_args(), + dest, + ) + self.run_command(cmd_args) + + +vcs.register(Subversion) diff --git a/ubuntu/venv/pip/_internal/vcs/versioncontrol.py b/ubuntu/venv/pip/_internal/vcs/versioncontrol.py new file mode 100644 index 0000000..7cfd568 --- /dev/null +++ b/ubuntu/venv/pip/_internal/vcs/versioncontrol.py @@ -0,0 +1,700 @@ +"""Handles all VCS (version control) support""" + +from __future__ import absolute_import + +import errno +import logging +import os +import shutil +import sys + +from pip._vendor import pkg_resources +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + hide_value, + rmtree, +) +from pip._internal.utils.subprocess import call_subprocess, make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, Iterator, List, Mapping, Optional, Text, Tuple, + Type, Union + ) + from pip._internal.utils.ui import SpinnerInterface + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs + + AuthInfo = Tuple[Optional[str], Optional[str]] + + +__all__ = ['vcs'] + + +logger = logging.getLogger(__name__) + + +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): + # type: (str, str, str, Optional[str]) -> str + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. + """ + egg_project_name = pkg_resources.to_filename(project_name) + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def find_path_to_setup_from_repo_root(location, repo_root): + # type: (str, str) -> Optional[str] + """ + Find the path to `setup.py` by searching up the filesystem from `location`. + Return the path to `setup.py` relative to `repo_root`. + Return None if `setup.py` is in `repo_root` or cannot be found. + """ + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + if samefile(repo_root, location): + return None + + return os.path.relpath(location, repo_root) + + +class RemoteNotFoundError(Exception): + pass + + +class RevOptions(object): + + """ + Encapsulates a VCS-specific revision to install, along with any VCS + install options. + + Instances of this class should be treated as if immutable. + """ + + def __init__( + self, + vc_class, # type: Type[VersionControl] + rev=None, # type: Optional[str] + extra_args=None, # type: Optional[CommandArgs] + ): + # type: (...) -> None + """ + Args: + vc_class: a VersionControl subclass. + rev: the name of the revision to install. + extra_args: a list of extra options. + """ + if extra_args is None: + extra_args = [] + + self.extra_args = extra_args + self.rev = rev + self.vc_class = vc_class + self.branch_name = None # type: Optional[str] + + def __repr__(self): + # type: () -> str + return ''.format(self.vc_class.name, self.rev) + + @property + def arg_rev(self): + # type: () -> Optional[str] + if self.rev is None: + return self.vc_class.default_arg_rev + + return self.rev + + def to_args(self): + # type: () -> CommandArgs + """ + Return the VCS-specific command arguments. + """ + args = [] # type: CommandArgs + rev = self.arg_rev + if rev is not None: + args += self.vc_class.get_base_rev_args(rev) + args += self.extra_args + + return args + + def to_display(self): + # type: () -> str + if not self.rev: + return '' + + return ' (to revision {})'.format(self.rev) + + def make_new(self, rev): + # type: (str) -> RevOptions + """ + Make a copy of the current instance, but with a new rev. + + Args: + rev: the name of the revision for the new object. + """ + return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) + + +class VcsSupport(object): + _registry = {} # type: Dict[str, VersionControl] + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] + + def __init__(self): + # type: () -> None + # Register more schemes with urlparse for various version control + # systems + urllib_parse.uses_netloc.extend(self.schemes) + # Python >= 2.7.4, 3.3 doesn't have uses_fragment + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + # type: () -> Iterator[str] + return self._registry.__iter__() + + @property + def backends(self): + # type: () -> List[VersionControl] + return list(self._registry.values()) + + @property + def dirnames(self): + # type: () -> List[str] + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + # type: () -> List[str] + schemes = [] # type: List[str] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + # type: (Type[VersionControl]) -> None + if not hasattr(cls, 'name'): + logger.warning('Cannot register VCS %s', cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls() + logger.debug('Registered VCS backend: %s', cls.name) + + def unregister(self, name): + # type: (str) -> None + if name in self._registry: + del self._registry[name] + + def get_backend_for_dir(self, location): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object if a repository of that type is found + at the given directory. + """ + for vcs_backend in self._registry.values(): + if vcs_backend.controls_location(location): + logger.debug('Determine that %s uses VCS: %s', + location, vcs_backend.name) + return vcs_backend + return None + + def get_backend_for_scheme(self, scheme): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + for vcs_backend in self._registry.values(): + if scheme in vcs_backend.schemes: + return vcs_backend + return None + + def get_backend(self, name): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + name = name.lower() + return self._registry.get(name) + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + repo_name = '' + # List of supported schemes for this Version Control + schemes = () # type: Tuple[str, ...] + # Iterable of environment variable names to pass to call_subprocess(). + unset_environ = () # type: Tuple[str, ...] + default_arg_rev = None # type: Optional[str] + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + # type: (str) -> bool + """ + Return whether the vcs prefix (e.g. "git+") should be added to a + repository's remote url when used in a requirement. + """ + return not remote_url.lower().startswith('{}:'.format(cls.name)) + + @classmethod + def get_subdirectory(cls, location): + # type: (str) -> Optional[str] + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + return None + + @classmethod + def get_requirement_revision(cls, repo_dir): + # type: (str) -> str + """ + Return the revision string that should be used in a requirement. + """ + return cls.get_revision(repo_dir) + + @classmethod + def get_src_requirement(cls, repo_dir, project_name): + # type: (str, str) -> Optional[str] + """ + Return the requirement string to use to redownload the files + currently at the given repository directory. + + Args: + project_name: the (unescaped) project name. + + The return value has a form similar to the following: + + {repository_url}@{revision}#egg={project_name} + """ + repo_url = cls.get_remote_url(repo_dir) + if repo_url is None: + return None + + if cls.should_add_vcs_url_prefix(repo_url): + repo_url = '{}+{}'.format(cls.name, repo_url) + + revision = cls.get_requirement_revision(repo_dir) + subdir = cls.get_subdirectory(repo_dir) + req = make_vcs_requirement_url(repo_url, revision, project_name, + subdir=subdir) + + return req + + @staticmethod + def get_base_rev_args(rev): + # type: (str) -> List[str] + """ + Return the base revision arguments for a vcs command. + + Args: + rev: the name of a revision to install. Cannot be None. + """ + raise NotImplementedError + + def is_immutable_rev_checkout(self, url, dest): + # type: (str, str) -> bool + """ + Return true if the commit hash checked out at dest matches + the revision in url. + + Always return False, if the VCS does not support immutable commit + hashes. + + This method does not check if there are local uncommitted changes + in dest after checkout, as pip currently has no use case for that. + """ + return False + + @classmethod + def make_rev_options(cls, rev=None, extra_args=None): + # type: (Optional[str], Optional[CommandArgs]) -> RevOptions + """ + Return a RevOptions object. + + Args: + rev: the name of a revision to install. + extra_args: a list of extra options. + """ + return RevOptions(cls, rev, extra_args=extra_args) + + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool + """ + posix absolute paths start with os.path.sep, + win32 ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or bool(drive) + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the repository at the url to the destination location + i.e. only download the files, without vcs informations + + :param url: the repository URL starting with a vcs prefix. + """ + raise NotImplementedError + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + # type: (str, str) -> Tuple[str, Tuple[Optional[str], Optional[str]]] + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). + """ + scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + """ + Return the RevOptions "extra arguments" to use in obtain(). + """ + return [] + + def get_url_rev_options(self, url): + # type: (HiddenText) -> Tuple[HiddenText, RevOptions] + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) + username, secret_password = user_pass + password = None # type: Optional[HiddenText] + if secret_password is not None: + password = hide_value(secret_password) + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return hide_url(secret_url), rev_options + + @staticmethod + def normalize_url(url): + # type: (str) -> str + """ + Normalize a URL for comparison by unquoting it and removing any + trailing slash. + """ + return urllib_parse.unquote(url).rstrip('/') + + @classmethod + def compare_urls(cls, url1, url2): + # type: (str, str) -> bool + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (cls.normalize_url(url1) == cls.normalize_url(url2)) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Switch the repo at ``dest`` to point to ``URL``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Update an already-existing repo to the given ``rev_options``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + @classmethod + def is_commit_id_equal(cls, dest, name): + # type: (str, Optional[str]) -> bool + """ + Return whether the id of the current commit equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + raise NotImplementedError + + def obtain(self, dest, url): + # type: (str, HiddenText) -> None + """ + Install or update in editable mode the package represented by this + VersionControl object. + + :param dest: the repository directory in which to install or update. + :param url: the repository URL starting with a vcs prefix. + """ + url, rev_options = self.get_url_rev_options(url) + + if not os.path.exists(dest): + self.fetch_new(dest, url, rev_options) + return + + rev_display = rev_options.to_display() + if self.is_repository_directory(dest): + existing_url = self.get_remote_url(dest) + if self.compare_urls(existing_url, url.secret): + logger.debug( + '%s in %s exists, and has correct URL (%s)', + self.repo_name.title(), + display_path(dest), + url, + ) + if not self.is_commit_id_equal(dest, rev_options.rev): + logger.info( + 'Updating %s %s%s', + display_path(dest), + self.repo_name, + rev_display, + ) + self.update(dest, url, rev_options) + else: + logger.info('Skipping because already up-to-date.') + return + + logger.warning( + '%s %s in %s exists with URL %s', + self.name, + self.repo_name, + display_path(dest), + existing_url, + ) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', + ('s', 'i', 'w', 'b')) + else: + logger.warning( + 'Directory %s already exists, and is not a %s %s.', + dest, + self.name, + self.repo_name, + ) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) + + logger.warning( + 'The plan is to install the %s repository %s', + self.name, + url, + ) + response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) + + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". + if response == 's': + logger.info( + 'Switching %s %s to %s%s', + self.repo_name, + display_path(dest), + url, + rev_display, + ) + self.switch(dest, url, rev_options) + + def unpack(self, location, url): + # type: (str, HiddenText) -> None + """ + Clean up current location and download the url repository + (and vcs infos) into location + + :param url: the repository URL starting with a vcs prefix. + """ + if os.path.exists(location): + rmtree(location) + self.obtain(location, url=url) + + @classmethod + def get_remote_url(cls, location): + # type: (str) -> str + """ + Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + raise NotImplementedError + + @classmethod + def get_revision(cls, location): + # type: (str) -> str + """ + Return the current commit id of the files at the given location. + """ + raise NotImplementedError + + @classmethod + def run_command( + cls, + cmd, # type: Union[List[str], CommandArgs] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True # type: bool + ): + # type: (...) -> Text + """ + Run a VCS subcommand + This is simply a wrapper around call_subprocess that adds the VCS + command name, and checks that the VCS is available + """ + cmd = make_command(cls.name, *cmd) + try: + return call_subprocess(cmd, show_stdout, cwd, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, + spinner=spinner, + log_failed_cmd=log_failed_cmd) + except OSError as e: + # errno.ENOENT = no such file or directory + # In other words, the VCS executable isn't available + if e.errno == errno.ENOENT: + raise BadCommand( + 'Cannot find command %r - do you have ' + '%r installed and in your ' + 'PATH?' % (cls.name, cls.name)) + else: + raise # re-raise exception if a different error occurred + + @classmethod + def is_repository_directory(cls, path): + # type: (str) -> bool + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + + @classmethod + def controls_location(cls, location): + # type: (str) -> bool + """ + Check if a location is controlled by the vcs. + It is meant to be overridden to implement smarter detection + mechanisms for specific vcs. + + This can do more than is_repository_directory() alone. For example, + the Git override checks that Git is actually available. + """ + return cls.is_repository_directory(location) diff --git a/ubuntu/venv/pip/_internal/wheel_builder.py b/ubuntu/venv/pip/_internal/wheel_builder.py new file mode 100644 index 0000000..7c7820d --- /dev/null +++ b/ubuntu/venv/pip/_internal/wheel_builder.py @@ -0,0 +1,305 @@ +"""Orchestrator for building wheels from InstallRequirements. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +import logging +import os.path +import re +import shutil + +from pip._internal.models.link import Link +from pip._internal.operations.build.wheel import build_wheel_pep517 +from pip._internal.operations.build.wheel_legacy import build_wheel_legacy +from pip._internal.utils.logging import indent_log +from pip._internal.utils.misc import ensure_dir, hash_file, is_wheel_installed +from pip._internal.utils.setuptools_build import make_setuptools_clean_args +from pip._internal.utils.subprocess import call_subprocess +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Iterable, List, Optional, Pattern, Tuple, + ) + + from pip._internal.cache import WheelCache + from pip._internal.req.req_install import InstallRequirement + + BinaryAllowedPredicate = Callable[[InstallRequirement], bool] + BuildResult = Tuple[List[InstallRequirement], List[InstallRequirement]] + +logger = logging.getLogger(__name__) + + +def _contains_egg_info( + s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + # type: (str, Pattern[str]) -> bool + """Determine whether the string looks like an egg_info. + + :param s: The string to parse. E.g. foo-2.1 + """ + return bool(_egg_info_re.search(s)) + + +def _should_build( + req, # type: InstallRequirement + need_wheel, # type: bool + check_binary_allowed, # type: BinaryAllowedPredicate +): + # type: (...) -> bool + """Return whether an InstallRequirement should be built into a wheel.""" + if req.constraint: + # never build requirements that are merely constraints + return False + if req.is_wheel: + if need_wheel: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return False + + if need_wheel: + # i.e. pip wheel, not pip install + return True + + # From this point, this concerns the pip install command only + # (need_wheel=False). + + if not req.use_pep517 and not is_wheel_installed(): + # we don't build legacy requirements if wheel is not installed + return False + + if req.editable or not req.source_dir: + return False + + if not check_binary_allowed(req): + logger.info( + "Skipping wheel build for %s, due to binaries " + "being disabled for it.", req.name, + ) + return False + + return True + + +def should_build_for_wheel_command( + req, # type: InstallRequirement +): + # type: (...) -> bool + return _should_build( + req, need_wheel=True, check_binary_allowed=_always_true + ) + + +def should_build_for_install_command( + req, # type: InstallRequirement + check_binary_allowed, # type: BinaryAllowedPredicate +): + # type: (...) -> bool + return _should_build( + req, need_wheel=False, check_binary_allowed=check_binary_allowed + ) + + +def _should_cache( + req, # type: InstallRequirement +): + # type: (...) -> Optional[bool] + """ + Return whether a built InstallRequirement can be stored in the persistent + wheel cache, assuming the wheel cache is available, and _should_build() + has determined a wheel needs to be built. + """ + if not should_build_for_install_command( + req, check_binary_allowed=_always_true + ): + # never cache if pip install would not have built + # (editable mode, etc) + return False + + if req.link and req.link.is_vcs: + # VCS checkout. Do not cache + # unless it points to an immutable commit hash. + assert not req.editable + assert req.source_dir + vcs_backend = vcs.get_backend_for_scheme(req.link.scheme) + assert vcs_backend + if vcs_backend.is_immutable_rev_checkout(req.link.url, req.source_dir): + return True + return False + + base, ext = req.link.splitext() + if _contains_egg_info(base): + return True + + # Otherwise, do not cache. + return False + + +def _get_cache_dir( + req, # type: InstallRequirement + wheel_cache, # type: WheelCache +): + # type: (...) -> str + """Return the persistent or temporary cache directory where the built + wheel need to be stored. + """ + cache_available = bool(wheel_cache.cache_dir) + if cache_available and _should_cache(req): + cache_dir = wheel_cache.get_path_for_link(req.link) + else: + cache_dir = wheel_cache.get_ephem_path_for_link(req.link) + return cache_dir + + +def _always_true(_): + # type: (Any) -> bool + return True + + +def _build_one( + req, # type: InstallRequirement + output_dir, # type: str + build_options, # type: List[str] + global_options, # type: List[str] +): + # type: (...) -> Optional[str] + """Build one wheel. + + :return: The filename of the built wheel, or None if the build failed. + """ + try: + ensure_dir(output_dir) + except OSError as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + return None + + # Install build deps into temporary directory (PEP 518) + with req.build_env: + return _build_one_inside_env( + req, output_dir, build_options, global_options + ) + + +def _build_one_inside_env( + req, # type: InstallRequirement + output_dir, # type: str + build_options, # type: List[str] + global_options, # type: List[str] +): + # type: (...) -> Optional[str] + with TempDirectory(kind="wheel") as temp_dir: + if req.use_pep517: + wheel_path = build_wheel_pep517( + name=req.name, + backend=req.pep517_backend, + metadata_directory=req.metadata_directory, + build_options=build_options, + tempd=temp_dir.path, + ) + else: + wheel_path = build_wheel_legacy( + name=req.name, + setup_py_path=req.setup_py_path, + source_dir=req.unpacked_source_directory, + global_options=global_options, + build_options=build_options, + tempd=temp_dir.path, + ) + + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) + try: + wheel_hash, length = hash_file(wheel_path) + shutil.move(wheel_path, dest_path) + logger.info('Created wheel for %s: ' + 'filename=%s size=%d sha256=%s', + req.name, wheel_name, length, + wheel_hash.hexdigest()) + logger.info('Stored in directory: %s', output_dir) + return dest_path + except Exception as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + # Ignore return, we can't do anything else useful. + if not req.use_pep517: + _clean_one_legacy(req, global_options) + return None + + +def _clean_one_legacy(req, global_options): + # type: (InstallRequirement, List[str]) -> bool + clean_args = make_setuptools_clean_args( + req.setup_py_path, + global_options=global_options, + ) + + logger.info('Running setup.py clean for %s', req.name) + try: + call_subprocess(clean_args, cwd=req.source_dir) + return True + except Exception: + logger.error('Failed cleaning build dir for %s', req.name) + return False + + +def build( + requirements, # type: Iterable[InstallRequirement] + wheel_cache, # type: WheelCache + build_options, # type: List[str] + global_options, # type: List[str] +): + # type: (...) -> BuildResult + """Build wheels. + + :return: The list of InstallRequirement that succeeded to build and + the list of InstallRequirement that failed to build. + """ + if not requirements: + return [], [] + + # Build the wheels. + logger.info( + 'Building wheels for collected packages: %s', + ', '.join(req.name for req in requirements), + ) + + with indent_log(): + build_successes, build_failures = [], [] + for req in requirements: + cache_dir = _get_cache_dir(req, wheel_cache) + wheel_file = _build_one( + req, cache_dir, build_options, global_options + ) + if wheel_file: + # Update the link for this. + req.link = Link(path_to_url(wheel_file)) + req.local_file_path = req.link.file_path + assert req.link.is_wheel + build_successes.append(req) + else: + build_failures.append(req) + + # notify success/failure + if build_successes: + logger.info( + 'Successfully built %s', + ' '.join([req.name for req in build_successes]), + ) + if build_failures: + logger.info( + 'Failed to build %s', + ' '.join([req.name for req in build_failures]), + ) + # Return a list of requirements that failed to build + return build_successes, build_failures diff --git a/ubuntu/venv/pip/_vendor/__init__.py b/ubuntu/venv/pip/_vendor/__init__.py new file mode 100644 index 0000000..e02eaef --- /dev/null +++ b/ubuntu/venv/pip/_vendor/__init__.py @@ -0,0 +1,119 @@ +""" +pip._vendor is for vendoring dependencies of pip to prevent needing pip to +depend on something external. + +Files inside of pip._vendor should be considered immutable and should only be +updated to versions from upstream. +""" +from __future__ import absolute_import + +import glob +import os.path +import sys + +# Downstream redistributors which have debundled our dependencies should also +# patch this value to be true. This will trigger the additional patching +# to cause things like "six" to be available as pip. +DEBUNDLED = True + +# By default, look in this directory for a bunch of .whl files which we will +# add to the beginning of sys.path before attempting to import anything. This +# is done to support downstream re-distributors like Debian and Fedora who +# wish to create their own Wheels for our dependencies to aid in debundling. +prefix = getattr(sys, "base_prefix", sys.prefix) +if prefix.startswith('/usr/lib/pypy'): + prefix = '/usr' +WHEEL_DIR = os.path.abspath(os.path.join(prefix, 'share', 'python-wheels')) + + +# Define a small helper function to alias our vendored modules to the real ones +# if the vendored ones do not exist. This idea of this was taken from +# https://github.com/kennethreitz/requests/pull/2567. +def vendored(modulename): + vendored_name = "{0}.{1}".format(__name__, modulename) + + try: + __import__(modulename, globals(), locals(), level=0) + except ImportError: + # We can just silently allow import failures to pass here. If we + # got to this point it means that ``import pip._vendor.whatever`` + # failed and so did ``import whatever``. Since we're importing this + # upfront in an attempt to alias imports, not erroring here will + # just mean we get a regular import error whenever pip *actually* + # tries to import one of these modules to use it, which actually + # gives us a better error message than we would have otherwise + # gotten. + pass + else: + sys.modules[vendored_name] = sys.modules[modulename] + base, head = vendored_name.rsplit(".", 1) + setattr(sys.modules[base], head, sys.modules[modulename]) + + +# If we're operating in a debundled setup, then we want to go ahead and trigger +# the aliasing of our vendored libraries as well as looking for wheels to add +# to our sys.path. This will cause all of this code to be a no-op typically +# however downstream redistributors can enable it in a consistent way across +# all platforms. +if DEBUNDLED: + # Actually look inside of WHEEL_DIR to find .whl files and add them to the + # front of our sys.path. + sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path + + # Actually alias all of our vendored dependencies. + vendored("appdirs") + vendored("cachecontrol") + vendored("colorama") + vendored("contextlib2") + vendored("distlib") + vendored("distro") + vendored("html5lib") + vendored("six") + vendored("six.moves") + vendored("six.moves.urllib") + vendored("six.moves.urllib.parse") + vendored("packaging") + vendored("packaging.version") + vendored("packaging.specifiers") + vendored("pep517") + vendored("pkg_resources") + vendored("progress") + vendored("retrying") + vendored("requests") + vendored("requests.exceptions") + vendored("requests.packages") + vendored("requests.packages.urllib3") + vendored("requests.packages.urllib3._collections") + vendored("requests.packages.urllib3.connection") + vendored("requests.packages.urllib3.connectionpool") + vendored("requests.packages.urllib3.contrib") + vendored("requests.packages.urllib3.contrib.ntlmpool") + vendored("requests.packages.urllib3.contrib.pyopenssl") + vendored("requests.packages.urllib3.exceptions") + vendored("requests.packages.urllib3.fields") + vendored("requests.packages.urllib3.filepost") + vendored("requests.packages.urllib3.packages") + try: + vendored("requests.packages.urllib3.packages.ordered_dict") + vendored("requests.packages.urllib3.packages.six") + except ImportError: + # Debian already unbundles these from requests. + pass + vendored("requests.packages.urllib3.packages.ssl_match_hostname") + vendored("requests.packages.urllib3.packages.ssl_match_hostname." + "_implementation") + vendored("requests.packages.urllib3.poolmanager") + vendored("requests.packages.urllib3.request") + vendored("requests.packages.urllib3.response") + vendored("requests.packages.urllib3.util") + vendored("requests.packages.urllib3.util.connection") + vendored("requests.packages.urllib3.util.request") + vendored("requests.packages.urllib3.util.response") + vendored("requests.packages.urllib3.util.retry") + vendored("requests.packages.urllib3.util.ssl_") + vendored("requests.packages.urllib3.util.timeout") + vendored("requests.packages.urllib3.util.url") + vendored("toml") + vendored("toml.encoder") + vendored("toml.decoder") + vendored("urllib3") diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/AUTHORS.txt b/ubuntu/venv/pkg_resources-0.0.0.dist-info/AUTHORS.txt new file mode 100644 index 0000000..72c87d7 --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/AUTHORS.txt @@ -0,0 +1,562 @@ +A_Rog +Aakanksha Agrawal <11389424+rasponic@users.noreply.github.com> +Abhinav Sagar <40603139+abhinavsagar@users.noreply.github.com> +ABHYUDAY PRATAP SINGH +abs51295 +AceGentile +Adam Chainz +Adam Tse +Adam Tse +Adam Wentz +admin +Adrien Morison +ahayrapetyan +Ahilya +AinsworthK +Akash Srivastava +Alan Yee +Albert Tugushev +Albert-Guan +albertg +Aleks Bunin +Alethea Flowers +Alex Gaynor +Alex Grönholm +Alex Loosley +Alex Morega +Alex Stachowiak +Alexander Shtyrov +Alexandre Conrad +Alexey Popravka +Alexey Popravka +Alli +Ami Fischman +Ananya Maiti +Anatoly Techtonik +Anders Kaseorg +Andreas Lutro +Andrei Geacar +Andrew Gaul +Andrey Bulgakov +Andrés Delfino <34587441+andresdelfino@users.noreply.github.com> +Andrés Delfino +Andy Freeland +Andy Freeland +Andy Kluger +Ani Hayrapetyan +Aniruddha Basak +Anish Tambe +Anrs Hu +Anthony Sottile +Antoine Musso +Anton Ovchinnikov +Anton Patrushev +Antonio Alvarado Hernandez +Antony Lee +Antti Kaihola +Anubhav Patel +Anuj Godase +AQNOUCH Mohammed +AraHaan +Arindam Choudhury +Armin Ronacher +Artem +Ashley Manton +Ashwin Ramaswami +atse +Atsushi Odagiri +Avner Cohen +Baptiste Mispelon +Barney Gale +barneygale +Bartek Ogryczak +Bastian Venthur +Ben Darnell +Ben Hoyt +Ben Rosser +Bence Nagy +Benjamin Peterson +Benjamin VanEvery +Benoit Pierre +Berker Peksag +Bernardo B. Marques +Bernhard M. Wiedemann +Bertil Hatt +Bogdan Opanchuk +BorisZZZ +Brad Erickson +Bradley Ayers +Brandon L. Reiss +Brandt Bucher +Brett Randall +Brian Cristante <33549821+brcrista@users.noreply.github.com> +Brian Cristante +Brian Rosner +BrownTruck +Bruno Oliveira +Bruno Renié +Bstrdsmkr +Buck Golemon +burrows +Bussonnier Matthias +c22 +Caleb Martinez +Calvin Smith +Carl Meyer +Carlos Liam +Carol Willing +Carter Thayer +Cass +Chandrasekhar Atina +Chih-Hsuan Yen +Chih-Hsuan Yen +Chris Brinker +Chris Hunt +Chris Jerdonek +Chris McDonough +Chris Wolfe +Christian Heimes +Christian Oudard +Christopher Hunt +Christopher Snyder +Clark Boylan +Clay McClure +Cody +Cody Soyland +Colin Watson +Connor Osborn +Cooper Lees +Cooper Ry Lees +Cory Benfield +Cory Wright +Craig Kerstiens +Cristian Sorinel +Curtis Doty +cytolentino +Damian Quiroga +Dan Black +Dan Savilonis +Dan Sully +daniel +Daniel Collins +Daniel Hahler +Daniel Holth +Daniel Jost +Daniel Shaulov +Daniele Esposti +Daniele Procida +Danny Hermes +Dav Clark +Dave Abrahams +Dave Jones +David Aguilar +David Black +David Bordeynik +David Bordeynik +David Caro +David Evans +David Linke +David Pursehouse +David Tucker +David Wales +Davidovich +derwolfe +Desetude +Diego Caraballo +DiegoCaraballo +Dmitry Gladkov +Domen Kožar +Donald Stufft +Dongweiming +Douglas Thor +DrFeathers +Dustin Ingram +Dwayne Bailey +Ed Morley <501702+edmorley@users.noreply.github.com> +Ed Morley +Eitan Adler +ekristina +elainechan +Eli Schwartz +Eli Schwartz +Emil Burzo +Emil Styrke +Endoh Takanao +enoch +Erdinc Mutlu +Eric Gillingham +Eric Hanchrow +Eric Hopper +Erik M. Bray +Erik Rose +Ernest W Durbin III +Ernest W. Durbin III +Erwin Janssen +Eugene Vereshchagin +everdimension +Felix Yan +fiber-space +Filip Kokosiński +Florian Briand +Florian Rathgeber +Francesco +Francesco Montesano +Frost Ming +Gabriel Curio +Gabriel de Perthuis +Garry Polley +gdanielson +Geoffrey Lehée +Geoffrey Sneddon +George Song +Georgi Valkov +Giftlin Rajaiah +gizmoguy1 +gkdoc <40815324+gkdoc@users.noreply.github.com> +Gopinath M <31352222+mgopi1990@users.noreply.github.com> +GOTO Hayato <3532528+gh640@users.noreply.github.com> +gpiks +Guilherme Espada +Guy Rozendorn +gzpan123 +Hanjun Kim +Hari Charan +Harsh Vardhan +Herbert Pfennig +Hsiaoming Yang +Hugo +Hugo Lopes Tavares +Hugo van Kemenade +hugovk +Hynek Schlawack +Ian Bicking +Ian Cordasco +Ian Lee +Ian Stapleton Cordasco +Ian Wienand +Ian Wienand +Igor Kuzmitshov +Igor Sobreira +Ilya Baryshev +INADA Naoki +Ionel Cristian Mărieș +Ionel Maries Cristian +Ivan Pozdeev +Jacob Kim +jakirkham +Jakub Stasiak +Jakub Vysoky +Jakub Wilk +James Cleveland +James Cleveland +James Firth +James Polley +Jan Pokorný +Jannis Leidel +jarondl +Jason R. Coombs +Jay Graves +Jean-Christophe Fillion-Robin +Jeff Barber +Jeff Dairiki +Jelmer Vernooij +jenix21 +Jeremy Stanley +Jeremy Zafran +Jiashuo Li +Jim Garrison +Jivan Amara +John Paton +John-Scott Atlakson +johnthagen +johnthagen +Jon Banafato +Jon Dufresne +Jon Parise +Jonas Nockert +Jonathan Herbert +Joost Molenaar +Jorge Niedbalski +Joseph Long +Josh Bronson +Josh Hansen +Josh Schneier +Juanjo Bazán +Julian Berman +Julian Gethmann +Julien Demoor +jwg4 +Jyrki Pulliainen +Kai Chen +Kamal Bin Mustafa +kaustav haldar +keanemind +Keith Maxwell +Kelsey Hightower +Kenneth Belitzky +Kenneth Reitz +Kenneth Reitz +Kevin Burke +Kevin Carter +Kevin Frommelt +Kevin R Patterson +Kexuan Sun +Kit Randel +kpinc +Krishna Oza +Kumar McMillan +Kyle Persohn +lakshmanaram +Laszlo Kiss-Kollar +Laurent Bristiel +Laurie Opperman +Leon Sasson +Lev Givon +Lincoln de Sousa +Lipis +Loren Carvalho +Lucas Cimon +Ludovic Gasc +Luke Macken +Luo Jiebin +luojiebin +luz.paz +László Kiss Kollár +László Kiss Kollár +Marc Abramowitz +Marc Tamlyn +Marcus Smith +Mariatta +Mark Kohler +Mark Williams +Mark Williams +Markus Hametner +Masaki +Masklinn +Matej Stuchlik +Mathew Jennings +Mathieu Bridon +Matt Good +Matt Maker +Matt Robenolt +matthew +Matthew Einhorn +Matthew Gilliard +Matthew Iversen +Matthew Trumbell +Matthew Willson +Matthias Bussonnier +mattip +Maxim Kurnikov +Maxime Rouyrre +mayeut +mbaluna <44498973+mbaluna@users.noreply.github.com> +mdebi <17590103+mdebi@users.noreply.github.com> +memoselyk +Michael +Michael Aquilina +Michael E. Karpeles +Michael Klich +Michael Williamson +michaelpacer +Mickaël Schoentgen +Miguel Araujo Perez +Mihir Singh +Mike +Mike Hendricks +Min RK +MinRK +Miro Hrončok +Monica Baluna +montefra +Monty Taylor +Nate Coraor +Nathaniel J. Smith +Nehal J Wani +Neil Botelho +Nick Coghlan +Nick Stenning +Nick Timkovich +Nicolas Bock +Nikhil Benesch +Nitesh Sharma +Nowell Strite +NtaleGrey +nvdv +Ofekmeister +ofrinevo +Oliver Jeeves +Oliver Tonnhofer +Olivier Girardot +Olivier Grisel +Ollie Rutherfurd +OMOTO Kenji +Omry Yadan +Oren Held +Oscar Benjamin +Oz N Tiram +Pachwenko <32424503+Pachwenko@users.noreply.github.com> +Patrick Dubroy +Patrick Jenkins +Patrick Lawson +patricktokeeffe +Patrik Kopkan +Paul Kehrer +Paul Moore +Paul Nasrat +Paul Oswald +Paul van der Linden +Paulus Schoutsen +Pavithra Eswaramoorthy <33131404+QueenCoffee@users.noreply.github.com> +Pawel Jasinski +Pekka Klärck +Peter Lisák +Peter Waller +petr-tik +Phaneendra Chiruvella +Phil Freo +Phil Pennock +Phil Whelan +Philip Jägenstedt +Philip Molloy +Philippe Ombredanne +Pi Delport +Pierre-Yves Rofes +pip +Prabakaran Kumaresshan +Prabhjyotsing Surjit Singh Sodhi +Prabhu Marappan +Pradyun Gedam +Pratik Mallya +Preet Thakkar +Preston Holmes +Przemek Wrzos +Pulkit Goyal <7895pulkit@gmail.com> +Qiangning Hong +Quentin Pradet +R. David Murray +Rafael Caricio +Ralf Schmitt +Razzi Abuissa +rdb +Remi Rampin +Remi Rampin +Rene Dudfield +Riccardo Magliocchetti +Richard Jones +RobberPhex +Robert Collins +Robert McGibbon +Robert T. McGibbon +robin elisha robinson +Roey Berman +Rohan Jain +Rohan Jain +Rohan Jain +Roman Bogorodskiy +Romuald Brunet +Ronny Pfannschmidt +Rory McCann +Ross Brattain +Roy Wellington Ⅳ +Roy Wellington Ⅳ +Ryan Wooden +ryneeverett +Sachi King +Salvatore Rinchiera +Savio Jomton +schlamar +Scott Kitterman +Sean +seanj +Sebastian Jordan +Sebastian Schaetz +Segev Finer +SeongSoo Cho +Sergey Vasilyev +Seth Woodworth +Shlomi Fish +Shovan Maity +Simeon Visser +Simon Cross +Simon Pichugin +sinoroc +Sorin Sbarnea +Stavros Korokithakis +Stefan Scherfke +Stephan Erb +stepshal +Steve (Gadget) Barnes +Steve Barnes +Steve Dower +Steve Kowalik +Steven Myint +stonebig +Stéphane Bidoul (ACSONE) +Stéphane Bidoul +Stéphane Klein +Sumana Harihareswara +Sviatoslav Sydorenko +Sviatoslav Sydorenko +Swat009 +Takayuki SHIMIZUKAWA +tbeswick +Thijs Triemstra +Thomas Fenzl +Thomas Grainger +Thomas Guettler +Thomas Johansson +Thomas Kluyver +Thomas Smith +Tim D. Smith +Tim Gates +Tim Harder +Tim Heap +tim smith +tinruufu +Tom Forbes +Tom Freudenheim +Tom V +Tomas Orsava +Tomer Chachamu +Tony Beswick +Tony Zhaocheng Tan +TonyBeswick +toonarmycaptain +Toshio Kuratomi +Travis Swicegood +Tzu-ping Chung +Valentin Haenel +Victor Stinner +victorvpaulo +Viktor Szépe +Ville Skyttä +Vinay Sajip +Vincent Philippon +Vinicyus Macedo <7549205+vinicyusmacedo@users.noreply.github.com> +Vitaly Babiy +Vladimir Rutsky +W. Trevor King +Wil Tan +Wilfred Hughes +William ML Leslie +William T Olson +Wilson Mo +wim glenn +Wolfgang Maier +Xavier Fernandez +Xavier Fernandez +xoviat +xtreak +YAMAMOTO Takashi +Yen Chi Hsuan +Yeray Diaz Diaz +Yoval P +Yu Jian +Yuan Jing Vincent Yan +Zearin +Zearin +Zhiping Deng +Zvezdan Petkovic +Łukasz Langa +Семён Марьясин diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/INSTALLER b/ubuntu/venv/pkg_resources-0.0.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/LICENSE.txt b/ubuntu/venv/pkg_resources-0.0.0.dist-info/LICENSE.txt new file mode 100644 index 0000000..737fec5 --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/METADATA b/ubuntu/venv/pkg_resources-0.0.0.dist-info/METADATA new file mode 100644 index 0000000..cf6c930 --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/METADATA @@ -0,0 +1,13 @@ +Metadata-Version: 2.1 +Name: pkg_resources +Version: 0.0.0 +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Platform: UNKNOWN + +UNKNOWN + + diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/RECORD b/ubuntu/venv/pkg_resources-0.0.0.dist-info/RECORD new file mode 100644 index 0000000..2bbae1d --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/RECORD @@ -0,0 +1,38 @@ +pkg_resources-0.0.0.dist-info/AUTHORS.txt,sha256=RtqU9KfonVGhI48DAA4-yTOBUhBtQTjFhaDzHoyh7uU,21518 +pkg_resources-0.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pkg_resources-0.0.0.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090 +pkg_resources-0.0.0.dist-info/METADATA,sha256=V9_WPOtD1FnuKrTGv6Ique7kAOn2lasvT8W0_iMCCCk,177 +pkg_resources-0.0.0.dist-info/RECORD,, +pkg_resources-0.0.0.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 +pkg_resources/__init__.py,sha256=0IssxXPnaDKpYZRra8Ime0JG4hwosQljItGD0bnIkGk,108349 +pkg_resources/__pycache__/__init__.cpython-38.pyc,, +pkg_resources/__pycache__/py31compat.cpython-38.pyc,, +pkg_resources/_vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +pkg_resources/_vendor/__pycache__/__init__.cpython-38.pyc,, +pkg_resources/_vendor/__pycache__/appdirs.cpython-38.pyc,, +pkg_resources/_vendor/__pycache__/pyparsing.cpython-38.pyc,, +pkg_resources/_vendor/__pycache__/six.cpython-38.pyc,, +pkg_resources/_vendor/appdirs.py,sha256=MievUEuv3l_mQISH5SF0shDk_BNhHHzYiAPrT3ITN4I,24701 +pkg_resources/_vendor/packaging/__about__.py,sha256=zkcCPTN_6TcLW0Nrlg0176-R1QQ_WVPTm8sz1R4-HjM,720 +pkg_resources/_vendor/packaging/__init__.py,sha256=_vNac5TrzwsrzbOFIbF-5cHqc_Y2aPT2D7zrIR06BOo,513 +pkg_resources/_vendor/packaging/__pycache__/__about__.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/__init__.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/_compat.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/_structures.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/markers.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/requirements.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/specifiers.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/utils.cpython-38.pyc,, +pkg_resources/_vendor/packaging/__pycache__/version.cpython-38.pyc,, +pkg_resources/_vendor/packaging/_compat.py,sha256=Vi_A0rAQeHbU-a9X0tt1yQm9RqkgQbDSxzRw8WlU9kA,860 +pkg_resources/_vendor/packaging/_structures.py,sha256=RImECJ4c_wTlaTYYwZYLHEiebDMaAJmK1oPARhw1T5o,1416 +pkg_resources/_vendor/packaging/markers.py,sha256=uEcBBtGvzqltgnArqb9c4RrcInXezDLos14zbBHhWJo,8248 +pkg_resources/_vendor/packaging/requirements.py,sha256=SikL2UynbsT0qtY9ltqngndha_sfo0w6XGFhAhoSoaQ,4355 +pkg_resources/_vendor/packaging/specifiers.py,sha256=SAMRerzO3fK2IkFZCaZkuwZaL_EGqHNOz4pni4vhnN0,28025 +pkg_resources/_vendor/packaging/utils.py,sha256=3m6WvPm6NNxE8rkTGmn0r75B_GZSGg7ikafxHsBN1WA,421 +pkg_resources/_vendor/packaging/version.py,sha256=OwGnxYfr2ghNzYx59qWIBkrK3SnB6n-Zfd1XaLpnnM0,11556 +pkg_resources/_vendor/pyparsing.py,sha256=tmrp-lu-qO1i75ZzIN5A12nKRRD1Cm4Vpk-5LR9rims,232055 +pkg_resources/_vendor/six.py,sha256=A6hdJZVjI3t_geebZ9BzUvwRrIXo0lfwzQlM2LcKyas,30098 +pkg_resources/extern/__init__.py,sha256=cHiEfHuLmm6rs5Ve_ztBfMI7Lr31vss-D4wkqF5xzlI,2498 +pkg_resources/extern/__pycache__/__init__.cpython-38.pyc,, +pkg_resources/py31compat.py,sha256=-WQ0e4c3RG_acdhwC3gLiXhP_lg4G5q7XYkZkQg0gxU,558 diff --git a/ubuntu/venv/pkg_resources-0.0.0.dist-info/WHEEL b/ubuntu/venv/pkg_resources-0.0.0.dist-info/WHEEL new file mode 100644 index 0000000..ef99c6c --- /dev/null +++ b/ubuntu/venv/pkg_resources-0.0.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/ubuntu/venv/pkg_resources/__init__.py b/ubuntu/venv/pkg_resources/__init__.py new file mode 100644 index 0000000..2f5aa64 --- /dev/null +++ b/ubuntu/venv/pkg_resources/__init__.py @@ -0,0 +1,3296 @@ +# coding: utf-8 +""" +Package resource API +-------------------- + +A resource is a logical file contained within a package, or a logical +subdirectory thereof. The package resource API expects resource names +to have their path parts separated with ``/``, *not* whatever the local +path separator is. Do not use os.path operations to manipulate resource +names being passed into the API. + +The package resource API is designed to work with normal filesystem packages, +.egg files, and unpacked .egg files. It can also work in a limited way with +.zip files and with custom PEP 302 loaders that support the ``get_data()`` +method. +""" + +from __future__ import absolute_import + +import sys +import os +import io +import time +import re +import types +import zipfile +import zipimport +import warnings +import stat +import functools +import pkgutil +import operator +import platform +import collections +import plistlib +import email.parser +import errno +import tempfile +import textwrap +import itertools +import inspect +import ntpath +import posixpath +from pkgutil import get_importer + +try: + import _imp +except ImportError: + # Python 3.2 compatibility + import imp as _imp + +try: + FileExistsError +except NameError: + FileExistsError = OSError + +from pkg_resources.extern import six +from pkg_resources.extern.six.moves import urllib, map, filter + +# capture these to bypass sandboxing +from os import utime +try: + from os import mkdir, rename, unlink + WRITE_SUPPORT = True +except ImportError: + # no write support, probably under GAE + WRITE_SUPPORT = False + +from os import open as os_open +from os.path import isdir, split + +try: + import importlib.machinery as importlib_machinery + # access attribute to force import under delayed import mechanisms. + importlib_machinery.__name__ +except ImportError: + importlib_machinery = None + +from . import py31compat +from pkg_resources.extern import appdirs +from pkg_resources.extern import packaging +__import__('pkg_resources.extern.packaging.version') +__import__('pkg_resources.extern.packaging.specifiers') +__import__('pkg_resources.extern.packaging.requirements') +__import__('pkg_resources.extern.packaging.markers') + + +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 5): + raise RuntimeError("Python 3.5 or later is required") + +if six.PY2: + # Those builtin exceptions are only defined in Python 3 + PermissionError = None + NotADirectoryError = None + +# declare some globals that will be defined later to +# satisfy the linters. +require = None +working_set = None +add_activation_listener = None +resources_stream = None +cleanup_resources = None +resource_dir = None +resource_stream = None +set_extraction_path = None +resource_isdir = None +resource_string = None +iter_entry_points = None +resource_listdir = None +resource_filename = None +resource_exists = None +_distribution_finders = None +_namespace_handlers = None +_namespace_packages = None + + +class PEP440Warning(RuntimeWarning): + """ + Used when there is an issue with a version or specifier not complying with + PEP 440. + """ + + +def parse_version(v): + try: + return packaging.version.Version(v) + except packaging.version.InvalidVersion: + return packaging.version.LegacyVersion(v) + + +_state_vars = {} + + +def _declare_state(vartype, **kw): + globals().update(kw) + _state_vars.update(dict.fromkeys(kw, vartype)) + + +def __getstate__(): + state = {} + g = globals() + for k, v in _state_vars.items(): + state[k] = g['_sget_' + v](g[k]) + return state + + +def __setstate__(state): + g = globals() + for k, v in state.items(): + g['_sset_' + _state_vars[k]](k, g[k], v) + return state + + +def _sget_dict(val): + return val.copy() + + +def _sset_dict(key, ob, state): + ob.clear() + ob.update(state) + + +def _sget_object(val): + return val.__getstate__() + + +def _sset_object(key, ob, state): + ob.__setstate__(state) + + +_sget_none = _sset_none = lambda *args: None + + +def get_supported_platform(): + """Return this platform's maximum compatible version. + + distutils.util.get_platform() normally reports the minimum version + of Mac OS X that would be required to *use* extensions produced by + distutils. But what we want when checking compatibility is to know the + version of Mac OS X that we are *running*. To allow usage of packages that + explicitly require a newer version of Mac OS X, we must also know the + current version of the OS. + + If this condition occurs for any other platform with a version in its + platform strings, this function should be extended accordingly. + """ + plat = get_build_platform() + m = macosVersionString.match(plat) + if m is not None and sys.platform == "darwin": + try: + plat = 'macosx-%s-%s' % ('.'.join(_macosx_vers()[:2]), m.group(3)) + except ValueError: + # not Mac OS X + pass + return plat + + +__all__ = [ + # Basic resource access and distribution/entry point discovery + 'require', 'run_script', 'get_provider', 'get_distribution', + 'load_entry_point', 'get_entry_map', 'get_entry_info', + 'iter_entry_points', + 'resource_string', 'resource_stream', 'resource_filename', + 'resource_listdir', 'resource_exists', 'resource_isdir', + + # Environmental control + 'declare_namespace', 'working_set', 'add_activation_listener', + 'find_distributions', 'set_extraction_path', 'cleanup_resources', + 'get_default_cache', + + # Primary implementation classes + 'Environment', 'WorkingSet', 'ResourceManager', + 'Distribution', 'Requirement', 'EntryPoint', + + # Exceptions + 'ResolutionError', 'VersionConflict', 'DistributionNotFound', + 'UnknownExtra', 'ExtractionError', + + # Warnings + 'PEP440Warning', + + # Parsing functions and string utilities + 'parse_requirements', 'parse_version', 'safe_name', 'safe_version', + 'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections', + 'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker', + + # filesystem utilities + 'ensure_directory', 'normalize_path', + + # Distribution "precedence" constants + 'EGG_DIST', 'BINARY_DIST', 'SOURCE_DIST', 'CHECKOUT_DIST', 'DEVELOP_DIST', + + # "Provider" interfaces, implementations, and registration/lookup APIs + 'IMetadataProvider', 'IResourceProvider', 'FileMetadata', + 'PathMetadata', 'EggMetadata', 'EmptyProvider', 'empty_provider', + 'NullProvider', 'EggProvider', 'DefaultProvider', 'ZipProvider', + 'register_finder', 'register_namespace_handler', 'register_loader_type', + 'fixup_namespace_packages', 'get_importer', + + # Warnings + 'PkgResourcesDeprecationWarning', + + # Deprecated/backward compatibility only + 'run_main', 'AvailableDistributions', +] + + +class ResolutionError(Exception): + """Abstract base for dependency resolution errors""" + + def __repr__(self): + return self.__class__.__name__ + repr(self.args) + + +class VersionConflict(ResolutionError): + """ + An already-installed version conflicts with the requested version. + + Should be initialized with the installed Distribution and the requested + Requirement. + """ + + _template = "{self.dist} is installed but {self.req} is required" + + @property + def dist(self): + return self.args[0] + + @property + def req(self): + return self.args[1] + + def report(self): + return self._template.format(**locals()) + + def with_context(self, required_by): + """ + If required_by is non-empty, return a version of self that is a + ContextualVersionConflict. + """ + if not required_by: + return self + args = self.args + (required_by,) + return ContextualVersionConflict(*args) + + +class ContextualVersionConflict(VersionConflict): + """ + A VersionConflict that accepts a third parameter, the set of the + requirements that required the installed Distribution. + """ + + _template = VersionConflict._template + ' by {self.required_by}' + + @property + def required_by(self): + return self.args[2] + + +class DistributionNotFound(ResolutionError): + """A requested distribution was not found""" + + _template = ("The '{self.req}' distribution was not found " + "and is required by {self.requirers_str}") + + @property + def req(self): + return self.args[0] + + @property + def requirers(self): + return self.args[1] + + @property + def requirers_str(self): + if not self.requirers: + return 'the application' + return ', '.join(self.requirers) + + def report(self): + return self._template.format(**locals()) + + def __str__(self): + return self.report() + + +class UnknownExtra(ResolutionError): + """Distribution doesn't have an "extra feature" of the given name""" + + +_provider_factories = {} + +PY_MAJOR = '{}.{}'.format(*sys.version_info) +EGG_DIST = 3 +BINARY_DIST = 2 +SOURCE_DIST = 1 +CHECKOUT_DIST = 0 +DEVELOP_DIST = -1 + + +def register_loader_type(loader_type, provider_factory): + """Register `provider_factory` to make providers for `loader_type` + + `loader_type` is the type or class of a PEP 302 ``module.__loader__``, + and `provider_factory` is a function that, passed a *module* object, + returns an ``IResourceProvider`` for that module. + """ + _provider_factories[loader_type] = provider_factory + + +def get_provider(moduleOrReq): + """Return an IResourceProvider for the named module or requirement""" + if isinstance(moduleOrReq, Requirement): + return working_set.find(moduleOrReq) or require(str(moduleOrReq))[0] + try: + module = sys.modules[moduleOrReq] + except KeyError: + __import__(moduleOrReq) + module = sys.modules[moduleOrReq] + loader = getattr(module, '__loader__', None) + return _find_adapter(_provider_factories, loader)(module) + + +def _macosx_vers(_cache=[]): + if not _cache: + version = platform.mac_ver()[0] + # fallback for MacPorts + if version == '': + plist = '/System/Library/CoreServices/SystemVersion.plist' + if os.path.exists(plist): + if hasattr(plistlib, 'readPlist'): + plist_content = plistlib.readPlist(plist) + if 'ProductVersion' in plist_content: + version = plist_content['ProductVersion'] + + _cache.append(version.split('.')) + return _cache[0] + + +def _macosx_arch(machine): + return {'PowerPC': 'ppc', 'Power_Macintosh': 'ppc'}.get(machine, machine) + + +def get_build_platform(): + """Return this platform's string for platform-specific distributions + + XXX Currently this is the same as ``distutils.util.get_platform()``, but it + needs some hacks for Linux and Mac OS X. + """ + from sysconfig import get_platform + + plat = get_platform() + if sys.platform == "darwin" and not plat.startswith('macosx-'): + try: + version = _macosx_vers() + machine = os.uname()[4].replace(" ", "_") + return "macosx-%d.%d-%s" % ( + int(version[0]), int(version[1]), + _macosx_arch(machine), + ) + except ValueError: + # if someone is running a non-Mac darwin system, this will fall + # through to the default implementation + pass + return plat + + +macosVersionString = re.compile(r"macosx-(\d+)\.(\d+)-(.*)") +darwinVersionString = re.compile(r"darwin-(\d+)\.(\d+)\.(\d+)-(.*)") +# XXX backward compat +get_platform = get_build_platform + + +def compatible_platforms(provided, required): + """Can code for the `provided` platform run on the `required` platform? + + Returns true if either platform is ``None``, or the platforms are equal. + + XXX Needs compatibility checks for Linux and other unixy OSes. + """ + if provided is None or required is None or provided == required: + # easy case + return True + + # Mac OS X special cases + reqMac = macosVersionString.match(required) + if reqMac: + provMac = macosVersionString.match(provided) + + # is this a Mac package? + if not provMac: + # this is backwards compatibility for packages built before + # setuptools 0.6. All packages built after this point will + # use the new macosx designation. + provDarwin = darwinVersionString.match(provided) + if provDarwin: + dversion = int(provDarwin.group(1)) + macosversion = "%s.%s" % (reqMac.group(1), reqMac.group(2)) + if dversion == 7 and macosversion >= "10.3" or \ + dversion == 8 and macosversion >= "10.4": + return True + # egg isn't macosx or legacy darwin + return False + + # are they the same major version and machine type? + if provMac.group(1) != reqMac.group(1) or \ + provMac.group(3) != reqMac.group(3): + return False + + # is the required OS major update >= the provided one? + if int(provMac.group(2)) > int(reqMac.group(2)): + return False + + return True + + # XXX Linux and other platforms' special cases should go here + return False + + +def run_script(dist_spec, script_name): + """Locate distribution `dist_spec` and run its `script_name` script""" + ns = sys._getframe(1).f_globals + name = ns['__name__'] + ns.clear() + ns['__name__'] = name + require(dist_spec)[0].run_script(script_name, ns) + + +# backward compatibility +run_main = run_script + + +def get_distribution(dist): + """Return a current distribution object for a Requirement or string""" + if isinstance(dist, six.string_types): + dist = Requirement.parse(dist) + if isinstance(dist, Requirement): + dist = get_provider(dist) + if not isinstance(dist, Distribution): + raise TypeError("Expected string, Requirement, or Distribution", dist) + return dist + + +def load_entry_point(dist, group, name): + """Return `name` entry point of `group` for `dist` or raise ImportError""" + return get_distribution(dist).load_entry_point(group, name) + + +def get_entry_map(dist, group=None): + """Return the entry point map for `group`, or the full entry map""" + return get_distribution(dist).get_entry_map(group) + + +def get_entry_info(dist, group, name): + """Return the EntryPoint object for `group`+`name`, or ``None``""" + return get_distribution(dist).get_entry_info(group, name) + + +class IMetadataProvider: + def has_metadata(name): + """Does the package's distribution contain the named metadata?""" + + def get_metadata(name): + """The named metadata resource as a string""" + + def get_metadata_lines(name): + """Yield named metadata resource as list of non-blank non-comment lines + + Leading and trailing whitespace is stripped from each line, and lines + with ``#`` as the first non-blank character are omitted.""" + + def metadata_isdir(name): + """Is the named metadata a directory? (like ``os.path.isdir()``)""" + + def metadata_listdir(name): + """List of metadata names in the directory (like ``os.listdir()``)""" + + def run_script(script_name, namespace): + """Execute the named script in the supplied namespace dictionary""" + + +class IResourceProvider(IMetadataProvider): + """An object that provides access to package resources""" + + def get_resource_filename(manager, resource_name): + """Return a true filesystem path for `resource_name` + + `manager` must be an ``IResourceManager``""" + + def get_resource_stream(manager, resource_name): + """Return a readable file-like object for `resource_name` + + `manager` must be an ``IResourceManager``""" + + def get_resource_string(manager, resource_name): + """Return a string containing the contents of `resource_name` + + `manager` must be an ``IResourceManager``""" + + def has_resource(resource_name): + """Does the package contain the named resource?""" + + def resource_isdir(resource_name): + """Is the named resource a directory? (like ``os.path.isdir()``)""" + + def resource_listdir(resource_name): + """List of resource names in the directory (like ``os.listdir()``)""" + + +class WorkingSet: + """A collection of active distributions on sys.path (or a similar list)""" + + def __init__(self, entries=None): + """Create working set from list of path entries (default=sys.path)""" + self.entries = [] + self.entry_keys = {} + self.by_key = {} + self.callbacks = [] + + if entries is None: + entries = sys.path + + for entry in entries: + self.add_entry(entry) + + @classmethod + def _build_master(cls): + """ + Prepare the master working set. + """ + ws = cls() + try: + from __main__ import __requires__ + except ImportError: + # The main program does not list any requirements + return ws + + # ensure the requirements are met + try: + ws.require(__requires__) + except VersionConflict: + return cls._build_from_requirements(__requires__) + + return ws + + @classmethod + def _build_from_requirements(cls, req_spec): + """ + Build a working set from a requirement spec. Rewrites sys.path. + """ + # try it without defaults already on sys.path + # by starting with an empty path + ws = cls([]) + reqs = parse_requirements(req_spec) + dists = ws.resolve(reqs, Environment()) + for dist in dists: + ws.add(dist) + + # add any missing entries from sys.path + for entry in sys.path: + if entry not in ws.entries: + ws.add_entry(entry) + + # then copy back to sys.path + sys.path[:] = ws.entries + return ws + + def add_entry(self, entry): + """Add a path item to ``.entries``, finding any distributions on it + + ``find_distributions(entry, True)`` is used to find distributions + corresponding to the path entry, and they are added. `entry` is + always appended to ``.entries``, even if it is already present. + (This is because ``sys.path`` can contain the same value more than + once, and the ``.entries`` of the ``sys.path`` WorkingSet should always + equal ``sys.path``.) + """ + self.entry_keys.setdefault(entry, []) + self.entries.append(entry) + for dist in find_distributions(entry, True): + self.add(dist, entry, False) + + def __contains__(self, dist): + """True if `dist` is the active distribution for its project""" + return self.by_key.get(dist.key) == dist + + def find(self, req): + """Find a distribution matching requirement `req` + + If there is an active distribution for the requested project, this + returns it as long as it meets the version requirement specified by + `req`. But, if there is an active distribution for the project and it + does *not* meet the `req` requirement, ``VersionConflict`` is raised. + If there is no active distribution for the requested project, ``None`` + is returned. + """ + dist = self.by_key.get(req.key) + if dist is not None and dist not in req: + # XXX add more info + raise VersionConflict(dist, req) + return dist + + def iter_entry_points(self, group, name=None): + """Yield entry point objects from `group` matching `name` + + If `name` is None, yields all entry points in `group` from all + distributions in the working set, otherwise only ones matching + both `group` and `name` are yielded (in distribution order). + """ + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) + + def run_script(self, requires, script_name): + """Locate distribution for `requires` and run `script_name` script""" + ns = sys._getframe(1).f_globals + name = ns['__name__'] + ns.clear() + ns['__name__'] = name + self.require(requires)[0].run_script(script_name, ns) + + def __iter__(self): + """Yield distributions for non-duplicate projects in the working set + + The yield order is the order in which the items' path entries were + added to the working set. + """ + seen = {} + for item in self.entries: + if item not in self.entry_keys: + # workaround a cache issue + continue + + for key in self.entry_keys[item]: + if key not in seen: + seen[key] = 1 + yield self.by_key[key] + + def add(self, dist, entry=None, insert=True, replace=False): + """Add `dist` to working set, associated with `entry` + + If `entry` is unspecified, it defaults to the ``.location`` of `dist`. + On exit from this routine, `entry` is added to the end of the working + set's ``.entries`` (if it wasn't already present). + + `dist` is only added to the working set if it's for a project that + doesn't already have a distribution in the set, unless `replace=True`. + If it's added, any callbacks registered with the ``subscribe()`` method + will be called. + """ + if insert: + dist.insert_on(self.entries, entry, replace=replace) + + if entry is None: + entry = dist.location + keys = self.entry_keys.setdefault(entry, []) + keys2 = self.entry_keys.setdefault(dist.location, []) + if not replace and dist.key in self.by_key: + # ignore hidden distros + return + + self.by_key[dist.key] = dist + if dist.key not in keys: + keys.append(dist.key) + if dist.key not in keys2: + keys2.append(dist.key) + self._added_new(dist) + + def resolve(self, requirements, env=None, installer=None, + replace_conflicting=False, extras=None): + """List all distributions needed to (recursively) meet `requirements` + + `requirements` must be a sequence of ``Requirement`` objects. `env`, + if supplied, should be an ``Environment`` instance. If + not supplied, it defaults to all distributions available within any + entry or distribution in the working set. `installer`, if supplied, + will be invoked with each requirement that cannot be met by an + already-installed distribution; it should return a ``Distribution`` or + ``None``. + + Unless `replace_conflicting=True`, raises a VersionConflict exception + if + any requirements are found on the path that have the correct name but + the wrong version. Otherwise, if an `installer` is supplied it will be + invoked to obtain the correct version of the requirement and activate + it. + + `extras` is a list of the extras to be used with these requirements. + This is important because extra requirements may look like `my_req; + extra = "my_extra"`, which would otherwise be interpreted as a purely + optional requirement. Instead, we want to be able to assert that these + requirements are truly required. + """ + + # set up the stack + requirements = list(requirements)[::-1] + # set of processed requirements + processed = {} + # key -> dist + best = {} + to_activate = [] + + req_extras = _ReqExtras() + + # Mapping of requirement to set of distributions that required it; + # useful for reporting info about conflicts. + required_by = collections.defaultdict(set) + + while requirements: + # process dependencies breadth-first + req = requirements.pop(0) + if req in processed: + # Ignore cyclic or redundant dependencies + continue + + if not req_extras.markers_pass(req, extras): + continue + + dist = best.get(req.key) + if dist is None: + # Find the best distribution and add it to the map + dist = self.by_key.get(req.key) + if dist is None or (dist not in req and replace_conflicting): + ws = self + if env is None: + if dist is None: + env = Environment(self.entries) + else: + # Use an empty environment and workingset to avoid + # any further conflicts with the conflicting + # distribution + env = Environment([]) + ws = WorkingSet([]) + dist = best[req.key] = env.best_match( + req, ws, installer, + replace_conflicting=replace_conflicting + ) + if dist is None: + requirers = required_by.get(req, None) + raise DistributionNotFound(req, requirers) + to_activate.append(dist) + if dist not in req: + # Oops, the "best" so far conflicts with a dependency + dependent_req = required_by[req] + raise VersionConflict(dist, req).with_context(dependent_req) + + # push the new requirements onto the stack + new_requirements = dist.requires(req.extras)[::-1] + requirements.extend(new_requirements) + + # Register the new requirements needed by req + for new_requirement in new_requirements: + required_by[new_requirement].add(req.project_name) + req_extras[new_requirement] = req.extras + + processed[req] = True + + # return list of distros to activate + return to_activate + + def find_plugins( + self, plugin_env, full_env=None, installer=None, fallback=True): + """Find all activatable distributions in `plugin_env` + + Example usage:: + + distributions, errors = working_set.find_plugins( + Environment(plugin_dirlist) + ) + # add plugins+libs to sys.path + map(working_set.add, distributions) + # display errors + print('Could not load', errors) + + The `plugin_env` should be an ``Environment`` instance that contains + only distributions that are in the project's "plugin directory" or + directories. The `full_env`, if supplied, should be an ``Environment`` + contains all currently-available distributions. If `full_env` is not + supplied, one is created automatically from the ``WorkingSet`` this + method is called on, which will typically mean that every directory on + ``sys.path`` will be scanned for distributions. + + `installer` is a standard installer callback as used by the + ``resolve()`` method. The `fallback` flag indicates whether we should + attempt to resolve older versions of a plugin if the newest version + cannot be resolved. + + This method returns a 2-tuple: (`distributions`, `error_info`), where + `distributions` is a list of the distributions found in `plugin_env` + that were loadable, along with any other distributions that are needed + to resolve their dependencies. `error_info` is a dictionary mapping + unloadable plugin distributions to an exception instance describing the + error that occurred. Usually this will be a ``DistributionNotFound`` or + ``VersionConflict`` instance. + """ + + plugin_projects = list(plugin_env) + # scan project names in alphabetic order + plugin_projects.sort() + + error_info = {} + distributions = {} + + if full_env is None: + env = Environment(self.entries) + env += plugin_env + else: + env = full_env + plugin_env + + shadow_set = self.__class__([]) + # put all our entries in shadow_set + list(map(shadow_set.add, self)) + + for project_name in plugin_projects: + + for dist in plugin_env[project_name]: + + req = [dist.as_requirement()] + + try: + resolvees = shadow_set.resolve(req, env, installer) + + except ResolutionError as v: + # save error info + error_info[dist] = v + if fallback: + # try the next older version of project + continue + else: + # give up on this project, keep going + break + + else: + list(map(shadow_set.add, resolvees)) + distributions.update(dict.fromkeys(resolvees)) + + # success, no need to try any more versions of this project + break + + distributions = list(distributions) + distributions.sort() + + return distributions, error_info + + def require(self, *requirements): + """Ensure that distributions matching `requirements` are activated + + `requirements` must be a string or a (possibly-nested) sequence + thereof, specifying the distributions and versions required. The + return value is a sequence of the distributions that needed to be + activated to fulfill the requirements; all relevant distributions are + included, even if they were already activated in this working set. + """ + needed = self.resolve(parse_requirements(requirements)) + + for dist in needed: + self.add(dist) + + return needed + + def subscribe(self, callback, existing=True): + """Invoke `callback` for all distributions + + If `existing=True` (default), + call on all existing ones, as well. + """ + if callback in self.callbacks: + return + self.callbacks.append(callback) + if not existing: + return + for dist in self: + callback(dist) + + def _added_new(self, dist): + for callback in self.callbacks: + callback(dist) + + def __getstate__(self): + return ( + self.entries[:], self.entry_keys.copy(), self.by_key.copy(), + self.callbacks[:] + ) + + def __setstate__(self, e_k_b_c): + entries, keys, by_key, callbacks = e_k_b_c + self.entries = entries[:] + self.entry_keys = keys.copy() + self.by_key = by_key.copy() + self.callbacks = callbacks[:] + + +class _ReqExtras(dict): + """ + Map each requirement to the extras that demanded it. + """ + + def markers_pass(self, req, extras=None): + """ + Evaluate markers for req against each extra that + demanded it. + + Return False if the req has a marker and fails + evaluation. Otherwise, return True. + """ + extra_evals = ( + req.marker.evaluate({'extra': extra}) + for extra in self.get(req, ()) + (extras or (None,)) + ) + return not req.marker or any(extra_evals) + + +class Environment: + """Searchable snapshot of distributions on a search path""" + + def __init__( + self, search_path=None, platform=get_supported_platform(), + python=PY_MAJOR): + """Snapshot distributions available on a search path + + Any distributions found on `search_path` are added to the environment. + `search_path` should be a sequence of ``sys.path`` items. If not + supplied, ``sys.path`` is used. + + `platform` is an optional string specifying the name of the platform + that platform-specific distributions must be compatible with. If + unspecified, it defaults to the current platform. `python` is an + optional string naming the desired version of Python (e.g. ``'3.6'``); + it defaults to the current version. + + You may explicitly set `platform` (and/or `python`) to ``None`` if you + wish to map *all* distributions, not just those compatible with the + running platform or Python version. + """ + self._distmap = {} + self.platform = platform + self.python = python + self.scan(search_path) + + def can_add(self, dist): + """Is distribution `dist` acceptable for this environment? + + The distribution must match the platform and python version + requirements specified when this environment was created, or False + is returned. + """ + py_compat = ( + self.python is None + or dist.py_version is None + or dist.py_version == self.python + ) + return py_compat and compatible_platforms(dist.platform, self.platform) + + def remove(self, dist): + """Remove `dist` from the environment""" + self._distmap[dist.key].remove(dist) + + def scan(self, search_path=None): + """Scan `search_path` for distributions usable in this environment + + Any distributions found are added to the environment. + `search_path` should be a sequence of ``sys.path`` items. If not + supplied, ``sys.path`` is used. Only distributions conforming to + the platform/python version defined at initialization are added. + """ + if search_path is None: + search_path = sys.path + + for item in search_path: + for dist in find_distributions(item): + self.add(dist) + + def __getitem__(self, project_name): + """Return a newest-to-oldest list of distributions for `project_name` + + Uses case-insensitive `project_name` comparison, assuming all the + project's distributions use their project's name converted to all + lowercase as their key. + + """ + distribution_key = project_name.lower() + return self._distmap.get(distribution_key, []) + + def add(self, dist): + """Add `dist` if we ``can_add()`` it and it has not already been added + """ + if self.can_add(dist) and dist.has_version(): + dists = self._distmap.setdefault(dist.key, []) + if dist not in dists: + dists.append(dist) + dists.sort(key=operator.attrgetter('hashcmp'), reverse=True) + + def best_match( + self, req, working_set, installer=None, replace_conflicting=False): + """Find distribution best matching `req` and usable on `working_set` + + This calls the ``find(req)`` method of the `working_set` to see if a + suitable distribution is already active. (This may raise + ``VersionConflict`` if an unsuitable version of the project is already + active in the specified `working_set`.) If a suitable distribution + isn't active, this method returns the newest distribution in the + environment that meets the ``Requirement`` in `req`. If no suitable + distribution is found, and `installer` is supplied, then the result of + calling the environment's ``obtain(req, installer)`` method will be + returned. + """ + try: + dist = working_set.find(req) + except VersionConflict: + if not replace_conflicting: + raise + dist = None + if dist is not None: + return dist + for dist in self[req.key]: + if dist in req: + return dist + # try to download/install + return self.obtain(req, installer) + + def obtain(self, requirement, installer=None): + """Obtain a distribution matching `requirement` (e.g. via download) + + Obtain a distro that matches requirement (e.g. via download). In the + base ``Environment`` class, this routine just returns + ``installer(requirement)``, unless `installer` is None, in which case + None is returned instead. This method is a hook that allows subclasses + to attempt other ways of obtaining a distribution before falling back + to the `installer` argument.""" + if installer is not None: + return installer(requirement) + + def __iter__(self): + """Yield the unique project names of the available distributions""" + for key in self._distmap.keys(): + if self[key]: + yield key + + def __iadd__(self, other): + """In-place addition of a distribution or environment""" + if isinstance(other, Distribution): + self.add(other) + elif isinstance(other, Environment): + for project in other: + for dist in other[project]: + self.add(dist) + else: + raise TypeError("Can't add %r to environment" % (other,)) + return self + + def __add__(self, other): + """Add an environment or distribution to an environment""" + new = self.__class__([], platform=None, python=None) + for env in self, other: + new += env + return new + + +# XXX backward compatibility +AvailableDistributions = Environment + + +class ExtractionError(RuntimeError): + """An error occurred extracting a resource + + The following attributes are available from instances of this exception: + + manager + The resource manager that raised this exception + + cache_path + The base directory for resource extraction + + original_error + The exception instance that caused extraction to fail + """ + + +class ResourceManager: + """Manage resource extraction and packages""" + extraction_path = None + + def __init__(self): + self.cached_files = {} + + def resource_exists(self, package_or_requirement, resource_name): + """Does the named resource exist?""" + return get_provider(package_or_requirement).has_resource(resource_name) + + def resource_isdir(self, package_or_requirement, resource_name): + """Is the named resource an existing directory?""" + return get_provider(package_or_requirement).resource_isdir( + resource_name + ) + + def resource_filename(self, package_or_requirement, resource_name): + """Return a true filesystem path for specified resource""" + return get_provider(package_or_requirement).get_resource_filename( + self, resource_name + ) + + def resource_stream(self, package_or_requirement, resource_name): + """Return a readable file-like object for specified resource""" + return get_provider(package_or_requirement).get_resource_stream( + self, resource_name + ) + + def resource_string(self, package_or_requirement, resource_name): + """Return specified resource as a string""" + return get_provider(package_or_requirement).get_resource_string( + self, resource_name + ) + + def resource_listdir(self, package_or_requirement, resource_name): + """List the contents of the named resource directory""" + return get_provider(package_or_requirement).resource_listdir( + resource_name + ) + + def extraction_error(self): + """Give an error message for problems extracting file(s)""" + + old_exc = sys.exc_info()[1] + cache_path = self.extraction_path or get_default_cache() + + tmpl = textwrap.dedent(""" + Can't extract file(s) to egg cache + + The following error occurred while trying to extract file(s) + to the Python egg cache: + + {old_exc} + + The Python egg cache directory is currently set to: + + {cache_path} + + Perhaps your account does not have write access to this directory? + You can change the cache directory by setting the PYTHON_EGG_CACHE + environment variable to point to an accessible directory. + """).lstrip() + err = ExtractionError(tmpl.format(**locals())) + err.manager = self + err.cache_path = cache_path + err.original_error = old_exc + raise err + + def get_cache_path(self, archive_name, names=()): + """Return absolute location in cache for `archive_name` and `names` + + The parent directory of the resulting path will be created if it does + not already exist. `archive_name` should be the base filename of the + enclosing egg (which may not be the name of the enclosing zipfile!), + including its ".egg" extension. `names`, if provided, should be a + sequence of path name parts "under" the egg's extraction location. + + This method should only be called by resource providers that need to + obtain an extraction location, and only for names they intend to + extract, as it tracks the generated names for possible cleanup later. + """ + extract_path = self.extraction_path or get_default_cache() + target_path = os.path.join(extract_path, archive_name + '-tmp', *names) + try: + _bypass_ensure_directory(target_path) + except Exception: + self.extraction_error() + + self._warn_unsafe_extraction_path(extract_path) + + self.cached_files[target_path] = 1 + return target_path + + @staticmethod + def _warn_unsafe_extraction_path(path): + """ + If the default extraction path is overridden and set to an insecure + location, such as /tmp, it opens up an opportunity for an attacker to + replace an extracted file with an unauthorized payload. Warn the user + if a known insecure location is used. + + See Distribute #375 for more details. + """ + if os.name == 'nt' and not path.startswith(os.environ['windir']): + # On Windows, permissions are generally restrictive by default + # and temp directories are not writable by other users, so + # bypass the warning. + return + mode = os.stat(path).st_mode + if mode & stat.S_IWOTH or mode & stat.S_IWGRP: + msg = ( + "%s is writable by group/others and vulnerable to attack " + "when " + "used with get_resource_filename. Consider a more secure " + "location (set with .set_extraction_path or the " + "PYTHON_EGG_CACHE environment variable)." % path + ) + warnings.warn(msg, UserWarning) + + def postprocess(self, tempname, filename): + """Perform any platform-specific postprocessing of `tempname` + + This is where Mac header rewrites should be done; other platforms don't + have anything special they should do. + + Resource providers should call this method ONLY after successfully + extracting a compressed resource. They must NOT call it on resources + that are already in the filesystem. + + `tempname` is the current (temporary) name of the file, and `filename` + is the name it will be renamed to by the caller after this routine + returns. + """ + + if os.name == 'posix': + # Make the resource executable + mode = ((os.stat(tempname).st_mode) | 0o555) & 0o7777 + os.chmod(tempname, mode) + + def set_extraction_path(self, path): + """Set the base path where resources will be extracted to, if needed. + + If you do not call this routine before any extractions take place, the + path defaults to the return value of ``get_default_cache()``. (Which + is based on the ``PYTHON_EGG_CACHE`` environment variable, with various + platform-specific fallbacks. See that routine's documentation for more + details.) + + Resources are extracted to subdirectories of this path based upon + information given by the ``IResourceProvider``. You may set this to a + temporary directory, but then you must call ``cleanup_resources()`` to + delete the extracted files when done. There is no guarantee that + ``cleanup_resources()`` will be able to remove all extracted files. + + (Note: you may not change the extraction path for a given resource + manager once resources have been extracted, unless you first call + ``cleanup_resources()``.) + """ + if self.cached_files: + raise ValueError( + "Can't change extraction path, files already extracted" + ) + + self.extraction_path = path + + def cleanup_resources(self, force=False): + """ + Delete all extracted resource files and directories, returning a list + of the file and directory names that could not be successfully removed. + This function does not have any concurrency protection, so it should + generally only be called when the extraction path is a temporary + directory exclusive to a single process. This method is not + automatically called; you must call it explicitly or register it as an + ``atexit`` function if you wish to ensure cleanup of a temporary + directory used for extractions. + """ + # XXX + + +def get_default_cache(): + """ + Return the ``PYTHON_EGG_CACHE`` environment variable + or a platform-relevant user cache dir for an app + named "Python-Eggs". + """ + return ( + os.environ.get('PYTHON_EGG_CACHE') + or appdirs.user_cache_dir(appname='Python-Eggs') + ) + + +def safe_name(name): + """Convert an arbitrary string to a standard distribution name + + Any runs of non-alphanumeric/. characters are replaced with a single '-'. + """ + return re.sub('[^A-Za-z0-9.]+', '-', name) + + +def safe_version(version): + """ + Convert an arbitrary string to a standard version string + """ + try: + # normalize the version + return str(packaging.version.Version(version)) + except packaging.version.InvalidVersion: + version = version.replace(' ', '.') + return re.sub('[^A-Za-z0-9.]+', '-', version) + + +def safe_extra(extra): + """Convert an arbitrary string to a standard 'extra' name + + Any runs of non-alphanumeric characters are replaced with a single '_', + and the result is always lowercased. + """ + return re.sub('[^A-Za-z0-9.-]+', '_', extra).lower() + + +def to_filename(name): + """Convert a project or version name to its filename-escaped form + + Any '-' characters are currently replaced with '_'. + """ + return name.replace('-', '_') + + +def invalid_marker(text): + """ + Validate text as a PEP 508 environment marker; return an exception + if invalid or False otherwise. + """ + try: + evaluate_marker(text) + except SyntaxError as e: + e.filename = None + e.lineno = None + return e + return False + + +def evaluate_marker(text, extra=None): + """ + Evaluate a PEP 508 environment marker. + Return a boolean indicating the marker result in this environment. + Raise SyntaxError if marker is invalid. + + This implementation uses the 'pyparsing' module. + """ + try: + marker = packaging.markers.Marker(text) + return marker.evaluate() + except packaging.markers.InvalidMarker as e: + raise SyntaxError(e) + + +class NullProvider: + """Try to implement resources and metadata for arbitrary PEP 302 loaders""" + + egg_name = None + egg_info = None + loader = None + + def __init__(self, module): + self.loader = getattr(module, '__loader__', None) + self.module_path = os.path.dirname(getattr(module, '__file__', '')) + + def get_resource_filename(self, manager, resource_name): + return self._fn(self.module_path, resource_name) + + def get_resource_stream(self, manager, resource_name): + return io.BytesIO(self.get_resource_string(manager, resource_name)) + + def get_resource_string(self, manager, resource_name): + return self._get(self._fn(self.module_path, resource_name)) + + def has_resource(self, resource_name): + return self._has(self._fn(self.module_path, resource_name)) + + def _get_metadata_path(self, name): + return self._fn(self.egg_info, name) + + def has_metadata(self, name): + if not self.egg_info: + return self.egg_info + + path = self._get_metadata_path(name) + return self._has(path) + + def get_metadata(self, name): + if not self.egg_info: + return "" + path = self._get_metadata_path(name) + value = self._get(path) + if six.PY2: + return value + try: + return value.decode('utf-8') + except UnicodeDecodeError as exc: + # Include the path in the error message to simplify + # troubleshooting, and without changing the exception type. + exc.reason += ' in {} file at path: {}'.format(name, path) + raise + + def get_metadata_lines(self, name): + return yield_lines(self.get_metadata(name)) + + def resource_isdir(self, resource_name): + return self._isdir(self._fn(self.module_path, resource_name)) + + def metadata_isdir(self, name): + return self.egg_info and self._isdir(self._fn(self.egg_info, name)) + + def resource_listdir(self, resource_name): + return self._listdir(self._fn(self.module_path, resource_name)) + + def metadata_listdir(self, name): + if self.egg_info: + return self._listdir(self._fn(self.egg_info, name)) + return [] + + def run_script(self, script_name, namespace): + script = 'scripts/' + script_name + if not self.has_metadata(script): + raise ResolutionError( + "Script {script!r} not found in metadata at {self.egg_info!r}" + .format(**locals()), + ) + script_text = self.get_metadata(script).replace('\r\n', '\n') + script_text = script_text.replace('\r', '\n') + script_filename = self._fn(self.egg_info, script) + namespace['__file__'] = script_filename + if os.path.exists(script_filename): + source = open(script_filename).read() + code = compile(source, script_filename, 'exec') + exec(code, namespace, namespace) + else: + from linecache import cache + cache[script_filename] = ( + len(script_text), 0, script_text.split('\n'), script_filename + ) + script_code = compile(script_text, script_filename, 'exec') + exec(script_code, namespace, namespace) + + def _has(self, path): + raise NotImplementedError( + "Can't perform this operation for unregistered loader type" + ) + + def _isdir(self, path): + raise NotImplementedError( + "Can't perform this operation for unregistered loader type" + ) + + def _listdir(self, path): + raise NotImplementedError( + "Can't perform this operation for unregistered loader type" + ) + + def _fn(self, base, resource_name): + self._validate_resource_path(resource_name) + if resource_name: + return os.path.join(base, *resource_name.split('/')) + return base + + @staticmethod + def _validate_resource_path(path): + """ + Validate the resource paths according to the docs. + https://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access + + >>> warned = getfixture('recwarn') + >>> warnings.simplefilter('always') + >>> vrp = NullProvider._validate_resource_path + >>> vrp('foo/bar.txt') + >>> bool(warned) + False + >>> vrp('../foo/bar.txt') + >>> bool(warned) + True + >>> warned.clear() + >>> vrp('/foo/bar.txt') + >>> bool(warned) + True + >>> vrp('foo/../../bar.txt') + >>> bool(warned) + True + >>> warned.clear() + >>> vrp('foo/f../bar.txt') + >>> bool(warned) + False + + Windows path separators are straight-up disallowed. + >>> vrp(r'\\foo/bar.txt') + Traceback (most recent call last): + ... + ValueError: Use of .. or absolute path in a resource path \ +is not allowed. + + >>> vrp(r'C:\\foo/bar.txt') + Traceback (most recent call last): + ... + ValueError: Use of .. or absolute path in a resource path \ +is not allowed. + + Blank values are allowed + + >>> vrp('') + >>> bool(warned) + False + + Non-string values are not. + + >>> vrp(None) + Traceback (most recent call last): + ... + AttributeError: ... + """ + invalid = ( + os.path.pardir in path.split(posixpath.sep) or + posixpath.isabs(path) or + ntpath.isabs(path) + ) + if not invalid: + return + + msg = "Use of .. or absolute path in a resource path is not allowed." + + # Aggressively disallow Windows absolute paths + if ntpath.isabs(path) and not posixpath.isabs(path): + raise ValueError(msg) + + # for compatibility, warn; in future + # raise ValueError(msg) + warnings.warn( + msg[:-1] + " and will raise exceptions in a future release.", + DeprecationWarning, + stacklevel=4, + ) + + def _get(self, path): + if hasattr(self.loader, 'get_data'): + return self.loader.get_data(path) + raise NotImplementedError( + "Can't perform this operation for loaders without 'get_data()'" + ) + + +register_loader_type(object, NullProvider) + + +class EggProvider(NullProvider): + """Provider based on a virtual filesystem""" + + def __init__(self, module): + NullProvider.__init__(self, module) + self._setup_prefix() + + def _setup_prefix(self): + # we assume here that our metadata may be nested inside a "basket" + # of multiple eggs; that's why we use module_path instead of .archive + path = self.module_path + old = None + while path != old: + if _is_egg_path(path): + self.egg_name = os.path.basename(path) + self.egg_info = os.path.join(path, 'EGG-INFO') + self.egg_root = path + break + old = path + path, base = os.path.split(path) + + +class DefaultProvider(EggProvider): + """Provides access to package resources in the filesystem""" + + def _has(self, path): + return os.path.exists(path) + + def _isdir(self, path): + return os.path.isdir(path) + + def _listdir(self, path): + return os.listdir(path) + + def get_resource_stream(self, manager, resource_name): + return open(self._fn(self.module_path, resource_name), 'rb') + + def _get(self, path): + with open(path, 'rb') as stream: + return stream.read() + + @classmethod + def _register(cls): + loader_names = 'SourceFileLoader', 'SourcelessFileLoader', + for name in loader_names: + loader_cls = getattr(importlib_machinery, name, type(None)) + register_loader_type(loader_cls, cls) + + +DefaultProvider._register() + + +class EmptyProvider(NullProvider): + """Provider that returns nothing for all requests""" + + module_path = None + + _isdir = _has = lambda self, path: False + + def _get(self, path): + return '' + + def _listdir(self, path): + return [] + + def __init__(self): + pass + + +empty_provider = EmptyProvider() + + +class ZipManifests(dict): + """ + zip manifest builder + """ + + @classmethod + def build(cls, path): + """ + Build a dictionary similar to the zipimport directory + caches, except instead of tuples, store ZipInfo objects. + + Use a platform-specific path separator (os.sep) for the path keys + for compatibility with pypy on Windows. + """ + with zipfile.ZipFile(path) as zfile: + items = ( + ( + name.replace('/', os.sep), + zfile.getinfo(name), + ) + for name in zfile.namelist() + ) + return dict(items) + + load = build + + +class MemoizedZipManifests(ZipManifests): + """ + Memoized zipfile manifests. + """ + manifest_mod = collections.namedtuple('manifest_mod', 'manifest mtime') + + def load(self, path): + """ + Load a manifest at path or return a suitable manifest already loaded. + """ + path = os.path.normpath(path) + mtime = os.stat(path).st_mtime + + if path not in self or self[path].mtime != mtime: + manifest = self.build(path) + self[path] = self.manifest_mod(manifest, mtime) + + return self[path].manifest + + +class ZipProvider(EggProvider): + """Resource support for zips and eggs""" + + eagers = None + _zip_manifests = MemoizedZipManifests() + + def __init__(self, module): + EggProvider.__init__(self, module) + self.zip_pre = self.loader.archive + os.sep + + def _zipinfo_name(self, fspath): + # Convert a virtual filename (full path to file) into a zipfile subpath + # usable with the zipimport directory cache for our target archive + fspath = fspath.rstrip(os.sep) + if fspath == self.loader.archive: + return '' + if fspath.startswith(self.zip_pre): + return fspath[len(self.zip_pre):] + raise AssertionError( + "%s is not a subpath of %s" % (fspath, self.zip_pre) + ) + + def _parts(self, zip_path): + # Convert a zipfile subpath into an egg-relative path part list. + # pseudo-fs path + fspath = self.zip_pre + zip_path + if fspath.startswith(self.egg_root + os.sep): + return fspath[len(self.egg_root) + 1:].split(os.sep) + raise AssertionError( + "%s is not a subpath of %s" % (fspath, self.egg_root) + ) + + @property + def zipinfo(self): + return self._zip_manifests.load(self.loader.archive) + + def get_resource_filename(self, manager, resource_name): + if not self.egg_name: + raise NotImplementedError( + "resource_filename() only supported for .egg, not .zip" + ) + # no need to lock for extraction, since we use temp names + zip_path = self._resource_to_zip(resource_name) + eagers = self._get_eager_resources() + if '/'.join(self._parts(zip_path)) in eagers: + for name in eagers: + self._extract_resource(manager, self._eager_to_zip(name)) + return self._extract_resource(manager, zip_path) + + @staticmethod + def _get_date_and_size(zip_stat): + size = zip_stat.file_size + # ymdhms+wday, yday, dst + date_time = zip_stat.date_time + (0, 0, -1) + # 1980 offset already done + timestamp = time.mktime(date_time) + return timestamp, size + + def _extract_resource(self, manager, zip_path): + + if zip_path in self._index(): + for name in self._index()[zip_path]: + last = self._extract_resource( + manager, os.path.join(zip_path, name) + ) + # return the extracted directory name + return os.path.dirname(last) + + timestamp, size = self._get_date_and_size(self.zipinfo[zip_path]) + + if not WRITE_SUPPORT: + raise IOError('"os.rename" and "os.unlink" are not supported ' + 'on this platform') + try: + + real_path = manager.get_cache_path( + self.egg_name, self._parts(zip_path) + ) + + if self._is_current(real_path, zip_path): + return real_path + + outf, tmpnam = _mkstemp( + ".$extract", + dir=os.path.dirname(real_path), + ) + os.write(outf, self.loader.get_data(zip_path)) + os.close(outf) + utime(tmpnam, (timestamp, timestamp)) + manager.postprocess(tmpnam, real_path) + + try: + rename(tmpnam, real_path) + + except os.error: + if os.path.isfile(real_path): + if self._is_current(real_path, zip_path): + # the file became current since it was checked above, + # so proceed. + return real_path + # Windows, del old file and retry + elif os.name == 'nt': + unlink(real_path) + rename(tmpnam, real_path) + return real_path + raise + + except os.error: + # report a user-friendly error + manager.extraction_error() + + return real_path + + def _is_current(self, file_path, zip_path): + """ + Return True if the file_path is current for this zip_path + """ + timestamp, size = self._get_date_and_size(self.zipinfo[zip_path]) + if not os.path.isfile(file_path): + return False + stat = os.stat(file_path) + if stat.st_size != size or stat.st_mtime != timestamp: + return False + # check that the contents match + zip_contents = self.loader.get_data(zip_path) + with open(file_path, 'rb') as f: + file_contents = f.read() + return zip_contents == file_contents + + def _get_eager_resources(self): + if self.eagers is None: + eagers = [] + for name in ('native_libs.txt', 'eager_resources.txt'): + if self.has_metadata(name): + eagers.extend(self.get_metadata_lines(name)) + self.eagers = eagers + return self.eagers + + def _index(self): + try: + return self._dirindex + except AttributeError: + ind = {} + for path in self.zipinfo: + parts = path.split(os.sep) + while parts: + parent = os.sep.join(parts[:-1]) + if parent in ind: + ind[parent].append(parts[-1]) + break + else: + ind[parent] = [parts.pop()] + self._dirindex = ind + return ind + + def _has(self, fspath): + zip_path = self._zipinfo_name(fspath) + return zip_path in self.zipinfo or zip_path in self._index() + + def _isdir(self, fspath): + return self._zipinfo_name(fspath) in self._index() + + def _listdir(self, fspath): + return list(self._index().get(self._zipinfo_name(fspath), ())) + + def _eager_to_zip(self, resource_name): + return self._zipinfo_name(self._fn(self.egg_root, resource_name)) + + def _resource_to_zip(self, resource_name): + return self._zipinfo_name(self._fn(self.module_path, resource_name)) + + +register_loader_type(zipimport.zipimporter, ZipProvider) + + +class FileMetadata(EmptyProvider): + """Metadata handler for standalone PKG-INFO files + + Usage:: + + metadata = FileMetadata("/path/to/PKG-INFO") + + This provider rejects all data and metadata requests except for PKG-INFO, + which is treated as existing, and will be the contents of the file at + the provided location. + """ + + def __init__(self, path): + self.path = path + + def _get_metadata_path(self, name): + return self.path + + def has_metadata(self, name): + return name == 'PKG-INFO' and os.path.isfile(self.path) + + def get_metadata(self, name): + if name != 'PKG-INFO': + raise KeyError("No metadata except PKG-INFO is available") + + with io.open(self.path, encoding='utf-8', errors="replace") as f: + metadata = f.read() + self._warn_on_replacement(metadata) + return metadata + + def _warn_on_replacement(self, metadata): + # Python 2.7 compat for: replacement_char = '�' + replacement_char = b'\xef\xbf\xbd'.decode('utf-8') + if replacement_char in metadata: + tmpl = "{self.path} could not be properly decoded in UTF-8" + msg = tmpl.format(**locals()) + warnings.warn(msg) + + def get_metadata_lines(self, name): + return yield_lines(self.get_metadata(name)) + + +class PathMetadata(DefaultProvider): + """Metadata provider for egg directories + + Usage:: + + # Development eggs: + + egg_info = "/path/to/PackageName.egg-info" + base_dir = os.path.dirname(egg_info) + metadata = PathMetadata(base_dir, egg_info) + dist_name = os.path.splitext(os.path.basename(egg_info))[0] + dist = Distribution(basedir, project_name=dist_name, metadata=metadata) + + # Unpacked egg directories: + + egg_path = "/path/to/PackageName-ver-pyver-etc.egg" + metadata = PathMetadata(egg_path, os.path.join(egg_path,'EGG-INFO')) + dist = Distribution.from_filename(egg_path, metadata=metadata) + """ + + def __init__(self, path, egg_info): + self.module_path = path + self.egg_info = egg_info + + +class EggMetadata(ZipProvider): + """Metadata provider for .egg files""" + + def __init__(self, importer): + """Create a metadata provider from a zipimporter""" + + self.zip_pre = importer.archive + os.sep + self.loader = importer + if importer.prefix: + self.module_path = os.path.join(importer.archive, importer.prefix) + else: + self.module_path = importer.archive + self._setup_prefix() + + +_declare_state('dict', _distribution_finders={}) + + +def register_finder(importer_type, distribution_finder): + """Register `distribution_finder` to find distributions in sys.path items + + `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item + handler), and `distribution_finder` is a callable that, passed a path + item and the importer instance, yields ``Distribution`` instances found on + that path item. See ``pkg_resources.find_on_path`` for an example.""" + _distribution_finders[importer_type] = distribution_finder + + +def find_distributions(path_item, only=False): + """Yield distributions accessible via `path_item`""" + importer = get_importer(path_item) + finder = _find_adapter(_distribution_finders, importer) + return finder(importer, path_item, only) + + +def find_eggs_in_zip(importer, path_item, only=False): + """ + Find eggs in zip files; possibly multiple nested eggs. + """ + if importer.archive.endswith('.whl'): + # wheels are not supported with this finder + # they don't have PKG-INFO metadata, and won't ever contain eggs + return + metadata = EggMetadata(importer) + if metadata.has_metadata('PKG-INFO'): + yield Distribution.from_filename(path_item, metadata=metadata) + if only: + # don't yield nested distros + return + for subitem in metadata.resource_listdir(''): + if _is_egg_path(subitem): + subpath = os.path.join(path_item, subitem) + dists = find_eggs_in_zip(zipimport.zipimporter(subpath), subpath) + for dist in dists: + yield dist + elif subitem.lower().endswith('.dist-info'): + subpath = os.path.join(path_item, subitem) + submeta = EggMetadata(zipimport.zipimporter(subpath)) + submeta.egg_info = subpath + yield Distribution.from_location(path_item, subitem, submeta) + + +register_finder(zipimport.zipimporter, find_eggs_in_zip) + + +def find_nothing(importer, path_item, only=False): + return () + + +register_finder(object, find_nothing) + + +def _by_version_descending(names): + """ + Given a list of filenames, return them in descending order + by version number. + + >>> names = 'bar', 'foo', 'Python-2.7.10.egg', 'Python-2.7.2.egg' + >>> _by_version_descending(names) + ['Python-2.7.10.egg', 'Python-2.7.2.egg', 'foo', 'bar'] + >>> names = 'Setuptools-1.2.3b1.egg', 'Setuptools-1.2.3.egg' + >>> _by_version_descending(names) + ['Setuptools-1.2.3.egg', 'Setuptools-1.2.3b1.egg'] + >>> names = 'Setuptools-1.2.3b1.egg', 'Setuptools-1.2.3.post1.egg' + >>> _by_version_descending(names) + ['Setuptools-1.2.3.post1.egg', 'Setuptools-1.2.3b1.egg'] + """ + def _by_version(name): + """ + Parse each component of the filename + """ + name, ext = os.path.splitext(name) + parts = itertools.chain(name.split('-'), [ext]) + return [packaging.version.parse(part) for part in parts] + + return sorted(names, key=_by_version, reverse=True) + + +def find_on_path(importer, path_item, only=False): + """Yield distributions accessible on a sys.path directory""" + path_item = _normalize_cached(path_item) + + if _is_unpacked_egg(path_item): + yield Distribution.from_filename( + path_item, metadata=PathMetadata( + path_item, os.path.join(path_item, 'EGG-INFO') + ) + ) + return + + entries = safe_listdir(path_item) + + # for performance, before sorting by version, + # screen entries for only those that will yield + # distributions + filtered = ( + entry + for entry in entries + if dist_factory(path_item, entry, only) + ) + + # scan for .egg and .egg-info in directory + path_item_entries = _by_version_descending(filtered) + for entry in path_item_entries: + fullpath = os.path.join(path_item, entry) + factory = dist_factory(path_item, entry, only) + for dist in factory(fullpath): + yield dist + + +def dist_factory(path_item, entry, only): + """ + Return a dist_factory for a path_item and entry + """ + lower = entry.lower() + is_meta = any(map(lower.endswith, ('.egg-info', '.dist-info'))) + return ( + distributions_from_metadata + if is_meta else + find_distributions + if not only and _is_egg_path(entry) else + resolve_egg_link + if not only and lower.endswith('.egg-link') else + NoDists() + ) + + +class NoDists: + """ + >>> bool(NoDists()) + False + + >>> list(NoDists()('anything')) + [] + """ + def __bool__(self): + return False + if six.PY2: + __nonzero__ = __bool__ + + def __call__(self, fullpath): + return iter(()) + + +def safe_listdir(path): + """ + Attempt to list contents of path, but suppress some exceptions. + """ + try: + return os.listdir(path) + except (PermissionError, NotADirectoryError): + pass + except OSError as e: + # Ignore the directory if does not exist, not a directory or + # permission denied + ignorable = ( + e.errno in (errno.ENOTDIR, errno.EACCES, errno.ENOENT) + # Python 2 on Windows needs to be handled this way :( + or getattr(e, "winerror", None) == 267 + ) + if not ignorable: + raise + return () + + +def distributions_from_metadata(path): + root = os.path.dirname(path) + if os.path.isdir(path): + if len(os.listdir(path)) == 0: + # empty metadata dir; skip + return + metadata = PathMetadata(root, path) + else: + metadata = FileMetadata(path) + entry = os.path.basename(path) + yield Distribution.from_location( + root, entry, metadata, precedence=DEVELOP_DIST, + ) + + +def non_empty_lines(path): + """ + Yield non-empty lines from file at path + """ + with open(path) as f: + for line in f: + line = line.strip() + if line: + yield line + + +def resolve_egg_link(path): + """ + Given a path to an .egg-link, resolve distributions + present in the referenced path. + """ + referenced_paths = non_empty_lines(path) + resolved_paths = ( + os.path.join(os.path.dirname(path), ref) + for ref in referenced_paths + ) + dist_groups = map(find_distributions, resolved_paths) + return next(dist_groups, ()) + + +register_finder(pkgutil.ImpImporter, find_on_path) + +if hasattr(importlib_machinery, 'FileFinder'): + register_finder(importlib_machinery.FileFinder, find_on_path) + +_declare_state('dict', _namespace_handlers={}) +_declare_state('dict', _namespace_packages={}) + + +def register_namespace_handler(importer_type, namespace_handler): + """Register `namespace_handler` to declare namespace packages + + `importer_type` is the type or class of a PEP 302 "Importer" (sys.path item + handler), and `namespace_handler` is a callable like this:: + + def namespace_handler(importer, path_entry, moduleName, module): + # return a path_entry to use for child packages + + Namespace handlers are only called if the importer object has already + agreed that it can handle the relevant path item, and they should only + return a subpath if the module __path__ does not already contain an + equivalent subpath. For an example namespace handler, see + ``pkg_resources.file_ns_handler``. + """ + _namespace_handlers[importer_type] = namespace_handler + + +def _handle_ns(packageName, path_item): + """Ensure that named package includes a subpath of path_item (if needed)""" + + importer = get_importer(path_item) + if importer is None: + return None + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + + if loader is None: + return None + module = sys.modules.get(packageName) + if module is None: + module = sys.modules[packageName] = types.ModuleType(packageName) + module.__path__ = [] + _set_parent_ns(packageName) + elif not hasattr(module, '__path__'): + raise TypeError("Not a package:", packageName) + handler = _find_adapter(_namespace_handlers, importer) + subpath = handler(importer, path_item, packageName, module) + if subpath is not None: + path = module.__path__ + path.append(subpath) + loader.load_module(packageName) + _rebuild_mod_path(path, packageName, module) + return subpath + + +def _rebuild_mod_path(orig_path, package_name, module): + """ + Rebuild module.__path__ ensuring that all entries are ordered + corresponding to their sys.path order + """ + sys_path = [_normalize_cached(p) for p in sys.path] + + def safe_sys_path_index(entry): + """ + Workaround for #520 and #513. + """ + try: + return sys_path.index(entry) + except ValueError: + return float('inf') + + def position_in_sys_path(path): + """ + Return the ordinal of the path based on its position in sys.path + """ + path_parts = path.split(os.sep) + module_parts = package_name.count('.') + 1 + parts = path_parts[:-module_parts] + return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) + + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] + + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path + + +def declare_namespace(packageName): + """Declare that package 'packageName' is a namespace package""" + + _imp.acquire_lock() + try: + if packageName in _namespace_packages: + return + + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: + declare_namespace(parent) + if parent not in _namespace_packages: + __import__(parent) + try: + path = sys.modules[parent].__path__ + except AttributeError: + raise TypeError("Not a package:", parent) + + # Track what packages are namespaces, so when new path items are added, + # they can be updated + _namespace_packages.setdefault(parent or None, []).append(packageName) + _namespace_packages.setdefault(packageName, []) + + for path_item in path: + # Ensure all the parent's path items are reflected in the child, + # if they apply + _handle_ns(packageName, path_item) + + finally: + _imp.release_lock() + + +def fixup_namespace_packages(path_item, parent=None): + """Ensure that previously-declared namespace packages include path_item""" + _imp.acquire_lock() + try: + for package in _namespace_packages.get(parent, ()): + subpath = _handle_ns(package, path_item) + if subpath: + fixup_namespace_packages(subpath, package) + finally: + _imp.release_lock() + + +def file_ns_handler(importer, path_item, packageName, module): + """Compute an ns-package subpath for a filesystem or zipfile importer""" + + subpath = os.path.join(path_item, packageName.split('.')[-1]) + normalized = _normalize_cached(subpath) + for item in module.__path__: + if _normalize_cached(item) == normalized: + break + else: + # Only return the path if it's not already there + return subpath + + +register_namespace_handler(pkgutil.ImpImporter, file_ns_handler) +register_namespace_handler(zipimport.zipimporter, file_ns_handler) + +if hasattr(importlib_machinery, 'FileFinder'): + register_namespace_handler(importlib_machinery.FileFinder, file_ns_handler) + + +def null_ns_handler(importer, path_item, packageName, module): + return None + + +register_namespace_handler(object, null_ns_handler) + + +def normalize_path(filename): + """Normalize a file/dir name for comparison purposes""" + return os.path.normcase(os.path.realpath(os.path.normpath(_cygwin_patch(filename)))) + + +def _cygwin_patch(filename): # pragma: nocover + """ + Contrary to POSIX 2008, on Cygwin, getcwd (3) contains + symlink components. Using + os.path.abspath() works around this limitation. A fix in os.getcwd() + would probably better, in Cygwin even more so, except + that this seems to be by design... + """ + return os.path.abspath(filename) if sys.platform == 'cygwin' else filename + + +def _normalize_cached(filename, _cache={}): + try: + return _cache[filename] + except KeyError: + _cache[filename] = result = normalize_path(filename) + return result + + +def _is_egg_path(path): + """ + Determine if given path appears to be an egg. + """ + return path.lower().endswith('.egg') + + +def _is_unpacked_egg(path): + """ + Determine if given path appears to be an unpacked egg. + """ + return ( + _is_egg_path(path) and + os.path.isfile(os.path.join(path, 'EGG-INFO', 'PKG-INFO')) + ) + + +def _set_parent_ns(packageName): + parts = packageName.split('.') + name = parts.pop() + if parts: + parent = '.'.join(parts) + setattr(sys.modules[parent], name, sys.modules[packageName]) + + +def yield_lines(strs): + """Yield non-empty/non-comment lines of a string or sequence""" + if isinstance(strs, six.string_types): + for s in strs.splitlines(): + s = s.strip() + # skip blank lines/comments + if s and not s.startswith('#'): + yield s + else: + for ss in strs: + for s in yield_lines(ss): + yield s + + +MODULE = re.compile(r"\w+(\.\w+)*$").match +EGG_NAME = re.compile( + r""" + (?P[^-]+) ( + -(?P[^-]+) ( + -py(?P[^-]+) ( + -(?P.+) + )? + )? + )? + """, + re.VERBOSE | re.IGNORECASE, +).match + + +class EntryPoint: + """Object representing an advertised importable object""" + + def __init__(self, name, module_name, attrs=(), extras=(), dist=None): + if not MODULE(module_name): + raise ValueError("Invalid module name", module_name) + self.name = name + self.module_name = module_name + self.attrs = tuple(attrs) + self.extras = tuple(extras) + self.dist = dist + + def __str__(self): + s = "%s = %s" % (self.name, self.module_name) + if self.attrs: + s += ':' + '.'.join(self.attrs) + if self.extras: + s += ' [%s]' % ','.join(self.extras) + return s + + def __repr__(self): + return "EntryPoint.parse(%r)" % str(self) + + def load(self, require=True, *args, **kwargs): + """ + Require packages for this EntryPoint, then resolve it. + """ + if not require or args or kwargs: + warnings.warn( + "Parameters to load are deprecated. Call .resolve and " + ".require separately.", + PkgResourcesDeprecationWarning, + stacklevel=2, + ) + if require: + self.require(*args, **kwargs) + return self.resolve() + + def resolve(self): + """ + Resolve the entry point from its module and attrs. + """ + module = __import__(self.module_name, fromlist=['__name__'], level=0) + try: + return functools.reduce(getattr, self.attrs, module) + except AttributeError as exc: + raise ImportError(str(exc)) + + def require(self, env=None, installer=None): + if self.extras and not self.dist: + raise UnknownExtra("Can't require() without a distribution", self) + + # Get the requirements for this entry point with all its extras and + # then resolve them. We have to pass `extras` along when resolving so + # that the working set knows what extras we want. Otherwise, for + # dist-info distributions, the working set will assume that the + # requirements for that extra are purely optional and skip over them. + reqs = self.dist.requires(self.extras) + items = working_set.resolve(reqs, env, installer, extras=self.extras) + list(map(working_set.add, items)) + + pattern = re.compile( + r'\s*' + r'(?P.+?)\s*' + r'=\s*' + r'(?P[\w.]+)\s*' + r'(:\s*(?P[\w.]+))?\s*' + r'(?P\[.*\])?\s*$' + ) + + @classmethod + def parse(cls, src, dist=None): + """Parse a single entry point from string `src` + + Entry point syntax follows the form:: + + name = some.module:some.attr [extra1, extra2] + + The entry name and module name are required, but the ``:attrs`` and + ``[extras]`` parts are optional + """ + m = cls.pattern.match(src) + if not m: + msg = "EntryPoint must be in 'name=module:attrs [extras]' format" + raise ValueError(msg, src) + res = m.groupdict() + extras = cls._parse_extras(res['extras']) + attrs = res['attr'].split('.') if res['attr'] else () + return cls(res['name'], res['module'], attrs, extras, dist) + + @classmethod + def _parse_extras(cls, extras_spec): + if not extras_spec: + return () + req = Requirement.parse('x' + extras_spec) + if req.specs: + raise ValueError() + return req.extras + + @classmethod + def parse_group(cls, group, lines, dist=None): + """Parse an entry point group""" + if not MODULE(group): + raise ValueError("Invalid group name", group) + this = {} + for line in yield_lines(lines): + ep = cls.parse(line, dist) + if ep.name in this: + raise ValueError("Duplicate entry point", group, ep.name) + this[ep.name] = ep + return this + + @classmethod + def parse_map(cls, data, dist=None): + """Parse a map of entry point groups""" + if isinstance(data, dict): + data = data.items() + else: + data = split_sections(data) + maps = {} + for group, lines in data: + if group is None: + if not lines: + continue + raise ValueError("Entry points must be listed in groups") + group = group.strip() + if group in maps: + raise ValueError("Duplicate group name", group) + maps[group] = cls.parse_group(group, lines, dist) + return maps + + +def _remove_md5_fragment(location): + if not location: + return '' + parsed = urllib.parse.urlparse(location) + if parsed[-1].startswith('md5='): + return urllib.parse.urlunparse(parsed[:-1] + ('',)) + return location + + +def _version_from_file(lines): + """ + Given an iterable of lines from a Metadata file, return + the value of the Version field, if present, or None otherwise. + """ + def is_version_line(line): + return line.lower().startswith('version:') + version_lines = filter(is_version_line, lines) + line = next(iter(version_lines), '') + _, _, value = line.partition(':') + return safe_version(value.strip()) or None + + +class Distribution: + """Wrap an actual or potential sys.path entry w/metadata""" + PKG_INFO = 'PKG-INFO' + + def __init__( + self, location=None, metadata=None, project_name=None, + version=None, py_version=PY_MAJOR, platform=None, + precedence=EGG_DIST): + self.project_name = safe_name(project_name or 'Unknown') + if version is not None: + self._version = safe_version(version) + self.py_version = py_version + self.platform = platform + self.location = location + self.precedence = precedence + self._provider = metadata or empty_provider + + @classmethod + def from_location(cls, location, basename, metadata=None, **kw): + project_name, version, py_version, platform = [None] * 4 + basename, ext = os.path.splitext(basename) + if ext.lower() in _distributionImpl: + cls = _distributionImpl[ext.lower()] + + match = EGG_NAME(basename) + if match: + project_name, version, py_version, platform = match.group( + 'name', 'ver', 'pyver', 'plat' + ) + return cls( + location, metadata, project_name=project_name, version=version, + py_version=py_version, platform=platform, **kw + )._reload_version() + + def _reload_version(self): + return self + + @property + def hashcmp(self): + return ( + self.parsed_version, + self.precedence, + self.key, + _remove_md5_fragment(self.location), + self.py_version or '', + self.platform or '', + ) + + def __hash__(self): + return hash(self.hashcmp) + + def __lt__(self, other): + return self.hashcmp < other.hashcmp + + def __le__(self, other): + return self.hashcmp <= other.hashcmp + + def __gt__(self, other): + return self.hashcmp > other.hashcmp + + def __ge__(self, other): + return self.hashcmp >= other.hashcmp + + def __eq__(self, other): + if not isinstance(other, self.__class__): + # It's not a Distribution, so they are not equal + return False + return self.hashcmp == other.hashcmp + + def __ne__(self, other): + return not self == other + + # These properties have to be lazy so that we don't have to load any + # metadata until/unless it's actually needed. (i.e., some distributions + # may not know their name or version without loading PKG-INFO) + + @property + def key(self): + try: + return self._key + except AttributeError: + self._key = key = self.project_name.lower() + return key + + @property + def parsed_version(self): + if not hasattr(self, "_parsed_version"): + self._parsed_version = parse_version(self.version) + + return self._parsed_version + + def _warn_legacy_version(self): + LV = packaging.version.LegacyVersion + is_legacy = isinstance(self._parsed_version, LV) + if not is_legacy: + return + + # While an empty version is technically a legacy version and + # is not a valid PEP 440 version, it's also unlikely to + # actually come from someone and instead it is more likely that + # it comes from setuptools attempting to parse a filename and + # including it in the list. So for that we'll gate this warning + # on if the version is anything at all or not. + if not self.version: + return + + tmpl = textwrap.dedent(""" + '{project_name} ({version})' is being parsed as a legacy, + non PEP 440, + version. You may find odd behavior and sort order. + In particular it will be sorted as less than 0.0. It + is recommended to migrate to PEP 440 compatible + versions. + """).strip().replace('\n', ' ') + + warnings.warn(tmpl.format(**vars(self)), PEP440Warning) + + @property + def version(self): + try: + return self._version + except AttributeError: + version = self._get_version() + if version is None: + path = self._get_metadata_path_for_display(self.PKG_INFO) + msg = ( + "Missing 'Version:' header and/or {} file at path: {}" + ).format(self.PKG_INFO, path) + raise ValueError(msg, self) + + return version + + @property + def _dep_map(self): + """ + A map of extra to its list of (direct) requirements + for this distribution, including the null extra. + """ + try: + return self.__dep_map + except AttributeError: + self.__dep_map = self._filter_extras(self._build_dep_map()) + return self.__dep_map + + @staticmethod + def _filter_extras(dm): + """ + Given a mapping of extras to dependencies, strip off + environment markers and filter out any dependencies + not matching the markers. + """ + for extra in list(filter(None, dm)): + new_extra = extra + reqs = dm.pop(extra) + new_extra, _, marker = extra.partition(':') + fails_marker = marker and ( + invalid_marker(marker) + or not evaluate_marker(marker) + ) + if fails_marker: + reqs = [] + new_extra = safe_extra(new_extra) or None + + dm.setdefault(new_extra, []).extend(reqs) + return dm + + def _build_dep_map(self): + dm = {} + for name in 'requires.txt', 'depends.txt': + for extra, reqs in split_sections(self._get_metadata(name)): + dm.setdefault(extra, []).extend(parse_requirements(reqs)) + return dm + + def requires(self, extras=()): + """List of Requirements needed for this distro if `extras` are used""" + dm = self._dep_map + deps = [] + deps.extend(dm.get(None, ())) + for ext in extras: + try: + deps.extend(dm[safe_extra(ext)]) + except KeyError: + raise UnknownExtra( + "%s has no such extra feature %r" % (self, ext) + ) + return deps + + def _get_metadata_path_for_display(self, name): + """ + Return the path to the given metadata file, if available. + """ + try: + # We need to access _get_metadata_path() on the provider object + # directly rather than through this class's __getattr__() + # since _get_metadata_path() is marked private. + path = self._provider._get_metadata_path(name) + + # Handle exceptions e.g. in case the distribution's metadata + # provider doesn't support _get_metadata_path(). + except Exception: + return '[could not detect]' + + return path + + def _get_metadata(self, name): + if self.has_metadata(name): + for line in self.get_metadata_lines(name): + yield line + + def _get_version(self): + lines = self._get_metadata(self.PKG_INFO) + version = _version_from_file(lines) + + return version + + def activate(self, path=None, replace=False): + """Ensure distribution is importable on `path` (default=sys.path)""" + if path is None: + path = sys.path + self.insert_on(path, replace=replace) + if path is sys.path: + fixup_namespace_packages(self.location) + for pkg in self._get_metadata('namespace_packages.txt'): + if pkg in sys.modules: + declare_namespace(pkg) + + def egg_name(self): + """Return what this distribution's standard .egg filename should be""" + filename = "%s-%s-py%s" % ( + to_filename(self.project_name), to_filename(self.version), + self.py_version or PY_MAJOR + ) + + if self.platform: + filename += '-' + self.platform + return filename + + def __repr__(self): + if self.location: + return "%s (%s)" % (self, self.location) + else: + return str(self) + + def __str__(self): + try: + version = getattr(self, 'version', None) + except ValueError: + version = None + version = version or "[unknown version]" + return "%s %s" % (self.project_name, version) + + def __getattr__(self, attr): + """Delegate all unrecognized public attributes to .metadata provider""" + if attr.startswith('_'): + raise AttributeError(attr) + return getattr(self._provider, attr) + + def __dir__(self): + return list( + set(super(Distribution, self).__dir__()) + | set( + attr for attr in self._provider.__dir__() + if not attr.startswith('_') + ) + ) + + if not hasattr(object, '__dir__'): + # python 2.7 not supported + del __dir__ + + @classmethod + def from_filename(cls, filename, metadata=None, **kw): + return cls.from_location( + _normalize_cached(filename), os.path.basename(filename), metadata, + **kw + ) + + def as_requirement(self): + """Return a ``Requirement`` that matches this distribution exactly""" + if isinstance(self.parsed_version, packaging.version.Version): + spec = "%s==%s" % (self.project_name, self.parsed_version) + else: + spec = "%s===%s" % (self.project_name, self.parsed_version) + + return Requirement.parse(spec) + + def load_entry_point(self, group, name): + """Return the `name` entry point of `group` or raise ImportError""" + ep = self.get_entry_info(group, name) + if ep is None: + raise ImportError("Entry point %r not found" % ((group, name),)) + return ep.load() + + def get_entry_map(self, group=None): + """Return the entry point map for `group`, or the full entry map""" + try: + ep_map = self._ep_map + except AttributeError: + ep_map = self._ep_map = EntryPoint.parse_map( + self._get_metadata('entry_points.txt'), self + ) + if group is not None: + return ep_map.get(group, {}) + return ep_map + + def get_entry_info(self, group, name): + """Return the EntryPoint object for `group`+`name`, or ``None``""" + return self.get_entry_map(group).get(name) + + def insert_on(self, path, loc=None, replace=False): + """Ensure self.location is on path + + If replace=False (default): + - If location is already in path anywhere, do nothing. + - Else: + - If it's an egg and its parent directory is on path, + insert just ahead of the parent. + - Else: add to the end of path. + If replace=True: + - If location is already on path anywhere (not eggs) + or higher priority than its parent (eggs) + do nothing. + - Else: + - If it's an egg and its parent directory is on path, + insert just ahead of the parent, + removing any lower-priority entries. + - Else: add it to the front of path. + """ + + loc = loc or self.location + if not loc: + return + + nloc = _normalize_cached(loc) + bdir = os.path.dirname(nloc) + npath = [(p and _normalize_cached(p) or p) for p in path] + + for p, item in enumerate(npath): + if item == nloc: + if replace: + break + else: + # don't modify path (even removing duplicates) if + # found and not replace + return + elif item == bdir and self.precedence == EGG_DIST: + # if it's an .egg, give it precedence over its directory + # UNLESS it's already been added to sys.path and replace=False + if (not replace) and nloc in npath[p:]: + return + if path is sys.path: + self.check_version_conflict() + path.insert(p, loc) + npath.insert(p, nloc) + break + else: + if path is sys.path: + self.check_version_conflict() + if replace: + path.insert(0, loc) + else: + path.append(loc) + return + + # p is the spot where we found or inserted loc; now remove duplicates + while True: + try: + np = npath.index(nloc, p + 1) + except ValueError: + break + else: + del npath[np], path[np] + # ha! + p = np + + return + + def check_version_conflict(self): + if self.key == 'setuptools': + # ignore the inevitable setuptools self-conflicts :( + return + + nsp = dict.fromkeys(self._get_metadata('namespace_packages.txt')) + loc = normalize_path(self.location) + for modname in self._get_metadata('top_level.txt'): + if (modname not in sys.modules or modname in nsp + or modname in _namespace_packages): + continue + if modname in ('pkg_resources', 'setuptools', 'site'): + continue + fn = getattr(sys.modules[modname], '__file__', None) + if fn and (normalize_path(fn).startswith(loc) or + fn.startswith(self.location)): + continue + issue_warning( + "Module %s was already imported from %s, but %s is being added" + " to sys.path" % (modname, fn, self.location), + ) + + def has_version(self): + try: + self.version + except ValueError: + issue_warning("Unbuilt egg for " + repr(self)) + return False + return True + + def clone(self, **kw): + """Copy this distribution, substituting in any changed keyword args""" + names = 'project_name version py_version platform location precedence' + for attr in names.split(): + kw.setdefault(attr, getattr(self, attr, None)) + kw.setdefault('metadata', self._provider) + return self.__class__(**kw) + + @property + def extras(self): + return [dep for dep in self._dep_map if dep] + + +class EggInfoDistribution(Distribution): + def _reload_version(self): + """ + Packages installed by distutils (e.g. numpy or scipy), + which uses an old safe_version, and so + their version numbers can get mangled when + converted to filenames (e.g., 1.11.0.dev0+2329eae to + 1.11.0.dev0_2329eae). These distributions will not be + parsed properly + downstream by Distribution and safe_version, so + take an extra step and try to get the version number from + the metadata file itself instead of the filename. + """ + md_version = self._get_version() + if md_version: + self._version = md_version + return self + + +class DistInfoDistribution(Distribution): + """ + Wrap an actual or potential sys.path entry + w/metadata, .dist-info style. + """ + PKG_INFO = 'METADATA' + EQEQ = re.compile(r"([\(,])\s*(\d.*?)\s*([,\)])") + + @property + def _parsed_pkg_info(self): + """Parse and cache metadata""" + try: + return self._pkg_info + except AttributeError: + metadata = self.get_metadata(self.PKG_INFO) + self._pkg_info = email.parser.Parser().parsestr(metadata) + return self._pkg_info + + @property + def _dep_map(self): + try: + return self.__dep_map + except AttributeError: + self.__dep_map = self._compute_dependencies() + return self.__dep_map + + def _compute_dependencies(self): + """Recompute this distribution's dependencies.""" + dm = self.__dep_map = {None: []} + + reqs = [] + # Including any condition expressions + for req in self._parsed_pkg_info.get_all('Requires-Dist') or []: + reqs.extend(parse_requirements(req)) + + def reqs_for_extra(extra): + for req in reqs: + if not req.marker or req.marker.evaluate({'extra': extra}): + yield req + + common = frozenset(reqs_for_extra(None)) + dm[None].extend(common) + + for extra in self._parsed_pkg_info.get_all('Provides-Extra') or []: + s_extra = safe_extra(extra.strip()) + dm[s_extra] = list(frozenset(reqs_for_extra(extra)) - common) + + return dm + + +_distributionImpl = { + '.egg': Distribution, + '.egg-info': EggInfoDistribution, + '.dist-info': DistInfoDistribution, +} + + +def issue_warning(*args, **kw): + level = 1 + g = globals() + try: + # find the first stack frame that is *not* code in + # the pkg_resources module, to use for the warning + while sys._getframe(level).f_globals is g: + level += 1 + except ValueError: + pass + warnings.warn(stacklevel=level + 1, *args, **kw) + + +class RequirementParseError(ValueError): + def __str__(self): + return ' '.join(self.args) + + +def parse_requirements(strs): + """Yield ``Requirement`` objects for each specification in `strs` + + `strs` must be a string, or a (possibly-nested) iterable thereof. + """ + # create a steppable iterator, so we can handle \-continuations + lines = iter(yield_lines(strs)) + + for line in lines: + # Drop comments -- a hash without a space may be in a URL. + if ' #' in line: + line = line[:line.find(' #')] + # If there is a line continuation, drop it, and append the next line. + if line.endswith('\\'): + line = line[:-2].strip() + try: + line += next(lines) + except StopIteration: + return + yield Requirement(line) + + +class Requirement(packaging.requirements.Requirement): + def __init__(self, requirement_string): + """DO NOT CALL THIS UNDOCUMENTED METHOD; use Requirement.parse()!""" + try: + super(Requirement, self).__init__(requirement_string) + except packaging.requirements.InvalidRequirement as e: + raise RequirementParseError(str(e)) + self.unsafe_name = self.name + project_name = safe_name(self.name) + self.project_name, self.key = project_name, project_name.lower() + self.specs = [ + (spec.operator, spec.version) for spec in self.specifier] + self.extras = tuple(map(safe_extra, self.extras)) + self.hashCmp = ( + self.key, + self.url, + self.specifier, + frozenset(self.extras), + str(self.marker) if self.marker else None, + ) + self.__hash = hash(self.hashCmp) + + def __eq__(self, other): + return ( + isinstance(other, Requirement) and + self.hashCmp == other.hashCmp + ) + + def __ne__(self, other): + return not self == other + + def __contains__(self, item): + if isinstance(item, Distribution): + if item.key != self.key: + return False + + item = item.version + + # Allow prereleases always in order to match the previous behavior of + # this method. In the future this should be smarter and follow PEP 440 + # more accurately. + return self.specifier.contains(item, prereleases=True) + + def __hash__(self): + return self.__hash + + def __repr__(self): + return "Requirement.parse(%r)" % str(self) + + @staticmethod + def parse(s): + req, = parse_requirements(s) + return req + + +def _always_object(classes): + """ + Ensure object appears in the mro even + for old-style classes. + """ + if object not in classes: + return classes + (object,) + return classes + + +def _find_adapter(registry, ob): + """Return an adapter factory for `ob` from `registry`""" + types = _always_object(inspect.getmro(getattr(ob, '__class__', type(ob)))) + for t in types: + if t in registry: + return registry[t] + + +def ensure_directory(path): + """Ensure that the parent directory of `path` exists""" + dirname = os.path.dirname(path) + py31compat.makedirs(dirname, exist_ok=True) + + +def _bypass_ensure_directory(path): + """Sandbox-bypassing version of ensure_directory()""" + if not WRITE_SUPPORT: + raise IOError('"os.mkdir" not supported on this platform.') + dirname, filename = split(path) + if dirname and filename and not isdir(dirname): + _bypass_ensure_directory(dirname) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass + + +def split_sections(s): + """Split a string or iterable thereof into (section, content) pairs + + Each ``section`` is a stripped version of the section header ("[section]") + and each ``content`` is a list of stripped lines excluding blank lines and + comment-only lines. If there are any such lines before the first section + header, they're returned in a first ``section`` of ``None``. + """ + section = None + content = [] + for line in yield_lines(s): + if line.startswith("["): + if line.endswith("]"): + if section or content: + yield section, content + section = line[1:-1].strip() + content = [] + else: + raise ValueError("Invalid section heading", line) + else: + content.append(line) + + # wrap up last segment + yield section, content + + +def _mkstemp(*args, **kw): + old_open = os.open + try: + # temporarily bypass sandboxing + os.open = os_open + return tempfile.mkstemp(*args, **kw) + finally: + # and then put it back + os.open = old_open + + +# Silence the PEP440Warning by default, so that end users don't get hit by it +# randomly just because they use pkg_resources. We want to append the rule +# because we want earlier uses of filterwarnings to take precedence over this +# one. +warnings.filterwarnings("ignore", category=PEP440Warning, append=True) + + +# from jaraco.functools 1.3 +def _call_aside(f, *args, **kwargs): + f(*args, **kwargs) + return f + + +@_call_aside +def _initialize(g=globals()): + "Set up global resource manager (deliberately not state-saved)" + manager = ResourceManager() + g['_manager'] = manager + g.update( + (name, getattr(manager, name)) + for name in dir(manager) + if not name.startswith('_') + ) + + +@_call_aside +def _initialize_master_working_set(): + """ + Prepare the master working set and make the ``require()`` + API available. + + This function has explicit effects on the global state + of pkg_resources. It is intended to be invoked once at + the initialization of this module. + + Invocation by other packages is unsupported and done + at their own risk. + """ + working_set = WorkingSet._build_master() + _declare_state('object', working_set=working_set) + + require = working_set.require + iter_entry_points = working_set.iter_entry_points + add_activation_listener = working_set.subscribe + run_script = working_set.run_script + # backward compatibility + run_main = run_script + # Activate all distributions already on sys.path with replace=False and + # ensure that all distributions added to the working set in the future + # (e.g. by calling ``require()``) will get activated as well, + # with higher priority (replace=True). + tuple( + dist.activate(replace=False) + for dist in working_set + ) + add_activation_listener( + lambda dist: dist.activate(replace=True), + existing=False, + ) + working_set.entries = [] + # match order + list(map(working_set.add_entry, sys.path)) + globals().update(locals()) + +class PkgResourcesDeprecationWarning(Warning): + """ + Base class for warning about deprecations in ``pkg_resources`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/ubuntu/venv/pkg_resources/_vendor/__init__.py b/ubuntu/venv/pkg_resources/_vendor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ubuntu/venv/pkg_resources/_vendor/appdirs.py b/ubuntu/venv/pkg_resources/_vendor/appdirs.py new file mode 100644 index 0000000..ae67001 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/appdirs.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2005-2010 ActiveState Software Inc. +# Copyright (c) 2013 Eddy Petrișor + +"""Utilities for determining application-specific dirs. + +See for details and usage. +""" +# Dev Notes: +# - MSDN on where to store app data files: +# http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120 +# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html +# - XDG spec for Un*x: http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html + +__version_info__ = (1, 4, 3) +__version__ = '.'.join(map(str, __version_info__)) + + +import sys +import os + +PY3 = sys.version_info[0] == 3 + +if PY3: + unicode = str + +if sys.platform.startswith('java'): + import platform + os_name = platform.java_ver()[3][0] + if os_name.startswith('Windows'): # "Windows XP", "Windows 7", etc. + system = 'win32' + elif os_name.startswith('Mac'): # "Mac OS X", etc. + system = 'darwin' + else: # "Linux", "SunOS", "FreeBSD", etc. + # Setting this to "linux2" is not ideal, but only Windows or Mac + # are actually checked for and the rest of the module expects + # *sys.platform* style strings. + system = 'linux2' +else: + system = sys.platform + + + +def user_data_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + + for a discussion of issues. + + Typical user data directories are: + Mac OS X: ~/Library/Application Support/ + Unix: ~/.local/share/ # or in $XDG_DATA_HOME, if defined + Win XP (not roaming): C:\Documents and Settings\\Application Data\\ + Win XP (roaming): C:\Documents and Settings\\Local Settings\Application Data\\ + Win 7 (not roaming): C:\Users\\AppData\Local\\ + Win 7 (roaming): C:\Users\\AppData\Roaming\\ + + For Unix, we follow the XDG spec and support $XDG_DATA_HOME. + That means, by default "~/.local/share/". + """ + if system == "win32": + if appauthor is None: + appauthor = appname + const = roaming and "CSIDL_APPDATA" or "CSIDL_LOCAL_APPDATA" + path = os.path.normpath(_get_win_folder(const)) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('~/Library/Application Support/') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_DATA_HOME', os.path.expanduser("~/.local/share")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_data_dir(appname=None, appauthor=None, version=None, multipath=False): + r"""Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of data dirs should be + returned. By default, the first item from XDG_DATA_DIRS is + returned, or '/usr/local/share/', + if XDG_DATA_DIRS is not set + + Typical site data directories are: + Mac OS X: /Library/Application Support/ + Unix: /usr/local/share/ or /usr/share/ + Win XP: C:\Documents and Settings\All Users\Application Data\\ + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + Win 7: C:\ProgramData\\ # Hidden, but writeable on Win 7. + + For Unix, this is using the $XDG_DATA_DIRS[0] default. + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_COMMON_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + elif system == 'darwin': + path = os.path.expanduser('/Library/Application Support') + if appname: + path = os.path.join(path, appname) + else: + # XDG default for $XDG_DATA_DIRS + # only first, if multipath is False + path = os.getenv('XDG_DATA_DIRS', + os.pathsep.join(['/usr/local/share', '/usr/share'])) + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + if appname and version: + path = os.path.join(path, version) + return path + + +def user_config_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific config dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + + for a discussion of issues. + + Typical user config directories are: + Mac OS X: same as user_data_dir + Unix: ~/.config/ # or in $XDG_CONFIG_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow the XDG spec and support $XDG_CONFIG_HOME. + That means, by default "~/.config/". + """ + if system in ["win32", "darwin"]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_CONFIG_HOME', os.path.expanduser("~/.config")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def site_config_dir(appname=None, appauthor=None, version=None, multipath=False): + r"""Return full path to the user-shared data dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "multipath" is an optional parameter only applicable to *nix + which indicates that the entire list of config dirs should be + returned. By default, the first item from XDG_CONFIG_DIRS is + returned, or '/etc/xdg/', if XDG_CONFIG_DIRS is not set + + Typical site config directories are: + Mac OS X: same as site_data_dir + Unix: /etc/xdg/ or $XDG_CONFIG_DIRS[i]/ for each value in + $XDG_CONFIG_DIRS + Win *: same as site_data_dir + Vista: (Fail! "C:\ProgramData" is a hidden *system* directory on Vista.) + + For Unix, this is using the $XDG_CONFIG_DIRS[0] default, if multipath=False + + WARNING: Do not use this on Windows. See the Vista-Fail note above for why. + """ + if system in ["win32", "darwin"]: + path = site_data_dir(appname, appauthor) + if appname and version: + path = os.path.join(path, version) + else: + # XDG default for $XDG_CONFIG_DIRS + # only first, if multipath is False + path = os.getenv('XDG_CONFIG_DIRS', '/etc/xdg') + pathlist = [os.path.expanduser(x.rstrip(os.sep)) for x in path.split(os.pathsep)] + if appname: + if version: + appname = os.path.join(appname, version) + pathlist = [os.sep.join([x, appname]) for x in pathlist] + + if multipath: + path = os.pathsep.join(pathlist) + else: + path = pathlist[0] + return path + + +def user_cache_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific cache dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Cache" to the base app data dir for Windows. See + discussion below. + + Typical user cache directories are: + Mac OS X: ~/Library/Caches/ + Unix: ~/.cache/ (XDG default) + Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Cache + Vista: C:\Users\\AppData\Local\\\Cache + + On Windows the only suggestion in the MSDN docs is that local settings go in + the `CSIDL_LOCAL_APPDATA` directory. This is identical to the non-roaming + app data dir (the default returned by `user_data_dir` above). Apps typically + put cache data somewhere *under* the given dir here. Some examples: + ...\Mozilla\Firefox\Profiles\\Cache + ...\Acme\SuperApp\Cache\1.0 + OPINION: This function appends "Cache" to the `CSIDL_LOCAL_APPDATA` value. + This can be disabled with the `opinion=False` option. + """ + if system == "win32": + if appauthor is None: + appauthor = appname + path = os.path.normpath(_get_win_folder("CSIDL_LOCAL_APPDATA")) + if appname: + if appauthor is not False: + path = os.path.join(path, appauthor, appname) + else: + path = os.path.join(path, appname) + if opinion: + path = os.path.join(path, "Cache") + elif system == 'darwin': + path = os.path.expanduser('~/Library/Caches') + if appname: + path = os.path.join(path, appname) + else: + path = os.getenv('XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def user_state_dir(appname=None, appauthor=None, version=None, roaming=False): + r"""Return full path to the user-specific state dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "roaming" (boolean, default False) can be set True to use the Windows + roaming appdata directory. That means that for users on a Windows + network setup for roaming profiles, this user data will be + sync'd on login. See + + for a discussion of issues. + + Typical user state directories are: + Mac OS X: same as user_data_dir + Unix: ~/.local/state/ # or in $XDG_STATE_HOME, if defined + Win *: same as user_data_dir + + For Unix, we follow this Debian proposal + to extend the XDG spec and support $XDG_STATE_HOME. + + That means, by default "~/.local/state/". + """ + if system in ["win32", "darwin"]: + path = user_data_dir(appname, appauthor, None, roaming) + else: + path = os.getenv('XDG_STATE_HOME', os.path.expanduser("~/.local/state")) + if appname: + path = os.path.join(path, appname) + if appname and version: + path = os.path.join(path, version) + return path + + +def user_log_dir(appname=None, appauthor=None, version=None, opinion=True): + r"""Return full path to the user-specific log dir for this application. + + "appname" is the name of application. + If None, just the system directory is returned. + "appauthor" (only used on Windows) is the name of the + appauthor or distributing body for this application. Typically + it is the owning company name. This falls back to appname. You may + pass False to disable it. + "version" is an optional version path element to append to the + path. You might want to use this if you want multiple versions + of your app to be able to run independently. If used, this + would typically be ".". + Only applied when appname is present. + "opinion" (boolean) can be False to disable the appending of + "Logs" to the base app data dir for Windows, and "log" to the + base cache dir for Unix. See discussion below. + + Typical user log directories are: + Mac OS X: ~/Library/Logs/ + Unix: ~/.cache//log # or under $XDG_CACHE_HOME if defined + Win XP: C:\Documents and Settings\\Local Settings\Application Data\\\Logs + Vista: C:\Users\\AppData\Local\\\Logs + + On Windows the only suggestion in the MSDN docs is that local settings + go in the `CSIDL_LOCAL_APPDATA` directory. (Note: I'm interested in + examples of what some windows apps use for a logs dir.) + + OPINION: This function appends "Logs" to the `CSIDL_LOCAL_APPDATA` + value for Windows and appends "log" to the user cache dir for Unix. + This can be disabled with the `opinion=False` option. + """ + if system == "darwin": + path = os.path.join( + os.path.expanduser('~/Library/Logs'), + appname) + elif system == "win32": + path = user_data_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "Logs") + else: + path = user_cache_dir(appname, appauthor, version) + version = False + if opinion: + path = os.path.join(path, "log") + if appname and version: + path = os.path.join(path, version) + return path + + +class AppDirs(object): + """Convenience wrapper for getting application dirs.""" + def __init__(self, appname=None, appauthor=None, version=None, + roaming=False, multipath=False): + self.appname = appname + self.appauthor = appauthor + self.version = version + self.roaming = roaming + self.multipath = multipath + + @property + def user_data_dir(self): + return user_data_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_data_dir(self): + return site_data_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_config_dir(self): + return user_config_dir(self.appname, self.appauthor, + version=self.version, roaming=self.roaming) + + @property + def site_config_dir(self): + return site_config_dir(self.appname, self.appauthor, + version=self.version, multipath=self.multipath) + + @property + def user_cache_dir(self): + return user_cache_dir(self.appname, self.appauthor, + version=self.version) + + @property + def user_state_dir(self): + return user_state_dir(self.appname, self.appauthor, + version=self.version) + + @property + def user_log_dir(self): + return user_log_dir(self.appname, self.appauthor, + version=self.version) + + +#---- internal support stuff + +def _get_win_folder_from_registry(csidl_name): + """This is a fallback technique at best. I'm not sure if using the + registry for this guarantees us the correct answer for all CSIDL_* + names. + """ + if PY3: + import winreg as _winreg + else: + import _winreg + + shell_folder_name = { + "CSIDL_APPDATA": "AppData", + "CSIDL_COMMON_APPDATA": "Common AppData", + "CSIDL_LOCAL_APPDATA": "Local AppData", + }[csidl_name] + + key = _winreg.OpenKey( + _winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders" + ) + dir, type = _winreg.QueryValueEx(key, shell_folder_name) + return dir + + +def _get_win_folder_with_pywin32(csidl_name): + from win32com.shell import shellcon, shell + dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0) + # Try to make this a unicode path because SHGetFolderPath does + # not return unicode strings when there is unicode data in the + # path. + try: + dir = unicode(dir) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + try: + import win32api + dir = win32api.GetShortPathName(dir) + except ImportError: + pass + except UnicodeError: + pass + return dir + + +def _get_win_folder_with_ctypes(csidl_name): + import ctypes + + csidl_const = { + "CSIDL_APPDATA": 26, + "CSIDL_COMMON_APPDATA": 35, + "CSIDL_LOCAL_APPDATA": 28, + }[csidl_name] + + buf = ctypes.create_unicode_buffer(1024) + ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in buf: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf2 = ctypes.create_unicode_buffer(1024) + if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + buf = buf2 + + return buf.value + +def _get_win_folder_with_jna(csidl_name): + import array + from com.sun import jna + from com.sun.jna.platform import win32 + + buf_size = win32.WinDef.MAX_PATH * 2 + buf = array.zeros('c', buf_size) + shell = win32.Shell32.INSTANCE + shell.SHGetFolderPath(None, getattr(win32.ShlObj, csidl_name), None, win32.ShlObj.SHGFP_TYPE_CURRENT, buf) + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + # Downgrade to short path name if have highbit chars. See + # . + has_high_char = False + for c in dir: + if ord(c) > 255: + has_high_char = True + break + if has_high_char: + buf = array.zeros('c', buf_size) + kernel = win32.Kernel32.INSTANCE + if kernel.GetShortPathName(dir, buf, buf_size): + dir = jna.Native.toString(buf.tostring()).rstrip("\0") + + return dir + +if system == "win32": + try: + import win32com.shell + _get_win_folder = _get_win_folder_with_pywin32 + except ImportError: + try: + from ctypes import windll + _get_win_folder = _get_win_folder_with_ctypes + except ImportError: + try: + import com.sun.jna + _get_win_folder = _get_win_folder_with_jna + except ImportError: + _get_win_folder = _get_win_folder_from_registry + + +#---- self test code + +if __name__ == "__main__": + appname = "MyApp" + appauthor = "MyCompany" + + props = ("user_data_dir", + "user_config_dir", + "user_cache_dir", + "user_state_dir", + "user_log_dir", + "site_data_dir", + "site_config_dir") + + print("-- app dirs %s --" % __version__) + + print("-- app dirs (with optional 'version')") + dirs = AppDirs(appname, appauthor, version="1.0") + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'version')") + dirs = AppDirs(appname, appauthor) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (without optional 'appauthor')") + dirs = AppDirs(appname) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) + + print("\n-- app dirs (with disabled 'appauthor')") + dirs = AppDirs(appname, appauthor=False) + for prop in props: + print("%s: %s" % (prop, getattr(dirs, prop))) diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/__about__.py b/ubuntu/venv/pkg_resources/_vendor/packaging/__about__.py new file mode 100644 index 0000000..95d330e --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/__about__.py @@ -0,0 +1,21 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] + +__title__ = "packaging" +__summary__ = "Core utilities for Python packages" +__uri__ = "https://github.com/pypa/packaging" + +__version__ = "16.8" + +__author__ = "Donald Stufft and individual contributors" +__email__ = "donald@stufft.io" + +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2014-2016 %s" % __author__ diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/__init__.py b/ubuntu/venv/pkg_resources/_vendor/packaging/__init__.py new file mode 100644 index 0000000..5ee6220 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/__init__.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +from .__about__ import ( + __author__, __copyright__, __email__, __license__, __summary__, __title__, + __uri__, __version__ +) + +__all__ = [ + "__title__", "__summary__", "__uri__", "__version__", "__author__", + "__email__", "__license__", "__copyright__", +] diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/_compat.py b/ubuntu/venv/pkg_resources/_vendor/packaging/_compat.py new file mode 100644 index 0000000..210bb80 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/_compat.py @@ -0,0 +1,30 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import sys + + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + +# flake8: noqa + +if PY3: + string_types = str, +else: + string_types = basestring, + + +def with_metaclass(meta, *bases): + """ + Create a base class with a metaclass. + """ + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/_structures.py b/ubuntu/venv/pkg_resources/_vendor/packaging/_structures.py new file mode 100644 index 0000000..ccc2786 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/_structures.py @@ -0,0 +1,68 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + + +class Infinity(object): + + def __repr__(self): + return "Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return True + + def __ge__(self, other): + return True + + def __neg__(self): + return NegativeInfinity + +Infinity = Infinity() + + +class NegativeInfinity(object): + + def __repr__(self): + return "-Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __lt__(self, other): + return True + + def __le__(self, other): + return True + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ne__(self, other): + return not isinstance(other, self.__class__) + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + + def __neg__(self): + return Infinity + +NegativeInfinity = NegativeInfinity() diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/markers.py b/ubuntu/venv/pkg_resources/_vendor/packaging/markers.py new file mode 100644 index 0000000..892e578 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/markers.py @@ -0,0 +1,301 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from pkg_resources.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from pkg_resources.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from pkg_resources.extern.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", + "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class UndefinedEnvironmentName(ValueError): + """ + A name was attempted to be used that does not exist inside of the + environment. + """ + + +class Node(object): + + def __init__(self, value): + self.value = value + + def __str__(self): + return str(self.value) + + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + def serialize(self): + raise NotImplementedError + + +class Variable(Node): + + def serialize(self): + return str(self) + + +class Value(Node): + + def serialize(self): + return '"{0}"'.format(self) + + +class Op(Node): + + def serialize(self): + return str(self) + + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("os.name") | # PEP-345 + L("sys.platform") | # PEP-345 + L("platform.version") | # PEP-345 + L("platform.machine") | # PEP-345 + L("platform.python_implementation") | # PEP-345 + L("python_implementation") | # undocumented setuptools legacy + L("extra") +) +ALIASES = { + 'os.name': 'os_name', + 'sys.platform': 'sys_platform', + 'platform.version': 'platform_version', + 'platform.machine': 'platform_machine', + 'platform.python_implementation': 'platform_python_implementation', + 'python_implementation': 'platform_python_implementation' +} +VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) + +MARKER_OP = VERSION_CMP | L("not in") | L("in") +MARKER_OP.setParseAction(lambda s, l, t: Op(t[0])) + +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) + +BOOLOP = L("and") | L("or") + +MARKER_VAR = VARIABLE | MARKER_VALUE + +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) + +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() + +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) + +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 and + isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return " ".join([m.serialize() for m in marker]) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op.serialize(), rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + + oper = _operators.get(op.serialize()) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + + return oper(lhs, rhs) + + +_undefined = object() + + +def _get_env(environment, name): + value = environment.get(name, _undefined) + + if value is _undefined: + raise UndefinedEnvironmentName( + "{0!r} does not exist in evaluation environment.".format(name) + ) + + return value + + +def _evaluate_markers(markers, environment): + groups = [[]] + + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + + if isinstance(lhs, Variable): + lhs_value = _get_env(environment, lhs.value) + rhs_value = rhs.value + else: + lhs_value = lhs.value + rhs_value = _get_env(environment, rhs.value) + + groups[-1].append(_eval_op(lhs_value, op, rhs_value)) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException as e: + err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( + marker, marker[e.loc:e.loc + 8]) + raise InvalidMarker(err_str) + + def __str__(self): + return _format_marker(self._markers) + + def __repr__(self): + return "".format(str(self)) + + def evaluate(self, environment=None): + """Evaluate a marker. + + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + + return _evaluate_markers(self._markers, current_environment) diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/requirements.py b/ubuntu/venv/pkg_resources/_vendor/packaging/requirements.py new file mode 100644 index 0000000..0c8c4a3 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/requirements.py @@ -0,0 +1,127 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import string +import re + +from pkg_resources.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException +from pkg_resources.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine +from pkg_resources.extern.pyparsing import Literal as L # noqa +from pkg_resources.extern.six.moves.urllib import parse as urlparse + +from .markers import MARKER_EXPR, Marker +from .specifiers import LegacySpecifier, Specifier, SpecifierSet + + +class InvalidRequirement(ValueError): + """ + An invalid requirement was found, users should refer to PEP 508. + """ + + +ALPHANUM = Word(string.ascii_letters + string.digits) + +LBRACKET = L("[").suppress() +RBRACKET = L("]").suppress() +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +COMMA = L(",").suppress() +SEMICOLON = L(";").suppress() +AT = L("@").suppress() + +PUNCTUATION = Word("-_.") +IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM) +IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) + +NAME = IDENTIFIER("name") +EXTRA = IDENTIFIER + +URI = Regex(r'[^ ]+')("url") +URL = (AT + URI) + +EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) +EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") + +VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) +VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) + +VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY +VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), + joinString=",", adjacent=False)("_raw_spec") +_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') + +VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") +VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) + +MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") +MARKER_EXPR.setParseAction( + lambda s, l, t: Marker(s[t._original_start:t._original_end]) +) +MARKER_SEPERATOR = SEMICOLON +MARKER = MARKER_SEPERATOR + MARKER_EXPR + +VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) +URL_AND_MARKER = URL + Optional(MARKER) + +NAMED_REQUIREMENT = \ + NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) + +REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd + + +class Requirement(object): + """Parse a requirement. + + Parse a given requirement string into its parts, such as name, specifier, + URL, and extras. Raises InvalidRequirement on a badly-formed requirement + string. + """ + + # TODO: Can we test whether something is contained within a requirement? + # If so how do we do that? Do we need to test against the _name_ of + # the thing as well as the version? What about the markers? + # TODO: Can we normalize the name and extra name? + + def __init__(self, requirement_string): + try: + req = REQUIREMENT.parseString(requirement_string) + except ParseException as e: + raise InvalidRequirement( + "Invalid requirement, parse error at \"{0!r}\"".format( + requirement_string[e.loc:e.loc + 8])) + + self.name = req.name + if req.url: + parsed_url = urlparse.urlparse(req.url) + if not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc): + raise InvalidRequirement("Invalid URL given") + self.url = req.url + else: + self.url = None + self.extras = set(req.extras.asList() if req.extras else []) + self.specifier = SpecifierSet(req.specifier) + self.marker = req.marker if req.marker else None + + def __str__(self): + parts = [self.name] + + if self.extras: + parts.append("[{0}]".format(",".join(sorted(self.extras)))) + + if self.specifier: + parts.append(str(self.specifier)) + + if self.url: + parts.append("@ {0}".format(self.url)) + + if self.marker: + parts.append("; {0}".format(self.marker)) + + return "".join(parts) + + def __repr__(self): + return "".format(str(self)) diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/specifiers.py b/ubuntu/venv/pkg_resources/_vendor/packaging/specifiers.py new file mode 100644 index 0000000..7f5a76c --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/specifiers.py @@ -0,0 +1,774 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import abc +import functools +import itertools +import re + +from ._compat import string_types, with_metaclass +from .version import Version, LegacyVersion, parse + + +class InvalidSpecifier(ValueError): + """ + An invalid specifier was found, users should refer to PEP 440. + """ + + +class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): + + @abc.abstractmethod + def __str__(self): + """ + Returns the str representation of this Specifier like object. This + should be representative of the Specifier itself. + """ + + @abc.abstractmethod + def __hash__(self): + """ + Returns a hash value for this Specifier like object. + """ + + @abc.abstractmethod + def __eq__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are equal. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Returns a boolean representing whether or not the two Specifier like + objects are not equal. + """ + + @abc.abstractproperty + def prereleases(self): + """ + Returns whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @prereleases.setter + def prereleases(self, value): + """ + Sets whether or not pre-releases as a whole are allowed by this + specifier. + """ + + @abc.abstractmethod + def contains(self, item, prereleases=None): + """ + Determines if the given item is contained within this specifier. + """ + + @abc.abstractmethod + def filter(self, iterable, prereleases=None): + """ + Takes an iterable of items and filters them so that only items which + are contained within this specifier are allowed in it. + """ + + +class _IndividualSpecifier(BaseSpecifier): + + _operators = {} + + def __init__(self, spec="", prereleases=None): + match = self._regex.search(spec) + if not match: + raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) + + self._spec = ( + match.group("operator").strip(), + match.group("version").strip(), + ) + + # Store whether or not this Specifier should accept prereleases + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "<{0}({1!r}{2})>".format( + self.__class__.__name__, + str(self), + pre, + ) + + def __str__(self): + return "{0}{1}".format(*self._spec) + + def __hash__(self): + return hash(self._spec) + + def __eq__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec == other._spec + + def __ne__(self, other): + if isinstance(other, string_types): + try: + other = self.__class__(other) + except InvalidSpecifier: + return NotImplemented + elif not isinstance(other, self.__class__): + return NotImplemented + + return self._spec != other._spec + + def _get_operator(self, op): + return getattr(self, "_compare_{0}".format(self._operators[op])) + + def _coerce_version(self, version): + if not isinstance(version, (LegacyVersion, Version)): + version = parse(version) + return version + + @property + def operator(self): + return self._spec[0] + + @property + def version(self): + return self._spec[1] + + @property + def prereleases(self): + return self._prereleases + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Determine if prereleases are to be allowed or not. + if prereleases is None: + prereleases = self.prereleases + + # Normalize item to a Version or LegacyVersion, this allows us to have + # a shortcut for ``"2.0" in Specifier(">=2") + item = self._coerce_version(item) + + # Determine if we should be supporting prereleases in this specifier + # or not, if we do not support prereleases than we can short circuit + # logic if this version is a prereleases. + if item.is_prerelease and not prereleases: + return False + + # Actually do the comparison to determine if this item is contained + # within this Specifier or not. + return self._get_operator(self.operator)(item, self.version) + + def filter(self, iterable, prereleases=None): + yielded = False + found_prereleases = [] + + kw = {"prereleases": prereleases if prereleases is not None else True} + + # Attempt to iterate over all the values in the iterable and if any of + # them match, yield them. + for version in iterable: + parsed_version = self._coerce_version(version) + + if self.contains(parsed_version, **kw): + # If our version is a prerelease, and we were not set to allow + # prereleases, then we'll store it for later incase nothing + # else matches this specifier. + if (parsed_version.is_prerelease and not + (prereleases or self.prereleases)): + found_prereleases.append(version) + # Either this is not a prerelease, or we should have been + # accepting prereleases from the begining. + else: + yielded = True + yield version + + # Now that we've iterated over everything, determine if we've yielded + # any values, and if we have not and we have any prereleases stored up + # then we will go ahead and yield the prereleases. + if not yielded and found_prereleases: + for version in found_prereleases: + yield version + + +class LegacySpecifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P(==|!=|<=|>=|<|>)) + \s* + (?P + [^,;\s)]* # Since this is a "legacy" specifier, and the version + # string can be just about anything, we match everything + # except for whitespace, a semi-colon for marker support, + # a closing paren since versions can be enclosed in + # them, and a comma since it's a version separator. + ) + """ + ) + + _regex = re.compile( + r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + + _operators = { + "==": "equal", + "!=": "not_equal", + "<=": "less_than_equal", + ">=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + } + + def _coerce_version(self, version): + if not isinstance(version, LegacyVersion): + version = LegacyVersion(str(version)) + return version + + def _compare_equal(self, prospective, spec): + return prospective == self._coerce_version(spec) + + def _compare_not_equal(self, prospective, spec): + return prospective != self._coerce_version(spec) + + def _compare_less_than_equal(self, prospective, spec): + return prospective <= self._coerce_version(spec) + + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= self._coerce_version(spec) + + def _compare_less_than(self, prospective, spec): + return prospective < self._coerce_version(spec) + + def _compare_greater_than(self, prospective, spec): + return prospective > self._coerce_version(spec) + + +def _require_version_compare(fn): + @functools.wraps(fn) + def wrapped(self, prospective, spec): + if not isinstance(prospective, Version): + return False + return fn(self, prospective, spec) + return wrapped + + +class Specifier(_IndividualSpecifier): + + _regex_str = ( + r""" + (?P(~=|==|!=|<=|>=|<|>|===)) + (?P + (?: + # The identity operators allow for an escape hatch that will + # do an exact string match of the version you wish to install. + # This will not be parsed by PEP 440 and we cannot determine + # any semantic meaning from it. This operator is discouraged + # but included entirely as an escape hatch. + (?<====) # Only match for the identity operator + \s* + [^\s]* # We just match everything, except for whitespace + # since we are only testing for strict identity. + ) + | + (?: + # The (non)equality operators allow for wild card and local + # versions to be specified so we have to define these two + # operators separately to enable that. + (?<===|!=) # Only match for equals and not equals + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)* # release + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + + # You cannot use a wild card and a dev or local version + # together so group them with a | and make them optional. + (?: + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local + | + \.\* # Wild card syntax of .* + )? + ) + | + (?: + # The compatible operator requires at least two digits in the + # release segment. + (?<=~=) # Only match for the compatible operator + + \s* + v? + (?:[0-9]+!)? # epoch + [0-9]+(?:\.[0-9]+)+ # release (We have a + instead of a *) + (?: # pre release + [-_\.]? + (a|b|c|rc|alpha|beta|pre|preview) + [-_\.]? + [0-9]* + )? + (?: # post release + (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*) + )? + (?:[-_\.]?dev[-_\.]?[0-9]*)? # dev release + ) + | + (?: + # All other operators only allow a sub set of what the + # (non)equality operators do. Specifically they do not allow + # local versions to be specified nor do they allow the prefix + # matching wild cards. + (?=": "greater_than_equal", + "<": "less_than", + ">": "greater_than", + "===": "arbitrary", + } + + @_require_version_compare + def _compare_compatible(self, prospective, spec): + # Compatible releases have an equivalent combination of >= and ==. That + # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to + # implement this in terms of the other specifiers instead of + # implementing it ourselves. The only thing we need to do is construct + # the other specifiers. + + # We want everything but the last item in the version, but we want to + # ignore post and dev releases and we want to treat the pre-release as + # it's own separate segment. + prefix = ".".join( + list( + itertools.takewhile( + lambda x: (not x.startswith("post") and not + x.startswith("dev")), + _version_split(spec), + ) + )[:-1] + ) + + # Add the prefix notation to the end of our string + prefix += ".*" + + return (self._get_operator(">=")(prospective, spec) and + self._get_operator("==")(prospective, prefix)) + + @_require_version_compare + def _compare_equal(self, prospective, spec): + # We need special logic to handle prefix matching + if spec.endswith(".*"): + # In the case of prefix matching we want to ignore local segment. + prospective = Version(prospective.public) + # Split the spec out by dots, and pretend that there is an implicit + # dot in between a release segment and a pre-release segment. + spec = _version_split(spec[:-2]) # Remove the trailing .* + + # Split the prospective version out by dots, and pretend that there + # is an implicit dot in between a release segment and a pre-release + # segment. + prospective = _version_split(str(prospective)) + + # Shorten the prospective version to be the same length as the spec + # so that we can determine if the specifier is a prefix of the + # prospective version or not. + prospective = prospective[:len(spec)] + + # Pad out our two sides with zeros so that they both equal the same + # length. + spec, prospective = _pad_version(spec, prospective) + else: + # Convert our spec string into a Version + spec = Version(spec) + + # If the specifier does not have a local segment, then we want to + # act as if the prospective version also does not have a local + # segment. + if not spec.local: + prospective = Version(prospective.public) + + return prospective == spec + + @_require_version_compare + def _compare_not_equal(self, prospective, spec): + return not self._compare_equal(prospective, spec) + + @_require_version_compare + def _compare_less_than_equal(self, prospective, spec): + return prospective <= Version(spec) + + @_require_version_compare + def _compare_greater_than_equal(self, prospective, spec): + return prospective >= Version(spec) + + @_require_version_compare + def _compare_less_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is less than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective < spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a pre-release version, that we do not accept pre-release + # versions for the version mentioned in the specifier (e.g. <3.1 should + # not match 3.1.dev0, but should match 3.0.dev0). + if not spec.is_prerelease and prospective.is_prerelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # less than the spec version *and* it's not a pre-release of the same + # version in the spec. + return True + + @_require_version_compare + def _compare_greater_than(self, prospective, spec): + # Convert our spec to a Version instance, since we'll want to work with + # it as a version. + spec = Version(spec) + + # Check to see if the prospective version is greater than the spec + # version. If it's not we can short circuit and just return False now + # instead of doing extra unneeded work. + if not prospective > spec: + return False + + # This special case is here so that, unless the specifier itself + # includes is a post-release version, that we do not accept + # post-release versions for the version mentioned in the specifier + # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0). + if not spec.is_postrelease and prospective.is_postrelease: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # Ensure that we do not allow a local version of the version mentioned + # in the specifier, which is techincally greater than, to match. + if prospective.local is not None: + if Version(prospective.base_version) == Version(spec.base_version): + return False + + # If we've gotten to here, it means that prospective version is both + # greater than the spec version *and* it's not a pre-release of the + # same version in the spec. + return True + + def _compare_arbitrary(self, prospective, spec): + return str(prospective).lower() == str(spec).lower() + + @property + def prereleases(self): + # If there is an explicit prereleases set for this, then we'll just + # blindly use that. + if self._prereleases is not None: + return self._prereleases + + # Look at all of our specifiers and determine if they are inclusive + # operators, and if they are if they are including an explicit + # prerelease. + operator, version = self._spec + if operator in ["==", ">=", "<=", "~=", "==="]: + # The == specifier can include a trailing .*, if it does we + # want to remove before parsing. + if operator == "==" and version.endswith(".*"): + version = version[:-2] + + # Parse the version, and if it is a pre-release than this + # specifier allows pre-releases. + if parse(version).is_prerelease: + return True + + return False + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + +_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") + + +def _version_split(version): + result = [] + for item in version.split("."): + match = _prefix_regex.search(item) + if match: + result.extend(match.groups()) + else: + result.append(item) + return result + + +def _pad_version(left, right): + left_split, right_split = [], [] + + # Get the release segment of our versions + left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left))) + right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) + + # Get the rest of our versions + left_split.append(left[len(left_split[0]):]) + right_split.append(right[len(right_split[0]):]) + + # Insert our padding + left_split.insert( + 1, + ["0"] * max(0, len(right_split[0]) - len(left_split[0])), + ) + right_split.insert( + 1, + ["0"] * max(0, len(left_split[0]) - len(right_split[0])), + ) + + return ( + list(itertools.chain(*left_split)), + list(itertools.chain(*right_split)), + ) + + +class SpecifierSet(BaseSpecifier): + + def __init__(self, specifiers="", prereleases=None): + # Split on , to break each indidivual specifier into it's own item, and + # strip each item to remove leading/trailing whitespace. + specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] + + # Parsed each individual specifier, attempting first to make it a + # Specifier and falling back to a LegacySpecifier. + parsed = set() + for specifier in specifiers: + try: + parsed.add(Specifier(specifier)) + except InvalidSpecifier: + parsed.add(LegacySpecifier(specifier)) + + # Turn our parsed specifiers into a frozen set and save them for later. + self._specs = frozenset(parsed) + + # Store our prereleases value so we can use it later to determine if + # we accept prereleases or not. + self._prereleases = prereleases + + def __repr__(self): + pre = ( + ", prereleases={0!r}".format(self.prereleases) + if self._prereleases is not None + else "" + ) + + return "".format(str(self), pre) + + def __str__(self): + return ",".join(sorted(str(s) for s in self._specs)) + + def __hash__(self): + return hash(self._specs) + + def __and__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + specifier = SpecifierSet() + specifier._specs = frozenset(self._specs | other._specs) + + if self._prereleases is None and other._prereleases is not None: + specifier._prereleases = other._prereleases + elif self._prereleases is not None and other._prereleases is None: + specifier._prereleases = self._prereleases + elif self._prereleases == other._prereleases: + specifier._prereleases = self._prereleases + else: + raise ValueError( + "Cannot combine SpecifierSets with True and False prerelease " + "overrides." + ) + + return specifier + + def __eq__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs == other._specs + + def __ne__(self, other): + if isinstance(other, string_types): + other = SpecifierSet(other) + elif isinstance(other, _IndividualSpecifier): + other = SpecifierSet(str(other)) + elif not isinstance(other, SpecifierSet): + return NotImplemented + + return self._specs != other._specs + + def __len__(self): + return len(self._specs) + + def __iter__(self): + return iter(self._specs) + + @property + def prereleases(self): + # If we have been given an explicit prerelease modifier, then we'll + # pass that through here. + if self._prereleases is not None: + return self._prereleases + + # If we don't have any specifiers, and we don't have a forced value, + # then we'll just return None since we don't know if this should have + # pre-releases or not. + if not self._specs: + return None + + # Otherwise we'll see if any of the given specifiers accept + # prereleases, if any of them do we'll return True, otherwise False. + return any(s.prereleases for s in self._specs) + + @prereleases.setter + def prereleases(self, value): + self._prereleases = value + + def __contains__(self, item): + return self.contains(item) + + def contains(self, item, prereleases=None): + # Ensure that our item is a Version or LegacyVersion instance. + if not isinstance(item, (LegacyVersion, Version)): + item = parse(item) + + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # We can determine if we're going to allow pre-releases by looking to + # see if any of the underlying items supports them. If none of them do + # and this item is a pre-release then we do not allow it and we can + # short circuit that here. + # Note: This means that 1.0.dev1 would not be contained in something + # like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0 + if not prereleases and item.is_prerelease: + return False + + # We simply dispatch to the underlying specs here to make sure that the + # given version is contained within all of them. + # Note: This use of all() here means that an empty set of specifiers + # will always return True, this is an explicit design decision. + return all( + s.contains(item, prereleases=prereleases) + for s in self._specs + ) + + def filter(self, iterable, prereleases=None): + # Determine if we're forcing a prerelease or not, if we're not forcing + # one for this particular filter call, then we'll use whatever the + # SpecifierSet thinks for whether or not we should support prereleases. + if prereleases is None: + prereleases = self.prereleases + + # If we have any specifiers, then we want to wrap our iterable in the + # filter method for each one, this will act as a logical AND amongst + # each specifier. + if self._specs: + for spec in self._specs: + iterable = spec.filter(iterable, prereleases=bool(prereleases)) + return iterable + # If we do not have any specifiers, then we need to have a rough filter + # which will filter out any pre-releases, unless there are no final + # releases, and which will filter out LegacyVersion in general. + else: + filtered = [] + found_prereleases = [] + + for item in iterable: + # Ensure that we some kind of Version class for this item. + if not isinstance(item, (LegacyVersion, Version)): + parsed_version = parse(item) + else: + parsed_version = item + + # Filter out any item which is parsed as a LegacyVersion + if isinstance(parsed_version, LegacyVersion): + continue + + # Store any item which is a pre-release for later unless we've + # already found a final version or we are accepting prereleases + if parsed_version.is_prerelease and not prereleases: + if not filtered: + found_prereleases.append(item) + else: + filtered.append(item) + + # If we've found no items except for pre-releases, then we'll go + # ahead and use the pre-releases + if not filtered and found_prereleases and prereleases is None: + return found_prereleases + + return filtered diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/utils.py b/ubuntu/venv/pkg_resources/_vendor/packaging/utils.py new file mode 100644 index 0000000..942387c --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/utils.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import re + + +_canonicalize_regex = re.compile(r"[-_.]+") + + +def canonicalize_name(name): + # This is taken from PEP 503. + return _canonicalize_regex.sub("-", name).lower() diff --git a/ubuntu/venv/pkg_resources/_vendor/packaging/version.py b/ubuntu/venv/pkg_resources/_vendor/packaging/version.py new file mode 100644 index 0000000..83b5ee8 --- /dev/null +++ b/ubuntu/venv/pkg_resources/_vendor/packaging/version.py @@ -0,0 +1,393 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import collections +import itertools +import re + +from ._structures import Infinity + + +__all__ = [ + "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" +] + + +_Version = collections.namedtuple( + "_Version", + ["epoch", "release", "dev", "pre", "post", "local"], +) + + +def parse(version): + """ + Parse the given version string and return either a :class:`Version` object + or a :class:`LegacyVersion` object depending on if the given version is + a valid PEP 440 version or a legacy version. + """ + try: + return Version(version) + except InvalidVersion: + return LegacyVersion(version) + + +class InvalidVersion(ValueError): + """ + An invalid version was found, users should refer to PEP 440. + """ + + +class _BaseVersion(object): + + def __hash__(self): + return hash(self._key) + + def __lt__(self, other): + return self._compare(other, lambda s, o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s, o: s <= o) + + def __eq__(self, other): + return self._compare(other, lambda s, o: s == o) + + def __ge__(self, other): + return self._compare(other, lambda s, o: s >= o) + + def __gt__(self, other): + return self._compare(other, lambda s, o: s > o) + + def __ne__(self, other): + return self._compare(other, lambda s, o: s != o) + + def _compare(self, other, method): + if not isinstance(other, _BaseVersion): + return NotImplemented + + return method(self._key, other._key) + + +class LegacyVersion(_BaseVersion): + + def __init__(self, version): + self._version = str(version) + self._key = _legacy_cmpkey(self._version) + + def __str__(self): + return self._version + + def __repr__(self): + return "".format(repr(str(self))) + + @property + def public(self): + return self._version + + @property + def base_version(self): + return self._version + + @property + def local(self): + return None + + @property + def is_prerelease(self): + return False + + @property + def is_postrelease(self): + return False + + +_legacy_version_component_re = re.compile( + r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, +) + +_legacy_version_replacement_map = { + "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", +} + + +def _parse_version_parts(s): + for part in _legacy_version_component_re.split(s): + part = _legacy_version_replacement_map.get(part, part) + + if not part or part == ".": + continue + + if part[:1] in "0123456789": + # pad for numeric comparison + yield part.zfill(8) + else: + yield "*" + part + + # ensure that alpha/beta/candidate are before final + yield "*final" + + +def _legacy_cmpkey(version): + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch + # greater than or equal to 0. This will effectively put the LegacyVersion, + # which uses the defacto standard originally implemented by setuptools, + # as before all PEP 440 versions. + epoch = -1 + + # This scheme is taken from pkg_resources.parse_version setuptools prior to + # it's adoption of the packaging library. + parts = [] + for part in _parse_version_parts(version.lower()): + if part.startswith("*"): + # remove "-" before a prerelease tag + if part < "*final": + while parts and parts[-1] == "*final-": + parts.pop() + + # remove trailing zeros from each series of numeric parts + while parts and parts[-1] == "00000000": + parts.pop() + + parts.append(part) + parts = tuple(parts) + + return epoch, parts + +# Deliberately not anchored to the start and end of the string, to make it +# easier for 3rd party code to reuse +VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                          # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+
+class Version(_BaseVersion):
+
+    _regex = re.compile(
+        r"^\s*" + VERSION_PATTERN + r"\s*$",
+        re.VERBOSE | re.IGNORECASE,
+    )
+
+    def __init__(self, version):
+        # Validate the version and parse it into pieces
+        match = self._regex.search(version)
+        if not match:
+            raise InvalidVersion("Invalid version: '{0}'".format(version))
+
+        # Store the parsed out pieces of the version
+        self._version = _Version(
+            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+            release=tuple(int(i) for i in match.group("release").split(".")),
+            pre=_parse_letter_version(
+                match.group("pre_l"),
+                match.group("pre_n"),
+            ),
+            post=_parse_letter_version(
+                match.group("post_l"),
+                match.group("post_n1") or match.group("post_n2"),
+            ),
+            dev=_parse_letter_version(
+                match.group("dev_l"),
+                match.group("dev_n"),
+            ),
+            local=_parse_local_version(match.group("local")),
+        )
+
+        # Generate a key which will be used for sorting
+        self._key = _cmpkey(
+            self._version.epoch,
+            self._version.release,
+            self._version.pre,
+            self._version.post,
+            self._version.dev,
+            self._version.local,
+        )
+
+    def __repr__(self):
+        return "".format(repr(str(self)))
+
+    def __str__(self):
+        parts = []
+
+        # Epoch
+        if self._version.epoch != 0:
+            parts.append("{0}!".format(self._version.epoch))
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self._version.release))
+
+        # Pre-release
+        if self._version.pre is not None:
+            parts.append("".join(str(x) for x in self._version.pre))
+
+        # Post-release
+        if self._version.post is not None:
+            parts.append(".post{0}".format(self._version.post[1]))
+
+        # Development release
+        if self._version.dev is not None:
+            parts.append(".dev{0}".format(self._version.dev[1]))
+
+        # Local version segment
+        if self._version.local is not None:
+            parts.append(
+                "+{0}".format(".".join(str(x) for x in self._version.local))
+            )
+
+        return "".join(parts)
+
+    @property
+    def public(self):
+        return str(self).split("+", 1)[0]
+
+    @property
+    def base_version(self):
+        parts = []
+
+        # Epoch
+        if self._version.epoch != 0:
+            parts.append("{0}!".format(self._version.epoch))
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self._version.release))
+
+        return "".join(parts)
+
+    @property
+    def local(self):
+        version_string = str(self)
+        if "+" in version_string:
+            return version_string.split("+", 1)[1]
+
+    @property
+    def is_prerelease(self):
+        return bool(self._version.dev or self._version.pre)
+
+    @property
+    def is_postrelease(self):
+        return bool(self._version.post)
+
+
+def _parse_letter_version(letter, number):
+    if letter:
+        # We consider there to be an implicit 0 in a pre-release if there is
+        # not a numeral associated with it.
+        if number is None:
+            number = 0
+
+        # We normalize any letters to their lower case form
+        letter = letter.lower()
+
+        # We consider some words to be alternate spellings of other words and
+        # in those cases we want to normalize the spellings to our preferred
+        # spelling.
+        if letter == "alpha":
+            letter = "a"
+        elif letter == "beta":
+            letter = "b"
+        elif letter in ["c", "pre", "preview"]:
+            letter = "rc"
+        elif letter in ["rev", "r"]:
+            letter = "post"
+
+        return letter, int(number)
+    if not letter and number:
+        # We assume if we are given a number, but we are not given a letter
+        # then this is using the implicit post release syntax (e.g. 1.0-1)
+        letter = "post"
+
+        return letter, int(number)
+
+
+_local_version_seperators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local):
+    """
+    Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+    """
+    if local is not None:
+        return tuple(
+            part.lower() if not part.isdigit() else int(part)
+            for part in _local_version_seperators.split(local)
+        )
+
+
+def _cmpkey(epoch, release, pre, post, dev, local):
+    # When we compare a release version, we want to compare it with all of the
+    # trailing zeros removed. So we'll use a reverse the list, drop all the now
+    # leading zeros until we come to something non zero, then take the rest
+    # re-reverse it back into the correct order and make it a tuple and use
+    # that for our sorting key.
+    release = tuple(
+        reversed(list(
+            itertools.dropwhile(
+                lambda x: x == 0,
+                reversed(release),
+            )
+        ))
+    )
+
+    # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+    # We'll do this by abusing the pre segment, but we _only_ want to do this
+    # if there is not a pre or a post segment. If we have one of those then
+    # the normal sorting rules will handle this case correctly.
+    if pre is None and post is None and dev is not None:
+        pre = -Infinity
+    # Versions without a pre-release (except as noted above) should sort after
+    # those with one.
+    elif pre is None:
+        pre = Infinity
+
+    # Versions without a post segment should sort before those with one.
+    if post is None:
+        post = -Infinity
+
+    # Versions without a development segment should sort after those with one.
+    if dev is None:
+        dev = Infinity
+
+    if local is None:
+        # Versions without a local segment should sort before those with one.
+        local = -Infinity
+    else:
+        # Versions with a local segment need that segment parsed to implement
+        # the sorting rules in PEP440.
+        # - Alpha numeric segments sort before numeric segments
+        # - Alpha numeric segments sort lexicographically
+        # - Numeric segments sort numerically
+        # - Shorter versions sort before longer versions when the prefixes
+        #   match exactly
+        local = tuple(
+            (i, "") if isinstance(i, int) else (-Infinity, i)
+            for i in local
+        )
+
+    return epoch, release, pre, post, dev, local
diff --git a/ubuntu/venv/pkg_resources/_vendor/pyparsing.py b/ubuntu/venv/pkg_resources/_vendor/pyparsing.py
new file mode 100644
index 0000000..cf75e1e
--- /dev/null
+++ b/ubuntu/venv/pkg_resources/_vendor/pyparsing.py
@@ -0,0 +1,5742 @@
+# module pyparsing.py
+#
+# Copyright (c) 2003-2018  Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form 
+C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements 
+(L{'+'} operator gives L{And} expressions, strings are auto-converted to
+L{Literal} expressions)::
+
+    from pyparsing import Word, alphas
+
+    # define grammar of a greeting
+    greet = Word(alphas) + "," + Word(alphas) + "!"
+
+    hello = "Hello, World!"
+    print (hello, "->", greet.parseString(hello))
+
+The program outputs the following::
+
+    Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
+ - quoted strings
+ - embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
+"""
+
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
+__author__ = "Paul McGuire "
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import traceback
+import types
+from datetime import datetime
+
+try:
+    from _thread import RLock
+except ImportError:
+    from threading import RLock
+
+try:
+    # Python 3
+    from collections.abc import Iterable
+    from collections.abc import MutableMapping
+except ImportError:
+    # Python 2.7
+    from collections import Iterable
+    from collections import MutableMapping
+
+try:
+    from collections import OrderedDict as _OrderedDict
+except ImportError:
+    try:
+        from ordereddict import OrderedDict as _OrderedDict
+    except ImportError:
+        _OrderedDict = None
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+'CloseMatch', 'tokenMap', 'pyparsing_common',
+]
+
+system_version = tuple(sys.version_info)[:3]
+PY_3 = system_version[0] == 3
+if PY_3:
+    _MAX_INT = sys.maxsize
+    basestring = str
+    unichr = chr
+    _ustr = str
+
+    # build list of single arg builtins, that can be used as parse actions
+    singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+    _MAX_INT = sys.maxint
+    range = xrange
+
+    def _ustr(obj):
+        """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+           str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+           then < returns the unicode object | encodes it with the default encoding | ... >.
+        """
+        if isinstance(obj,unicode):
+            return obj
+
+        try:
+            # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+            # it won't break any existing code.
+            return str(obj)
+
+        except UnicodeEncodeError:
+            # Else encode it
+            ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
+            xmlcharref = Regex(r'&#\d+;')
+            xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
+            return xmlcharref.transformString(ret)
+
+    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+    singleArgBuiltins = []
+    import __builtin__
+    for fname in "sum len sorted reversed list tuple set any all min max".split():
+        try:
+            singleArgBuiltins.append(getattr(__builtin__,fname))
+        except AttributeError:
+            continue
+            
+_generatorType = type((y for y in range(1)))
+ 
+def _xml_escape(data):
+    """Escape &, <, >, ", ', etc. in a string of data."""
+
+    # ampersand must be replaced first
+    from_symbols = '&><"\''
+    to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+    for from_,to_ in zip(from_symbols, to_symbols):
+        data = data.replace(from_, to_)
+    return data
+
+class _Constants(object):
+    pass
+
+alphas     = string.ascii_uppercase + string.ascii_lowercase
+nums       = "0123456789"
+hexnums    = nums + "ABCDEFabcdef"
+alphanums  = alphas + nums
+_bslash    = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+    """base exception class for all parsing runtime exceptions"""
+    # Performance tuning: we construct a *lot* of these, so keep this
+    # constructor as small and fast as possible
+    def __init__( self, pstr, loc=0, msg=None, elem=None ):
+        self.loc = loc
+        if msg is None:
+            self.msg = pstr
+            self.pstr = ""
+        else:
+            self.msg = msg
+            self.pstr = pstr
+        self.parserElement = elem
+        self.args = (pstr, loc, msg)
+
+    @classmethod
+    def _from_exception(cls, pe):
+        """
+        internal factory method to simplify creating one type of ParseException 
+        from another - avoids having __init__ signature conflicts among subclasses
+        """
+        return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+    def __getattr__( self, aname ):
+        """supported attributes by name are:
+            - lineno - returns the line number of the exception text
+            - col - returns the column number of the exception text
+            - line - returns the line containing the exception text
+        """
+        if( aname == "lineno" ):
+            return lineno( self.loc, self.pstr )
+        elif( aname in ("col", "column") ):
+            return col( self.loc, self.pstr )
+        elif( aname == "line" ):
+            return line( self.loc, self.pstr )
+        else:
+            raise AttributeError(aname)
+
+    def __str__( self ):
+        return "%s (at char %d), (line:%d, col:%d)" % \
+                ( self.msg, self.loc, self.lineno, self.column )
+    def __repr__( self ):
+        return _ustr(self)
+    def markInputline( self, markerString = ">!<" ):
+        """Extracts the exception line from the input string, and marks
+           the location of the exception with a special symbol.
+        """
+        line_str = self.line
+        line_column = self.column - 1
+        if markerString:
+            line_str = "".join((line_str[:line_column],
+                                markerString, line_str[line_column:]))
+        return line_str.strip()
+    def __dir__(self):
+        return "lineno col line".split() + dir(type(self))
+
+class ParseException(ParseBaseException):
+    """
+    Exception thrown when parse expressions don't match class;
+    supported attributes by name are:
+     - lineno - returns the line number of the exception text
+     - col - returns the column number of the exception text
+     - line - returns the line containing the exception text
+        
+    Example::
+        try:
+            Word(nums).setName("integer").parseString("ABC")
+        except ParseException as pe:
+            print(pe)
+            print("column: {}".format(pe.col))
+            
+    prints::
+       Expected integer (at char 0), (line:1, col:1)
+        column: 1
+    """
+    pass
+
+class ParseFatalException(ParseBaseException):
+    """user-throwable exception thrown when inconsistent parse content
+       is found; stops all parsing immediately"""
+    pass
+
+class ParseSyntaxException(ParseFatalException):
+    """just like L{ParseFatalException}, but thrown internally when an
+       L{ErrorStop} ('-' operator) indicates that parsing is to stop 
+       immediately because an unbacktrackable syntax error has been found"""
+    pass
+
+#~ class ReparseException(ParseBaseException):
+    #~ """Experimental class - parse actions can raise this exception to cause
+       #~ pyparsing to reparse the input string:
+        #~ - with a modified input string, and/or
+        #~ - with a modified start location
+       #~ Set the values of the ReparseException in the constructor, and raise the
+       #~ exception in a parse action to cause pyparsing to use the new string/location.
+       #~ Setting the values as None causes no change to be made.
+       #~ """
+    #~ def __init_( self, newstring, restartLoc ):
+        #~ self.newParseText = newstring
+        #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+    """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive"""
+    def __init__( self, parseElementList ):
+        self.parseElementTrace = parseElementList
+
+    def __str__( self ):
+        return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+    def __init__(self,p1,p2):
+        self.tup = (p1,p2)
+    def __getitem__(self,i):
+        return self.tup[i]
+    def __repr__(self):
+        return repr(self.tup[0])
+    def setOffset(self,i):
+        self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+    """
+    Structured parse results, to provide multiple means of access to the parsed data:
+       - as a list (C{len(results)})
+       - by list index (C{results[0], results[1]}, etc.)
+       - by attribute (C{results.} - see L{ParserElement.setResultsName})
+
+    Example::
+        integer = Word(nums)
+        date_str = (integer.setResultsName("year") + '/' 
+                        + integer.setResultsName("month") + '/' 
+                        + integer.setResultsName("day"))
+        # equivalent form:
+        # date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+        # parseString returns a ParseResults object
+        result = date_str.parseString("1999/12/31")
+
+        def test(s, fn=repr):
+            print("%s -> %s" % (s, fn(eval(s))))
+        test("list(result)")
+        test("result[0]")
+        test("result['month']")
+        test("result.day")
+        test("'month' in result")
+        test("'minutes' in result")
+        test("result.dump()", str)
+    prints::
+        list(result) -> ['1999', '/', '12', '/', '31']
+        result[0] -> '1999'
+        result['month'] -> '12'
+        result.day -> '31'
+        'month' in result -> True
+        'minutes' in result -> False
+        result.dump() -> ['1999', '/', '12', '/', '31']
+        - day: 31
+        - month: 12
+        - year: 1999
+    """
+    def __new__(cls, toklist=None, name=None, asList=True, modal=True ):
+        if isinstance(toklist, cls):
+            return toklist
+        retobj = object.__new__(cls)
+        retobj.__doinit = True
+        return retobj
+
+    # Performance tuning: we construct a *lot* of these, so keep this
+    # constructor as small and fast as possible
+    def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ):
+        if self.__doinit:
+            self.__doinit = False
+            self.__name = None
+            self.__parent = None
+            self.__accumNames = {}
+            self.__asList = asList
+            self.__modal = modal
+            if toklist is None:
+                toklist = []
+            if isinstance(toklist, list):
+                self.__toklist = toklist[:]
+            elif isinstance(toklist, _generatorType):
+                self.__toklist = list(toklist)
+            else:
+                self.__toklist = [toklist]
+            self.__tokdict = dict()
+
+        if name is not None and name:
+            if not modal:
+                self.__accumNames[name] = 0
+            if isinstance(name,int):
+                name = _ustr(name) # will always return a str, but use _ustr for consistency
+            self.__name = name
+            if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+                if isinstance(toklist,basestring):
+                    toklist = [ toklist ]
+                if asList:
+                    if isinstance(toklist,ParseResults):
+                        self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+                    else:
+                        self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+                    self[name].__name = name
+                else:
+                    try:
+                        self[name] = toklist[0]
+                    except (KeyError,TypeError,IndexError):
+                        self[name] = toklist
+
+    def __getitem__( self, i ):
+        if isinstance( i, (int,slice) ):
+            return self.__toklist[i]
+        else:
+            if i not in self.__accumNames:
+                return self.__tokdict[i][-1][0]
+            else:
+                return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+    def __setitem__( self, k, v, isinstance=isinstance ):
+        if isinstance(v,_ParseResultsWithOffset):
+            self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+            sub = v[0]
+        elif isinstance(k,(int,slice)):
+            self.__toklist[k] = v
+            sub = v
+        else:
+            self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+            sub = v
+        if isinstance(sub,ParseResults):
+            sub.__parent = wkref(self)
+
+    def __delitem__( self, i ):
+        if isinstance(i,(int,slice)):
+            mylen = len( self.__toklist )
+            del self.__toklist[i]
+
+            # convert int to slice
+            if isinstance(i, int):
+                if i < 0:
+                    i += mylen
+                i = slice(i, i+1)
+            # get removed indices
+            removed = list(range(*i.indices(mylen)))
+            removed.reverse()
+            # fixup indices in token dictionary
+            for name,occurrences in self.__tokdict.items():
+                for j in removed:
+                    for k, (value, position) in enumerate(occurrences):
+                        occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+        else:
+            del self.__tokdict[i]
+
+    def __contains__( self, k ):
+        return k in self.__tokdict
+
+    def __len__( self ): return len( self.__toklist )
+    def __bool__(self): return ( not not self.__toklist )
+    __nonzero__ = __bool__
+    def __iter__( self ): return iter( self.__toklist )
+    def __reversed__( self ): return iter( self.__toklist[::-1] )
+    def _iterkeys( self ):
+        if hasattr(self.__tokdict, "iterkeys"):
+            return self.__tokdict.iterkeys()
+        else:
+            return iter(self.__tokdict)
+
+    def _itervalues( self ):
+        return (self[k] for k in self._iterkeys())
+            
+    def _iteritems( self ):
+        return ((k, self[k]) for k in self._iterkeys())
+
+    if PY_3:
+        keys = _iterkeys       
+        """Returns an iterator of all named result keys (Python 3.x only)."""
+
+        values = _itervalues
+        """Returns an iterator of all named result values (Python 3.x only)."""
+
+        items = _iteritems
+        """Returns an iterator of all named result key-value tuples (Python 3.x only)."""
+
+    else:
+        iterkeys = _iterkeys
+        """Returns an iterator of all named result keys (Python 2.x only)."""
+
+        itervalues = _itervalues
+        """Returns an iterator of all named result values (Python 2.x only)."""
+
+        iteritems = _iteritems
+        """Returns an iterator of all named result key-value tuples (Python 2.x only)."""
+
+        def keys( self ):
+            """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.iterkeys())
+
+        def values( self ):
+            """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.itervalues())
+                
+        def items( self ):
+            """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.iteritems())
+
+    def haskeys( self ):
+        """Since keys() returns an iterator, this method is helpful in bypassing
+           code that looks for the existence of any defined results names."""
+        return bool(self.__tokdict)
+        
+    def pop( self, *args, **kwargs):
+        """
+        Removes and returns item at specified index (default=C{last}).
+        Supports both C{list} and C{dict} semantics for C{pop()}. If passed no
+        argument or an integer argument, it will use C{list} semantics
+        and pop tokens from the list of parsed tokens. If passed a 
+        non-integer argument (most likely a string), it will use C{dict}
+        semantics and pop the corresponding value from any defined 
+        results names. A second default return value argument is 
+        supported, just as in C{dict.pop()}.
+
+        Example::
+            def remove_first(tokens):
+                tokens.pop(0)
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+            print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321']
+
+            label = Word(alphas)
+            patt = label("LABEL") + OneOrMore(Word(nums))
+            print(patt.parseString("AAB 123 321").dump())
+
+            # Use pop() in a parse action to remove named result (note that corresponding value is not
+            # removed from list form of results)
+            def remove_LABEL(tokens):
+                tokens.pop("LABEL")
+                return tokens
+            patt.addParseAction(remove_LABEL)
+            print(patt.parseString("AAB 123 321").dump())
+        prints::
+            ['AAB', '123', '321']
+            - LABEL: AAB
+
+            ['AAB', '123', '321']
+        """
+        if not args:
+            args = [-1]
+        for k,v in kwargs.items():
+            if k == 'default':
+                args = (args[0], v)
+            else:
+                raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+        if (isinstance(args[0], int) or 
+                        len(args) == 1 or 
+                        args[0] in self):
+            index = args[0]
+            ret = self[index]
+            del self[index]
+            return ret
+        else:
+            defaultvalue = args[1]
+            return defaultvalue
+
+    def get(self, key, defaultValue=None):
+        """
+        Returns named result matching the given key, or if there is no
+        such name, then returns the given C{defaultValue} or C{None} if no
+        C{defaultValue} is specified.
+
+        Similar to C{dict.get()}.
+        
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            result = date_str.parseString("1999/12/31")
+            print(result.get("year")) # -> '1999'
+            print(result.get("hour", "not specified")) # -> 'not specified'
+            print(result.get("hour")) # -> None
+        """
+        if key in self:
+            return self[key]
+        else:
+            return defaultValue
+
+    def insert( self, index, insStr ):
+        """
+        Inserts new element at location index in the list of parsed tokens.
+        
+        Similar to C{list.insert()}.
+
+        Example::
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+            # use a parse action to insert the parse location in the front of the parsed results
+            def insert_locn(locn, tokens):
+                tokens.insert(0, locn)
+            print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321']
+        """
+        self.__toklist.insert(index, insStr)
+        # fixup indices in token dictionary
+        for name,occurrences in self.__tokdict.items():
+            for k, (value, position) in enumerate(occurrences):
+                occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+    def append( self, item ):
+        """
+        Add single element to end of ParseResults list of elements.
+
+        Example::
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+            
+            # use a parse action to compute the sum of the parsed integers, and add it to the end
+            def append_sum(tokens):
+                tokens.append(sum(map(int, tokens)))
+            print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444]
+        """
+        self.__toklist.append(item)
+
+    def extend( self, itemseq ):
+        """
+        Add sequence of elements to end of ParseResults list of elements.
+
+        Example::
+            patt = OneOrMore(Word(alphas))
+            
+            # use a parse action to append the reverse of the matched strings, to make a palindrome
+            def make_palindrome(tokens):
+                tokens.extend(reversed([t[::-1] for t in tokens]))
+                return ''.join(tokens)
+            print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl'
+        """
+        if isinstance(itemseq, ParseResults):
+            self += itemseq
+        else:
+            self.__toklist.extend(itemseq)
+
+    def clear( self ):
+        """
+        Clear all elements and results names.
+        """
+        del self.__toklist[:]
+        self.__tokdict.clear()
+
+    def __getattr__( self, name ):
+        try:
+            return self[name]
+        except KeyError:
+            return ""
+            
+        if name in self.__tokdict:
+            if name not in self.__accumNames:
+                return self.__tokdict[name][-1][0]
+            else:
+                return ParseResults([ v[0] for v in self.__tokdict[name] ])
+        else:
+            return ""
+
+    def __add__( self, other ):
+        ret = self.copy()
+        ret += other
+        return ret
+
+    def __iadd__( self, other ):
+        if other.__tokdict:
+            offset = len(self.__toklist)
+            addoffset = lambda a: offset if a<0 else a+offset
+            otheritems = other.__tokdict.items()
+            otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+                                for (k,vlist) in otheritems for v in vlist]
+            for k,v in otherdictitems:
+                self[k] = v
+                if isinstance(v[0],ParseResults):
+                    v[0].__parent = wkref(self)
+            
+        self.__toklist += other.__toklist
+        self.__accumNames.update( other.__accumNames )
+        return self
+
+    def __radd__(self, other):
+        if isinstance(other,int) and other == 0:
+            # useful for merging many ParseResults using sum() builtin
+            return self.copy()
+        else:
+            # this may raise a TypeError - so be it
+            return other + self
+        
+    def __repr__( self ):
+        return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+    def __str__( self ):
+        return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+    def _asStringList( self, sep='' ):
+        out = []
+        for item in self.__toklist:
+            if out and sep:
+                out.append(sep)
+            if isinstance( item, ParseResults ):
+                out += item._asStringList()
+            else:
+                out.append( _ustr(item) )
+        return out
+
+    def asList( self ):
+        """
+        Returns the parse results as a nested list of matching tokens, all converted to strings.
+
+        Example::
+            patt = OneOrMore(Word(alphas))
+            result = patt.parseString("sldkj lsdkj sldkj")
+            # even though the result prints in string-like form, it is actually a pyparsing ParseResults
+            print(type(result), result) # ->  ['sldkj', 'lsdkj', 'sldkj']
+            
+            # Use asList() to create an actual list
+            result_list = result.asList()
+            print(type(result_list), result_list) # ->  ['sldkj', 'lsdkj', 'sldkj']
+        """
+        return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+    def asDict( self ):
+        """
+        Returns the named parse results as a nested dictionary.
+
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+            
+            result = date_str.parseString('12/31/1999')
+            print(type(result), repr(result)) # ->  (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]})
+            
+            result_dict = result.asDict()
+            print(type(result_dict), repr(result_dict)) # ->  {'day': '1999', 'year': '12', 'month': '31'}
+
+            # even though a ParseResults supports dict-like access, sometime you just need to have a dict
+            import json
+            print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable
+            print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"}
+        """
+        if PY_3:
+            item_fn = self.items
+        else:
+            item_fn = self.iteritems
+            
+        def toItem(obj):
+            if isinstance(obj, ParseResults):
+                if obj.haskeys():
+                    return obj.asDict()
+                else:
+                    return [toItem(v) for v in obj]
+            else:
+                return obj
+                
+        return dict((k,toItem(v)) for k,v in item_fn())
+
+    def copy( self ):
+        """
+        Returns a new copy of a C{ParseResults} object.
+        """
+        ret = ParseResults( self.__toklist )
+        ret.__tokdict = self.__tokdict.copy()
+        ret.__parent = self.__parent
+        ret.__accumNames.update( self.__accumNames )
+        ret.__name = self.__name
+        return ret
+
+    def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+        """
+        (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.
+        """
+        nl = "\n"
+        out = []
+        namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+                                                            for v in vlist)
+        nextLevelIndent = indent + "  "
+
+        # collapse out indents if formatting is not desired
+        if not formatted:
+            indent = ""
+            nextLevelIndent = ""
+            nl = ""
+
+        selfTag = None
+        if doctag is not None:
+            selfTag = doctag
+        else:
+            if self.__name:
+                selfTag = self.__name
+
+        if not selfTag:
+            if namedItemsOnly:
+                return ""
+            else:
+                selfTag = "ITEM"
+
+        out += [ nl, indent, "<", selfTag, ">" ]
+
+        for i,res in enumerate(self.__toklist):
+            if isinstance(res,ParseResults):
+                if i in namedItems:
+                    out += [ res.asXML(namedItems[i],
+                                        namedItemsOnly and doctag is None,
+                                        nextLevelIndent,
+                                        formatted)]
+                else:
+                    out += [ res.asXML(None,
+                                        namedItemsOnly and doctag is None,
+                                        nextLevelIndent,
+                                        formatted)]
+            else:
+                # individual token, see if there is a name for it
+                resTag = None
+                if i in namedItems:
+                    resTag = namedItems[i]
+                if not resTag:
+                    if namedItemsOnly:
+                        continue
+                    else:
+                        resTag = "ITEM"
+                xmlBodyText = _xml_escape(_ustr(res))
+                out += [ nl, nextLevelIndent, "<", resTag, ">",
+                                                xmlBodyText,
+                                                "" ]
+
+        out += [ nl, indent, "" ]
+        return "".join(out)
+
+    def __lookup(self,sub):
+        for k,vlist in self.__tokdict.items():
+            for v,loc in vlist:
+                if sub is v:
+                    return k
+        return None
+
+    def getName(self):
+        r"""
+        Returns the results name for this token expression. Useful when several 
+        different expressions might match at a particular location.
+
+        Example::
+            integer = Word(nums)
+            ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d")
+            house_number_expr = Suppress('#') + Word(nums, alphanums)
+            user_data = (Group(house_number_expr)("house_number") 
+                        | Group(ssn_expr)("ssn")
+                        | Group(integer)("age"))
+            user_info = OneOrMore(user_data)
+            
+            result = user_info.parseString("22 111-22-3333 #221B")
+            for item in result:
+                print(item.getName(), ':', item[0])
+        prints::
+            age : 22
+            ssn : 111-22-3333
+            house_number : 221B
+        """
+        if self.__name:
+            return self.__name
+        elif self.__parent:
+            par = self.__parent()
+            if par:
+                return par.__lookup(self)
+            else:
+                return None
+        elif (len(self) == 1 and
+               len(self.__tokdict) == 1 and
+               next(iter(self.__tokdict.values()))[0][1] in (0,-1)):
+            return next(iter(self.__tokdict.keys()))
+        else:
+            return None
+
+    def dump(self, indent='', depth=0, full=True):
+        """
+        Diagnostic method for listing out the contents of a C{ParseResults}.
+        Accepts an optional C{indent} argument so that this string can be embedded
+        in a nested display of other data.
+
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+            
+            result = date_str.parseString('12/31/1999')
+            print(result.dump())
+        prints::
+            ['12', '/', '31', '/', '1999']
+            - day: 1999
+            - month: 31
+            - year: 12
+        """
+        out = []
+        NL = '\n'
+        out.append( indent+_ustr(self.asList()) )
+        if full:
+            if self.haskeys():
+                items = sorted((str(k), v) for k,v in self.items())
+                for k,v in items:
+                    if out:
+                        out.append(NL)
+                    out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
+                    if isinstance(v,ParseResults):
+                        if v:
+                            out.append( v.dump(indent,depth+1) )
+                        else:
+                            out.append(_ustr(v))
+                    else:
+                        out.append(repr(v))
+            elif any(isinstance(vv,ParseResults) for vv in self):
+                v = self
+                for i,vv in enumerate(v):
+                    if isinstance(vv,ParseResults):
+                        out.append("\n%s%s[%d]:\n%s%s%s" % (indent,('  '*(depth)),i,indent,('  '*(depth+1)),vv.dump(indent,depth+1) ))
+                    else:
+                        out.append("\n%s%s[%d]:\n%s%s%s" % (indent,('  '*(depth)),i,indent,('  '*(depth+1)),_ustr(vv)))
+            
+        return "".join(out)
+
+    def pprint(self, *args, **kwargs):
+        """
+        Pretty-printer for parsed results as a list, using the C{pprint} module.
+        Accepts additional positional or keyword args as defined for the 
+        C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})
+
+        Example::
+            ident = Word(alphas, alphanums)
+            num = Word(nums)
+            func = Forward()
+            term = ident | num | Group('(' + func + ')')
+            func <<= ident + Group(Optional(delimitedList(term)))
+            result = func.parseString("fna a,b,(fnb c,d,200),100")
+            result.pprint(width=40)
+        prints::
+            ['fna',
+             ['a',
+              'b',
+              ['(', 'fnb', ['c', 'd', '200'], ')'],
+              '100']]
+        """
+        pprint.pprint(self.asList(), *args, **kwargs)
+
+    # add support for pickle protocol
+    def __getstate__(self):
+        return ( self.__toklist,
+                 ( self.__tokdict.copy(),
+                   self.__parent is not None and self.__parent() or None,
+                   self.__accumNames,
+                   self.__name ) )
+
+    def __setstate__(self,state):
+        self.__toklist = state[0]
+        (self.__tokdict,
+         par,
+         inAccumNames,
+         self.__name) = state[1]
+        self.__accumNames = {}
+        self.__accumNames.update(inAccumNames)
+        if par is not None:
+            self.__parent = wkref(par)
+        else:
+            self.__parent = None
+
+    def __getnewargs__(self):
+        return self.__toklist, self.__name, self.__asList, self.__modal
+
+    def __dir__(self):
+        return (dir(type(self)) + list(self.keys()))
+
+MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+    """Returns current column within a string, counting newlines as line separators.
+   The first column is number 1.
+
+   Note: the default parsing behavior is to expand tabs in the input string
+   before starting the parsing process.  See L{I{ParserElement.parseString}} for more information
+   on parsing strings containing C{}s, and suggested methods to maintain a
+   consistent view of the parsed string, the parse location, and line and column
+   positions within the parsed string.
+   """
+    s = strg
+    return 1 if 0} for more information
+   on parsing strings containing C{}s, and suggested methods to maintain a
+   consistent view of the parsed string, the parse location, and line and column
+   positions within the parsed string.
+   """
+    return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+    """Returns the line of text containing loc within a string, counting newlines as line separators.
+       """
+    lastCR = strg.rfind("\n", 0, loc)
+    nextCR = strg.find("\n", loc)
+    if nextCR >= 0:
+        return strg[lastCR+1:nextCR]
+    else:
+        return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+    print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+    print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+    print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+    """'Do-nothing' debug action, to suppress debugging output during parsing."""
+    pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+    #~ if func in singleArgBuiltins:
+        #~ return lambda s,l,t: func(t)
+    #~ limit = 0
+    #~ foundArity = False
+    #~ def wrapper(*args):
+        #~ nonlocal limit,foundArity
+        #~ while 1:
+            #~ try:
+                #~ ret = func(*args[limit:])
+                #~ foundArity = True
+                #~ return ret
+            #~ except TypeError:
+                #~ if limit == maxargs or foundArity:
+                    #~ raise
+                #~ limit += 1
+                #~ continue
+    #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+    if func in singleArgBuiltins:
+        return lambda s,l,t: func(t)
+    limit = [0]
+    foundArity = [False]
+    
+    # traceback return data structure changed in Py3.5 - normalize back to plain tuples
+    if system_version[:2] >= (3,5):
+        def extract_stack(limit=0):
+            # special handling for Python 3.5.0 - extra deep call stack by 1
+            offset = -3 if system_version == (3,5,0) else -2
+            frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+            return [frame_summary[:2]]
+        def extract_tb(tb, limit=0):
+            frames = traceback.extract_tb(tb, limit=limit)
+            frame_summary = frames[-1]
+            return [frame_summary[:2]]
+    else:
+        extract_stack = traceback.extract_stack
+        extract_tb = traceback.extract_tb
+    
+    # synthesize what would be returned by traceback.extract_stack at the call to 
+    # user's parse action 'func', so that we don't incur call penalty at parse time
+    
+    LINE_DIFF = 6
+    # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND 
+    # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
+    this_line = extract_stack(limit=2)[-1]
+    pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF)
+
+    def wrapper(*args):
+        while 1:
+            try:
+                ret = func(*args[limit[0]:])
+                foundArity[0] = True
+                return ret
+            except TypeError:
+                # re-raise TypeErrors if they did not come from our arity testing
+                if foundArity[0]:
+                    raise
+                else:
+                    try:
+                        tb = sys.exc_info()[-1]
+                        if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth:
+                            raise
+                    finally:
+                        del tb
+
+                if limit[0] <= maxargs:
+                    limit[0] += 1
+                    continue
+                raise
+
+    # copy func name to wrapper for sensible debug output
+    func_name = ""
+    try:
+        func_name = getattr(func, '__name__', 
+                            getattr(func, '__class__').__name__)
+    except Exception:
+        func_name = str(func)
+    wrapper.__name__ = func_name
+
+    return wrapper
+
+class ParserElement(object):
+    """Abstract base level parser element class."""
+    DEFAULT_WHITE_CHARS = " \n\t\r"
+    verbose_stacktrace = False
+
+    @staticmethod
+    def setDefaultWhitespaceChars( chars ):
+        r"""
+        Overrides the default whitespace chars
+
+        Example::
+            # default whitespace chars are space,  and newline
+            OneOrMore(Word(alphas)).parseString("abc def\nghi jkl")  # -> ['abc', 'def', 'ghi', 'jkl']
+            
+            # change to just treat newline as significant
+            ParserElement.setDefaultWhitespaceChars(" \t")
+            OneOrMore(Word(alphas)).parseString("abc def\nghi jkl")  # -> ['abc', 'def']
+        """
+        ParserElement.DEFAULT_WHITE_CHARS = chars
+
+    @staticmethod
+    def inlineLiteralsUsing(cls):
+        """
+        Set class to be used for inclusion of string literals into a parser.
+        
+        Example::
+            # default literal class used is Literal
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '/', '12', '/', '31']
+
+
+            # change to Suppress
+            ParserElement.inlineLiteralsUsing(Suppress)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '12', '31']
+        """
+        ParserElement._literalStringClass = cls
+
+    def __init__( self, savelist=False ):
+        self.parseAction = list()
+        self.failAction = None
+        #~ self.name = ""  # don't define self.name, let subclasses try/except upcall
+        self.strRepr = None
+        self.resultsName = None
+        self.saveAsList = savelist
+        self.skipWhitespace = True
+        self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+        self.copyDefaultWhiteChars = True
+        self.mayReturnEmpty = False # used when checking for left-recursion
+        self.keepTabs = False
+        self.ignoreExprs = list()
+        self.debug = False
+        self.streamlined = False
+        self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+        self.errmsg = ""
+        self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+        self.debugActions = ( None, None, None ) #custom debug actions
+        self.re = None
+        self.callPreparse = True # used to avoid redundant calls to preParse
+        self.callDuringTry = False
+
+    def copy( self ):
+        """
+        Make a copy of this C{ParserElement}.  Useful for defining different parse actions
+        for the same parsing pattern, using copies of the original parse element.
+        
+        Example::
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K")
+            integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+            
+            print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M"))
+        prints::
+            [5120, 100, 655360, 268435456]
+        Equivalent form of C{expr.copy()} is just C{expr()}::
+            integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+        """
+        cpy = copy.copy( self )
+        cpy.parseAction = self.parseAction[:]
+        cpy.ignoreExprs = self.ignoreExprs[:]
+        if self.copyDefaultWhiteChars:
+            cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+        return cpy
+
+    def setName( self, name ):
+        """
+        Define name for this expression, makes debugging and exception messages clearer.
+        
+        Example::
+            Word(nums).parseString("ABC")  # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1)
+            Word(nums).setName("integer").parseString("ABC")  # -> Exception: Expected integer (at char 0), (line:1, col:1)
+        """
+        self.name = name
+        self.errmsg = "Expected " + self.name
+        if hasattr(self,"exception"):
+            self.exception.msg = self.errmsg
+        return self
+
+    def setResultsName( self, name, listAllMatches=False ):
+        """
+        Define name for referencing matching tokens as a nested attribute
+        of the returned parse results.
+        NOTE: this returns a *copy* of the original C{ParserElement} object;
+        this is so that the client can define a basic element, such as an
+        integer, and reference it in multiple places with different names.
+
+        You can also set results names using the abbreviated syntax,
+        C{expr("name")} in place of C{expr.setResultsName("name")} - 
+        see L{I{__call__}<__call__>}.
+
+        Example::
+            date_str = (integer.setResultsName("year") + '/' 
+                        + integer.setResultsName("month") + '/' 
+                        + integer.setResultsName("day"))
+
+            # equivalent form:
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+        """
+        newself = self.copy()
+        if name.endswith("*"):
+            name = name[:-1]
+            listAllMatches=True
+        newself.resultsName = name
+        newself.modalResults = not listAllMatches
+        return newself
+
+    def setBreak(self,breakFlag = True):
+        """Method to invoke the Python pdb debugger when this element is
+           about to be parsed. Set C{breakFlag} to True to enable, False to
+           disable.
+        """
+        if breakFlag:
+            _parseMethod = self._parse
+            def breaker(instring, loc, doActions=True, callPreParse=True):
+                import pdb
+                pdb.set_trace()
+                return _parseMethod( instring, loc, doActions, callPreParse )
+            breaker._originalParseMethod = _parseMethod
+            self._parse = breaker
+        else:
+            if hasattr(self._parse,"_originalParseMethod"):
+                self._parse = self._parse._originalParseMethod
+        return self
+
+    def setParseAction( self, *fns, **kwargs ):
+        """
+        Define one or more actions to perform when successfully matching parse element definition.
+        Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+        C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+         - s   = the original string being parsed (see note below)
+         - loc = the location of the matching substring
+         - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+        If the functions in fns modify the tokens, they can return them as the return
+        value from fn, and the modified list of tokens will replace the original.
+        Otherwise, fn does not need to return any value.
+
+        Optional keyword arguments:
+         - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing
+
+        Note: the default parsing behavior is to expand tabs in the input string
+        before starting the parsing process.  See L{I{parseString}} for more information
+        on parsing strings containing C{}s, and suggested methods to maintain a
+        consistent view of the parsed string, the parse location, and line and column
+        positions within the parsed string.
+        
+        Example::
+            integer = Word(nums)
+            date_str = integer + '/' + integer + '/' + integer
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '/', '12', '/', '31']
+
+            # use parse action to convert to ints at parse time
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            date_str = integer + '/' + integer + '/' + integer
+
+            # note that integer fields are now ints, not strings
+            date_str.parseString("1999/12/31")  # -> [1999, '/', 12, '/', 31]
+        """
+        self.parseAction = list(map(_trim_arity, list(fns)))
+        self.callDuringTry = kwargs.get("callDuringTry", False)
+        return self
+
+    def addParseAction( self, *fns, **kwargs ):
+        """
+        Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}.
+        
+        See examples in L{I{copy}}.
+        """
+        self.parseAction += list(map(_trim_arity, list(fns)))
+        self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+        return self
+
+    def addCondition(self, *fns, **kwargs):
+        """Add a boolean predicate function to expression's list of parse actions. See 
+        L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, 
+        functions passed to C{addCondition} need to return boolean success/fail of the condition.
+
+        Optional keyword arguments:
+         - message = define a custom message to be used in the raised exception
+         - fatal   = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException
+         
+        Example::
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            year_int = integer.copy()
+            year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")
+            date_str = year_int + '/' + integer + '/' + integer
+
+            result = date_str.parseString("1999/12/31")  # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1)
+        """
+        msg = kwargs.get("message", "failed user-defined condition")
+        exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException
+        for fn in fns:
+            def pa(s,l,t):
+                if not bool(_trim_arity(fn)(s,l,t)):
+                    raise exc_type(s,l,msg)
+            self.parseAction.append(pa)
+        self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+        return self
+
+    def setFailAction( self, fn ):
+        """Define action to perform if parsing fails at this expression.
+           Fail acton fn is a callable function that takes the arguments
+           C{fn(s,loc,expr,err)} where:
+            - s = string being parsed
+            - loc = location where expression match was attempted and failed
+            - expr = the parse expression that failed
+            - err = the exception thrown
+           The function returns no value.  It may throw C{L{ParseFatalException}}
+           if it is desired to stop parsing immediately."""
+        self.failAction = fn
+        return self
+
+    def _skipIgnorables( self, instring, loc ):
+        exprsFound = True
+        while exprsFound:
+            exprsFound = False
+            for e in self.ignoreExprs:
+                try:
+                    while 1:
+                        loc,dummy = e._parse( instring, loc )
+                        exprsFound = True
+                except ParseException:
+                    pass
+        return loc
+
+    def preParse( self, instring, loc ):
+        if self.ignoreExprs:
+            loc = self._skipIgnorables( instring, loc )
+
+        if self.skipWhitespace:
+            wt = self.whiteChars
+            instrlen = len(instring)
+            while loc < instrlen and instring[loc] in wt:
+                loc += 1
+
+        return loc
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        return loc, []
+
+    def postParse( self, instring, loc, tokenlist ):
+        return tokenlist
+
+    #~ @profile
+    def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+        debugging = ( self.debug ) #and doActions )
+
+        if debugging or self.failAction:
+            #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+            if (self.debugActions[0] ):
+                self.debugActions[0]( instring, loc, self )
+            if callPreParse and self.callPreparse:
+                preloc = self.preParse( instring, loc )
+            else:
+                preloc = loc
+            tokensStart = preloc
+            try:
+                try:
+                    loc,tokens = self.parseImpl( instring, preloc, doActions )
+                except IndexError:
+                    raise ParseException( instring, len(instring), self.errmsg, self )
+            except ParseBaseException as err:
+                #~ print ("Exception raised:", err)
+                if self.debugActions[2]:
+                    self.debugActions[2]( instring, tokensStart, self, err )
+                if self.failAction:
+                    self.failAction( instring, tokensStart, self, err )
+                raise
+        else:
+            if callPreParse and self.callPreparse:
+                preloc = self.preParse( instring, loc )
+            else:
+                preloc = loc
+            tokensStart = preloc
+            if self.mayIndexError or preloc >= len(instring):
+                try:
+                    loc,tokens = self.parseImpl( instring, preloc, doActions )
+                except IndexError:
+                    raise ParseException( instring, len(instring), self.errmsg, self )
+            else:
+                loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+        tokens = self.postParse( instring, loc, tokens )
+
+        retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+        if self.parseAction and (doActions or self.callDuringTry):
+            if debugging:
+                try:
+                    for fn in self.parseAction:
+                        tokens = fn( instring, tokensStart, retTokens )
+                        if tokens is not None:
+                            retTokens = ParseResults( tokens,
+                                                      self.resultsName,
+                                                      asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+                                                      modal=self.modalResults )
+                except ParseBaseException as err:
+                    #~ print "Exception raised in user parse action:", err
+                    if (self.debugActions[2] ):
+                        self.debugActions[2]( instring, tokensStart, self, err )
+                    raise
+            else:
+                for fn in self.parseAction:
+                    tokens = fn( instring, tokensStart, retTokens )
+                    if tokens is not None:
+                        retTokens = ParseResults( tokens,
+                                                  self.resultsName,
+                                                  asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+                                                  modal=self.modalResults )
+        if debugging:
+            #~ print ("Matched",self,"->",retTokens.asList())
+            if (self.debugActions[1] ):
+                self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+        return loc, retTokens
+
+    def tryParse( self, instring, loc ):
+        try:
+            return self._parse( instring, loc, doActions=False )[0]
+        except ParseFatalException:
+            raise ParseException( instring, loc, self.errmsg, self)
+    
+    def canParseNext(self, instring, loc):
+        try:
+            self.tryParse(instring, loc)
+        except (ParseException, IndexError):
+            return False
+        else:
+            return True
+
+    class _UnboundedCache(object):
+        def __init__(self):
+            cache = {}
+            self.not_in_cache = not_in_cache = object()
+
+            def get(self, key):
+                return cache.get(key, not_in_cache)
+
+            def set(self, key, value):
+                cache[key] = value
+
+            def clear(self):
+                cache.clear()
+                
+            def cache_len(self):
+                return len(cache)
+
+            self.get = types.MethodType(get, self)
+            self.set = types.MethodType(set, self)
+            self.clear = types.MethodType(clear, self)
+            self.__len__ = types.MethodType(cache_len, self)
+
+    if _OrderedDict is not None:
+        class _FifoCache(object):
+            def __init__(self, size):
+                self.not_in_cache = not_in_cache = object()
+
+                cache = _OrderedDict()
+
+                def get(self, key):
+                    return cache.get(key, not_in_cache)
+
+                def set(self, key, value):
+                    cache[key] = value
+                    while len(cache) > size:
+                        try:
+                            cache.popitem(False)
+                        except KeyError:
+                            pass
+
+                def clear(self):
+                    cache.clear()
+
+                def cache_len(self):
+                    return len(cache)
+
+                self.get = types.MethodType(get, self)
+                self.set = types.MethodType(set, self)
+                self.clear = types.MethodType(clear, self)
+                self.__len__ = types.MethodType(cache_len, self)
+
+    else:
+        class _FifoCache(object):
+            def __init__(self, size):
+                self.not_in_cache = not_in_cache = object()
+
+                cache = {}
+                key_fifo = collections.deque([], size)
+
+                def get(self, key):
+                    return cache.get(key, not_in_cache)
+
+                def set(self, key, value):
+                    cache[key] = value
+                    while len(key_fifo) > size:
+                        cache.pop(key_fifo.popleft(), None)
+                    key_fifo.append(key)
+
+                def clear(self):
+                    cache.clear()
+                    key_fifo.clear()
+
+                def cache_len(self):
+                    return len(cache)
+
+                self.get = types.MethodType(get, self)
+                self.set = types.MethodType(set, self)
+                self.clear = types.MethodType(clear, self)
+                self.__len__ = types.MethodType(cache_len, self)
+
+    # argument cache for optimizing repeated calls when backtracking through recursive expressions
+    packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
+    packrat_cache_lock = RLock()
+    packrat_cache_stats = [0, 0]
+
+    # this method gets repeatedly called during backtracking with the same arguments -
+    # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+    def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+        HIT, MISS = 0, 1
+        lookup = (self, instring, loc, callPreParse, doActions)
+        with ParserElement.packrat_cache_lock:
+            cache = ParserElement.packrat_cache
+            value = cache.get(lookup)
+            if value is cache.not_in_cache:
+                ParserElement.packrat_cache_stats[MISS] += 1
+                try:
+                    value = self._parseNoCache(instring, loc, doActions, callPreParse)
+                except ParseBaseException as pe:
+                    # cache a copy of the exception, without the traceback
+                    cache.set(lookup, pe.__class__(*pe.args))
+                    raise
+                else:
+                    cache.set(lookup, (value[0], value[1].copy()))
+                    return value
+            else:
+                ParserElement.packrat_cache_stats[HIT] += 1
+                if isinstance(value, Exception):
+                    raise value
+                return (value[0], value[1].copy())
+
+    _parse = _parseNoCache
+
+    @staticmethod
+    def resetCache():
+        ParserElement.packrat_cache.clear()
+        ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats)
+
+    _packratEnabled = False
+    @staticmethod
+    def enablePackrat(cache_size_limit=128):
+        """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+           Repeated parse attempts at the same string location (which happens
+           often in many complex grammars) can immediately return a cached value,
+           instead of re-executing parsing/validating code.  Memoizing is done of
+           both valid results and parsing exceptions.
+           
+           Parameters:
+            - cache_size_limit - (default=C{128}) - if an integer value is provided
+              will limit the size of the packrat cache; if None is passed, then
+              the cache size will be unbounded; if 0 is passed, the cache will
+              be effectively disabled.
+            
+           This speedup may break existing programs that use parse actions that
+           have side-effects.  For this reason, packrat parsing is disabled when
+           you first import pyparsing.  To activate the packrat feature, your
+           program must call the class method C{ParserElement.enablePackrat()}.  If
+           your program uses C{psyco} to "compile as you go", you must call
+           C{enablePackrat} before calling C{psyco.full()}.  If you do not do this,
+           Python will crash.  For best results, call C{enablePackrat()} immediately
+           after importing pyparsing.
+           
+           Example::
+               import pyparsing
+               pyparsing.ParserElement.enablePackrat()
+        """
+        if not ParserElement._packratEnabled:
+            ParserElement._packratEnabled = True
+            if cache_size_limit is None:
+                ParserElement.packrat_cache = ParserElement._UnboundedCache()
+            else:
+                ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit)
+            ParserElement._parse = ParserElement._parseCache
+
+    def parseString( self, instring, parseAll=False ):
+        """
+        Execute the parse expression with the given string.
+        This is the main interface to the client code, once the complete
+        expression has been built.
+
+        If you want the grammar to require that the entire input string be
+        successfully parsed, then set C{parseAll} to True (equivalent to ending
+        the grammar with C{L{StringEnd()}}).
+
+        Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+        in order to report proper column numbers in parse actions.
+        If the input string contains tabs and
+        the grammar uses parse actions that use the C{loc} argument to index into the
+        string being parsed, you can ensure you have a consistent view of the input
+        string by:
+         - calling C{parseWithTabs} on your grammar before calling C{parseString}
+           (see L{I{parseWithTabs}})
+         - define your parse action using the full C{(s,loc,toks)} signature, and
+           reference the input string using the parse action's C{s} argument
+         - explictly expand the tabs in your input string before calling
+           C{parseString}
+        
+        Example::
+            Word('a').parseString('aaaaabaaa')  # -> ['aaaaa']
+            Word('a').parseString('aaaaabaaa', parseAll=True)  # -> Exception: Expected end of text
+        """
+        ParserElement.resetCache()
+        if not self.streamlined:
+            self.streamline()
+            #~ self.saveAsList = True
+        for e in self.ignoreExprs:
+            e.streamline()
+        if not self.keepTabs:
+            instring = instring.expandtabs()
+        try:
+            loc, tokens = self._parse( instring, 0 )
+            if parseAll:
+                loc = self.preParse( instring, loc )
+                se = Empty() + StringEnd()
+                se._parse( instring, loc )
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+        else:
+            return tokens
+
+    def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+        """
+        Scan the input string for expression matches.  Each match will return the
+        matching tokens, start location, and end location.  May be called with optional
+        C{maxMatches} argument, to clip scanning after 'n' matches are found.  If
+        C{overlap} is specified, then overlapping matches will be reported.
+
+        Note that the start and end locations are reported relative to the string
+        being parsed.  See L{I{parseString}} for more information on parsing
+        strings with embedded tabs.
+
+        Example::
+            source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
+            print(source)
+            for tokens,start,end in Word(alphas).scanString(source):
+                print(' '*start + '^'*(end-start))
+                print(' '*start + tokens[0])
+        
+        prints::
+        
+            sldjf123lsdjjkf345sldkjf879lkjsfd987
+            ^^^^^
+            sldjf
+                    ^^^^^^^
+                    lsdjjkf
+                              ^^^^^^
+                              sldkjf
+                                       ^^^^^^
+                                       lkjsfd
+        """
+        if not self.streamlined:
+            self.streamline()
+        for e in self.ignoreExprs:
+            e.streamline()
+
+        if not self.keepTabs:
+            instring = _ustr(instring).expandtabs()
+        instrlen = len(instring)
+        loc = 0
+        preparseFn = self.preParse
+        parseFn = self._parse
+        ParserElement.resetCache()
+        matches = 0
+        try:
+            while loc <= instrlen and matches < maxMatches:
+                try:
+                    preloc = preparseFn( instring, loc )
+                    nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+                except ParseException:
+                    loc = preloc+1
+                else:
+                    if nextLoc > loc:
+                        matches += 1
+                        yield tokens, preloc, nextLoc
+                        if overlap:
+                            nextloc = preparseFn( instring, loc )
+                            if nextloc > loc:
+                                loc = nextLoc
+                            else:
+                                loc += 1
+                        else:
+                            loc = nextLoc
+                    else:
+                        loc = preloc+1
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def transformString( self, instring ):
+        """
+        Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+        be returned from a parse action.  To use C{transformString}, define a grammar and
+        attach a parse action to it that modifies the returned token list.
+        Invoking C{transformString()} on a target string will then scan for matches,
+        and replace the matched text patterns according to the logic in the parse
+        action.  C{transformString()} returns the resulting transformed string.
+        
+        Example::
+            wd = Word(alphas)
+            wd.setParseAction(lambda toks: toks[0].title())
+            
+            print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york."))
+        Prints::
+            Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
+        """
+        out = []
+        lastE = 0
+        # force preservation of s, to minimize unwanted transformation of string, and to
+        # keep string locs straight between transformString and scanString
+        self.keepTabs = True
+        try:
+            for t,s,e in self.scanString( instring ):
+                out.append( instring[lastE:s] )
+                if t:
+                    if isinstance(t,ParseResults):
+                        out += t.asList()
+                    elif isinstance(t,list):
+                        out += t
+                    else:
+                        out.append(t)
+                lastE = e
+            out.append(instring[lastE:])
+            out = [o for o in out if o]
+            return "".join(map(_ustr,_flatten(out)))
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def searchString( self, instring, maxMatches=_MAX_INT ):
+        """
+        Another extension to C{L{scanString}}, simplifying the access to the tokens found
+        to match the given parse expression.  May be called with optional
+        C{maxMatches} argument, to clip searching after 'n' matches are found.
+        
+        Example::
+            # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters
+            cap_word = Word(alphas.upper(), alphas.lower())
+            
+            print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+
+            # the sum() builtin can be used to merge results into a single ParseResults object
+            print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")))
+        prints::
+            [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
+            ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
+        """
+        try:
+            return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False):
+        """
+        Generator method to split a string using the given expression as a separator.
+        May be called with optional C{maxsplit} argument, to limit the number of splits;
+        and the optional C{includeSeparators} argument (default=C{False}), if the separating
+        matching text should be included in the split results.
+        
+        Example::        
+            punc = oneOf(list(".,;:/-!?"))
+            print(list(punc.split("This, this?, this sentence, is badly punctuated!")))
+        prints::
+            ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
+        """
+        splits = 0
+        last = 0
+        for t,s,e in self.scanString(instring, maxMatches=maxsplit):
+            yield instring[last:s]
+            if includeSeparators:
+                yield t[0]
+            last = e
+        yield instring[last:]
+
+    def __add__(self, other ):
+        """
+        Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement
+        converts them to L{Literal}s by default.
+        
+        Example::
+            greet = Word(alphas) + "," + Word(alphas) + "!"
+            hello = "Hello, World!"
+            print (hello, "->", greet.parseString(hello))
+        Prints::
+            Hello, World! -> ['Hello', ',', 'World', '!']
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return And( [ self, other ] )
+
+    def __radd__(self, other ):
+        """
+        Implementation of + operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other + self
+
+    def __sub__(self, other):
+        """
+        Implementation of - operator, returns C{L{And}} with error stop
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return self + And._ErrorStop() + other
+
+    def __rsub__(self, other ):
+        """
+        Implementation of - operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other - self
+
+    def __mul__(self,other):
+        """
+        Implementation of * operator, allows use of C{expr * 3} in place of
+        C{expr + expr + expr}.  Expressions may also me multiplied by a 2-integer
+        tuple, similar to C{{min,max}} multipliers in regular expressions.  Tuples
+        may also include C{None} as in:
+         - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+              to C{expr*n + L{ZeroOrMore}(expr)}
+              (read as "at least n instances of C{expr}")
+         - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+              (read as "0 to n instances of C{expr}")
+         - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+         - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+        Note that C{expr*(None,n)} does not raise an exception if
+        more than n exprs exist in the input stream; that is,
+        C{expr*(None,n)} does not enforce a maximum number of expr
+        occurrences.  If this behavior is desired, then write
+        C{expr*(None,n) + ~expr}
+        """
+        if isinstance(other,int):
+            minElements, optElements = other,0
+        elif isinstance(other,tuple):
+            other = (other + (None, None))[:2]
+            if other[0] is None:
+                other = (0, other[1])
+            if isinstance(other[0],int) and other[1] is None:
+                if other[0] == 0:
+                    return ZeroOrMore(self)
+                if other[0] == 1:
+                    return OneOrMore(self)
+                else:
+                    return self*other[0] + ZeroOrMore(self)
+            elif isinstance(other[0],int) and isinstance(other[1],int):
+                minElements, optElements = other
+                optElements -= minElements
+            else:
+                raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+        else:
+            raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+        if minElements < 0:
+            raise ValueError("cannot multiply ParserElement by negative value")
+        if optElements < 0:
+            raise ValueError("second tuple value must be greater or equal to first tuple value")
+        if minElements == optElements == 0:
+            raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+        if (optElements):
+            def makeOptionalList(n):
+                if n>1:
+                    return Optional(self + makeOptionalList(n-1))
+                else:
+                    return Optional(self)
+            if minElements:
+                if minElements == 1:
+                    ret = self + makeOptionalList(optElements)
+                else:
+                    ret = And([self]*minElements) + makeOptionalList(optElements)
+            else:
+                ret = makeOptionalList(optElements)
+        else:
+            if minElements == 1:
+                ret = self
+            else:
+                ret = And([self]*minElements)
+        return ret
+
+    def __rmul__(self, other):
+        return self.__mul__(other)
+
+    def __or__(self, other ):
+        """
+        Implementation of | operator - returns C{L{MatchFirst}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return MatchFirst( [ self, other ] )
+
+    def __ror__(self, other ):
+        """
+        Implementation of | operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other | self
+
+    def __xor__(self, other ):
+        """
+        Implementation of ^ operator - returns C{L{Or}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return Or( [ self, other ] )
+
+    def __rxor__(self, other ):
+        """
+        Implementation of ^ operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other ^ self
+
+    def __and__(self, other ):
+        """
+        Implementation of & operator - returns C{L{Each}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return Each( [ self, other ] )
+
+    def __rand__(self, other ):
+        """
+        Implementation of & operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other & self
+
+    def __invert__( self ):
+        """
+        Implementation of ~ operator - returns C{L{NotAny}}
+        """
+        return NotAny( self )
+
+    def __call__(self, name=None):
+        """
+        Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}.
+        
+        If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+        passed as C{True}.
+           
+        If C{name} is omitted, same as calling C{L{copy}}.
+
+        Example::
+            # these are equivalent
+            userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+            userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")             
+        """
+        if name is not None:
+            return self.setResultsName(name)
+        else:
+            return self.copy()
+
+    def suppress( self ):
+        """
+        Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+        cluttering up returned output.
+        """
+        return Suppress( self )
+
+    def leaveWhitespace( self ):
+        """
+        Disables the skipping of whitespace before matching the characters in the
+        C{ParserElement}'s defined pattern.  This is normally only used internally by
+        the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+        """
+        self.skipWhitespace = False
+        return self
+
+    def setWhitespaceChars( self, chars ):
+        """
+        Overrides the default whitespace chars
+        """
+        self.skipWhitespace = True
+        self.whiteChars = chars
+        self.copyDefaultWhiteChars = False
+        return self
+
+    def parseWithTabs( self ):
+        """
+        Overrides default behavior to expand C{}s to spaces before parsing the input string.
+        Must be called before C{parseString} when the input grammar contains elements that
+        match C{} characters.
+        """
+        self.keepTabs = True
+        return self
+
+    def ignore( self, other ):
+        """
+        Define expression to be ignored (e.g., comments) while doing pattern
+        matching; may be called repeatedly, to define multiple comment or other
+        ignorable patterns.
+        
+        Example::
+            patt = OneOrMore(Word(alphas))
+            patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj']
+            
+            patt.ignore(cStyleComment)
+            patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd']
+        """
+        if isinstance(other, basestring):
+            other = Suppress(other)
+
+        if isinstance( other, Suppress ):
+            if other not in self.ignoreExprs:
+                self.ignoreExprs.append(other)
+        else:
+            self.ignoreExprs.append( Suppress( other.copy() ) )
+        return self
+
+    def setDebugActions( self, startAction, successAction, exceptionAction ):
+        """
+        Enable display of debugging messages while doing pattern matching.
+        """
+        self.debugActions = (startAction or _defaultStartDebugAction,
+                             successAction or _defaultSuccessDebugAction,
+                             exceptionAction or _defaultExceptionDebugAction)
+        self.debug = True
+        return self
+
+    def setDebug( self, flag=True ):
+        """
+        Enable display of debugging messages while doing pattern matching.
+        Set C{flag} to True to enable, False to disable.
+
+        Example::
+            wd = Word(alphas).setName("alphaword")
+            integer = Word(nums).setName("numword")
+            term = wd | integer
+            
+            # turn on debugging for wd
+            wd.setDebug()
+
+            OneOrMore(term).parseString("abc 123 xyz 890")
+        
+        prints::
+            Match alphaword at loc 0(1,1)
+            Matched alphaword -> ['abc']
+            Match alphaword at loc 3(1,4)
+            Exception raised:Expected alphaword (at char 4), (line:1, col:5)
+            Match alphaword at loc 7(1,8)
+            Matched alphaword -> ['xyz']
+            Match alphaword at loc 11(1,12)
+            Exception raised:Expected alphaword (at char 12), (line:1, col:13)
+            Match alphaword at loc 15(1,16)
+            Exception raised:Expected alphaword (at char 15), (line:1, col:16)
+
+        The output shown is that produced by the default debug actions - custom debug actions can be
+        specified using L{setDebugActions}. Prior to attempting
+        to match the C{wd} expression, the debugging message C{"Match  at loc (,)"}
+        is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"}
+        message is shown. Also note the use of L{setName} to assign a human-readable name to the expression,
+        which makes debugging and exception messages easier to understand - for instance, the default
+        name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}.
+        """
+        if flag:
+            self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+        else:
+            self.debug = False
+        return self
+
+    def __str__( self ):
+        return self.name
+
+    def __repr__( self ):
+        return _ustr(self)
+
+    def streamline( self ):
+        self.streamlined = True
+        self.strRepr = None
+        return self
+
+    def checkRecursion( self, parseElementList ):
+        pass
+
+    def validate( self, validateTrace=[] ):
+        """
+        Check defined expressions for valid structure, check for infinite recursive definitions.
+        """
+        self.checkRecursion( [] )
+
+    def parseFile( self, file_or_filename, parseAll=False ):
+        """
+        Execute the parse expression on the given file or filename.
+        If a filename is specified (instead of a file object),
+        the entire file is opened, read, and closed before parsing.
+        """
+        try:
+            file_contents = file_or_filename.read()
+        except AttributeError:
+            with open(file_or_filename, "r") as f:
+                file_contents = f.read()
+        try:
+            return self.parseString(file_contents, parseAll)
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def __eq__(self,other):
+        if isinstance(other, ParserElement):
+            return self is other or vars(self) == vars(other)
+        elif isinstance(other, basestring):
+            return self.matches(other)
+        else:
+            return super(ParserElement,self)==other
+
+    def __ne__(self,other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(id(self))
+
+    def __req__(self,other):
+        return self == other
+
+    def __rne__(self,other):
+        return not (self == other)
+
+    def matches(self, testString, parseAll=True):
+        """
+        Method for quick testing of a parser against a test string. Good for simple 
+        inline microtests of sub expressions while building up larger parser.
+           
+        Parameters:
+         - testString - to test against this expression for a match
+         - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+            
+        Example::
+            expr = Word(nums)
+            assert expr.matches("100")
+        """
+        try:
+            self.parseString(_ustr(testString), parseAll=parseAll)
+            return True
+        except ParseBaseException:
+            return False
+                
+    def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False):
+        """
+        Execute the parse expression on a series of test strings, showing each
+        test, the parsed results or where the parse failed. Quick and easy way to
+        run a parse expression against a list of sample strings.
+           
+        Parameters:
+         - tests - a list of separate test strings, or a multiline string of test strings
+         - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests           
+         - comment - (default=C{'#'}) - expression for indicating embedded comments in the test 
+              string; pass None to disable comment filtering
+         - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline;
+              if False, only dump nested list
+         - printResults - (default=C{True}) prints test output to stdout
+         - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing
+
+        Returns: a (success, results) tuple, where success indicates that all tests succeeded
+        (or failed if C{failureTests} is True), and the results contain a list of lines of each 
+        test's output
+        
+        Example::
+            number_expr = pyparsing_common.number.copy()
+
+            result = number_expr.runTests('''
+                # unsigned integer
+                100
+                # negative integer
+                -100
+                # float with scientific notation
+                6.02e23
+                # integer with scientific notation
+                1e-12
+                ''')
+            print("Success" if result[0] else "Failed!")
+
+            result = number_expr.runTests('''
+                # stray character
+                100Z
+                # missing leading digit before '.'
+                -.100
+                # too many '.'
+                3.14.159
+                ''', failureTests=True)
+            print("Success" if result[0] else "Failed!")
+        prints::
+            # unsigned integer
+            100
+            [100]
+
+            # negative integer
+            -100
+            [-100]
+
+            # float with scientific notation
+            6.02e23
+            [6.02e+23]
+
+            # integer with scientific notation
+            1e-12
+            [1e-12]
+
+            Success
+            
+            # stray character
+            100Z
+               ^
+            FAIL: Expected end of text (at char 3), (line:1, col:4)
+
+            # missing leading digit before '.'
+            -.100
+            ^
+            FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1)
+
+            # too many '.'
+            3.14.159
+                ^
+            FAIL: Expected end of text (at char 4), (line:1, col:5)
+
+            Success
+
+        Each test string must be on a single line. If you want to test a string that spans multiple
+        lines, create a test like this::
+
+            expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines")
+        
+        (Note that this is a raw string literal, you must include the leading 'r'.)
+        """
+        if isinstance(tests, basestring):
+            tests = list(map(str.strip, tests.rstrip().splitlines()))
+        if isinstance(comment, basestring):
+            comment = Literal(comment)
+        allResults = []
+        comments = []
+        success = True
+        for t in tests:
+            if comment is not None and comment.matches(t, False) or comments and not t:
+                comments.append(t)
+                continue
+            if not t:
+                continue
+            out = ['\n'.join(comments), t]
+            comments = []
+            try:
+                t = t.replace(r'\n','\n')
+                result = self.parseString(t, parseAll=parseAll)
+                out.append(result.dump(full=fullDump))
+                success = success and not failureTests
+            except ParseBaseException as pe:
+                fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else ""
+                if '\n' in t:
+                    out.append(line(pe.loc, t))
+                    out.append(' '*(col(pe.loc,t)-1) + '^' + fatal)
+                else:
+                    out.append(' '*pe.loc + '^' + fatal)
+                out.append("FAIL: " + str(pe))
+                success = success and failureTests
+                result = pe
+            except Exception as exc:
+                out.append("FAIL-EXCEPTION: " + str(exc))
+                success = success and failureTests
+                result = exc
+
+            if printResults:
+                if fullDump:
+                    out.append('')
+                print('\n'.join(out))
+
+            allResults.append((t, result))
+        
+        return success, allResults
+
+        
+class Token(ParserElement):
+    """
+    Abstract C{ParserElement} subclass, for defining atomic matching patterns.
+    """
+    def __init__( self ):
+        super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+    """
+    An empty token, will always match.
+    """
+    def __init__( self ):
+        super(Empty,self).__init__()
+        self.name = "Empty"
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+
+
+class NoMatch(Token):
+    """
+    A token that will never match.
+    """
+    def __init__( self ):
+        super(NoMatch,self).__init__()
+        self.name = "NoMatch"
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+        self.errmsg = "Unmatchable token"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+    """
+    Token to exactly match a specified string.
+    
+    Example::
+        Literal('blah').parseString('blah')  # -> ['blah']
+        Literal('blah').parseString('blahfooblah')  # -> ['blah']
+        Literal('blah').parseString('bla')  # -> Exception: Expected "blah"
+    
+    For case-insensitive matching, use L{CaselessLiteral}.
+    
+    For keyword matching (force word break before and after the matched string),
+    use L{Keyword} or L{CaselessKeyword}.
+    """
+    def __init__( self, matchString ):
+        super(Literal,self).__init__()
+        self.match = matchString
+        self.matchLen = len(matchString)
+        try:
+            self.firstMatchChar = matchString[0]
+        except IndexError:
+            warnings.warn("null string passed to Literal; use Empty() instead",
+                            SyntaxWarning, stacklevel=2)
+            self.__class__ = Empty
+        self.name = '"%s"' % _ustr(self.match)
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = False
+        self.mayIndexError = False
+
+    # Performance tuning: this routine gets called a *lot*
+    # if this is a single character match string  and the first character matches,
+    # short-circuit as quickly as possible, and avoid calling startswith
+    #~ @profile
+    def parseImpl( self, instring, loc, doActions=True ):
+        if (instring[loc] == self.firstMatchChar and
+            (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+            return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement._literalStringClass = Literal
+
+class Keyword(Token):
+    """
+    Token to exactly match a specified string as a keyword, that is, it must be
+    immediately followed by a non-keyword character.  Compare with C{L{Literal}}:
+     - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+     - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+    Accepts two optional constructor arguments in addition to the keyword string:
+     - C{identChars} is a string of characters that would be valid identifier characters,
+          defaulting to all alphanumerics + "_" and "$"
+     - C{caseless} allows case-insensitive matching, default is C{False}.
+       
+    Example::
+        Keyword("start").parseString("start")  # -> ['start']
+        Keyword("start").parseString("starting")  # -> Exception
+
+    For case-insensitive matching, use L{CaselessKeyword}.
+    """
+    DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+    def __init__( self, matchString, identChars=None, caseless=False ):
+        super(Keyword,self).__init__()
+        if identChars is None:
+            identChars = Keyword.DEFAULT_KEYWORD_CHARS
+        self.match = matchString
+        self.matchLen = len(matchString)
+        try:
+            self.firstMatchChar = matchString[0]
+        except IndexError:
+            warnings.warn("null string passed to Keyword; use Empty() instead",
+                            SyntaxWarning, stacklevel=2)
+        self.name = '"%s"' % self.match
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = False
+        self.mayIndexError = False
+        self.caseless = caseless
+        if caseless:
+            self.caselessmatch = matchString.upper()
+            identChars = identChars.upper()
+        self.identChars = set(identChars)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.caseless:
+            if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+                 (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+                return loc+self.matchLen, self.match
+        else:
+            if (instring[loc] == self.firstMatchChar and
+                (self.matchLen==1 or instring.startswith(self.match,loc)) and
+                (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+                (loc == 0 or instring[loc-1] not in self.identChars) ):
+                return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+
+    def copy(self):
+        c = super(Keyword,self).copy()
+        c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+        return c
+
+    @staticmethod
+    def setDefaultKeywordChars( chars ):
+        """Overrides the default Keyword chars
+        """
+        Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+    """
+    Token to match a specified string, ignoring case of letters.
+    Note: the matched results will always be in the case of the given
+    match string, NOT the case of the input text.
+
+    Example::
+        OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD']
+        
+    (Contrast with example for L{CaselessKeyword}.)
+    """
+    def __init__( self, matchString ):
+        super(CaselessLiteral,self).__init__( matchString.upper() )
+        # Preserve the defining literal.
+        self.returnString = matchString
+        self.name = "'%s'" % self.returnString
+        self.errmsg = "Expected " + self.name
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if instring[ loc:loc+self.matchLen ].upper() == self.match:
+            return loc+self.matchLen, self.returnString
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+    """
+    Caseless version of L{Keyword}.
+
+    Example::
+        OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD']
+        
+    (Contrast with example for L{CaselessLiteral}.)
+    """
+    def __init__( self, matchString, identChars=None ):
+        super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+             (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+            return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class CloseMatch(Token):
+    """
+    A variation on L{Literal} which matches "close" matches, that is, 
+    strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters:
+     - C{match_string} - string to be matched
+     - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match
+    
+    The results from a successful parse will contain the matched text from the input string and the following named results:
+     - C{mismatches} - a list of the positions within the match_string where mismatches were found
+     - C{original} - the original match_string used to compare against the input string
+    
+    If C{mismatches} is an empty list, then the match was an exact match.
+    
+    Example::
+        patt = CloseMatch("ATCATCGAATGGA")
+        patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']})
+        patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1)
+
+        # exact match
+        patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']})
+
+        # close match allowing up to 2 mismatches
+        patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2)
+        patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']})
+    """
+    def __init__(self, match_string, maxMismatches=1):
+        super(CloseMatch,self).__init__()
+        self.name = match_string
+        self.match_string = match_string
+        self.maxMismatches = maxMismatches
+        self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches)
+        self.mayIndexError = False
+        self.mayReturnEmpty = False
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        start = loc
+        instrlen = len(instring)
+        maxloc = start + len(self.match_string)
+
+        if maxloc <= instrlen:
+            match_string = self.match_string
+            match_stringloc = 0
+            mismatches = []
+            maxMismatches = self.maxMismatches
+
+            for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)):
+                src,mat = s_m
+                if src != mat:
+                    mismatches.append(match_stringloc)
+                    if len(mismatches) > maxMismatches:
+                        break
+            else:
+                loc = match_stringloc + 1
+                results = ParseResults([instring[start:loc]])
+                results['original'] = self.match_string
+                results['mismatches'] = mismatches
+                return loc, results
+
+        raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Word(Token):
+    """
+    Token for matching words composed of allowed character sets.
+    Defined with string containing all allowed initial characters,
+    an optional string containing allowed body characters (if omitted,
+    defaults to the initial character set), and an optional minimum,
+    maximum, and/or exact length.  The default value for C{min} is 1 (a
+    minimum value < 1 is not valid); the default values for C{max} and C{exact}
+    are 0, meaning no maximum or exact length restriction. An optional
+    C{excludeChars} parameter can list characters that might be found in 
+    the input C{bodyChars} string; useful to define a word of all printables
+    except for one or two characters, for instance.
+    
+    L{srange} is useful for defining custom character set strings for defining 
+    C{Word} expressions, using range notation from regular expression character sets.
+    
+    A common mistake is to use C{Word} to match a specific literal string, as in 
+    C{Word("Address")}. Remember that C{Word} uses the string argument to define
+    I{sets} of matchable characters. This expression would match "Add", "AAA",
+    "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'.
+    To match an exact literal string, use L{Literal} or L{Keyword}.
+
+    pyparsing includes helper strings for building Words:
+     - L{alphas}
+     - L{nums}
+     - L{alphanums}
+     - L{hexnums}
+     - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.)
+     - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.)
+     - L{printables} (any non-whitespace character)
+
+    Example::
+        # a word composed of digits
+        integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9"))
+        
+        # a word with a leading capital, and zero or more lowercase
+        capital_word = Word(alphas.upper(), alphas.lower())
+
+        # hostnames are alphanumeric, with leading alpha, and '-'
+        hostname = Word(alphas, alphanums+'-')
+        
+        # roman numeral (not a strict parser, accepts invalid mix of characters)
+        roman = Word("IVXLCDM")
+        
+        # any string of non-whitespace characters, except for ','
+        csv_value = Word(printables, excludeChars=",")
+    """
+    def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+        super(Word,self).__init__()
+        if excludeChars:
+            initChars = ''.join(c for c in initChars if c not in excludeChars)
+            if bodyChars:
+                bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+        self.initCharsOrig = initChars
+        self.initChars = set(initChars)
+        if bodyChars :
+            self.bodyCharsOrig = bodyChars
+            self.bodyChars = set(bodyChars)
+        else:
+            self.bodyCharsOrig = initChars
+            self.bodyChars = set(initChars)
+
+        self.maxSpecified = max > 0
+
+        if min < 1:
+            raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.asKeyword = asKeyword
+
+        if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+            if self.bodyCharsOrig == self.initCharsOrig:
+                self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+            elif len(self.initCharsOrig) == 1:
+                self.reString = "%s[%s]*" % \
+                                      (re.escape(self.initCharsOrig),
+                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
+            else:
+                self.reString = "[%s][%s]*" % \
+                                      (_escapeRegexRangeChars(self.initCharsOrig),
+                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
+            if self.asKeyword:
+                self.reString = r"\b"+self.reString+r"\b"
+            try:
+                self.re = re.compile( self.reString )
+            except Exception:
+                self.re = None
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.re:
+            result = self.re.match(instring,loc)
+            if not result:
+                raise ParseException(instring, loc, self.errmsg, self)
+
+            loc = result.end()
+            return loc, result.group()
+
+        if not(instring[ loc ] in self.initChars):
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        start = loc
+        loc += 1
+        instrlen = len(instring)
+        bodychars = self.bodyChars
+        maxloc = start + self.maxLen
+        maxloc = min( maxloc, instrlen )
+        while loc < maxloc and instring[loc] in bodychars:
+            loc += 1
+
+        throwException = False
+        if loc - start < self.minLen:
+            throwException = True
+        if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+            throwException = True
+        if self.asKeyword:
+            if (start>0 and instring[start-1] in bodychars) or (loc4:
+                    return s[:4]+"..."
+                else:
+                    return s
+
+            if ( self.initCharsOrig != self.bodyCharsOrig ):
+                self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+            else:
+                self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+        return self.strRepr
+
+
+class Regex(Token):
+    r"""
+    Token for matching strings that match a given regular expression.
+    Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+    If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as 
+    named parse results.
+
+    Example::
+        realnum = Regex(r"[+-]?\d+\.\d*")
+        date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)')
+        # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
+        roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
+    """
+    compiledREtype = type(re.compile("[A-Z]"))
+    def __init__( self, pattern, flags=0):
+        """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+        super(Regex,self).__init__()
+
+        if isinstance(pattern, basestring):
+            if not pattern:
+                warnings.warn("null string passed to Regex; use Empty() instead",
+                        SyntaxWarning, stacklevel=2)
+
+            self.pattern = pattern
+            self.flags = flags
+
+            try:
+                self.re = re.compile(self.pattern, self.flags)
+                self.reString = self.pattern
+            except sre_constants.error:
+                warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+                    SyntaxWarning, stacklevel=2)
+                raise
+
+        elif isinstance(pattern, Regex.compiledREtype):
+            self.re = pattern
+            self.pattern = \
+            self.reString = str(pattern)
+            self.flags = flags
+            
+        else:
+            raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        result = self.re.match(instring,loc)
+        if not result:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        loc = result.end()
+        d = result.groupdict()
+        ret = ParseResults(result.group())
+        if d:
+            for k in d:
+                ret[k] = d[k]
+        return loc,ret
+
+    def __str__( self ):
+        try:
+            return super(Regex,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+        return self.strRepr
+
+
+class QuotedString(Token):
+    r"""
+    Token for matching strings that are delimited by quoting characters.
+    
+    Defined with the following parameters:
+        - quoteChar - string of one or more characters defining the quote delimiting string
+        - escChar - character to escape quotes, typically backslash (default=C{None})
+        - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None})
+        - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+        - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+        - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+        - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True})
+
+    Example::
+        qs = QuotedString('"')
+        print(qs.searchString('lsjdf "This is the quote" sldjf'))
+        complex_qs = QuotedString('{{', endQuoteChar='}}')
+        print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf'))
+        sql_qs = QuotedString('"', escQuote='""')
+        print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
+    prints::
+        [['This is the quote']]
+        [['This is the "quote"']]
+        [['This is the quote with "embedded" quotes']]
+    """
+    def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True):
+        super(QuotedString,self).__init__()
+
+        # remove white space from quote chars - wont work anyway
+        quoteChar = quoteChar.strip()
+        if not quoteChar:
+            warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+            raise SyntaxError()
+
+        if endQuoteChar is None:
+            endQuoteChar = quoteChar
+        else:
+            endQuoteChar = endQuoteChar.strip()
+            if not endQuoteChar:
+                warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+                raise SyntaxError()
+
+        self.quoteChar = quoteChar
+        self.quoteCharLen = len(quoteChar)
+        self.firstQuoteChar = quoteChar[0]
+        self.endQuoteChar = endQuoteChar
+        self.endQuoteCharLen = len(endQuoteChar)
+        self.escChar = escChar
+        self.escQuote = escQuote
+        self.unquoteResults = unquoteResults
+        self.convertWhitespaceEscapes = convertWhitespaceEscapes
+
+        if multiline:
+            self.flags = re.MULTILINE | re.DOTALL
+            self.pattern = r'%s(?:[^%s%s]' % \
+                ( re.escape(self.quoteChar),
+                  _escapeRegexRangeChars(self.endQuoteChar[0]),
+                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+        else:
+            self.flags = 0
+            self.pattern = r'%s(?:[^%s\n\r%s]' % \
+                ( re.escape(self.quoteChar),
+                  _escapeRegexRangeChars(self.endQuoteChar[0]),
+                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+        if len(self.endQuoteChar) > 1:
+            self.pattern += (
+                '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+                                               _escapeRegexRangeChars(self.endQuoteChar[i]))
+                                    for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+                )
+        if escQuote:
+            self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+        if escChar:
+            self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+            self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+        self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+        try:
+            self.re = re.compile(self.pattern, self.flags)
+            self.reString = self.pattern
+        except sre_constants.error:
+            warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+                SyntaxWarning, stacklevel=2)
+            raise
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+        if not result:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        loc = result.end()
+        ret = result.group()
+
+        if self.unquoteResults:
+
+            # strip off quotes
+            ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+            if isinstance(ret,basestring):
+                # replace escaped whitespace
+                if '\\' in ret and self.convertWhitespaceEscapes:
+                    ws_map = {
+                        r'\t' : '\t',
+                        r'\n' : '\n',
+                        r'\f' : '\f',
+                        r'\r' : '\r',
+                    }
+                    for wslit,wschar in ws_map.items():
+                        ret = ret.replace(wslit, wschar)
+
+                # replace escaped characters
+                if self.escChar:
+                    ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret)
+
+                # replace escaped quotes
+                if self.escQuote:
+                    ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+        return loc, ret
+
+    def __str__( self ):
+        try:
+            return super(QuotedString,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+        return self.strRepr
+
+
+class CharsNotIn(Token):
+    """
+    Token for matching words composed of characters I{not} in a given set (will
+    include whitespace in matched characters if not listed in the provided exclusion set - see example).
+    Defined with string containing all disallowed characters, and an optional
+    minimum, maximum, and/or exact length.  The default value for C{min} is 1 (a
+    minimum value < 1 is not valid); the default values for C{max} and C{exact}
+    are 0, meaning no maximum or exact length restriction.
+
+    Example::
+        # define a comma-separated-value as anything that is not a ','
+        csv_value = CharsNotIn(',')
+        print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213"))
+    prints::
+        ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
+    """
+    def __init__( self, notChars, min=1, max=0, exact=0 ):
+        super(CharsNotIn,self).__init__()
+        self.skipWhitespace = False
+        self.notChars = notChars
+
+        if min < 1:
+            raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = ( self.minLen == 0 )
+        self.mayIndexError = False
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if instring[loc] in self.notChars:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        start = loc
+        loc += 1
+        notchars = self.notChars
+        maxlen = min( start+self.maxLen, len(instring) )
+        while loc < maxlen and \
+              (instring[loc] not in notchars):
+            loc += 1
+
+        if loc - start < self.minLen:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        return loc, instring[start:loc]
+
+    def __str__( self ):
+        try:
+            return super(CharsNotIn, self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            if len(self.notChars) > 4:
+                self.strRepr = "!W:(%s...)" % self.notChars[:4]
+            else:
+                self.strRepr = "!W:(%s)" % self.notChars
+
+        return self.strRepr
+
+class White(Token):
+    """
+    Special matching class for matching whitespace.  Normally, whitespace is ignored
+    by pyparsing grammars.  This class is included when some whitespace structures
+    are significant.  Define with a string containing the whitespace characters to be
+    matched; default is C{" \\t\\r\\n"}.  Also takes optional C{min}, C{max}, and C{exact} arguments,
+    as defined for the C{L{Word}} class.
+    """
+    whiteStrs = {
+        " " : "",
+        "\t": "",
+        "\n": "",
+        "\r": "",
+        "\f": "",
+        }
+    def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+        super(White,self).__init__()
+        self.matchWhite = ws
+        self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+        #~ self.leaveWhitespace()
+        self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+        self.mayReturnEmpty = True
+        self.errmsg = "Expected " + self.name
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if not(instring[ loc ] in self.matchWhite):
+            raise ParseException(instring, loc, self.errmsg, self)
+        start = loc
+        loc += 1
+        maxloc = start + self.maxLen
+        maxloc = min( maxloc, len(instring) )
+        while loc < maxloc and instring[loc] in self.matchWhite:
+            loc += 1
+
+        if loc - start < self.minLen:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+    def __init__( self ):
+        super(_PositionToken,self).__init__()
+        self.name=self.__class__.__name__
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+    """
+    Token to advance to a specific column of input text; useful for tabular report scraping.
+    """
+    def __init__( self, colno ):
+        super(GoToColumn,self).__init__()
+        self.col = colno
+
+    def preParse( self, instring, loc ):
+        if col(loc,instring) != self.col:
+            instrlen = len(instring)
+            if self.ignoreExprs:
+                loc = self._skipIgnorables( instring, loc )
+            while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+                loc += 1
+        return loc
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        thiscol = col( loc, instring )
+        if thiscol > self.col:
+            raise ParseException( instring, loc, "Text not in expected column", self )
+        newloc = loc + self.col - thiscol
+        ret = instring[ loc: newloc ]
+        return newloc, ret
+
+
+class LineStart(_PositionToken):
+    """
+    Matches if current position is at the beginning of a line within the parse string
+    
+    Example::
+    
+        test = '''\
+        AAA this line
+        AAA and this line
+          AAA but not this one
+        B AAA and definitely not this one
+        '''
+
+        for t in (LineStart() + 'AAA' + restOfLine).searchString(test):
+            print(t)
+    
+    Prints::
+        ['AAA', ' this line']
+        ['AAA', ' and this line']    
+
+    """
+    def __init__( self ):
+        super(LineStart,self).__init__()
+        self.errmsg = "Expected start of line"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if col(loc, instring) == 1:
+            return loc, []
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class LineEnd(_PositionToken):
+    """
+    Matches if current position is at the end of a line within the parse string
+    """
+    def __init__( self ):
+        super(LineEnd,self).__init__()
+        self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+        self.errmsg = "Expected end of line"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if loc len(instring):
+            return loc, []
+        else:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+    """
+    Matches if the current position is at the beginning of a Word, and
+    is not preceded by any character in a given set of C{wordChars}
+    (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+    use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+    the string being parsed, or at the beginning of a line.
+    """
+    def __init__(self, wordChars = printables):
+        super(WordStart,self).__init__()
+        self.wordChars = set(wordChars)
+        self.errmsg = "Not at the start of a word"
+
+    def parseImpl(self, instring, loc, doActions=True ):
+        if loc != 0:
+            if (instring[loc-1] in self.wordChars or
+                instring[loc] not in self.wordChars):
+                raise ParseException(instring, loc, self.errmsg, self)
+        return loc, []
+
+class WordEnd(_PositionToken):
+    """
+    Matches if the current position is at the end of a Word, and
+    is not followed by any character in a given set of C{wordChars}
+    (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+    use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+    the string being parsed, or at the end of a line.
+    """
+    def __init__(self, wordChars = printables):
+        super(WordEnd,self).__init__()
+        self.wordChars = set(wordChars)
+        self.skipWhitespace = False
+        self.errmsg = "Not at the end of a word"
+
+    def parseImpl(self, instring, loc, doActions=True ):
+        instrlen = len(instring)
+        if instrlen>0 and loc maxExcLoc:
+                    maxException = err
+                    maxExcLoc = err.loc
+            except IndexError:
+                if len(instring) > maxExcLoc:
+                    maxException = ParseException(instring,len(instring),e.errmsg,self)
+                    maxExcLoc = len(instring)
+            else:
+                # save match among all matches, to retry longest to shortest
+                matches.append((loc2, e))
+
+        if matches:
+            matches.sort(key=lambda x: -x[0])
+            for _,e in matches:
+                try:
+                    return e._parse( instring, loc, doActions )
+                except ParseException as err:
+                    err.__traceback__ = None
+                    if err.loc > maxExcLoc:
+                        maxException = err
+                        maxExcLoc = err.loc
+
+        if maxException is not None:
+            maxException.msg = self.errmsg
+            raise maxException
+        else:
+            raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+    def __ixor__(self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        return self.append( other ) #Or( [ self, other ] )
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+    """
+    Requires that at least one C{ParseExpression} is found.
+    If two expressions match, the first one listed is the one that will match.
+    May be constructed using the C{'|'} operator.
+
+    Example::
+        # construct MatchFirst using '|' operator
+        
+        # watch the order of expressions to match
+        number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
+        print(number.searchString("123 3.1416 789")) #  Fail! -> [['123'], ['3'], ['1416'], ['789']]
+
+        # put more selective expression first
+        number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
+        print(number.searchString("123 3.1416 789")) #  Better -> [['123'], ['3.1416'], ['789']]
+    """
+    def __init__( self, exprs, savelist = False ):
+        super(MatchFirst,self).__init__(exprs, savelist)
+        if self.exprs:
+            self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+        else:
+            self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        maxExcLoc = -1
+        maxException = None
+        for e in self.exprs:
+            try:
+                ret = e._parse( instring, loc, doActions )
+                return ret
+            except ParseException as err:
+                if err.loc > maxExcLoc:
+                    maxException = err
+                    maxExcLoc = err.loc
+            except IndexError:
+                if len(instring) > maxExcLoc:
+                    maxException = ParseException(instring,len(instring),e.errmsg,self)
+                    maxExcLoc = len(instring)
+
+        # only got here if no expression matched, raise exception for match that made it the furthest
+        else:
+            if maxException is not None:
+                maxException.msg = self.errmsg
+                raise maxException
+            else:
+                raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+    def __ior__(self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        return self.append( other ) #MatchFirst( [ self, other ] )
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+    """
+    Requires all given C{ParseExpression}s to be found, but in any order.
+    Expressions may be separated by whitespace.
+    May be constructed using the C{'&'} operator.
+
+    Example::
+        color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
+        shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
+        integer = Word(nums)
+        shape_attr = "shape:" + shape_type("shape")
+        posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
+        color_attr = "color:" + color("color")
+        size_attr = "size:" + integer("size")
+
+        # use Each (using operator '&') to accept attributes in any order 
+        # (shape and posn are required, color and size are optional)
+        shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr)
+
+        shape_spec.runTests('''
+            shape: SQUARE color: BLACK posn: 100, 120
+            shape: CIRCLE size: 50 color: BLUE posn: 50,80
+            color:GREEN size:20 shape:TRIANGLE posn:20,40
+            '''
+            )
+    prints::
+        shape: SQUARE color: BLACK posn: 100, 120
+        ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
+        - color: BLACK
+        - posn: ['100', ',', '120']
+          - x: 100
+          - y: 120
+        - shape: SQUARE
+
+
+        shape: CIRCLE size: 50 color: BLUE posn: 50,80
+        ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']]
+        - color: BLUE
+        - posn: ['50', ',', '80']
+          - x: 50
+          - y: 80
+        - shape: CIRCLE
+        - size: 50
+
+
+        color: GREEN size: 20 shape: TRIANGLE posn: 20,40
+        ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']]
+        - color: GREEN
+        - posn: ['20', ',', '40']
+          - x: 20
+          - y: 40
+        - shape: TRIANGLE
+        - size: 20
+    """
+    def __init__( self, exprs, savelist = True ):
+        super(Each,self).__init__(exprs, savelist)
+        self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+        self.skipWhitespace = True
+        self.initExprGroups = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.initExprGroups:
+            self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+            opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+            opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+            self.optionals = opt1 + opt2
+            self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+            self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+            self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+            self.required += self.multirequired
+            self.initExprGroups = False
+        tmpLoc = loc
+        tmpReqd = self.required[:]
+        tmpOpt  = self.optionals[:]
+        matchOrder = []
+
+        keepMatching = True
+        while keepMatching:
+            tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+            failed = []
+            for e in tmpExprs:
+                try:
+                    tmpLoc = e.tryParse( instring, tmpLoc )
+                except ParseException:
+                    failed.append(e)
+                else:
+                    matchOrder.append(self.opt1map.get(id(e),e))
+                    if e in tmpReqd:
+                        tmpReqd.remove(e)
+                    elif e in tmpOpt:
+                        tmpOpt.remove(e)
+            if len(failed) == len(tmpExprs):
+                keepMatching = False
+
+        if tmpReqd:
+            missing = ", ".join(_ustr(e) for e in tmpReqd)
+            raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+        # add any unmatched Optionals, in case they have default values defined
+        matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+        resultlist = []
+        for e in matchOrder:
+            loc,results = e._parse(instring,loc,doActions)
+            resultlist.append(results)
+
+        finalResults = sum(resultlist, ParseResults([]))
+        return loc, finalResults
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+    """
+    Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.
+    """
+    def __init__( self, expr, savelist=False ):
+        super(ParseElementEnhance,self).__init__(savelist)
+        if isinstance( expr, basestring ):
+            if issubclass(ParserElement._literalStringClass, Token):
+                expr = ParserElement._literalStringClass(expr)
+            else:
+                expr = ParserElement._literalStringClass(Literal(expr))
+        self.expr = expr
+        self.strRepr = None
+        if expr is not None:
+            self.mayIndexError = expr.mayIndexError
+            self.mayReturnEmpty = expr.mayReturnEmpty
+            self.setWhitespaceChars( expr.whiteChars )
+            self.skipWhitespace = expr.skipWhitespace
+            self.saveAsList = expr.saveAsList
+            self.callPreparse = expr.callPreparse
+            self.ignoreExprs.extend(expr.ignoreExprs)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.expr is not None:
+            return self.expr._parse( instring, loc, doActions, callPreParse=False )
+        else:
+            raise ParseException("",loc,self.errmsg,self)
+
+    def leaveWhitespace( self ):
+        self.skipWhitespace = False
+        self.expr = self.expr.copy()
+        if self.expr is not None:
+            self.expr.leaveWhitespace()
+        return self
+
+    def ignore( self, other ):
+        if isinstance( other, Suppress ):
+            if other not in self.ignoreExprs:
+                super( ParseElementEnhance, self).ignore( other )
+                if self.expr is not None:
+                    self.expr.ignore( self.ignoreExprs[-1] )
+        else:
+            super( ParseElementEnhance, self).ignore( other )
+            if self.expr is not None:
+                self.expr.ignore( self.ignoreExprs[-1] )
+        return self
+
+    def streamline( self ):
+        super(ParseElementEnhance,self).streamline()
+        if self.expr is not None:
+            self.expr.streamline()
+        return self
+
+    def checkRecursion( self, parseElementList ):
+        if self in parseElementList:
+            raise RecursiveGrammarException( parseElementList+[self] )
+        subRecCheckList = parseElementList[:] + [ self ]
+        if self.expr is not None:
+            self.expr.checkRecursion( subRecCheckList )
+
+    def validate( self, validateTrace=[] ):
+        tmp = validateTrace[:]+[self]
+        if self.expr is not None:
+            self.expr.validate(tmp)
+        self.checkRecursion( [] )
+
+    def __str__( self ):
+        try:
+            return super(ParseElementEnhance,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None and self.expr is not None:
+            self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+        return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+    """
+    Lookahead matching of the given parse expression.  C{FollowedBy}
+    does I{not} advance the parsing position within the input string, it only
+    verifies that the specified parse expression matches at the current
+    position.  C{FollowedBy} always returns a null token list.
+
+    Example::
+        # use FollowedBy to match a label only if it is followed by a ':'
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        
+        OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint()
+    prints::
+        [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
+    """
+    def __init__( self, expr ):
+        super(FollowedBy,self).__init__(expr)
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        self.expr.tryParse( instring, loc )
+        return loc, []
+
+
+class NotAny(ParseElementEnhance):
+    """
+    Lookahead to disallow matching with the given parse expression.  C{NotAny}
+    does I{not} advance the parsing position within the input string, it only
+    verifies that the specified parse expression does I{not} match at the current
+    position.  Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny}
+    always returns a null token list.  May be constructed using the '~' operator.
+
+    Example::
+        
+    """
+    def __init__( self, expr ):
+        super(NotAny,self).__init__(expr)
+        #~ self.leaveWhitespace()
+        self.skipWhitespace = False  # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+        self.mayReturnEmpty = True
+        self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.expr.canParseNext(instring, loc):
+            raise ParseException(instring, loc, self.errmsg, self)
+        return loc, []
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+        return self.strRepr
+
+class _MultipleMatch(ParseElementEnhance):
+    def __init__( self, expr, stopOn=None):
+        super(_MultipleMatch, self).__init__(expr)
+        self.saveAsList = True
+        ender = stopOn
+        if isinstance(ender, basestring):
+            ender = ParserElement._literalStringClass(ender)
+        self.not_ender = ~ender if ender is not None else None
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        self_expr_parse = self.expr._parse
+        self_skip_ignorables = self._skipIgnorables
+        check_ender = self.not_ender is not None
+        if check_ender:
+            try_not_ender = self.not_ender.tryParse
+        
+        # must be at least one (but first see if we are the stopOn sentinel;
+        # if so, fail)
+        if check_ender:
+            try_not_ender(instring, loc)
+        loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False )
+        try:
+            hasIgnoreExprs = (not not self.ignoreExprs)
+            while 1:
+                if check_ender:
+                    try_not_ender(instring, loc)
+                if hasIgnoreExprs:
+                    preloc = self_skip_ignorables( instring, loc )
+                else:
+                    preloc = loc
+                loc, tmptokens = self_expr_parse( instring, preloc, doActions )
+                if tmptokens or tmptokens.haskeys():
+                    tokens += tmptokens
+        except (ParseException,IndexError):
+            pass
+
+        return loc, tokens
+        
+class OneOrMore(_MultipleMatch):
+    """
+    Repetition of one or more of the given expression.
+    
+    Parameters:
+     - expr - expression that must match one or more times
+     - stopOn - (default=C{None}) - expression for a terminating sentinel
+          (only required if the sentinel would ordinarily match the repetition 
+          expression)          
+
+    Example::
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+        text = "shape: SQUARE posn: upper left color: BLACK"
+        OneOrMore(attr_expr).parseString(text).pprint()  # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']]
+
+        # use stopOn attribute for OneOrMore to avoid reading label string as part of the data
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
+        
+        # could also be written as
+        (attr_expr * (1,)).parseString(text).pprint()
+    """
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+        return self.strRepr
+
+class ZeroOrMore(_MultipleMatch):
+    """
+    Optional repetition of zero or more of the given expression.
+    
+    Parameters:
+     - expr - expression that must match zero or more times
+     - stopOn - (default=C{None}) - expression for a terminating sentinel
+          (only required if the sentinel would ordinarily match the repetition 
+          expression)          
+
+    Example: similar to L{OneOrMore}
+    """
+    def __init__( self, expr, stopOn=None):
+        super(ZeroOrMore,self).__init__(expr, stopOn=stopOn)
+        self.mayReturnEmpty = True
+        
+    def parseImpl( self, instring, loc, doActions=True ):
+        try:
+            return super(ZeroOrMore, self).parseImpl(instring, loc, doActions)
+        except (ParseException,IndexError):
+            return loc, []
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+        return self.strRepr
+
+class _NullToken(object):
+    def __bool__(self):
+        return False
+    __nonzero__ = __bool__
+    def __str__(self):
+        return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+    """
+    Optional matching of the given expression.
+
+    Parameters:
+     - expr - expression that must match zero or more times
+     - default (optional) - value to be returned if the optional expression is not found.
+
+    Example::
+        # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
+        zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4)))
+        zip.runTests('''
+            # traditional ZIP code
+            12345
+            
+            # ZIP+4 form
+            12101-0001
+            
+            # invalid ZIP
+            98765-
+            ''')
+    prints::
+        # traditional ZIP code
+        12345
+        ['12345']
+
+        # ZIP+4 form
+        12101-0001
+        ['12101-0001']
+
+        # invalid ZIP
+        98765-
+             ^
+        FAIL: Expected end of text (at char 5), (line:1, col:6)
+    """
+    def __init__( self, expr, default=_optionalNotMatched ):
+        super(Optional,self).__init__( expr, savelist=False )
+        self.saveAsList = self.expr.saveAsList
+        self.defaultValue = default
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        try:
+            loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+        except (ParseException,IndexError):
+            if self.defaultValue is not _optionalNotMatched:
+                if self.expr.resultsName:
+                    tokens = ParseResults([ self.defaultValue ])
+                    tokens[self.expr.resultsName] = self.defaultValue
+                else:
+                    tokens = [ self.defaultValue ]
+            else:
+                tokens = []
+        return loc, tokens
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "[" + _ustr(self.expr) + "]"
+
+        return self.strRepr
+
+class SkipTo(ParseElementEnhance):
+    """
+    Token for skipping over all undefined text until the matched expression is found.
+
+    Parameters:
+     - expr - target expression marking the end of the data to be skipped
+     - include - (default=C{False}) if True, the target expression is also parsed 
+          (the skipped text and target expression are returned as a 2-element list).
+     - ignore - (default=C{None}) used to define grammars (typically quoted strings and 
+          comments) that might contain false matches to the target expression
+     - failOn - (default=C{None}) define expressions that are not allowed to be 
+          included in the skipped test; if found before the target expression is found, 
+          the SkipTo is not a match
+
+    Example::
+        report = '''
+            Outstanding Issues Report - 1 Jan 2000
+
+               # | Severity | Description                               |  Days Open
+            -----+----------+-------------------------------------------+-----------
+             101 | Critical | Intermittent system crash                 |          6
+              94 | Cosmetic | Spelling error on Login ('log|n')         |         14
+              79 | Minor    | System slow when running too many reports |         47
+            '''
+        integer = Word(nums)
+        SEP = Suppress('|')
+        # use SkipTo to simply match everything up until the next SEP
+        # - ignore quoted strings, so that a '|' character inside a quoted string does not match
+        # - parse action will call token.strip() for each matched token, i.e., the description body
+        string_data = SkipTo(SEP, ignore=quotedString)
+        string_data.setParseAction(tokenMap(str.strip))
+        ticket_expr = (integer("issue_num") + SEP 
+                      + string_data("sev") + SEP 
+                      + string_data("desc") + SEP 
+                      + integer("days_open"))
+        
+        for tkt in ticket_expr.searchString(report):
+            print tkt.dump()
+    prints::
+        ['101', 'Critical', 'Intermittent system crash', '6']
+        - days_open: 6
+        - desc: Intermittent system crash
+        - issue_num: 101
+        - sev: Critical
+        ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
+        - days_open: 14
+        - desc: Spelling error on Login ('log|n')
+        - issue_num: 94
+        - sev: Cosmetic
+        ['79', 'Minor', 'System slow when running too many reports', '47']
+        - days_open: 47
+        - desc: System slow when running too many reports
+        - issue_num: 79
+        - sev: Minor
+    """
+    def __init__( self, other, include=False, ignore=None, failOn=None ):
+        super( SkipTo, self ).__init__( other )
+        self.ignoreExpr = ignore
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+        self.includeMatch = include
+        self.asList = False
+        if isinstance(failOn, basestring):
+            self.failOn = ParserElement._literalStringClass(failOn)
+        else:
+            self.failOn = failOn
+        self.errmsg = "No match found for "+_ustr(self.expr)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        startloc = loc
+        instrlen = len(instring)
+        expr = self.expr
+        expr_parse = self.expr._parse
+        self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None
+        self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None
+        
+        tmploc = loc
+        while tmploc <= instrlen:
+            if self_failOn_canParseNext is not None:
+                # break if failOn expression matches
+                if self_failOn_canParseNext(instring, tmploc):
+                    break
+                    
+            if self_ignoreExpr_tryParse is not None:
+                # advance past ignore expressions
+                while 1:
+                    try:
+                        tmploc = self_ignoreExpr_tryParse(instring, tmploc)
+                    except ParseBaseException:
+                        break
+            
+            try:
+                expr_parse(instring, tmploc, doActions=False, callPreParse=False)
+            except (ParseException, IndexError):
+                # no match, advance loc in string
+                tmploc += 1
+            else:
+                # matched skipto expr, done
+                break
+
+        else:
+            # ran off the end of the input string without matching skipto expr, fail
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        # build up return values
+        loc = tmploc
+        skiptext = instring[startloc:loc]
+        skipresult = ParseResults(skiptext)
+        
+        if self.includeMatch:
+            loc, mat = expr_parse(instring,loc,doActions,callPreParse=False)
+            skipresult += mat
+
+        return loc, skipresult
+
+class Forward(ParseElementEnhance):
+    """
+    Forward declaration of an expression to be defined later -
+    used for recursive grammars, such as algebraic infix notation.
+    When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+    Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+    Specifically, '|' has a lower precedence than '<<', so that::
+        fwdExpr << a | b | c
+    will actually be evaluated as::
+        (fwdExpr << a) | b | c
+    thereby leaving b and c out as parseable alternatives.  It is recommended that you
+    explicitly group the values inserted into the C{Forward}::
+        fwdExpr << (a | b | c)
+    Converting to use the '<<=' operator instead will avoid this problem.
+
+    See L{ParseResults.pprint} for an example of a recursive parser created using
+    C{Forward}.
+    """
+    def __init__( self, other=None ):
+        super(Forward,self).__init__( other, savelist=False )
+
+    def __lshift__( self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass(other)
+        self.expr = other
+        self.strRepr = None
+        self.mayIndexError = self.expr.mayIndexError
+        self.mayReturnEmpty = self.expr.mayReturnEmpty
+        self.setWhitespaceChars( self.expr.whiteChars )
+        self.skipWhitespace = self.expr.skipWhitespace
+        self.saveAsList = self.expr.saveAsList
+        self.ignoreExprs.extend(self.expr.ignoreExprs)
+        return self
+        
+    def __ilshift__(self, other):
+        return self << other
+    
+    def leaveWhitespace( self ):
+        self.skipWhitespace = False
+        return self
+
+    def streamline( self ):
+        if not self.streamlined:
+            self.streamlined = True
+            if self.expr is not None:
+                self.expr.streamline()
+        return self
+
+    def validate( self, validateTrace=[] ):
+        if self not in validateTrace:
+            tmp = validateTrace[:]+[self]
+            if self.expr is not None:
+                self.expr.validate(tmp)
+        self.checkRecursion([])
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+        return self.__class__.__name__ + ": ..."
+
+        # stubbed out for now - creates awful memory and perf issues
+        self._revertClass = self.__class__
+        self.__class__ = _ForwardNoRecurse
+        try:
+            if self.expr is not None:
+                retString = _ustr(self.expr)
+            else:
+                retString = "None"
+        finally:
+            self.__class__ = self._revertClass
+        return self.__class__.__name__ + ": " + retString
+
+    def copy(self):
+        if self.expr is not None:
+            return super(Forward,self).copy()
+        else:
+            ret = Forward()
+            ret <<= self
+            return ret
+
+class _ForwardNoRecurse(Forward):
+    def __str__( self ):
+        return "..."
+
+class TokenConverter(ParseElementEnhance):
+    """
+    Abstract subclass of C{ParseExpression}, for converting parsed results.
+    """
+    def __init__( self, expr, savelist=False ):
+        super(TokenConverter,self).__init__( expr )#, savelist )
+        self.saveAsList = False
+
+class Combine(TokenConverter):
+    """
+    Converter to concatenate all matching tokens to a single string.
+    By default, the matching patterns must also be contiguous in the input string;
+    this can be disabled by specifying C{'adjacent=False'} in the constructor.
+
+    Example::
+        real = Word(nums) + '.' + Word(nums)
+        print(real.parseString('3.1416')) # -> ['3', '.', '1416']
+        # will also erroneously match the following
+        print(real.parseString('3. 1416')) # -> ['3', '.', '1416']
+
+        real = Combine(Word(nums) + '.' + Word(nums))
+        print(real.parseString('3.1416')) # -> ['3.1416']
+        # no match when there are internal spaces
+        print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...)
+    """
+    def __init__( self, expr, joinString="", adjacent=True ):
+        super(Combine,self).__init__( expr )
+        # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+        if adjacent:
+            self.leaveWhitespace()
+        self.adjacent = adjacent
+        self.skipWhitespace = True
+        self.joinString = joinString
+        self.callPreparse = True
+
+    def ignore( self, other ):
+        if self.adjacent:
+            ParserElement.ignore(self, other)
+        else:
+            super( Combine, self).ignore( other )
+        return self
+
+    def postParse( self, instring, loc, tokenlist ):
+        retToks = tokenlist.copy()
+        del retToks[:]
+        retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+        if self.resultsName and retToks.haskeys():
+            return [ retToks ]
+        else:
+            return retToks
+
+class Group(TokenConverter):
+    """
+    Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions.
+
+    Example::
+        ident = Word(alphas)
+        num = Word(nums)
+        term = ident | num
+        func = ident + Optional(delimitedList(term))
+        print(func.parseString("fn a,b,100"))  # -> ['fn', 'a', 'b', '100']
+
+        func = ident + Group(Optional(delimitedList(term)))
+        print(func.parseString("fn a,b,100"))  # -> ['fn', ['a', 'b', '100']]
+    """
+    def __init__( self, expr ):
+        super(Group,self).__init__( expr )
+        self.saveAsList = True
+
+    def postParse( self, instring, loc, tokenlist ):
+        return [ tokenlist ]
+
+class Dict(TokenConverter):
+    """
+    Converter to return a repetitive expression as a list, but also as a dictionary.
+    Each element can also be referenced using the first token in the expression as its key.
+    Useful for tabular report scraping when the first column can be used as a item key.
+
+    Example::
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+        text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+        attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        
+        # print attributes as plain groups
+        print(OneOrMore(attr_expr).parseString(text).dump())
+        
+        # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names
+        result = Dict(OneOrMore(Group(attr_expr))).parseString(text)
+        print(result.dump())
+        
+        # access named fields as dict entries, or output as dict
+        print(result['shape'])        
+        print(result.asDict())
+    prints::
+        ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
+
+        [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+        - color: light blue
+        - posn: upper left
+        - shape: SQUARE
+        - texture: burlap
+        SQUARE
+        {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'}
+    See more examples at L{ParseResults} of accessing fields by results name.
+    """
+    def __init__( self, expr ):
+        super(Dict,self).__init__( expr )
+        self.saveAsList = True
+
+    def postParse( self, instring, loc, tokenlist ):
+        for i,tok in enumerate(tokenlist):
+            if len(tok) == 0:
+                continue
+            ikey = tok[0]
+            if isinstance(ikey,int):
+                ikey = _ustr(tok[0]).strip()
+            if len(tok)==1:
+                tokenlist[ikey] = _ParseResultsWithOffset("",i)
+            elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+                tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+            else:
+                dictvalue = tok.copy() #ParseResults(i)
+                del dictvalue[0]
+                if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+                else:
+                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+        if self.resultsName:
+            return [ tokenlist ]
+        else:
+            return tokenlist
+
+
+class Suppress(TokenConverter):
+    """
+    Converter for ignoring the results of a parsed expression.
+
+    Example::
+        source = "a, b, c,d"
+        wd = Word(alphas)
+        wd_list1 = wd + ZeroOrMore(',' + wd)
+        print(wd_list1.parseString(source))
+
+        # often, delimiters that are useful during parsing are just in the
+        # way afterward - use Suppress to keep them out of the parsed output
+        wd_list2 = wd + ZeroOrMore(Suppress(',') + wd)
+        print(wd_list2.parseString(source))
+    prints::
+        ['a', ',', 'b', ',', 'c', ',', 'd']
+        ['a', 'b', 'c', 'd']
+    (See also L{delimitedList}.)
+    """
+    def postParse( self, instring, loc, tokenlist ):
+        return []
+
+    def suppress( self ):
+        return self
+
+
+class OnlyOnce(object):
+    """
+    Wrapper for parse actions, to ensure they are only called once.
+    """
+    def __init__(self, methodCall):
+        self.callable = _trim_arity(methodCall)
+        self.called = False
+    def __call__(self,s,l,t):
+        if not self.called:
+            results = self.callable(s,l,t)
+            self.called = True
+            return results
+        raise ParseException(s,l,"")
+    def reset(self):
+        self.called = False
+
+def traceParseAction(f):
+    """
+    Decorator for debugging parse actions. 
+    
+    When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".}
+    When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised.
+
+    Example::
+        wd = Word(alphas)
+
+        @traceParseAction
+        def remove_duplicate_chars(tokens):
+            return ''.join(sorted(set(''.join(tokens))))
+
+        wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
+        print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
+    prints::
+        >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
+        <3:
+            thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+        sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) )
+        try:
+            ret = f(*paArgs)
+        except Exception as exc:
+            sys.stderr.write( "< ['aa', 'bb', 'cc']
+        delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE']
+    """
+    dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+    if combine:
+        return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+    else:
+        return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+    """
+    Helper to define a counted list of expressions.
+    This helper defines a pattern of the form::
+        integer expr expr expr...
+    where the leading integer tells how many expr expressions follow.
+    The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+    
+    If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value.
+
+    Example::
+        countedArray(Word(alphas)).parseString('2 ab cd ef')  # -> ['ab', 'cd']
+
+        # in this parser, the leading integer value is given in binary,
+        # '10' indicating that 2 values are in the array
+        binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2))
+        countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef')  # -> ['ab', 'cd']
+    """
+    arrayExpr = Forward()
+    def countFieldParseAction(s,l,t):
+        n = t[0]
+        arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+        return []
+    if intExpr is None:
+        intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+    else:
+        intExpr = intExpr.copy()
+    intExpr.setName("arrayLen")
+    intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+    return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...')
+
+def _flatten(L):
+    ret = []
+    for i in L:
+        if isinstance(i,list):
+            ret.extend(_flatten(i))
+        else:
+            ret.append(i)
+    return ret
+
+def matchPreviousLiteral(expr):
+    """
+    Helper to define an expression that is indirectly defined from
+    the tokens matched in a previous expression, that is, it looks
+    for a 'repeat' of a previous expression.  For example::
+        first = Word(nums)
+        second = matchPreviousLiteral(first)
+        matchExpr = first + ":" + second
+    will match C{"1:1"}, but not C{"1:2"}.  Because this matches a
+    previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+    If this is not desired, use C{matchPreviousExpr}.
+    Do I{not} use with packrat parsing enabled.
+    """
+    rep = Forward()
+    def copyTokenToRepeater(s,l,t):
+        if t:
+            if len(t) == 1:
+                rep << t[0]
+            else:
+                # flatten t tokens
+                tflat = _flatten(t.asList())
+                rep << And(Literal(tt) for tt in tflat)
+        else:
+            rep << Empty()
+    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+    rep.setName('(prev) ' + _ustr(expr))
+    return rep
+
+def matchPreviousExpr(expr):
+    """
+    Helper to define an expression that is indirectly defined from
+    the tokens matched in a previous expression, that is, it looks
+    for a 'repeat' of a previous expression.  For example::
+        first = Word(nums)
+        second = matchPreviousExpr(first)
+        matchExpr = first + ":" + second
+    will match C{"1:1"}, but not C{"1:2"}.  Because this matches by
+    expressions, will I{not} match the leading C{"1:1"} in C{"1:10"};
+    the expressions are evaluated first, and then compared, so
+    C{"1"} is compared with C{"10"}.
+    Do I{not} use with packrat parsing enabled.
+    """
+    rep = Forward()
+    e2 = expr.copy()
+    rep <<= e2
+    def copyTokenToRepeater(s,l,t):
+        matchTokens = _flatten(t.asList())
+        def mustMatchTheseTokens(s,l,t):
+            theseTokens = _flatten(t.asList())
+            if  theseTokens != matchTokens:
+                raise ParseException("",0,"")
+        rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+    rep.setName('(prev) ' + _ustr(expr))
+    return rep
+
+def _escapeRegexRangeChars(s):
+    #~  escape these chars: ^-]
+    for c in r"\^-]":
+        s = s.replace(c,_bslash+c)
+    s = s.replace("\n",r"\n")
+    s = s.replace("\t",r"\t")
+    return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+    """
+    Helper to quickly define a set of alternative Literals, and makes sure to do
+    longest-first testing when there is a conflict, regardless of the input order,
+    but returns a C{L{MatchFirst}} for best performance.
+
+    Parameters:
+     - strs - a string of space-delimited literals, or a collection of string literals
+     - caseless - (default=C{False}) - treat all literals as caseless
+     - useRegex - (default=C{True}) - as an optimization, will generate a Regex
+          object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+          if creating a C{Regex} raises an exception)
+
+    Example::
+        comp_oper = oneOf("< = > <= >= !=")
+        var = Word(alphas)
+        number = Word(nums)
+        term = var | number
+        comparison_expr = term + comp_oper + term
+        print(comparison_expr.searchString("B = 12  AA=23 B<=AA AA>12"))
+    prints::
+        [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']]
+    """
+    if caseless:
+        isequal = ( lambda a,b: a.upper() == b.upper() )
+        masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+        parseElementClass = CaselessLiteral
+    else:
+        isequal = ( lambda a,b: a == b )
+        masks = ( lambda a,b: b.startswith(a) )
+        parseElementClass = Literal
+
+    symbols = []
+    if isinstance(strs,basestring):
+        symbols = strs.split()
+    elif isinstance(strs, Iterable):
+        symbols = list(strs)
+    else:
+        warnings.warn("Invalid argument to oneOf, expected string or iterable",
+                SyntaxWarning, stacklevel=2)
+    if not symbols:
+        return NoMatch()
+
+    i = 0
+    while i < len(symbols)-1:
+        cur = symbols[i]
+        for j,other in enumerate(symbols[i+1:]):
+            if ( isequal(other, cur) ):
+                del symbols[i+j+1]
+                break
+            elif ( masks(cur, other) ):
+                del symbols[i+j+1]
+                symbols.insert(i,other)
+                cur = other
+                break
+        else:
+            i += 1
+
+    if not caseless and useRegex:
+        #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+        try:
+            if len(symbols)==len("".join(symbols)):
+                return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols))
+            else:
+                return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols))
+        except Exception:
+            warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+                    SyntaxWarning, stacklevel=2)
+
+
+    # last resort, just use MatchFirst
+    return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols))
+
+def dictOf( key, value ):
+    """
+    Helper to easily and clearly define a dictionary by specifying the respective patterns
+    for the key and value.  Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+    in the proper order.  The key pattern can include delimiting markers or punctuation,
+    as long as they are suppressed, thereby leaving the significant key text.  The value
+    pattern can include named results, so that the C{Dict} results can include named token
+    fields.
+
+    Example::
+        text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+        attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        print(OneOrMore(attr_expr).parseString(text).dump())
+        
+        attr_label = label
+        attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)
+
+        # similar to Dict, but simpler call format
+        result = dictOf(attr_label, attr_value).parseString(text)
+        print(result.dump())
+        print(result['shape'])
+        print(result.shape)  # object attribute access works too
+        print(result.asDict())
+    prints::
+        [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+        - color: light blue
+        - posn: upper left
+        - shape: SQUARE
+        - texture: burlap
+        SQUARE
+        SQUARE
+        {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'}
+    """
+    return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+    """
+    Helper to return the original, untokenized text for a given expression.  Useful to
+    restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+    revert separate tokens with intervening whitespace back to the original matching
+    input text. By default, returns astring containing the original parsed text.  
+       
+    If the optional C{asString} argument is passed as C{False}, then the return value is a 
+    C{L{ParseResults}} containing any results names that were originally matched, and a 
+    single token containing the original matched text from the input string.  So if 
+    the expression passed to C{L{originalTextFor}} contains expressions with defined
+    results names, you must set C{asString} to C{False} if you want to preserve those
+    results name values.
+
+    Example::
+        src = "this is test  bold text  normal text "
+        for tag in ("b","i"):
+            opener,closer = makeHTMLTags(tag)
+            patt = originalTextFor(opener + SkipTo(closer) + closer)
+            print(patt.searchString(src)[0])
+    prints::
+        [' bold text ']
+        ['text']
+    """
+    locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+    endlocMarker = locMarker.copy()
+    endlocMarker.callPreparse = False
+    matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+    if asString:
+        extractText = lambda s,l,t: s[t._original_start:t._original_end]
+    else:
+        def extractText(s,l,t):
+            t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]]
+    matchExpr.setParseAction(extractText)
+    matchExpr.ignoreExprs = expr.ignoreExprs
+    return matchExpr
+
+def ungroup(expr): 
+    """
+    Helper to undo pyparsing's default grouping of And expressions, even
+    if all but one are non-empty.
+    """
+    return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+    """
+    Helper to decorate a returned token with its starting and ending locations in the input string.
+    This helper adds the following results names:
+     - locn_start = location where matched expression begins
+     - locn_end = location where matched expression ends
+     - value = the actual parsed results
+
+    Be careful if the input text contains C{} characters, you may want to call
+    C{L{ParserElement.parseWithTabs}}
+
+    Example::
+        wd = Word(alphas)
+        for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"):
+            print(match)
+    prints::
+        [[0, 'ljsdf', 5]]
+        [[8, 'lksdjjf', 15]]
+        [[18, 'lkkjj', 23]]
+    """
+    locator = Empty().setParseAction(lambda s,l,t: l)
+    return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty       = Empty().setName("empty")
+lineStart   = LineStart().setName("lineStart")
+lineEnd     = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd   = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+    r"""
+    Helper to easily define string ranges for use in Word construction.  Borrows
+    syntax from regexp '[]' string range definitions::
+        srange("[0-9]")   -> "0123456789"
+        srange("[a-z]")   -> "abcdefghijklmnopqrstuvwxyz"
+        srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+    The input string must be enclosed in []'s, and the returned string is the expanded
+    character set joined into a single string.
+    The values enclosed in the []'s may be:
+     - a single character
+     - an escaped character with a leading backslash (such as C{\-} or C{\]})
+     - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) 
+         (C{\0x##} is also supported for backwards compatibility) 
+     - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character)
+     - a range of any of the above, separated by a dash (C{'a-z'}, etc.)
+     - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.)
+    """
+    _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+    try:
+        return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+    except Exception:
+        return ""
+
+def matchOnlyAtCol(n):
+    """
+    Helper method for defining parse actions that require matching at a specific
+    column in the input text.
+    """
+    def verifyCol(strg,locn,toks):
+        if col(locn,strg) != n:
+            raise ParseException(strg,locn,"matched token not at column %d" % n)
+    return verifyCol
+
+def replaceWith(replStr):
+    """
+    Helper method for common parse actions that simply return a literal value.  Especially
+    useful when used with C{L{transformString}()}.
+
+    Example::
+        num = Word(nums).setParseAction(lambda toks: int(toks[0]))
+        na = oneOf("N/A NA").setParseAction(replaceWith(math.nan))
+        term = na | num
+        
+        OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234]
+    """
+    return lambda s,l,t: [replStr]
+
+def removeQuotes(s,l,t):
+    """
+    Helper parse action for removing quotation marks from parsed quoted strings.
+
+    Example::
+        # by default, quotation marks are included in parsed results
+        quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"]
+
+        # use removeQuotes to strip quotation marks from parsed results
+        quotedString.setParseAction(removeQuotes)
+        quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"]
+    """
+    return t[0][1:-1]
+
+def tokenMap(func, *args):
+    """
+    Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional 
+    args are passed, they are forwarded to the given function as additional arguments after
+    the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the
+    parsed data to an integer using base 16.
+
+    Example (compare the last to example in L{ParserElement.transformString}::
+        hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16))
+        hex_ints.runTests('''
+            00 11 22 aa FF 0a 0d 1a
+            ''')
+        
+        upperword = Word(alphas).setParseAction(tokenMap(str.upper))
+        OneOrMore(upperword).runTests('''
+            my kingdom for a horse
+            ''')
+
+        wd = Word(alphas).setParseAction(tokenMap(str.title))
+        OneOrMore(wd).setParseAction(' '.join).runTests('''
+            now is the winter of our discontent made glorious summer by this sun of york
+            ''')
+    prints::
+        00 11 22 aa FF 0a 0d 1a
+        [0, 17, 34, 170, 255, 10, 13, 26]
+
+        my kingdom for a horse
+        ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
+
+        now is the winter of our discontent made glorious summer by this sun of york
+        ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
+    """
+    def pa(s,l,t):
+        return [func(tokn, *args) for tokn in t]
+
+    try:
+        func_name = getattr(func, '__name__', 
+                            getattr(func, '__class__').__name__)
+    except Exception:
+        func_name = str(func)
+    pa.__name__ = func_name
+
+    return pa
+
+upcaseTokens = tokenMap(lambda t: _ustr(t).upper())
+"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}"""
+
+downcaseTokens = tokenMap(lambda t: _ustr(t).lower())
+"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}"""
+    
+def _makeTags(tagStr, xml):
+    """Internal helper to construct opening and closing tag expressions, given a tag name"""
+    if isinstance(tagStr,basestring):
+        resname = tagStr
+        tagStr = Keyword(tagStr, caseless=not xml)
+    else:
+        resname = tagStr.name
+
+    tagAttrName = Word(alphas,alphanums+"_-:")
+    if (xml):
+        tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+        openTag = Suppress("<") + tagStr("tag") + \
+                Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+    else:
+        printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+        tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+        openTag = Suppress("<") + tagStr("tag") + \
+                Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+                Optional( Suppress("=") + tagAttrValue ) ))) + \
+                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+    closeTag = Combine(_L("")
+
+    openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname)
+    closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("" % resname)
+    openTag.tag = resname
+    closeTag.tag = resname
+    return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+    """
+    Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches
+    tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values.
+
+    Example::
+        text = 'More info at the pyparsing wiki page'
+        # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple
+        a,a_end = makeHTMLTags("A")
+        link_expr = a + SkipTo(a_end)("link_text") + a_end
+        
+        for link in link_expr.searchString(text):
+            # attributes in the  tag (like "href" shown here) are also accessible as named results
+            print(link.link_text, '->', link.href)
+    prints::
+        pyparsing -> http://pyparsing.wikispaces.com
+    """
+    return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+    """
+    Helper to construct opening and closing tag expressions for XML, given a tag name. Matches
+    tags only in the given upper/lower case.
+
+    Example: similar to L{makeHTMLTags}
+    """
+    return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+    """
+    Helper to create a validating parse action to be used with start tags created
+    with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+    with a required attribute value, to avoid false matches on common tags such as
+    C{} or C{
}.
+
+    Call C{withAttribute} with a series of attribute names and values. Specify the list
+    of filter attributes names and values as:
+     - keyword arguments, as in C{(align="right")}, or
+     - as an explicit dict with C{**} operator, when an attribute name is also a Python
+          reserved word, as in C{**{"class":"Customer", "align":"right"}}
+     - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+    For attribute names with a namespace prefix, you must use the second form.  Attribute
+    names are matched insensitive to upper/lower case.
+       
+    If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+    To verify that the attribute exists, but without specifying a value, pass
+    C{withAttribute.ANY_VALUE} as the value.
+
+    Example::
+        html = '''
+            

+            Some text
+            
1 4 0 1 0

+            
1,3 2,3 1,1

+            
this has no type

+            

+                
+        '''
+        div,div_end = makeHTMLTags("div")
+
+        # only match div tag having a type attribute with value "grid"
+        div_grid = div().setParseAction(withAttribute(type="grid"))
+        grid_expr = div_grid + SkipTo(div | div_end)("body")
+        for grid_header in grid_expr.searchString(html):
+            print(grid_header.body)
+        
+        # construct a match with any div tag having a type attribute, regardless of the value
+        div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE))
+        div_expr = div_any_type + SkipTo(div | div_end)("body")
+        for div_header in div_expr.searchString(html):
+            print(div_header.body)
+    prints::
+        1 4 0 1 0
+
+        1 4 0 1 0
+        1,3 2,3 1,1
+    """
+    if args:
+        attrs = args[:]
+    else:
+        attrs = attrDict.items()
+    attrs = [(k,v) for k,v in attrs]
+    def pa(s,l,tokens):
+        for attrName,attrValue in attrs:
+            if attrName not in tokens:
+                raise ParseException(s,l,"no matching attribute " + attrName)
+            if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+                raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+                                            (attrName, tokens[attrName], attrValue))
+    return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+    """
+    Simplified version of C{L{withAttribute}} when matching on a div class - made
+    difficult because C{class} is a reserved word in Python.
+
+    Example::
+        html = '''
+            

+            Some text
+            
1 4 0 1 0

+            
1,3 2,3 1,1

+            
this <div> has no class

+            

+                
+        '''
+        div,div_end = makeHTMLTags("div")
+        div_grid = div().setParseAction(withClass("grid"))
+        
+        grid_expr = div_grid + SkipTo(div | div_end)("body")
+        for grid_header in grid_expr.searchString(html):
+            print(grid_header.body)
+        
+        div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE))
+        div_expr = div_any_type + SkipTo(div | div_end)("body")
+        for div_header in div_expr.searchString(html):
+            print(div_header.body)
+    prints::
+        1 4 0 1 0
+
+        1 4 0 1 0
+        1,3 2,3 1,1
+    """
+    classattr = "%s:class" % namespace if namespace else "class"
+    return withAttribute(**{classattr : classname})        
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+    """
+    Helper method for constructing grammars of expressions made up of
+    operators working in a precedence hierarchy.  Operators may be unary or
+    binary, left- or right-associative.  Parse actions can also be attached
+    to operator expressions. The generated parser will also recognize the use 
+    of parentheses to override operator precedences (see example below).
+    
+    Note: if you define a deep operator list, you may see performance issues
+    when using infixNotation. See L{ParserElement.enablePackrat} for a
+    mechanism to potentially improve your parser performance.
+
+    Parameters:
+     - baseExpr - expression representing the most basic element for the nested
+     - opList - list of tuples, one for each operator precedence level in the
+      expression grammar; each tuple is of the form
+      (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+       - opExpr is the pyparsing expression for the operator;
+          may also be a string, which will be converted to a Literal;
+          if numTerms is 3, opExpr is a tuple of two expressions, for the
+          two operators separating the 3 terms
+       - numTerms is the number of terms for this operator (must
+          be 1, 2, or 3)
+       - rightLeftAssoc is the indicator whether the operator is
+          right or left associative, using the pyparsing-defined
+          constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+       - parseAction is the parse action to be associated with
+          expressions matching this operator expression (the
+          parse action tuple member may be omitted); if the parse action
+          is passed a tuple or list of functions, this is equivalent to
+          calling C{setParseAction(*fn)} (L{ParserElement.setParseAction})
+     - lpar - expression for matching left-parentheses (default=C{Suppress('(')})
+     - rpar - expression for matching right-parentheses (default=C{Suppress(')')})
+
+    Example::
+        # simple example of four-function arithmetic with ints and variable names
+        integer = pyparsing_common.signed_integer
+        varname = pyparsing_common.identifier 
+        
+        arith_expr = infixNotation(integer | varname,
+            [
+            ('-', 1, opAssoc.RIGHT),
+            (oneOf('* /'), 2, opAssoc.LEFT),
+            (oneOf('+ -'), 2, opAssoc.LEFT),
+            ])
+        
+        arith_expr.runTests('''
+            5+3*6
+            (5+3)*6
+            -2--11
+            ''', fullDump=False)
+    prints::
+        5+3*6
+        [[5, '+', [3, '*', 6]]]
+
+        (5+3)*6
+        [[[5, '+', 3], '*', 6]]
+
+        -2--11
+        [[['-', 2], '-', ['-', 11]]]
+    """
+    ret = Forward()
+    lastExpr = baseExpr | ( lpar + ret + rpar )
+    for i,operDef in enumerate(opList):
+        opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+        termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr
+        if arity == 3:
+            if opExpr is None or len(opExpr) != 2:
+                raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+            opExpr1, opExpr2 = opExpr
+        thisExpr = Forward().setName(termName)
+        if rightLeftAssoc == opAssoc.LEFT:
+            if arity == 1:
+                matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+            elif arity == 2:
+                if opExpr is not None:
+                    matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+                else:
+                    matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+            elif arity == 3:
+                matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+                            Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+            else:
+                raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+        elif rightLeftAssoc == opAssoc.RIGHT:
+            if arity == 1:
+                # try to avoid LR with this extra test
+                if not isinstance(opExpr, Optional):
+                    opExpr = Optional(opExpr)
+                matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+            elif arity == 2:
+                if opExpr is not None:
+                    matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+                else:
+                    matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+            elif arity == 3:
+                matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+                            Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+            else:
+                raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+        else:
+            raise ValueError("operator must indicate right or left associativity")
+        if pa:
+            if isinstance(pa, (tuple, list)):
+                matchExpr.setParseAction(*pa)
+            else:
+                matchExpr.setParseAction(pa)
+        thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
+        lastExpr = thisExpr
+    ret <<= lastExpr
+    return ret
+
+operatorPrecedence = infixNotation
+"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release."""
+
+dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes")
+sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes")
+quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'|
+                       Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal")
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+    """
+    Helper method for defining nested lists enclosed in opening and closing
+    delimiters ("(" and ")" are the default).
+
+    Parameters:
+     - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression
+     - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression
+     - content - expression for items within the nested lists (default=C{None})
+     - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString})
+
+    If an expression is not provided for the content argument, the nested
+    expression will capture all whitespace-delimited content between delimiters
+    as a list of separate values.
+
+    Use the C{ignoreExpr} argument to define expressions that may contain
+    opening or closing characters that should not be treated as opening
+    or closing characters for nesting, such as quotedString or a comment
+    expression.  Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+    The default is L{quotedString}, but if no expressions are to be ignored,
+    then pass C{None} for this argument.
+
+    Example::
+        data_type = oneOf("void int short long char float double")
+        decl_data_type = Combine(data_type + Optional(Word('*')))
+        ident = Word(alphas+'_', alphanums+'_')
+        number = pyparsing_common.number
+        arg = Group(decl_data_type + ident)
+        LPAR,RPAR = map(Suppress, "()")
+
+        code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment))
+
+        c_function = (decl_data_type("type") 
+                      + ident("name")
+                      + LPAR + Optional(delimitedList(arg), [])("args") + RPAR 
+                      + code_body("body"))
+        c_function.ignore(cStyleComment)
+        
+        source_code = '''
+            int is_odd(int x) { 
+                return (x%2); 
+            }
+                
+            int dec_to_hex(char hchar) { 
+                if (hchar >= '0' && hchar <= '9') { 
+                    return (ord(hchar)-ord('0')); 
+                } else { 
+                    return (10+ord(hchar)-ord('A'));
+                } 
+            }
+        '''
+        for func in c_function.searchString(source_code):
+            print("%(name)s (%(type)s) args: %(args)s" % func)
+
+    prints::
+        is_odd (int) args: [['int', 'x']]
+        dec_to_hex (int) args: [['char', 'hchar']]
+    """
+    if opener == closer:
+        raise ValueError("opening and closing strings cannot be the same")
+    if content is None:
+        if isinstance(opener,basestring) and isinstance(closer,basestring):
+            if len(opener) == 1 and len(closer)==1:
+                if ignoreExpr is not None:
+                    content = (Combine(OneOrMore(~ignoreExpr +
+                                    CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+                else:
+                    content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+                                ).setParseAction(lambda t:t[0].strip()))
+            else:
+                if ignoreExpr is not None:
+                    content = (Combine(OneOrMore(~ignoreExpr + 
+                                    ~Literal(opener) + ~Literal(closer) +
+                                    CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+                else:
+                    content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+                                    CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+        else:
+            raise ValueError("opening and closing arguments must be strings if no content expression is given")
+    ret = Forward()
+    if ignoreExpr is not None:
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+    else:
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
+    ret.setName('nested %s%s expression' % (opener,closer))
+    return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+    """
+    Helper method for defining space-delimited indentation blocks, such as
+    those used to define block statements in Python source code.
+
+    Parameters:
+     - blockStatementExpr - expression defining syntax of statement that
+            is repeated within the indented block
+     - indentStack - list created by caller to manage indentation stack
+            (multiple statementWithIndentedBlock expressions within a single grammar
+            should share a common indentStack)
+     - indent - boolean indicating whether block must be indented beyond the
+            the current level; set to False for block of left-most statements
+            (default=C{True})
+
+    A valid block must contain at least one C{blockStatement}.
+
+    Example::
+        data = '''
+        def A(z):
+          A1
+          B = 100
+          G = A2
+          A2
+          A3
+        B
+        def BB(a,b,c):
+          BB1
+          def BBA():
+            bba1
+            bba2
+            bba3
+        C
+        D
+        def spam(x,y):
+             def eggs(z):
+                 pass
+        '''
+
+
+        indentStack = [1]
+        stmt = Forward()
+
+        identifier = Word(alphas, alphanums)
+        funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
+        func_body = indentedBlock(stmt, indentStack)
+        funcDef = Group( funcDecl + func_body )
+
+        rvalue = Forward()
+        funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
+        rvalue << (funcCall | identifier | Word(nums))
+        assignment = Group(identifier + "=" + rvalue)
+        stmt << ( funcDef | assignment | identifier )
+
+        module_body = OneOrMore(stmt)
+
+        parseTree = module_body.parseString(data)
+        parseTree.pprint()
+    prints::
+        [['def',
+          'A',
+          ['(', 'z', ')'],
+          ':',
+          [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],
+         'B',
+         ['def',
+          'BB',
+          ['(', 'a', 'b', 'c', ')'],
+          ':',
+          [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],
+         'C',
+         'D',
+         ['def',
+          'spam',
+          ['(', 'x', 'y', ')'],
+          ':',
+          [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] 
+    """
+    def checkPeerIndent(s,l,t):
+        if l >= len(s): return
+        curCol = col(l,s)
+        if curCol != indentStack[-1]:
+            if curCol > indentStack[-1]:
+                raise ParseFatalException(s,l,"illegal nesting")
+            raise ParseException(s,l,"not a peer entry")
+
+    def checkSubIndent(s,l,t):
+        curCol = col(l,s)
+        if curCol > indentStack[-1]:
+            indentStack.append( curCol )
+        else:
+            raise ParseException(s,l,"not a subentry")
+
+    def checkUnindent(s,l,t):
+        if l >= len(s): return
+        curCol = col(l,s)
+        if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+            raise ParseException(s,l,"not an unindent")
+        indentStack.pop()
+
+    NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+    INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')
+    PEER   = Empty().setParseAction(checkPeerIndent).setName('')
+    UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')
+    if indent:
+        smExpr = Group( Optional(NL) +
+            #~ FollowedBy(blockStatementExpr) +
+            INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+    else:
+        smExpr = Group( Optional(NL) +
+            (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+    blockStatementExpr.ignore(_bslash + LineEnd())
+    return smExpr.setName('indented block')
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag'))
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\''))
+commonHTMLEntity = Regex('&(?P' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity")
+def replaceHTMLEntity(t):
+    """Helper parser action to replace common HTML entities with their special characters"""
+    return _htmlEntityMap.get(t.entity)
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment")
+"Comment of the form C{/* ... */}"
+
+htmlComment = Regex(r"").setName("HTML comment")
+"Comment of the form C{}"
+
+restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line")
+dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment")
+"Comment of the form C{// ... (to end of line)}"
+
+cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment")
+"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}"
+
+javaStyleComment = cppStyleComment
+"Same as C{L{cppStyleComment}}"
+
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+"Comment of the form C{# ... (to end of line)}"
+
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+                                  Optional( Word(" \t") +
+                                            ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas.
+   This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}."""
+
+# some other useful expressions - using lower-case class name since we are really using this as a namespace
+class pyparsing_common:
+    """
+    Here are some common low-level expressions that may be useful in jump-starting parser development:
+     - numeric forms (L{integers}, L{reals}, L{scientific notation})
+     - common L{programming identifiers}
+     - network addresses (L{MAC}, L{IPv4}, L{IPv6})
+     - ISO8601 L{dates} and L{datetime}
+     - L{UUID}
+     - L{comma-separated list}
+    Parse actions:
+     - C{L{convertToInteger}}
+     - C{L{convertToFloat}}
+     - C{L{convertToDate}}
+     - C{L{convertToDatetime}}
+     - C{L{stripHTMLTags}}
+     - C{L{upcaseTokens}}
+     - C{L{downcaseTokens}}
+
+    Example::
+        pyparsing_common.number.runTests('''
+            # any int or real number, returned as the appropriate type
+            100
+            -100
+            +100
+            3.14159
+            6.02e23
+            1e-12
+            ''')
+
+        pyparsing_common.fnumber.runTests('''
+            # any int or real number, returned as float
+            100
+            -100
+            +100
+            3.14159
+            6.02e23
+            1e-12
+            ''')
+
+        pyparsing_common.hex_integer.runTests('''
+            # hex numbers
+            100
+            FF
+            ''')
+
+        pyparsing_common.fraction.runTests('''
+            # fractions
+            1/2
+            -3/4
+            ''')
+
+        pyparsing_common.mixed_integer.runTests('''
+            # mixed fractions
+            1
+            1/2
+            -3/4
+            1-3/4
+            ''')
+
+        import uuid
+        pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+        pyparsing_common.uuid.runTests('''
+            # uuid
+            12345678-1234-5678-1234-567812345678
+            ''')
+    prints::
+        # any int or real number, returned as the appropriate type
+        100
+        [100]
+
+        -100
+        [-100]
+
+        +100
+        [100]
+
+        3.14159
+        [3.14159]
+
+        6.02e23
+        [6.02e+23]
+
+        1e-12
+        [1e-12]
+
+        # any int or real number, returned as float
+        100
+        [100.0]
+
+        -100
+        [-100.0]
+
+        +100
+        [100.0]
+
+        3.14159
+        [3.14159]
+
+        6.02e23
+        [6.02e+23]
+
+        1e-12
+        [1e-12]
+
+        # hex numbers
+        100
+        [256]
+
+        FF
+        [255]
+
+        # fractions
+        1/2
+        [0.5]
+
+        -3/4
+        [-0.75]
+
+        # mixed fractions
+        1
+        [1]
+
+        1/2
+        [0.5]
+
+        -3/4
+        [-0.75]
+
+        1-3/4
+        [1.75]
+
+        # uuid
+        12345678-1234-5678-1234-567812345678
+        [UUID('12345678-1234-5678-1234-567812345678')]
+    """
+
+    convertToInteger = tokenMap(int)
+    """
+    Parse action for converting parsed integers to Python int
+    """
+
+    convertToFloat = tokenMap(float)
+    """
+    Parse action for converting parsed numbers to Python float
+    """
+
+    integer = Word(nums).setName("integer").setParseAction(convertToInteger)
+    """expression that parses an unsigned integer, returns an int"""
+
+    hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16))
+    """expression that parses a hexadecimal integer, returns an int"""
+
+    signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger)
+    """expression that parses an integer with optional leading sign, returns an int"""
+
+    fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction")
+    """fractional expression of an integer divided by an integer, returns a float"""
+    fraction.addParseAction(lambda t: t[0]/t[-1])
+
+    mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction")
+    """mixed integer of the form 'integer - fraction', with optional leading integer, returns float"""
+    mixed_integer.addParseAction(sum)
+
+    real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat)
+    """expression that parses a floating point number and returns a float"""
+
+    sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat)
+    """expression that parses a floating point number with optional scientific notation and returns a float"""
+
+    # streamlining this expression makes the docs nicer-looking
+    number = (sci_real | real | signed_integer).streamline()
+    """any numeric expression, returns the corresponding Python type"""
+
+    fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat)
+    """any int or real number, returned as float"""
+    
+    identifier = Word(alphas+'_', alphanums+'_').setName("identifier")
+    """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
+    
+    ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address")
+    "IPv4 address (C{0.0.0.0 - 255.255.255.255})"
+
+    _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer")
+    _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address")
+    _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address")
+    _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8)
+    _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address")
+    ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address")
+    "IPv6 address (long, short, or mixed form)"
+    
+    mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address")
+    "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
+
+    @staticmethod
+    def convertToDate(fmt="%Y-%m-%d"):
+        """
+        Helper to create a parse action for converting parsed date string to Python datetime.date
+
+        Params -
+         - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"})
+
+        Example::
+            date_expr = pyparsing_common.iso8601_date.copy()
+            date_expr.setParseAction(pyparsing_common.convertToDate())
+            print(date_expr.parseString("1999-12-31"))
+        prints::
+            [datetime.date(1999, 12, 31)]
+        """
+        def cvt_fn(s,l,t):
+            try:
+                return datetime.strptime(t[0], fmt).date()
+            except ValueError as ve:
+                raise ParseException(s, l, str(ve))
+        return cvt_fn
+
+    @staticmethod
+    def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"):
+        """
+        Helper to create a parse action for converting parsed datetime string to Python datetime.datetime
+
+        Params -
+         - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"})
+
+        Example::
+            dt_expr = pyparsing_common.iso8601_datetime.copy()
+            dt_expr.setParseAction(pyparsing_common.convertToDatetime())
+            print(dt_expr.parseString("1999-12-31T23:59:59.999"))
+        prints::
+            [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
+        """
+        def cvt_fn(s,l,t):
+            try:
+                return datetime.strptime(t[0], fmt)
+            except ValueError as ve:
+                raise ParseException(s, l, str(ve))
+        return cvt_fn
+
+    iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date")
+    "ISO8601 date (C{yyyy-mm-dd})"
+
+    iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime")
+    "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}"
+
+    uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID")
+    "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})"
+
+    _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress()
+    @staticmethod
+    def stripHTMLTags(s, l, tokens):
+        """
+        Parse action to remove HTML tags from web page HTML source
+
+        Example::
+            # strip HTML links from normal text 
+            text = 'More info at the pyparsing wiki page'
+            td,td_end = makeHTMLTags("TD")
+            table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end
+            
+            print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page'
+        """
+        return pyparsing_common._html_stripper.transformString(tokens[0])
+
+    _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') 
+                                        + Optional( White(" \t") ) ) ).streamline().setName("commaItem")
+    comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list")
+    """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
+
+    upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper()))
+    """Parse action to convert tokens to upper case."""
+
+    downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower()))
+    """Parse action to convert tokens to lower case."""
+
+
+if __name__ == "__main__":
+
+    selectToken    = CaselessLiteral("select")
+    fromToken      = CaselessLiteral("from")
+
+    ident          = Word(alphas, alphanums + "_$")
+
+    columnName     = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+    columnNameList = Group(delimitedList(columnName)).setName("columns")
+    columnSpec     = ('*' | columnNameList)
+
+    tableName      = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+    tableNameList  = Group(delimitedList(tableName)).setName("tables")
+    
+    simpleSQL      = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables")
+
+    # demo runTests method, including embedded comments in test string
+    simpleSQL.runTests("""
+        # '*' as column list and dotted table name
+        select * from SYS.XYZZY
+
+        # caseless match on "SELECT", and casts back to "select"
+        SELECT * from XYZZY, ABC
+
+        # list of column names, and mixed case SELECT keyword
+        Select AA,BB,CC from Sys.dual
+
+        # multiple tables
+        Select A, B, C from Sys.dual, Table2
+
+        # invalid SELECT keyword - should fail
+        Xelect A, B, C from Sys.dual
+
+        # incomplete command - should fail
+        Select
+
+        # invalid column name - should fail
+        Select ^^^ frox Sys.dual
+
+        """)
+
+    pyparsing_common.number.runTests("""
+        100
+        -100
+        +100
+        3.14159
+        6.02e23
+        1e-12
+        """)
+
+    # any int or real number, returned as float
+    pyparsing_common.fnumber.runTests("""
+        100
+        -100
+        +100
+        3.14159
+        6.02e23
+        1e-12
+        """)
+
+    pyparsing_common.hex_integer.runTests("""
+        100
+        FF
+        """)
+
+    import uuid
+    pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+    pyparsing_common.uuid.runTests("""
+        12345678-1234-5678-1234-567812345678
+        """)
diff --git a/ubuntu/venv/pkg_resources/_vendor/six.py b/ubuntu/venv/pkg_resources/_vendor/six.py
new file mode 100644
index 0000000..190c023
--- /dev/null
+++ b/ubuntu/venv/pkg_resources/_vendor/six.py
@@ -0,0 +1,868 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2015 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson "
+__version__ = "1.10.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)  # Invokes __set__.
+        try:
+            # This is a bit ugly, but it avoids running this again by
+            # removing this descriptor.
+            delattr(obj.__class__, self.name)
+        except AttributeError:
+            pass
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("URLopener", "urllib", "urllib.request"),
+    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    def create_unbound_method(func, cls):
+        return func
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
+    viewkeys = operator.methodcaller("keys")
+
+    viewvalues = operator.methodcaller("values")
+
+    viewitems = operator.methodcaller("items")
+else:
+    def iterkeys(d, **kw):
+        return d.iterkeys(**kw)
+
+    def itervalues(d, **kw):
+        return d.itervalues(**kw)
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
+    viewkeys = operator.methodcaller("viewkeys")
+
+    viewvalues = operator.methodcaller("viewvalues")
+
+    viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+
+    def u(s):
+        return s
+    unichr = chr
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+    _assertCountEqual = "assertCountEqual"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+
+    def byte2int(bs):
+        return ord(bs[0])
+
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    iterbytes = functools.partial(itertools.imap, ord)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+    _assertCountEqual = "assertItemsEqual"
+    _assertRaisesRegex = "assertRaisesRegexp"
+    _assertRegex = "assertRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+    return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+    return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+    return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+    def reraise(tp, value, tb=None):
+        if value is None:
+            value = tp()
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+if sys.version_info[:2] == (3, 2):
+    exec_("""def raise_from(value, from_value):
+    if from_value is None:
+        raise value
+    raise value from from_value
+""")
+elif sys.version_info[:2] > (3, 2):
+    exec_("""def raise_from(value, from_value):
+    raise value from from_value
+""")
+else:
+    def raise_from(value, from_value):
+        raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+if sys.version_info[:2] < (3, 3):
+    _print = print_
+
+    def print_(*args, **kwargs):
+        fp = kwargs.get("file", sys.stdout)
+        flush = kwargs.pop("flush", False)
+        _print(*args, **kwargs)
+        if flush and fp is not None:
+            fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        def wrapper(f):
+            f = functools.wraps(wrapped, assigned, updated)(f)
+            f.__wrapped__ = wrapped
+            return f
+        return wrapper
+else:
+    wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(meta):
+
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
+def python_2_unicode_compatible(klass):
+    """
+    A decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if PY2:
+        if '__str__' not in klass.__dict__:
+            raise ValueError("@python_2_unicode_compatible cannot be applied "
+                             "to %s because it doesn't define __str__()." %
+                             klass.__name__)
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+                importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)
diff --git a/ubuntu/venv/pkg_resources/extern/__init__.py b/ubuntu/venv/pkg_resources/extern/__init__.py
new file mode 100644
index 0000000..c1eb9e9
--- /dev/null
+++ b/ubuntu/venv/pkg_resources/extern/__init__.py
@@ -0,0 +1,73 @@
+import sys
+
+
+class VendorImporter:
+    """
+    A PEP 302 meta path importer for finding optionally-vendored
+    or otherwise naturally-installed packages from root_name.
+    """
+
+    def __init__(self, root_name, vendored_names=(), vendor_pkg=None):
+        self.root_name = root_name
+        self.vendored_names = set(vendored_names)
+        self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor')
+
+    @property
+    def search_path(self):
+        """
+        Search first the vendor package then as a natural package.
+        """
+        yield self.vendor_pkg + '.'
+        yield ''
+
+    def find_module(self, fullname, path=None):
+        """
+        Return self when fullname starts with root_name and the
+        target module is one vendored through this importer.
+        """
+        root, base, target = fullname.partition(self.root_name + '.')
+        if root:
+            return
+        if not any(map(target.startswith, self.vendored_names)):
+            return
+        return self
+
+    def load_module(self, fullname):
+        """
+        Iterate over the search path to locate and load fullname.
+        """
+        root, base, target = fullname.partition(self.root_name + '.')
+        for prefix in self.search_path:
+            try:
+                extant = prefix + target
+                __import__(extant)
+                mod = sys.modules[extant]
+                sys.modules[fullname] = mod
+                # mysterious hack:
+                # Remove the reference to the extant package/module
+                # on later Python versions to cause relative imports
+                # in the vendor package to resolve the same modules
+                # as those going through this importer.
+                if prefix and sys.version_info > (3, 3):
+                    del sys.modules[extant]
+                return mod
+            except ImportError:
+                pass
+        else:
+            raise ImportError(
+                "The '{target}' package is required; "
+                "normally this is bundled with this package so if you get "
+                "this warning, consult the packager of your "
+                "distribution.".format(**locals())
+            )
+
+    def install(self):
+        """
+        Install this importer into sys.meta_path if not already present.
+        """
+        if self not in sys.meta_path:
+            sys.meta_path.append(self)
+
+
+names = 'packaging', 'pyparsing', 'six', 'appdirs'
+VendorImporter(__name__, names).install()
diff --git a/ubuntu/venv/pkg_resources/py31compat.py b/ubuntu/venv/pkg_resources/py31compat.py
new file mode 100644
index 0000000..a381c42
--- /dev/null
+++ b/ubuntu/venv/pkg_resources/py31compat.py
@@ -0,0 +1,23 @@
+import os
+import errno
+import sys
+
+from .extern import six
+
+
+def _makedirs_31(path, exist_ok=False):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if not exist_ok or exc.errno != errno.EEXIST:
+            raise
+
+
+# rely on compatibility behavior until mode considerations
+#  and exists_ok considerations are disentangled.
+# See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663
+needs_makedirs = (
+    six.PY2 or
+    (3, 4) <= sys.version_info < (3, 4, 1)
+)
+makedirs = _makedirs_31 if needs_makedirs else os.makedirs
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/AUTHORS.txt b/ubuntu/venv/setuptools-44.0.0.dist-info/AUTHORS.txt
new file mode 100644
index 0000000..72c87d7
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/AUTHORS.txt
@@ -0,0 +1,562 @@
+A_Rog 
+Aakanksha Agrawal <11389424+rasponic@users.noreply.github.com>
+Abhinav Sagar <40603139+abhinavsagar@users.noreply.github.com>
+ABHYUDAY PRATAP SINGH 
+abs51295 
+AceGentile 
+Adam Chainz 
+Adam Tse 
+Adam Tse 
+Adam Wentz 
+admin 
+Adrien Morison 
+ahayrapetyan 
+Ahilya 
+AinsworthK 
+Akash Srivastava 
+Alan Yee 
+Albert Tugushev 
+Albert-Guan 
+albertg 
+Aleks Bunin 
+Alethea Flowers 
+Alex Gaynor 
+Alex Grönholm 
+Alex Loosley 
+Alex Morega 
+Alex Stachowiak 
+Alexander Shtyrov 
+Alexandre Conrad 
+Alexey Popravka 
+Alexey Popravka 
+Alli 
+Ami Fischman 
+Ananya Maiti 
+Anatoly Techtonik 
+Anders Kaseorg 
+Andreas Lutro 
+Andrei Geacar 
+Andrew Gaul 
+Andrey Bulgakov 
+Andrés Delfino <34587441+andresdelfino@users.noreply.github.com>
+Andrés Delfino 
+Andy Freeland 
+Andy Freeland 
+Andy Kluger 
+Ani Hayrapetyan 
+Aniruddha Basak 
+Anish Tambe 
+Anrs Hu 
+Anthony Sottile 
+Antoine Musso 
+Anton Ovchinnikov 
+Anton Patrushev 
+Antonio Alvarado Hernandez 
+Antony Lee 
+Antti Kaihola 
+Anubhav Patel 
+Anuj Godase 
+AQNOUCH Mohammed 
+AraHaan 
+Arindam Choudhury 
+Armin Ronacher 
+Artem 
+Ashley Manton 
+Ashwin Ramaswami 
+atse 
+Atsushi Odagiri 
+Avner Cohen 
+Baptiste Mispelon 
+Barney Gale 
+barneygale 
+Bartek Ogryczak 
+Bastian Venthur 
+Ben Darnell 
+Ben Hoyt 
+Ben Rosser 
+Bence Nagy 
+Benjamin Peterson 
+Benjamin VanEvery 
+Benoit Pierre 
+Berker Peksag 
+Bernardo B. Marques 
+Bernhard M. Wiedemann 
+Bertil Hatt 
+Bogdan Opanchuk 
+BorisZZZ 
+Brad Erickson 
+Bradley Ayers 
+Brandon L. Reiss 
+Brandt Bucher 
+Brett Randall 
+Brian Cristante <33549821+brcrista@users.noreply.github.com>
+Brian Cristante 
+Brian Rosner 
+BrownTruck 
+Bruno Oliveira 
+Bruno Renié 
+Bstrdsmkr 
+Buck Golemon 
+burrows 
+Bussonnier Matthias 
+c22 
+Caleb Martinez 
+Calvin Smith 
+Carl Meyer 
+Carlos Liam 
+Carol Willing 
+Carter Thayer 
+Cass 
+Chandrasekhar Atina 
+Chih-Hsuan Yen 
+Chih-Hsuan Yen 
+Chris Brinker 
+Chris Hunt 
+Chris Jerdonek 
+Chris McDonough 
+Chris Wolfe 
+Christian Heimes 
+Christian Oudard 
+Christopher Hunt 
+Christopher Snyder 
+Clark Boylan 
+Clay McClure 
+Cody 
+Cody Soyland 
+Colin Watson 
+Connor Osborn 
+Cooper Lees 
+Cooper Ry Lees 
+Cory Benfield 
+Cory Wright 
+Craig Kerstiens 
+Cristian Sorinel 
+Curtis Doty 
+cytolentino 
+Damian Quiroga 
+Dan Black 
+Dan Savilonis 
+Dan Sully 
+daniel 
+Daniel Collins 
+Daniel Hahler 
+Daniel Holth 
+Daniel Jost 
+Daniel Shaulov 
+Daniele Esposti 
+Daniele Procida 
+Danny Hermes 
+Dav Clark 
+Dave Abrahams 
+Dave Jones 
+David Aguilar 
+David Black 
+David Bordeynik 
+David Bordeynik 
+David Caro 
+David Evans 
+David Linke 
+David Pursehouse 
+David Tucker 
+David Wales 
+Davidovich 
+derwolfe 
+Desetude 
+Diego Caraballo 
+DiegoCaraballo 
+Dmitry Gladkov 
+Domen Kožar 
+Donald Stufft 
+Dongweiming 
+Douglas Thor 
+DrFeathers 
+Dustin Ingram 
+Dwayne Bailey 
+Ed Morley <501702+edmorley@users.noreply.github.com>
+Ed Morley 
+Eitan Adler 
+ekristina 
+elainechan 
+Eli Schwartz 
+Eli Schwartz 
+Emil Burzo 
+Emil Styrke 
+Endoh Takanao 
+enoch 
+Erdinc Mutlu 
+Eric Gillingham 
+Eric Hanchrow 
+Eric Hopper 
+Erik M. Bray 
+Erik Rose 
+Ernest W Durbin III 
+Ernest W. Durbin III 
+Erwin Janssen 
+Eugene Vereshchagin 
+everdimension 
+Felix Yan 
+fiber-space 
+Filip Kokosiński 
+Florian Briand 
+Florian Rathgeber 
+Francesco 
+Francesco Montesano 
+Frost Ming 
+Gabriel Curio 
+Gabriel de Perthuis 
+Garry Polley 
+gdanielson 
+Geoffrey Lehée 
+Geoffrey Sneddon 
+George Song 
+Georgi Valkov 
+Giftlin Rajaiah 
+gizmoguy1 
+gkdoc <40815324+gkdoc@users.noreply.github.com>
+Gopinath M <31352222+mgopi1990@users.noreply.github.com>
+GOTO Hayato <3532528+gh640@users.noreply.github.com>
+gpiks 
+Guilherme Espada 
+Guy Rozendorn 
+gzpan123 
+Hanjun Kim 
+Hari Charan 
+Harsh Vardhan 
+Herbert Pfennig 
+Hsiaoming Yang 
+Hugo 
+Hugo Lopes Tavares 
+Hugo van Kemenade 
+hugovk 
+Hynek Schlawack 
+Ian Bicking 
+Ian Cordasco 
+Ian Lee 
+Ian Stapleton Cordasco 
+Ian Wienand 
+Ian Wienand 
+Igor Kuzmitshov 
+Igor Sobreira 
+Ilya Baryshev 
+INADA Naoki 
+Ionel Cristian Mărieș 
+Ionel Maries Cristian 
+Ivan Pozdeev 
+Jacob Kim 
+jakirkham 
+Jakub Stasiak 
+Jakub Vysoky 
+Jakub Wilk 
+James Cleveland 
+James Cleveland 
+James Firth 
+James Polley 
+Jan Pokorný 
+Jannis Leidel 
+jarondl 
+Jason R. Coombs 
+Jay Graves 
+Jean-Christophe Fillion-Robin 
+Jeff Barber 
+Jeff Dairiki 
+Jelmer Vernooij 
+jenix21 
+Jeremy Stanley 
+Jeremy Zafran 
+Jiashuo Li 
+Jim Garrison 
+Jivan Amara 
+John Paton 
+John-Scott Atlakson 
+johnthagen 
+johnthagen 
+Jon Banafato 
+Jon Dufresne 
+Jon Parise 
+Jonas Nockert 
+Jonathan Herbert 
+Joost Molenaar 
+Jorge Niedbalski 
+Joseph Long 
+Josh Bronson 
+Josh Hansen 
+Josh Schneier 
+Juanjo Bazán 
+Julian Berman 
+Julian Gethmann 
+Julien Demoor 
+jwg4 
+Jyrki Pulliainen 
+Kai Chen 
+Kamal Bin Mustafa 
+kaustav haldar 
+keanemind 
+Keith Maxwell 
+Kelsey Hightower 
+Kenneth Belitzky 
+Kenneth Reitz 
+Kenneth Reitz 
+Kevin Burke 
+Kevin Carter 
+Kevin Frommelt 
+Kevin R Patterson 
+Kexuan Sun 
+Kit Randel 
+kpinc 
+Krishna Oza 
+Kumar McMillan 
+Kyle Persohn 
+lakshmanaram 
+Laszlo Kiss-Kollar 
+Laurent Bristiel 
+Laurie Opperman 
+Leon Sasson 
+Lev Givon 
+Lincoln de Sousa 
+Lipis 
+Loren Carvalho 
+Lucas Cimon 
+Ludovic Gasc 
+Luke Macken 
+Luo Jiebin 
+luojiebin 
+luz.paz 
+László Kiss Kollár 
+László Kiss Kollár 
+Marc Abramowitz 
+Marc Tamlyn 
+Marcus Smith 
+Mariatta 
+Mark Kohler 
+Mark Williams 
+Mark Williams 
+Markus Hametner 
+Masaki 
+Masklinn 
+Matej Stuchlik 
+Mathew Jennings 
+Mathieu Bridon 
+Matt Good 
+Matt Maker 
+Matt Robenolt 
+matthew 
+Matthew Einhorn 
+Matthew Gilliard 
+Matthew Iversen 
+Matthew Trumbell 
+Matthew Willson 
+Matthias Bussonnier 
+mattip 
+Maxim Kurnikov 
+Maxime Rouyrre 
+mayeut 
+mbaluna <44498973+mbaluna@users.noreply.github.com>
+mdebi <17590103+mdebi@users.noreply.github.com>
+memoselyk 
+Michael 
+Michael Aquilina 
+Michael E. Karpeles 
+Michael Klich 
+Michael Williamson 
+michaelpacer 
+Mickaël Schoentgen 
+Miguel Araujo Perez 
+Mihir Singh 
+Mike 
+Mike Hendricks 
+Min RK 
+MinRK 
+Miro Hrončok 
+Monica Baluna 
+montefra 
+Monty Taylor 
+Nate Coraor 
+Nathaniel J. Smith 
+Nehal J Wani 
+Neil Botelho 
+Nick Coghlan 
+Nick Stenning 
+Nick Timkovich 
+Nicolas Bock 
+Nikhil Benesch 
+Nitesh Sharma 
+Nowell Strite 
+NtaleGrey 
+nvdv 
+Ofekmeister 
+ofrinevo 
+Oliver Jeeves 
+Oliver Tonnhofer 
+Olivier Girardot 
+Olivier Grisel 
+Ollie Rutherfurd 
+OMOTO Kenji 
+Omry Yadan 
+Oren Held 
+Oscar Benjamin 
+Oz N Tiram 
+Pachwenko <32424503+Pachwenko@users.noreply.github.com>
+Patrick Dubroy 
+Patrick Jenkins 
+Patrick Lawson 
+patricktokeeffe 
+Patrik Kopkan 
+Paul Kehrer 
+Paul Moore 
+Paul Nasrat 
+Paul Oswald 
+Paul van der Linden 
+Paulus Schoutsen 
+Pavithra Eswaramoorthy <33131404+QueenCoffee@users.noreply.github.com>
+Pawel Jasinski 
+Pekka Klärck 
+Peter Lisák 
+Peter Waller 
+petr-tik 
+Phaneendra Chiruvella 
+Phil Freo 
+Phil Pennock 
+Phil Whelan 
+Philip Jägenstedt 
+Philip Molloy 
+Philippe Ombredanne 
+Pi Delport 
+Pierre-Yves Rofes 
+pip 
+Prabakaran Kumaresshan 
+Prabhjyotsing Surjit Singh Sodhi 
+Prabhu Marappan 
+Pradyun Gedam 
+Pratik Mallya 
+Preet Thakkar 
+Preston Holmes 
+Przemek Wrzos 
+Pulkit Goyal <7895pulkit@gmail.com>
+Qiangning Hong 
+Quentin Pradet 
+R. David Murray 
+Rafael Caricio 
+Ralf Schmitt 
+Razzi Abuissa 
+rdb 
+Remi Rampin 
+Remi Rampin 
+Rene Dudfield 
+Riccardo Magliocchetti 
+Richard Jones 
+RobberPhex 
+Robert Collins 
+Robert McGibbon 
+Robert T. McGibbon 
+robin elisha robinson 
+Roey Berman 
+Rohan Jain 
+Rohan Jain 
+Rohan Jain 
+Roman Bogorodskiy 
+Romuald Brunet 
+Ronny Pfannschmidt 
+Rory McCann 
+Ross Brattain 
+Roy Wellington Ⅳ 
+Roy Wellington Ⅳ 
+Ryan Wooden 
+ryneeverett 
+Sachi King 
+Salvatore Rinchiera 
+Savio Jomton 
+schlamar 
+Scott Kitterman 
+Sean 
+seanj 
+Sebastian Jordan 
+Sebastian Schaetz 
+Segev Finer 
+SeongSoo Cho 
+Sergey Vasilyev 
+Seth Woodworth 
+Shlomi Fish 
+Shovan Maity 
+Simeon Visser 
+Simon Cross 
+Simon Pichugin 
+sinoroc 
+Sorin Sbarnea 
+Stavros Korokithakis 
+Stefan Scherfke 
+Stephan Erb 
+stepshal 
+Steve (Gadget) Barnes 
+Steve Barnes 
+Steve Dower 
+Steve Kowalik 
+Steven Myint 
+stonebig 
+Stéphane Bidoul (ACSONE) 
+Stéphane Bidoul 
+Stéphane Klein 
+Sumana Harihareswara 
+Sviatoslav Sydorenko 
+Sviatoslav Sydorenko 
+Swat009 
+Takayuki SHIMIZUKAWA 
+tbeswick 
+Thijs Triemstra 
+Thomas Fenzl 
+Thomas Grainger 
+Thomas Guettler 
+Thomas Johansson 
+Thomas Kluyver 
+Thomas Smith 
+Tim D. Smith 
+Tim Gates 
+Tim Harder 
+Tim Heap 
+tim smith 
+tinruufu 
+Tom Forbes 
+Tom Freudenheim 
+Tom V 
+Tomas Orsava 
+Tomer Chachamu 
+Tony Beswick 
+Tony Zhaocheng Tan 
+TonyBeswick 
+toonarmycaptain 
+Toshio Kuratomi 
+Travis Swicegood 
+Tzu-ping Chung 
+Valentin Haenel 
+Victor Stinner 
+victorvpaulo 
+Viktor Szépe 
+Ville Skyttä 
+Vinay Sajip 
+Vincent Philippon 
+Vinicyus Macedo <7549205+vinicyusmacedo@users.noreply.github.com>
+Vitaly Babiy 
+Vladimir Rutsky 
+W. Trevor King 
+Wil Tan 
+Wilfred Hughes 
+William ML Leslie 
+William T Olson 
+Wilson Mo 
+wim glenn 
+Wolfgang Maier 
+Xavier Fernandez 
+Xavier Fernandez 
+xoviat 
+xtreak 
+YAMAMOTO Takashi 
+Yen Chi Hsuan 
+Yeray Diaz Diaz 
+Yoval P 
+Yu Jian 
+Yuan Jing Vincent Yan 
+Zearin 
+Zearin 
+Zhiping Deng 
+Zvezdan Petkovic 
+Łukasz Langa 
+Семён Марьясин 
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/INSTALLER b/ubuntu/venv/setuptools-44.0.0.dist-info/INSTALLER
new file mode 100644
index 0000000..a1b589e
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/INSTALLER
@@ -0,0 +1 @@
+pip
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/LICENSE.txt b/ubuntu/venv/setuptools-44.0.0.dist-info/LICENSE.txt
new file mode 100644
index 0000000..737fec5
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/LICENSE.txt
@@ -0,0 +1,20 @@
+Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/METADATA b/ubuntu/venv/setuptools-44.0.0.dist-info/METADATA
new file mode 100644
index 0000000..4adf953
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/METADATA
@@ -0,0 +1,82 @@
+Metadata-Version: 2.1
+Name: setuptools
+Version: 44.0.0
+Summary: Easily download, build, install, upgrade, and uninstall Python packages
+Home-page: https://github.com/pypa/setuptools
+Author: Python Packaging Authority
+Author-email: distutils-sig@python.org
+License: UNKNOWN
+Project-URL: Documentation, https://setuptools.readthedocs.io/
+Keywords: CPAN PyPI distutils eggs package management
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Archiving :: Packaging
+Classifier: Topic :: System :: Systems Administration
+Classifier: Topic :: Utilities
+Requires-Python: !=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7
+Description-Content-Type: text/x-rst; charset=UTF-8
+
+.. image:: https://img.shields.io/pypi/v/setuptools.svg
+   :target: https://pypi.org/project/setuptools
+
+.. image:: https://img.shields.io/readthedocs/setuptools/latest.svg
+    :target: https://setuptools.readthedocs.io
+
+.. image:: https://img.shields.io/travis/pypa/setuptools/master.svg?label=Linux%20CI&logo=travis&logoColor=white
+   :target: https://travis-ci.org/pypa/setuptools
+
+.. image:: https://img.shields.io/appveyor/ci/pypa/setuptools/master.svg?label=Windows%20CI&logo=appveyor&logoColor=white
+   :target: https://ci.appveyor.com/project/pypa/setuptools/branch/master
+
+.. image:: https://img.shields.io/codecov/c/github/pypa/setuptools/master.svg?logo=codecov&logoColor=white
+   :target: https://codecov.io/gh/pypa/setuptools
+
+.. image:: https://tidelift.com/badges/github/pypa/setuptools?style=flat
+   :target: https://tidelift.com/subscription/pkg/pypi-setuptools?utm_source=pypi-setuptools&utm_medium=readme
+
+.. image:: https://img.shields.io/pypi/pyversions/setuptools.svg
+
+See the `Installation Instructions
+`_ in the Python Packaging
+User's Guide for instructions on installing, upgrading, and uninstalling
+Setuptools.
+
+Questions and comments should be directed to the `distutils-sig
+mailing list `_.
+Bug reports and especially tested patches may be
+submitted directly to the `bug tracker
+`_.
+
+To report a security vulnerability, please use the
+`Tidelift security contact `_.
+Tidelift will coordinate the fix and disclosure.
+
+
+For Enterprise
+==============
+
+Available as part of the Tidelift Subscription.
+
+Setuptools and the maintainers of thousands of other packages are working with Tidelift to deliver one enterprise subscription that covers all of the open source you use.
+
+`Learn more `_.
+
+Code of Conduct
+===============
+
+Everyone interacting in the setuptools project's codebases, issue trackers,
+chat rooms, and mailing lists is expected to follow the
+`PyPA Code of Conduct `_.
+
+
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/RECORD b/ubuntu/venv/setuptools-44.0.0.dist-info/RECORD
new file mode 100644
index 0000000..4aa4bcd
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/RECORD
@@ -0,0 +1,163 @@
+../../../bin/easy_install,sha256=VHYKLkmInIcDgX7ejbAbrSoXLjN42kItcLNrhQx1qrw,262
+../../../bin/easy_install-3.8,sha256=VHYKLkmInIcDgX7ejbAbrSoXLjN42kItcLNrhQx1qrw,262
+__pycache__/easy_install.cpython-38.pyc,,
+easy_install.py,sha256=MDC9vt5AxDsXX5qcKlBz2TnW6Tpuv_AobnfhCJ9X3PM,126
+setuptools-44.0.0.dist-info/AUTHORS.txt,sha256=RtqU9KfonVGhI48DAA4-yTOBUhBtQTjFhaDzHoyh7uU,21518
+setuptools-44.0.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+setuptools-44.0.0.dist-info/LICENSE.txt,sha256=W6Ifuwlk-TatfRU2LR7W1JMcyMj5_y1NkRkOEJvnRDE,1090
+setuptools-44.0.0.dist-info/METADATA,sha256=L93fcafgVw4xoJUNG0lehyy0prVj-jU_JFxRh0ZUtos,3523
+setuptools-44.0.0.dist-info/RECORD,,
+setuptools-44.0.0.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110
+setuptools-44.0.0.dist-info/dependency_links.txt,sha256=HlkCFkoK5TbZ5EMLbLKYhLcY_E31kBWD8TqW2EgmatQ,239
+setuptools-44.0.0.dist-info/entry_points.txt,sha256=ZmIqlp-SBdsBS2cuetmU2NdSOs4DG0kxctUR9UJ8Xk0,3150
+setuptools-44.0.0.dist-info/top_level.txt,sha256=2HUXVVwA4Pff1xgTFr3GsTXXKaPaO6vlG6oNJ_4u4Tg,38
+setuptools-44.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+setuptools/__init__.py,sha256=WBpCcn2lvdckotabeae1TTYonPOcgCIF3raD2zRWzBc,7283
+setuptools/__pycache__/__init__.cpython-38.pyc,,
+setuptools/__pycache__/_deprecation_warning.cpython-38.pyc,,
+setuptools/__pycache__/_imp.cpython-38.pyc,,
+setuptools/__pycache__/archive_util.cpython-38.pyc,,
+setuptools/__pycache__/build_meta.cpython-38.pyc,,
+setuptools/__pycache__/config.cpython-38.pyc,,
+setuptools/__pycache__/dep_util.cpython-38.pyc,,
+setuptools/__pycache__/depends.cpython-38.pyc,,
+setuptools/__pycache__/dist.cpython-38.pyc,,
+setuptools/__pycache__/errors.cpython-38.pyc,,
+setuptools/__pycache__/extension.cpython-38.pyc,,
+setuptools/__pycache__/glob.cpython-38.pyc,,
+setuptools/__pycache__/installer.cpython-38.pyc,,
+setuptools/__pycache__/launch.cpython-38.pyc,,
+setuptools/__pycache__/lib2to3_ex.cpython-38.pyc,,
+setuptools/__pycache__/monkey.cpython-38.pyc,,
+setuptools/__pycache__/msvc.cpython-38.pyc,,
+setuptools/__pycache__/namespaces.cpython-38.pyc,,
+setuptools/__pycache__/package_index.cpython-38.pyc,,
+setuptools/__pycache__/py27compat.cpython-38.pyc,,
+setuptools/__pycache__/py31compat.cpython-38.pyc,,
+setuptools/__pycache__/py33compat.cpython-38.pyc,,
+setuptools/__pycache__/py34compat.cpython-38.pyc,,
+setuptools/__pycache__/sandbox.cpython-38.pyc,,
+setuptools/__pycache__/site-patch.cpython-38.pyc,,
+setuptools/__pycache__/ssl_support.cpython-38.pyc,,
+setuptools/__pycache__/unicode_utils.cpython-38.pyc,,
+setuptools/__pycache__/version.cpython-38.pyc,,
+setuptools/__pycache__/wheel.cpython-38.pyc,,
+setuptools/__pycache__/windows_support.cpython-38.pyc,,
+setuptools/_deprecation_warning.py,sha256=jU9-dtfv6cKmtQJOXN8nP1mm7gONw5kKEtiPtbwnZyI,218
+setuptools/_imp.py,sha256=jloslOkxrTKbobgemfP94YII0nhqiJzE1bRmCTZ1a5I,2223
+setuptools/_vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+setuptools/_vendor/__pycache__/__init__.cpython-38.pyc,,
+setuptools/_vendor/__pycache__/ordered_set.cpython-38.pyc,,
+setuptools/_vendor/__pycache__/pyparsing.cpython-38.pyc,,
+setuptools/_vendor/__pycache__/six.cpython-38.pyc,,
+setuptools/_vendor/ordered_set.py,sha256=dbaCcs27dyN9gnMWGF5nA_BrVn6Q-NrjKYJpV9_fgBs,15130
+setuptools/_vendor/packaging/__about__.py,sha256=CpuMSyh1V7adw8QMjWKkY3LtdqRUkRX4MgJ6nF4stM0,744
+setuptools/_vendor/packaging/__init__.py,sha256=6enbp5XgRfjBjsI9-bn00HjHf5TH21PDMOKkJW8xw-w,562
+setuptools/_vendor/packaging/__pycache__/__about__.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/__init__.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/_compat.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/_structures.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/markers.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/requirements.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/specifiers.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/tags.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/utils.cpython-38.pyc,,
+setuptools/_vendor/packaging/__pycache__/version.cpython-38.pyc,,
+setuptools/_vendor/packaging/_compat.py,sha256=Ugdm-qcneSchW25JrtMIKgUxfEEBcCAz6WrEeXeqz9o,865
+setuptools/_vendor/packaging/_structures.py,sha256=pVd90XcXRGwpZRB_qdFuVEibhCHpX_bL5zYr9-N0mc8,1416
+setuptools/_vendor/packaging/markers.py,sha256=-meFl9Fr9V8rF5Rduzgett5EHK9wBYRUqssAV2pj0lw,8268
+setuptools/_vendor/packaging/requirements.py,sha256=3dwIJekt8RRGCUbgxX8reeAbgmZYjb0wcCRtmH63kxI,4742
+setuptools/_vendor/packaging/specifiers.py,sha256=0ZzQpcUnvrQ6LjR-mQRLzMr8G6hdRv-mY0VSf_amFtI,27778
+setuptools/_vendor/packaging/tags.py,sha256=EPLXhO6GTD7_oiWEO1U0l0PkfR8R_xivpMDHXnsTlts,12933
+setuptools/_vendor/packaging/utils.py,sha256=VaTC0Ei7zO2xl9ARiWmz2YFLFt89PuuhLbAlXMyAGms,1520
+setuptools/_vendor/packaging/version.py,sha256=Npdwnb8OHedj_2L86yiUqscujb7w_i5gmSK1PhOAFzg,11978
+setuptools/_vendor/pyparsing.py,sha256=tmrp-lu-qO1i75ZzIN5A12nKRRD1Cm4Vpk-5LR9rims,232055
+setuptools/_vendor/six.py,sha256=A6hdJZVjI3t_geebZ9BzUvwRrIXo0lfwzQlM2LcKyas,30098
+setuptools/archive_util.py,sha256=kw8Ib_lKjCcnPKNbS7h8HztRVK0d5RacU3r_KRdVnmM,6592
+setuptools/build_meta.py,sha256=-9Nmj9YdbW4zX3TssPJZhsENrTa4fw3k86Jm1cdKMik,9597
+setuptools/cli-32.exe,sha256=dfEuovMNnA2HLa3jRfMPVi5tk4R7alCbpTvuxtCyw0Y,65536
+setuptools/cli-64.exe,sha256=KLABu5pyrnokJCv6skjXZ6GsXeyYHGcqOUT3oHI3Xpo,74752
+setuptools/cli.exe,sha256=dfEuovMNnA2HLa3jRfMPVi5tk4R7alCbpTvuxtCyw0Y,65536
+setuptools/command/__init__.py,sha256=QCAuA9whnq8Bnoc0bBaS6Lw_KAUO0DiHYZQXEMNn5hg,568
+setuptools/command/__pycache__/__init__.cpython-38.pyc,,
+setuptools/command/__pycache__/alias.cpython-38.pyc,,
+setuptools/command/__pycache__/bdist_egg.cpython-38.pyc,,
+setuptools/command/__pycache__/bdist_rpm.cpython-38.pyc,,
+setuptools/command/__pycache__/bdist_wininst.cpython-38.pyc,,
+setuptools/command/__pycache__/build_clib.cpython-38.pyc,,
+setuptools/command/__pycache__/build_ext.cpython-38.pyc,,
+setuptools/command/__pycache__/build_py.cpython-38.pyc,,
+setuptools/command/__pycache__/develop.cpython-38.pyc,,
+setuptools/command/__pycache__/dist_info.cpython-38.pyc,,
+setuptools/command/__pycache__/easy_install.cpython-38.pyc,,
+setuptools/command/__pycache__/egg_info.cpython-38.pyc,,
+setuptools/command/__pycache__/install.cpython-38.pyc,,
+setuptools/command/__pycache__/install_egg_info.cpython-38.pyc,,
+setuptools/command/__pycache__/install_lib.cpython-38.pyc,,
+setuptools/command/__pycache__/install_scripts.cpython-38.pyc,,
+setuptools/command/__pycache__/py36compat.cpython-38.pyc,,
+setuptools/command/__pycache__/register.cpython-38.pyc,,
+setuptools/command/__pycache__/rotate.cpython-38.pyc,,
+setuptools/command/__pycache__/saveopts.cpython-38.pyc,,
+setuptools/command/__pycache__/sdist.cpython-38.pyc,,
+setuptools/command/__pycache__/setopt.cpython-38.pyc,,
+setuptools/command/__pycache__/test.cpython-38.pyc,,
+setuptools/command/__pycache__/upload.cpython-38.pyc,,
+setuptools/command/__pycache__/upload_docs.cpython-38.pyc,,
+setuptools/command/alias.py,sha256=KjpE0sz_SDIHv3fpZcIQK-sCkJz-SrC6Gmug6b9Nkc8,2426
+setuptools/command/bdist_egg.py,sha256=nnfV8Ah8IRC_Ifv5Loa9FdxL66MVbyDXwy-foP810zM,18185
+setuptools/command/bdist_rpm.py,sha256=B7l0TnzCGb-0nLlm6rS00jWLkojASwVmdhW2w5Qz_Ak,1508
+setuptools/command/bdist_wininst.py,sha256=_6dz3lpB1tY200LxKPLM7qgwTCceOMgaWFF-jW2-pm0,637
+setuptools/command/build_clib.py,sha256=bQ9aBr-5ZSO-9fGsGsDLz0mnnFteHUZnftVLkhvHDq0,4484
+setuptools/command/build_ext.py,sha256=Ib42YUGksBswm2mL5xmQPF6NeTA6HcqrvAtEgFCv32A,13019
+setuptools/command/build_py.py,sha256=yWyYaaS9F3o9JbIczn064A5g1C5_UiKRDxGaTqYbtLE,9596
+setuptools/command/develop.py,sha256=MQlnGS6uP19erK2JCNOyQYoYyquk3PADrqrrinqqLtA,8184
+setuptools/command/dist_info.py,sha256=5t6kOfrdgALT-P3ogss6PF9k-Leyesueycuk3dUyZnI,960
+setuptools/command/easy_install.py,sha256=0lY8Agxe-7IgMtxgxFuOY1NrDlBzOUlpCKsvayXlTYY,89903
+setuptools/command/egg_info.py,sha256=0e_TXrMfpa8nGTO7GmJcmpPCMWzliZi6zt9aMchlumc,25578
+setuptools/command/install.py,sha256=8doMxeQEDoK4Eco0mO2WlXXzzp9QnsGJQ7Z7yWkZPG8,4705
+setuptools/command/install_egg_info.py,sha256=4zq_Ad3jE-EffParuyDEnvxU6efB-Xhrzdr8aB6Ln_8,3195
+setuptools/command/install_lib.py,sha256=9zdc-H5h6RPxjySRhOwi30E_WfcVva7gpfhZ5ata60w,5023
+setuptools/command/install_scripts.py,sha256=UD0rEZ6861mTYhIdzcsqKnUl8PozocXWl9VBQ1VTWnc,2439
+setuptools/command/launcher manifest.xml,sha256=xlLbjWrB01tKC0-hlVkOKkiSPbzMml2eOPtJ_ucCnbE,628
+setuptools/command/py36compat.py,sha256=SzjZcOxF7zdFUT47Zv2n7AM3H8koDys_0OpS-n9gIfc,4986
+setuptools/command/register.py,sha256=kk3DxXCb5lXTvqnhfwx2g6q7iwbUmgTyXUCaBooBOUk,468
+setuptools/command/rotate.py,sha256=co5C1EkI7P0GGT6Tqz-T2SIj2LBJTZXYELpmao6d4KQ,2164
+setuptools/command/saveopts.py,sha256=za7QCBcQimKKriWcoCcbhxPjUz30gSB74zuTL47xpP4,658
+setuptools/command/sdist.py,sha256=IL1LepD2h8qGKOFJ3rrQVbjNH_Q6ViD40l0QADr4MEU,8088
+setuptools/command/setopt.py,sha256=NTWDyx-gjDF-txf4dO577s7LOzHVoKR0Mq33rFxaRr8,5085
+setuptools/command/test.py,sha256=u2kXngIIdSYqtvwFlHiN6Iye1IB4TU6uadB2uiV1szw,9602
+setuptools/command/upload.py,sha256=XT3YFVfYPAmA5qhGg0euluU98ftxRUW-PzKcODMLxUs,462
+setuptools/command/upload_docs.py,sha256=oXiGplM_cUKLwE4CWWw98RzCufAu8tBhMC97GegFcms,7311
+setuptools/config.py,sha256=6SB2OY3qcooOJmG_rsK_s0pKBsorBlDpfMJUyzjQIGk,20575
+setuptools/dep_util.py,sha256=fgixvC1R7sH3r13ktyf7N0FALoqEXL1cBarmNpSEoWg,935
+setuptools/depends.py,sha256=qt2RWllArRvhnm8lxsyRpcthEZYp4GHQgREl1q0LkFw,5517
+setuptools/dist.py,sha256=xtXaNsOsE32MwwQqErzgXJF7jsTQz9GYFRrwnPFQ0J0,49865
+setuptools/errors.py,sha256=MVOcv381HNSajDgEUWzOQ4J6B5BHCBMSjHfaWcEwA1o,524
+setuptools/extension.py,sha256=uc6nHI-MxwmNCNPbUiBnybSyqhpJqjbhvOQ-emdvt_E,1729
+setuptools/extern/__init__.py,sha256=4q9gtShB1XFP6CisltsyPqtcfTO6ZM9Lu1QBl3l-qmo,2514
+setuptools/extern/__pycache__/__init__.cpython-38.pyc,,
+setuptools/glob.py,sha256=o75cHrOxYsvn854thSxE0x9k8JrKDuhP_rRXlVB00Q4,5084
+setuptools/gui-32.exe,sha256=XBr0bHMA6Hpz2s9s9Bzjl-PwXfa9nH4ie0rFn4V2kWA,65536
+setuptools/gui-64.exe,sha256=aYKMhX1IJLn4ULHgWX0sE0yREUt6B3TEHf_jOw6yNyE,75264
+setuptools/gui.exe,sha256=XBr0bHMA6Hpz2s9s9Bzjl-PwXfa9nH4ie0rFn4V2kWA,65536
+setuptools/installer.py,sha256=TCFRonRo01I79zo-ucf3Ymhj8TenPlmhMijN916aaJs,5337
+setuptools/launch.py,sha256=sd7ejwhBocCDx_wG9rIs0OaZ8HtmmFU8ZC6IR_S0Lvg,787
+setuptools/lib2to3_ex.py,sha256=t5e12hbR2pi9V4ezWDTB4JM-AISUnGOkmcnYHek3xjg,2013
+setuptools/monkey.py,sha256=FGc9fffh7gAxMLFmJs2DW_OYWpBjkdbNS2n14UAK4NA,5264
+setuptools/msvc.py,sha256=8baJ6aYgCA4TRdWQQi185qB9dnU8FaP4wgpbmd7VODs,46751
+setuptools/namespaces.py,sha256=F0Nrbv8KCT2OrO7rwa03om4N4GZKAlnce-rr-cgDQa8,3199
+setuptools/package_index.py,sha256=6pb-B1POtHyLycAbkDETk4fO-Qv8_sY-rjTXhUOoh6k,40605
+setuptools/py27compat.py,sha256=tvmer0Tn-wk_JummCkoM22UIjpjL-AQ8uUiOaqTs8sI,1496
+setuptools/py31compat.py,sha256=h2rtZghOfwoGYd8sQ0-auaKiF3TcL3qX0bX3VessqcE,838
+setuptools/py33compat.py,sha256=SMF9Z8wnGicTOkU1uRNwZ_kz5Z_bj29PUBbqdqeeNsc,1330
+setuptools/py34compat.py,sha256=KYOd6ybRxjBW8NJmYD8t_UyyVmysppFXqHpFLdslGXU,245
+setuptools/sandbox.py,sha256=9UbwfEL5QY436oMI1LtFWohhoZ-UzwHvGyZjUH_qhkw,14276
+setuptools/script (dev).tmpl,sha256=RUzQzCQUaXtwdLtYHWYbIQmOaES5Brqq1FvUA_tu-5I,218
+setuptools/script.tmpl,sha256=WGTt5piezO27c-Dbx6l5Q4T3Ff20A5z7872hv3aAhYY,138
+setuptools/site-patch.py,sha256=OumkIHMuoSenRSW1382kKWI1VAwxNE86E5W8iDd34FY,2302
+setuptools/ssl_support.py,sha256=nLjPUBBw7RTTx6O4RJZ5eAMGgjJG8beiDbkFXDZpLuM,8493
+setuptools/unicode_utils.py,sha256=NOiZ_5hD72A6w-4wVj8awHFM3n51Kmw1Ic_vx15XFqw,996
+setuptools/version.py,sha256=og_cuZQb0QI6ukKZFfZWPlr1HgJBPPn2vO2m_bI9ZTE,144
+setuptools/wheel.py,sha256=zct-SEj5_LoHg6XELt2cVRdulsUENenCdS1ekM7TlZA,8455
+setuptools/windows_support.py,sha256=5GrfqSP2-dLGJoZTq2g6dCKkyQxxa2n5IQiXlJCoYEE,714
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/WHEEL b/ubuntu/venv/setuptools-44.0.0.dist-info/WHEEL
new file mode 100644
index 0000000..ef99c6c
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.34.2)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/dependency_links.txt b/ubuntu/venv/setuptools-44.0.0.dist-info/dependency_links.txt
new file mode 100644
index 0000000..e87d021
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/dependency_links.txt
@@ -0,0 +1,2 @@
+https://files.pythonhosted.org/packages/source/c/certifi/certifi-2016.9.26.tar.gz#md5=baa81e951a29958563689d868ef1064d
+https://files.pythonhosted.org/packages/source/w/wincertstore/wincertstore-0.2.zip#md5=ae728f2f007185648d0c7a8679b361e2
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/entry_points.txt b/ubuntu/venv/setuptools-44.0.0.dist-info/entry_points.txt
new file mode 100644
index 0000000..0fed3f1
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/entry_points.txt
@@ -0,0 +1,68 @@
+[console_scripts]
+easy_install = setuptools.command.easy_install:main
+
+[distutils.commands]
+alias = setuptools.command.alias:alias
+bdist_egg = setuptools.command.bdist_egg:bdist_egg
+bdist_rpm = setuptools.command.bdist_rpm:bdist_rpm
+bdist_wininst = setuptools.command.bdist_wininst:bdist_wininst
+build_clib = setuptools.command.build_clib:build_clib
+build_ext = setuptools.command.build_ext:build_ext
+build_py = setuptools.command.build_py:build_py
+develop = setuptools.command.develop:develop
+dist_info = setuptools.command.dist_info:dist_info
+easy_install = setuptools.command.easy_install:easy_install
+egg_info = setuptools.command.egg_info:egg_info
+install = setuptools.command.install:install
+install_egg_info = setuptools.command.install_egg_info:install_egg_info
+install_lib = setuptools.command.install_lib:install_lib
+install_scripts = setuptools.command.install_scripts:install_scripts
+rotate = setuptools.command.rotate:rotate
+saveopts = setuptools.command.saveopts:saveopts
+sdist = setuptools.command.sdist:sdist
+setopt = setuptools.command.setopt:setopt
+test = setuptools.command.test:test
+upload_docs = setuptools.command.upload_docs:upload_docs
+
+[distutils.setup_keywords]
+convert_2to3_doctests = setuptools.dist:assert_string_list
+dependency_links = setuptools.dist:assert_string_list
+eager_resources = setuptools.dist:assert_string_list
+entry_points = setuptools.dist:check_entry_points
+exclude_package_data = setuptools.dist:check_package_data
+extras_require = setuptools.dist:check_extras
+include_package_data = setuptools.dist:assert_bool
+install_requires = setuptools.dist:check_requirements
+namespace_packages = setuptools.dist:check_nsp
+package_data = setuptools.dist:check_package_data
+packages = setuptools.dist:check_packages
+python_requires = setuptools.dist:check_specifier
+setup_requires = setuptools.dist:check_requirements
+test_loader = setuptools.dist:check_importable
+test_runner = setuptools.dist:check_importable
+test_suite = setuptools.dist:check_test_suite
+tests_require = setuptools.dist:check_requirements
+use_2to3 = setuptools.dist:assert_bool
+use_2to3_exclude_fixers = setuptools.dist:assert_string_list
+use_2to3_fixers = setuptools.dist:assert_string_list
+zip_safe = setuptools.dist:assert_bool
+
+[egg_info.writers]
+PKG-INFO = setuptools.command.egg_info:write_pkg_info
+dependency_links.txt = setuptools.command.egg_info:overwrite_arg
+depends.txt = setuptools.command.egg_info:warn_depends_obsolete
+eager_resources.txt = setuptools.command.egg_info:overwrite_arg
+entry_points.txt = setuptools.command.egg_info:write_entries
+namespace_packages.txt = setuptools.command.egg_info:overwrite_arg
+requires.txt = setuptools.command.egg_info:write_requirements
+top_level.txt = setuptools.command.egg_info:write_toplevel_names
+
+[setuptools.finalize_distribution_options]
+2to3_doctests = setuptools.dist:Distribution._finalize_2to3_doctests
+features = setuptools.dist:Distribution._finalize_feature_opts
+keywords = setuptools.dist:Distribution._finalize_setup_keywords
+parent_finalize = setuptools.dist:_Distribution.finalize_options
+
+[setuptools.installation]
+eggsecutable = setuptools.command.easy_install:bootstrap
+
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/top_level.txt b/ubuntu/venv/setuptools-44.0.0.dist-info/top_level.txt
new file mode 100644
index 0000000..4577c6a
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/top_level.txt
@@ -0,0 +1,3 @@
+easy_install
+pkg_resources
+setuptools
diff --git a/ubuntu/venv/setuptools-44.0.0.dist-info/zip-safe b/ubuntu/venv/setuptools-44.0.0.dist-info/zip-safe
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/ubuntu/venv/setuptools-44.0.0.dist-info/zip-safe
@@ -0,0 +1 @@
+
diff --git a/ubuntu/venv/setuptools/__init__.py b/ubuntu/venv/setuptools/__init__.py
new file mode 100644
index 0000000..a71b2bb
--- /dev/null
+++ b/ubuntu/venv/setuptools/__init__.py
@@ -0,0 +1,228 @@
+"""Extensions to the 'distutils' for large or complex distributions"""
+
+import os
+import sys
+import functools
+import distutils.core
+import distutils.filelist
+import re
+from distutils.errors import DistutilsOptionError
+from distutils.util import convert_path
+from fnmatch import fnmatchcase
+
+from ._deprecation_warning import SetuptoolsDeprecationWarning
+
+from setuptools.extern.six import PY3, string_types
+from setuptools.extern.six.moves import filter, map
+
+import setuptools.version
+from setuptools.extension import Extension
+from setuptools.dist import Distribution, Feature
+from setuptools.depends import Require
+from . import monkey
+
+__metaclass__ = type
+
+
+__all__ = [
+    'setup', 'Distribution', 'Feature', 'Command', 'Extension', 'Require',
+    'SetuptoolsDeprecationWarning',
+    'find_packages'
+]
+
+if PY3:
+  __all__.append('find_namespace_packages')
+
+__version__ = setuptools.version.__version__
+
+bootstrap_install_from = None
+
+# If we run 2to3 on .py files, should we also convert docstrings?
+# Default: yes; assume that we can detect doctests reliably
+run_2to3_on_doctests = True
+# Standard package names for fixer packages
+lib2to3_fixer_packages = ['lib2to3.fixes']
+
+
+class PackageFinder:
+    """
+    Generate a list of all Python packages found within a directory
+    """
+
+    @classmethod
+    def find(cls, where='.', exclude=(), include=('*',)):
+        """Return a list all Python packages found within directory 'where'
+
+        'where' is the root directory which will be searched for packages.  It
+        should be supplied as a "cross-platform" (i.e. URL-style) path; it will
+        be converted to the appropriate local path syntax.
+
+        'exclude' is a sequence of package names to exclude; '*' can be used
+        as a wildcard in the names, such that 'foo.*' will exclude all
+        subpackages of 'foo' (but not 'foo' itself).
+
+        'include' is a sequence of package names to include.  If it's
+        specified, only the named packages will be included.  If it's not
+        specified, all found packages will be included.  'include' can contain
+        shell style wildcard patterns just like 'exclude'.
+        """
+
+        return list(cls._find_packages_iter(
+            convert_path(where),
+            cls._build_filter('ez_setup', '*__pycache__', *exclude),
+            cls._build_filter(*include)))
+
+    @classmethod
+    def _find_packages_iter(cls, where, exclude, include):
+        """
+        All the packages found in 'where' that pass the 'include' filter, but
+        not the 'exclude' filter.
+        """
+        for root, dirs, files in os.walk(where, followlinks=True):
+            # Copy dirs to iterate over it, then empty dirs.
+            all_dirs = dirs[:]
+            dirs[:] = []
+
+            for dir in all_dirs:
+                full_path = os.path.join(root, dir)
+                rel_path = os.path.relpath(full_path, where)
+                package = rel_path.replace(os.path.sep, '.')
+
+                # Skip directory trees that are not valid packages
+                if ('.' in dir or not cls._looks_like_package(full_path)):
+                    continue
+
+                # Should this package be included?
+                if include(package) and not exclude(package):
+                    yield package
+
+                # Keep searching subdirectories, as there may be more packages
+                # down there, even if the parent was excluded.
+                dirs.append(dir)
+
+    @staticmethod
+    def _looks_like_package(path):
+        """Does a directory look like a package?"""
+        return os.path.isfile(os.path.join(path, '__init__.py'))
+
+    @staticmethod
+    def _build_filter(*patterns):
+        """
+        Given a list of patterns, return a callable that will be true only if
+        the input matches at least one of the patterns.
+        """
+        return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns)
+
+
+class PEP420PackageFinder(PackageFinder):
+    @staticmethod
+    def _looks_like_package(path):
+        return True
+
+
+find_packages = PackageFinder.find
+
+if PY3:
+  find_namespace_packages = PEP420PackageFinder.find
+
+
+def _install_setup_requires(attrs):
+    # Note: do not use `setuptools.Distribution` directly, as
+    # our PEP 517 backend patch `distutils.core.Distribution`.
+    dist = distutils.core.Distribution(dict(
+        (k, v) for k, v in attrs.items()
+        if k in ('dependency_links', 'setup_requires')
+    ))
+    # Honor setup.cfg's options.
+    dist.parse_config_files(ignore_option_errors=True)
+    if dist.setup_requires:
+        dist.fetch_build_eggs(dist.setup_requires)
+
+
+def setup(**attrs):
+    # Make sure we have any requirements needed to interpret 'attrs'.
+    _install_setup_requires(attrs)
+    return distutils.core.setup(**attrs)
+
+setup.__doc__ = distutils.core.setup.__doc__
+
+
+_Command = monkey.get_unpatched(distutils.core.Command)
+
+
+class Command(_Command):
+    __doc__ = _Command.__doc__
+
+    command_consumes_arguments = False
+
+    def __init__(self, dist, **kw):
+        """
+        Construct the command for dist, updating
+        vars(self) with any keyword parameters.
+        """
+        _Command.__init__(self, dist)
+        vars(self).update(kw)
+
+    def _ensure_stringlike(self, option, what, default=None):
+        val = getattr(self, option)
+        if val is None:
+            setattr(self, option, default)
+            return default
+        elif not isinstance(val, string_types):
+            raise DistutilsOptionError("'%s' must be a %s (got `%s`)"
+                                       % (option, what, val))
+        return val
+
+    def ensure_string_list(self, option):
+        r"""Ensure that 'option' is a list of strings.  If 'option' is
+        currently a string, we split it either on /,\s*/ or /\s+/, so
+        "foo bar baz", "foo,bar,baz", and "foo,   bar baz" all become
+        ["foo", "bar", "baz"].
+        """
+        val = getattr(self, option)
+        if val is None:
+            return
+        elif isinstance(val, string_types):
+            setattr(self, option, re.split(r',\s*|\s+', val))
+        else:
+            if isinstance(val, list):
+                ok = all(isinstance(v, string_types) for v in val)
+            else:
+                ok = False
+            if not ok:
+                raise DistutilsOptionError(
+                      "'%s' must be a list of strings (got %r)"
+                      % (option, val))
+
+    def reinitialize_command(self, command, reinit_subcommands=0, **kw):
+        cmd = _Command.reinitialize_command(self, command, reinit_subcommands)
+        vars(cmd).update(kw)
+        return cmd
+
+
+def _find_all_simple(path):
+    """
+    Find all files under 'path'
+    """
+    results = (
+        os.path.join(base, file)
+        for base, dirs, files in os.walk(path, followlinks=True)
+        for file in files
+    )
+    return filter(os.path.isfile, results)
+
+
+def findall(dir=os.curdir):
+    """
+    Find all files under 'dir' and return the list of full filenames.
+    Unless dir is '.', return full filenames with dir prepended.
+    """
+    files = _find_all_simple(dir)
+    if dir == os.curdir:
+        make_rel = functools.partial(os.path.relpath, start=dir)
+        files = map(make_rel, files)
+    return list(files)
+
+
+# Apply monkey patches
+monkey.patch_all()
diff --git a/ubuntu/venv/setuptools/_deprecation_warning.py b/ubuntu/venv/setuptools/_deprecation_warning.py
new file mode 100644
index 0000000..086b64d
--- /dev/null
+++ b/ubuntu/venv/setuptools/_deprecation_warning.py
@@ -0,0 +1,7 @@
+class SetuptoolsDeprecationWarning(Warning):
+    """
+    Base class for warning deprecations in ``setuptools``
+
+    This class is not derived from ``DeprecationWarning``, and as such is
+    visible by default.
+    """
diff --git a/ubuntu/venv/setuptools/_imp.py b/ubuntu/venv/setuptools/_imp.py
new file mode 100644
index 0000000..a3cce9b
--- /dev/null
+++ b/ubuntu/venv/setuptools/_imp.py
@@ -0,0 +1,73 @@
+"""
+Re-implementation of find_module and get_frozen_object
+from the deprecated imp module.
+"""
+
+import os
+import importlib.util
+import importlib.machinery
+
+from .py34compat import module_from_spec
+
+
+PY_SOURCE = 1
+PY_COMPILED = 2
+C_EXTENSION = 3
+C_BUILTIN = 6
+PY_FROZEN = 7
+
+
+def find_module(module, paths=None):
+    """Just like 'imp.find_module()', but with package support"""
+    spec = importlib.util.find_spec(module, paths)
+    if spec is None:
+        raise ImportError("Can't find %s" % module)
+    if not spec.has_location and hasattr(spec, 'submodule_search_locations'):
+        spec = importlib.util.spec_from_loader('__init__.py', spec.loader)
+
+    kind = -1
+    file = None
+    static = isinstance(spec.loader, type)
+    if spec.origin == 'frozen' or static and issubclass(
+            spec.loader, importlib.machinery.FrozenImporter):
+        kind = PY_FROZEN
+        path = None  # imp compabilty
+        suffix = mode = ''  # imp compability
+    elif spec.origin == 'built-in' or static and issubclass(
+            spec.loader, importlib.machinery.BuiltinImporter):
+        kind = C_BUILTIN
+        path = None  # imp compabilty
+        suffix = mode = ''  # imp compability
+    elif spec.has_location:
+        path = spec.origin
+        suffix = os.path.splitext(path)[1]
+        mode = 'r' if suffix in importlib.machinery.SOURCE_SUFFIXES else 'rb'
+
+        if suffix in importlib.machinery.SOURCE_SUFFIXES:
+            kind = PY_SOURCE
+        elif suffix in importlib.machinery.BYTECODE_SUFFIXES:
+            kind = PY_COMPILED
+        elif suffix in importlib.machinery.EXTENSION_SUFFIXES:
+            kind = C_EXTENSION
+
+        if kind in {PY_SOURCE, PY_COMPILED}:
+            file = open(path, mode)
+    else:
+        path = None
+        suffix = mode = ''
+
+    return file, path, (suffix, mode, kind)
+
+
+def get_frozen_object(module, paths=None):
+    spec = importlib.util.find_spec(module, paths)
+    if not spec:
+        raise ImportError("Can't find %s" % module)
+    return spec.loader.get_code(module)
+
+
+def get_module(module, paths, info):
+    spec = importlib.util.find_spec(module, paths)
+    if not spec:
+        raise ImportError("Can't find %s" % module)
+    return module_from_spec(spec)
diff --git a/ubuntu/venv/setuptools/_vendor/__init__.py b/ubuntu/venv/setuptools/_vendor/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ubuntu/venv/setuptools/_vendor/ordered_set.py b/ubuntu/venv/setuptools/_vendor/ordered_set.py
new file mode 100644
index 0000000..1487600
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/ordered_set.py
@@ -0,0 +1,488 @@
+"""
+An OrderedSet is a custom MutableSet that remembers its order, so that every
+entry has an index that can be looked up.
+
+Based on a recipe originally posted to ActiveState Recipes by Raymond Hettiger,
+and released under the MIT license.
+"""
+import itertools as it
+from collections import deque
+
+try:
+    # Python 3
+    from collections.abc import MutableSet, Sequence
+except ImportError:
+    # Python 2.7
+    from collections import MutableSet, Sequence
+
+SLICE_ALL = slice(None)
+__version__ = "3.1"
+
+
+def is_iterable(obj):
+    """
+    Are we being asked to look up a list of things, instead of a single thing?
+    We check for the `__iter__` attribute so that this can cover types that
+    don't have to be known by this module, such as NumPy arrays.
+
+    Strings, however, should be considered as atomic values to look up, not
+    iterables. The same goes for tuples, since they are immutable and therefore
+    valid entries.
+
+    We don't need to check for the Python 2 `unicode` type, because it doesn't
+    have an `__iter__` attribute anyway.
+    """
+    return (
+        hasattr(obj, "__iter__")
+        and not isinstance(obj, str)
+        and not isinstance(obj, tuple)
+    )
+
+
+class OrderedSet(MutableSet, Sequence):
+    """
+    An OrderedSet is a custom MutableSet that remembers its order, so that
+    every entry has an index that can be looked up.
+
+    Example:
+        >>> OrderedSet([1, 1, 2, 3, 2])
+        OrderedSet([1, 2, 3])
+    """
+
+    def __init__(self, iterable=None):
+        self.items = []
+        self.map = {}
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        """
+        Returns the number of unique elements in the ordered set
+
+        Example:
+            >>> len(OrderedSet([]))
+            0
+            >>> len(OrderedSet([1, 2]))
+            2
+        """
+        return len(self.items)
+
+    def __getitem__(self, index):
+        """
+        Get the item at a given index.
+
+        If `index` is a slice, you will get back that slice of items, as a
+        new OrderedSet.
+
+        If `index` is a list or a similar iterable, you'll get a list of
+        items corresponding to those indices. This is similar to NumPy's
+        "fancy indexing". The result is not an OrderedSet because you may ask
+        for duplicate indices, and the number of elements returned should be
+        the number of elements asked for.
+
+        Example:
+            >>> oset = OrderedSet([1, 2, 3])
+            >>> oset[1]
+            2
+        """
+        if isinstance(index, slice) and index == SLICE_ALL:
+            return self.copy()
+        elif is_iterable(index):
+            return [self.items[i] for i in index]
+        elif hasattr(index, "__index__") or isinstance(index, slice):
+            result = self.items[index]
+            if isinstance(result, list):
+                return self.__class__(result)
+            else:
+                return result
+        else:
+            raise TypeError("Don't know how to index an OrderedSet by %r" % index)
+
+    def copy(self):
+        """
+        Return a shallow copy of this object.
+
+        Example:
+            >>> this = OrderedSet([1, 2, 3])
+            >>> other = this.copy()
+            >>> this == other
+            True
+            >>> this is other
+            False
+        """
+        return self.__class__(self)
+
+    def __getstate__(self):
+        if len(self) == 0:
+            # The state can't be an empty list.
+            # We need to return a truthy value, or else __setstate__ won't be run.
+            #
+            # This could have been done more gracefully by always putting the state
+            # in a tuple, but this way is backwards- and forwards- compatible with
+            # previous versions of OrderedSet.
+            return (None,)
+        else:
+            return list(self)
+
+    def __setstate__(self, state):
+        if state == (None,):
+            self.__init__([])
+        else:
+            self.__init__(state)
+
+    def __contains__(self, key):
+        """
+        Test if the item is in this ordered set
+
+        Example:
+            >>> 1 in OrderedSet([1, 3, 2])
+            True
+            >>> 5 in OrderedSet([1, 3, 2])
+            False
+        """
+        return key in self.map
+
+    def add(self, key):
+        """
+        Add `key` as an item to this OrderedSet, then return its index.
+
+        If `key` is already in the OrderedSet, return the index it already
+        had.
+
+        Example:
+            >>> oset = OrderedSet()
+            >>> oset.append(3)
+            0
+            >>> print(oset)
+            OrderedSet([3])
+        """
+        if key not in self.map:
+            self.map[key] = len(self.items)
+            self.items.append(key)
+        return self.map[key]
+
+    append = add
+
+    def update(self, sequence):
+        """
+        Update the set with the given iterable sequence, then return the index
+        of the last element inserted.
+
+        Example:
+            >>> oset = OrderedSet([1, 2, 3])
+            >>> oset.update([3, 1, 5, 1, 4])
+            4
+            >>> print(oset)
+            OrderedSet([1, 2, 3, 5, 4])
+        """
+        item_index = None
+        try:
+            for item in sequence:
+                item_index = self.add(item)
+        except TypeError:
+            raise ValueError(
+                "Argument needs to be an iterable, got %s" % type(sequence)
+            )
+        return item_index
+
+    def index(self, key):
+        """
+        Get the index of a given entry, raising an IndexError if it's not
+        present.
+
+        `key` can be an iterable of entries that is not a string, in which case
+        this returns a list of indices.
+
+        Example:
+            >>> oset = OrderedSet([1, 2, 3])
+            >>> oset.index(2)
+            1
+        """
+        if is_iterable(key):
+            return [self.index(subkey) for subkey in key]
+        return self.map[key]
+
+    # Provide some compatibility with pd.Index
+    get_loc = index
+    get_indexer = index
+
+    def pop(self):
+        """
+        Remove and return the last element from the set.
+
+        Raises KeyError if the set is empty.
+
+        Example:
+            >>> oset = OrderedSet([1, 2, 3])
+            >>> oset.pop()
+            3
+        """
+        if not self.items:
+            raise KeyError("Set is empty")
+
+        elem = self.items[-1]
+        del self.items[-1]
+        del self.map[elem]
+        return elem
+
+    def discard(self, key):
+        """
+        Remove an element.  Do not raise an exception if absent.
+
+        The MutableSet mixin uses this to implement the .remove() method, which
+        *does* raise an error when asked to remove a non-existent item.
+
+        Example:
+            >>> oset = OrderedSet([1, 2, 3])
+            >>> oset.discard(2)
+            >>> print(oset)
+            OrderedSet([1, 3])
+            >>> oset.discard(2)
+            >>> print(oset)
+            OrderedSet([1, 3])
+        """
+        if key in self:
+            i = self.map[key]
+            del self.items[i]
+            del self.map[key]
+            for k, v in self.map.items():
+                if v >= i:
+                    self.map[k] = v - 1
+
+    def clear(self):
+        """
+        Remove all items from this OrderedSet.
+        """
+        del self.items[:]
+        self.map.clear()
+
+    def __iter__(self):
+        """
+        Example:
+            >>> list(iter(OrderedSet([1, 2, 3])))
+            [1, 2, 3]
+        """
+        return iter(self.items)
+
+    def __reversed__(self):
+        """
+        Example:
+            >>> list(reversed(OrderedSet([1, 2, 3])))
+            [3, 2, 1]
+        """
+        return reversed(self.items)
+
+    def __repr__(self):
+        if not self:
+            return "%s()" % (self.__class__.__name__,)
+        return "%s(%r)" % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        """
+        Returns true if the containers have the same items. If `other` is a
+        Sequence, then order is checked, otherwise it is ignored.
+
+        Example:
+            >>> oset = OrderedSet([1, 3, 2])
+            >>> oset == [1, 3, 2]
+            True
+            >>> oset == [1, 2, 3]
+            False
+            >>> oset == [2, 3]
+            False
+            >>> oset == OrderedSet([3, 2, 1])
+            False
+        """
+        # In Python 2 deque is not a Sequence, so treat it as one for
+        # consistent behavior with Python 3.
+        if isinstance(other, (Sequence, deque)):
+            # Check that this OrderedSet contains the same elements, in the
+            # same order, as the other object.
+            return list(self) == list(other)
+        try:
+            other_as_set = set(other)
+        except TypeError:
+            # If `other` can't be converted into a set, it's not equal.
+            return False
+        else:
+            return set(self) == other_as_set
+
+    def union(self, *sets):
+        """
+        Combines all unique items.
+        Each items order is defined by its first appearance.
+
+        Example:
+            >>> oset = OrderedSet.union(OrderedSet([3, 1, 4, 1, 5]), [1, 3], [2, 0])
+            >>> print(oset)
+            OrderedSet([3, 1, 4, 5, 2, 0])
+            >>> oset.union([8, 9])
+            OrderedSet([3, 1, 4, 5, 2, 0, 8, 9])
+            >>> oset | {10}
+            OrderedSet([3, 1, 4, 5, 2, 0, 10])
+        """
+        cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
+        containers = map(list, it.chain([self], sets))
+        items = it.chain.from_iterable(containers)
+        return cls(items)
+
+    def __and__(self, other):
+        # the parent implementation of this is backwards
+        return self.intersection(other)
+
+    def intersection(self, *sets):
+        """
+        Returns elements in common between all sets. Order is defined only
+        by the first set.
+
+        Example:
+            >>> oset = OrderedSet.intersection(OrderedSet([0, 1, 2, 3]), [1, 2, 3])
+            >>> print(oset)
+            OrderedSet([1, 2, 3])
+            >>> oset.intersection([2, 4, 5], [1, 2, 3, 4])
+            OrderedSet([2])
+            >>> oset.intersection()
+            OrderedSet([1, 2, 3])
+        """
+        cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
+        if sets:
+            common = set.intersection(*map(set, sets))
+            items = (item for item in self if item in common)
+        else:
+            items = self
+        return cls(items)
+
+    def difference(self, *sets):
+        """
+        Returns all elements that are in this set but not the others.
+
+        Example:
+            >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2]))
+            OrderedSet([1, 3])
+            >>> OrderedSet([1, 2, 3]).difference(OrderedSet([2]), OrderedSet([3]))
+            OrderedSet([1])
+            >>> OrderedSet([1, 2, 3]) - OrderedSet([2])
+            OrderedSet([1, 3])
+            >>> OrderedSet([1, 2, 3]).difference()
+            OrderedSet([1, 2, 3])
+        """
+        cls = self.__class__
+        if sets:
+            other = set.union(*map(set, sets))
+            items = (item for item in self if item not in other)
+        else:
+            items = self
+        return cls(items)
+
+    def issubset(self, other):
+        """
+        Report whether another set contains this set.
+
+        Example:
+            >>> OrderedSet([1, 2, 3]).issubset({1, 2})
+            False
+            >>> OrderedSet([1, 2, 3]).issubset({1, 2, 3, 4})
+            True
+            >>> OrderedSet([1, 2, 3]).issubset({1, 4, 3, 5})
+            False
+        """
+        if len(self) > len(other):  # Fast check for obvious cases
+            return False
+        return all(item in other for item in self)
+
+    def issuperset(self, other):
+        """
+        Report whether this set contains another set.
+
+        Example:
+            >>> OrderedSet([1, 2]).issuperset([1, 2, 3])
+            False
+            >>> OrderedSet([1, 2, 3, 4]).issuperset({1, 2, 3})
+            True
+            >>> OrderedSet([1, 4, 3, 5]).issuperset({1, 2, 3})
+            False
+        """
+        if len(self) < len(other):  # Fast check for obvious cases
+            return False
+        return all(item in self for item in other)
+
+    def symmetric_difference(self, other):
+        """
+        Return the symmetric difference of two OrderedSets as a new set.
+        That is, the new set will contain all elements that are in exactly
+        one of the sets.
+
+        Their order will be preserved, with elements from `self` preceding
+        elements from `other`.
+
+        Example:
+            >>> this = OrderedSet([1, 4, 3, 5, 7])
+            >>> other = OrderedSet([9, 7, 1, 3, 2])
+            >>> this.symmetric_difference(other)
+            OrderedSet([4, 5, 9, 2])
+        """
+        cls = self.__class__ if isinstance(self, OrderedSet) else OrderedSet
+        diff1 = cls(self).difference(other)
+        diff2 = cls(other).difference(self)
+        return diff1.union(diff2)
+
+    def _update_items(self, items):
+        """
+        Replace the 'items' list of this OrderedSet with a new one, updating
+        self.map accordingly.
+        """
+        self.items = items
+        self.map = {item: idx for (idx, item) in enumerate(items)}
+
+    def difference_update(self, *sets):
+        """
+        Update this OrderedSet to remove items from one or more other sets.
+
+        Example:
+            >>> this = OrderedSet([1, 2, 3])
+            >>> this.difference_update(OrderedSet([2, 4]))
+            >>> print(this)
+            OrderedSet([1, 3])
+
+            >>> this = OrderedSet([1, 2, 3, 4, 5])
+            >>> this.difference_update(OrderedSet([2, 4]), OrderedSet([1, 4, 6]))
+            >>> print(this)
+            OrderedSet([3, 5])
+        """
+        items_to_remove = set()
+        for other in sets:
+            items_to_remove |= set(other)
+        self._update_items([item for item in self.items if item not in items_to_remove])
+
+    def intersection_update(self, other):
+        """
+        Update this OrderedSet to keep only items in another set, preserving
+        their order in this set.
+
+        Example:
+            >>> this = OrderedSet([1, 4, 3, 5, 7])
+            >>> other = OrderedSet([9, 7, 1, 3, 2])
+            >>> this.intersection_update(other)
+            >>> print(this)
+            OrderedSet([1, 3, 7])
+        """
+        other = set(other)
+        self._update_items([item for item in self.items if item in other])
+
+    def symmetric_difference_update(self, other):
+        """
+        Update this OrderedSet to remove items from another set, then
+        add items from the other set that were not present in this set.
+
+        Example:
+            >>> this = OrderedSet([1, 4, 3, 5, 7])
+            >>> other = OrderedSet([9, 7, 1, 3, 2])
+            >>> this.symmetric_difference_update(other)
+            >>> print(this)
+            OrderedSet([4, 5, 9, 2])
+        """
+        items_to_add = [item for item in other if item not in self]
+        items_to_remove = set(other)
+        self._update_items(
+            [item for item in self.items if item not in items_to_remove] + items_to_add
+        )
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/__about__.py b/ubuntu/venv/setuptools/_vendor/packaging/__about__.py
new file mode 100644
index 0000000..dc95138
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/__about__.py
@@ -0,0 +1,27 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+__all__ = [
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
+]
+
+__title__ = "packaging"
+__summary__ = "Core utilities for Python packages"
+__uri__ = "https://github.com/pypa/packaging"
+
+__version__ = "19.2"
+
+__author__ = "Donald Stufft and individual contributors"
+__email__ = "donald@stufft.io"
+
+__license__ = "BSD or Apache License, Version 2.0"
+__copyright__ = "Copyright 2014-2019 %s" % __author__
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/__init__.py b/ubuntu/venv/setuptools/_vendor/packaging/__init__.py
new file mode 100644
index 0000000..a0cf67d
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/__init__.py
@@ -0,0 +1,26 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+from .__about__ import (
+    __author__,
+    __copyright__,
+    __email__,
+    __license__,
+    __summary__,
+    __title__,
+    __uri__,
+    __version__,
+)
+
+__all__ = [
+    "__title__",
+    "__summary__",
+    "__uri__",
+    "__version__",
+    "__author__",
+    "__email__",
+    "__license__",
+    "__copyright__",
+]
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/_compat.py b/ubuntu/venv/setuptools/_vendor/packaging/_compat.py
new file mode 100644
index 0000000..25da473
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/_compat.py
@@ -0,0 +1,31 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import sys
+
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+# flake8: noqa
+
+if PY3:
+    string_types = (str,)
+else:
+    string_types = (basestring,)
+
+
+def with_metaclass(meta, *bases):
+    """
+    Create a base class with a metaclass.
+    """
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(meta):
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+
+    return type.__new__(metaclass, "temporary_class", (), {})
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/_structures.py b/ubuntu/venv/setuptools/_vendor/packaging/_structures.py
new file mode 100644
index 0000000..68dcca6
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/_structures.py
@@ -0,0 +1,68 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+
+class Infinity(object):
+    def __repr__(self):
+        return "Infinity"
+
+    def __hash__(self):
+        return hash(repr(self))
+
+    def __lt__(self, other):
+        return False
+
+    def __le__(self, other):
+        return False
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__)
+
+    def __ne__(self, other):
+        return not isinstance(other, self.__class__)
+
+    def __gt__(self, other):
+        return True
+
+    def __ge__(self, other):
+        return True
+
+    def __neg__(self):
+        return NegativeInfinity
+
+
+Infinity = Infinity()
+
+
+class NegativeInfinity(object):
+    def __repr__(self):
+        return "-Infinity"
+
+    def __hash__(self):
+        return hash(repr(self))
+
+    def __lt__(self, other):
+        return True
+
+    def __le__(self, other):
+        return True
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__)
+
+    def __ne__(self, other):
+        return not isinstance(other, self.__class__)
+
+    def __gt__(self, other):
+        return False
+
+    def __ge__(self, other):
+        return False
+
+    def __neg__(self):
+        return Infinity
+
+
+NegativeInfinity = NegativeInfinity()
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/markers.py b/ubuntu/venv/setuptools/_vendor/packaging/markers.py
new file mode 100644
index 0000000..4bdfdb2
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/markers.py
@@ -0,0 +1,296 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import operator
+import os
+import platform
+import sys
+
+from setuptools.extern.pyparsing import ParseException, ParseResults, stringStart, stringEnd
+from setuptools.extern.pyparsing import ZeroOrMore, Group, Forward, QuotedString
+from setuptools.extern.pyparsing import Literal as L  # noqa
+
+from ._compat import string_types
+from .specifiers import Specifier, InvalidSpecifier
+
+
+__all__ = [
+    "InvalidMarker",
+    "UndefinedComparison",
+    "UndefinedEnvironmentName",
+    "Marker",
+    "default_environment",
+]
+
+
+class InvalidMarker(ValueError):
+    """
+    An invalid marker was found, users should refer to PEP 508.
+    """
+
+
+class UndefinedComparison(ValueError):
+    """
+    An invalid operation was attempted on a value that doesn't support it.
+    """
+
+
+class UndefinedEnvironmentName(ValueError):
+    """
+    A name was attempted to be used that does not exist inside of the
+    environment.
+    """
+
+
+class Node(object):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return str(self.value)
+
+    def __repr__(self):
+        return "<{0}({1!r})>".format(self.__class__.__name__, str(self))
+
+    def serialize(self):
+        raise NotImplementedError
+
+
+class Variable(Node):
+    def serialize(self):
+        return str(self)
+
+
+class Value(Node):
+    def serialize(self):
+        return '"{0}"'.format(self)
+
+
+class Op(Node):
+    def serialize(self):
+        return str(self)
+
+
+VARIABLE = (
+    L("implementation_version")
+    | L("platform_python_implementation")
+    | L("implementation_name")
+    | L("python_full_version")
+    | L("platform_release")
+    | L("platform_version")
+    | L("platform_machine")
+    | L("platform_system")
+    | L("python_version")
+    | L("sys_platform")
+    | L("os_name")
+    | L("os.name")
+    | L("sys.platform")  # PEP-345
+    | L("platform.version")  # PEP-345
+    | L("platform.machine")  # PEP-345
+    | L("platform.python_implementation")  # PEP-345
+    | L("python_implementation")  # PEP-345
+    | L("extra")  # undocumented setuptools legacy
+)
+ALIASES = {
+    "os.name": "os_name",
+    "sys.platform": "sys_platform",
+    "platform.version": "platform_version",
+    "platform.machine": "platform_machine",
+    "platform.python_implementation": "platform_python_implementation",
+    "python_implementation": "platform_python_implementation",
+}
+VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0])))
+
+VERSION_CMP = (
+    L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<")
+)
+
+MARKER_OP = VERSION_CMP | L("not in") | L("in")
+MARKER_OP.setParseAction(lambda s, l, t: Op(t[0]))
+
+MARKER_VALUE = QuotedString("'") | QuotedString('"')
+MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0]))
+
+BOOLOP = L("and") | L("or")
+
+MARKER_VAR = VARIABLE | MARKER_VALUE
+
+MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR)
+MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0]))
+
+LPAREN = L("(").suppress()
+RPAREN = L(")").suppress()
+
+MARKER_EXPR = Forward()
+MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN)
+MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR)
+
+MARKER = stringStart + MARKER_EXPR + stringEnd
+
+
+def _coerce_parse_result(results):
+    if isinstance(results, ParseResults):
+        return [_coerce_parse_result(i) for i in results]
+    else:
+        return results
+
+
+def _format_marker(marker, first=True):
+    assert isinstance(marker, (list, tuple, string_types))
+
+    # Sometimes we have a structure like [[...]] which is a single item list
+    # where the single item is itself it's own list. In that case we want skip
+    # the rest of this function so that we don't get extraneous () on the
+    # outside.
+    if (
+        isinstance(marker, list)
+        and len(marker) == 1
+        and isinstance(marker[0], (list, tuple))
+    ):
+        return _format_marker(marker[0])
+
+    if isinstance(marker, list):
+        inner = (_format_marker(m, first=False) for m in marker)
+        if first:
+            return " ".join(inner)
+        else:
+            return "(" + " ".join(inner) + ")"
+    elif isinstance(marker, tuple):
+        return " ".join([m.serialize() for m in marker])
+    else:
+        return marker
+
+
+_operators = {
+    "in": lambda lhs, rhs: lhs in rhs,
+    "not in": lambda lhs, rhs: lhs not in rhs,
+    "<": operator.lt,
+    "<=": operator.le,
+    "==": operator.eq,
+    "!=": operator.ne,
+    ">=": operator.ge,
+    ">": operator.gt,
+}
+
+
+def _eval_op(lhs, op, rhs):
+    try:
+        spec = Specifier("".join([op.serialize(), rhs]))
+    except InvalidSpecifier:
+        pass
+    else:
+        return spec.contains(lhs)
+
+    oper = _operators.get(op.serialize())
+    if oper is None:
+        raise UndefinedComparison(
+            "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs)
+        )
+
+    return oper(lhs, rhs)
+
+
+_undefined = object()
+
+
+def _get_env(environment, name):
+    value = environment.get(name, _undefined)
+
+    if value is _undefined:
+        raise UndefinedEnvironmentName(
+            "{0!r} does not exist in evaluation environment.".format(name)
+        )
+
+    return value
+
+
+def _evaluate_markers(markers, environment):
+    groups = [[]]
+
+    for marker in markers:
+        assert isinstance(marker, (list, tuple, string_types))
+
+        if isinstance(marker, list):
+            groups[-1].append(_evaluate_markers(marker, environment))
+        elif isinstance(marker, tuple):
+            lhs, op, rhs = marker
+
+            if isinstance(lhs, Variable):
+                lhs_value = _get_env(environment, lhs.value)
+                rhs_value = rhs.value
+            else:
+                lhs_value = lhs.value
+                rhs_value = _get_env(environment, rhs.value)
+
+            groups[-1].append(_eval_op(lhs_value, op, rhs_value))
+        else:
+            assert marker in ["and", "or"]
+            if marker == "or":
+                groups.append([])
+
+    return any(all(item) for item in groups)
+
+
+def format_full_version(info):
+    version = "{0.major}.{0.minor}.{0.micro}".format(info)
+    kind = info.releaselevel
+    if kind != "final":
+        version += kind[0] + str(info.serial)
+    return version
+
+
+def default_environment():
+    if hasattr(sys, "implementation"):
+        iver = format_full_version(sys.implementation.version)
+        implementation_name = sys.implementation.name
+    else:
+        iver = "0"
+        implementation_name = ""
+
+    return {
+        "implementation_name": implementation_name,
+        "implementation_version": iver,
+        "os_name": os.name,
+        "platform_machine": platform.machine(),
+        "platform_release": platform.release(),
+        "platform_system": platform.system(),
+        "platform_version": platform.version(),
+        "python_full_version": platform.python_version(),
+        "platform_python_implementation": platform.python_implementation(),
+        "python_version": ".".join(platform.python_version_tuple()[:2]),
+        "sys_platform": sys.platform,
+    }
+
+
+class Marker(object):
+    def __init__(self, marker):
+        try:
+            self._markers = _coerce_parse_result(MARKER.parseString(marker))
+        except ParseException as e:
+            err_str = "Invalid marker: {0!r}, parse error at {1!r}".format(
+                marker, marker[e.loc : e.loc + 8]
+            )
+            raise InvalidMarker(err_str)
+
+    def __str__(self):
+        return _format_marker(self._markers)
+
+    def __repr__(self):
+        return "".format(str(self))
+
+    def evaluate(self, environment=None):
+        """Evaluate a marker.
+
+        Return the boolean from evaluating the given marker against the
+        environment. environment is an optional argument to override all or
+        part of the determined environment.
+
+        The environment is determined from the current Python process.
+        """
+        current_environment = default_environment()
+        if environment is not None:
+            current_environment.update(environment)
+
+        return _evaluate_markers(self._markers, current_environment)
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/requirements.py b/ubuntu/venv/setuptools/_vendor/packaging/requirements.py
new file mode 100644
index 0000000..8a0c2cb
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/requirements.py
@@ -0,0 +1,138 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import string
+import re
+
+from setuptools.extern.pyparsing import stringStart, stringEnd, originalTextFor, ParseException
+from setuptools.extern.pyparsing import ZeroOrMore, Word, Optional, Regex, Combine
+from setuptools.extern.pyparsing import Literal as L  # noqa
+from setuptools.extern.six.moves.urllib import parse as urlparse
+
+from .markers import MARKER_EXPR, Marker
+from .specifiers import LegacySpecifier, Specifier, SpecifierSet
+
+
+class InvalidRequirement(ValueError):
+    """
+    An invalid requirement was found, users should refer to PEP 508.
+    """
+
+
+ALPHANUM = Word(string.ascii_letters + string.digits)
+
+LBRACKET = L("[").suppress()
+RBRACKET = L("]").suppress()
+LPAREN = L("(").suppress()
+RPAREN = L(")").suppress()
+COMMA = L(",").suppress()
+SEMICOLON = L(";").suppress()
+AT = L("@").suppress()
+
+PUNCTUATION = Word("-_.")
+IDENTIFIER_END = ALPHANUM | (ZeroOrMore(PUNCTUATION) + ALPHANUM)
+IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END))
+
+NAME = IDENTIFIER("name")
+EXTRA = IDENTIFIER
+
+URI = Regex(r"[^ ]+")("url")
+URL = AT + URI
+
+EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA)
+EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras")
+
+VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE)
+VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE)
+
+VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY
+VERSION_MANY = Combine(
+    VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False
+)("_raw_spec")
+_VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY))
+_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "")
+
+VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier")
+VERSION_SPEC.setParseAction(lambda s, l, t: t[1])
+
+MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker")
+MARKER_EXPR.setParseAction(
+    lambda s, l, t: Marker(s[t._original_start : t._original_end])
+)
+MARKER_SEPARATOR = SEMICOLON
+MARKER = MARKER_SEPARATOR + MARKER_EXPR
+
+VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER)
+URL_AND_MARKER = URL + Optional(MARKER)
+
+NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER)
+
+REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd
+# setuptools.extern.pyparsing isn't thread safe during initialization, so we do it eagerly, see
+# issue #104
+REQUIREMENT.parseString("x[]")
+
+
+class Requirement(object):
+    """Parse a requirement.
+
+    Parse a given requirement string into its parts, such as name, specifier,
+    URL, and extras. Raises InvalidRequirement on a badly-formed requirement
+    string.
+    """
+
+    # TODO: Can we test whether something is contained within a requirement?
+    #       If so how do we do that? Do we need to test against the _name_ of
+    #       the thing as well as the version? What about the markers?
+    # TODO: Can we normalize the name and extra name?
+
+    def __init__(self, requirement_string):
+        try:
+            req = REQUIREMENT.parseString(requirement_string)
+        except ParseException as e:
+            raise InvalidRequirement(
+                'Parse error at "{0!r}": {1}'.format(
+                    requirement_string[e.loc : e.loc + 8], e.msg
+                )
+            )
+
+        self.name = req.name
+        if req.url:
+            parsed_url = urlparse.urlparse(req.url)
+            if parsed_url.scheme == "file":
+                if urlparse.urlunparse(parsed_url) != req.url:
+                    raise InvalidRequirement("Invalid URL given")
+            elif not (parsed_url.scheme and parsed_url.netloc) or (
+                not parsed_url.scheme and not parsed_url.netloc
+            ):
+                raise InvalidRequirement("Invalid URL: {0}".format(req.url))
+            self.url = req.url
+        else:
+            self.url = None
+        self.extras = set(req.extras.asList() if req.extras else [])
+        self.specifier = SpecifierSet(req.specifier)
+        self.marker = req.marker if req.marker else None
+
+    def __str__(self):
+        parts = [self.name]
+
+        if self.extras:
+            parts.append("[{0}]".format(",".join(sorted(self.extras))))
+
+        if self.specifier:
+            parts.append(str(self.specifier))
+
+        if self.url:
+            parts.append("@ {0}".format(self.url))
+            if self.marker:
+                parts.append(" ")
+
+        if self.marker:
+            parts.append("; {0}".format(self.marker))
+
+        return "".join(parts)
+
+    def __repr__(self):
+        return "".format(str(self))
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/specifiers.py b/ubuntu/venv/setuptools/_vendor/packaging/specifiers.py
new file mode 100644
index 0000000..743576a
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/specifiers.py
@@ -0,0 +1,749 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import abc
+import functools
+import itertools
+import re
+
+from ._compat import string_types, with_metaclass
+from .version import Version, LegacyVersion, parse
+
+
+class InvalidSpecifier(ValueError):
+    """
+    An invalid specifier was found, users should refer to PEP 440.
+    """
+
+
+class BaseSpecifier(with_metaclass(abc.ABCMeta, object)):
+    @abc.abstractmethod
+    def __str__(self):
+        """
+        Returns the str representation of this Specifier like object. This
+        should be representative of the Specifier itself.
+        """
+
+    @abc.abstractmethod
+    def __hash__(self):
+        """
+        Returns a hash value for this Specifier like object.
+        """
+
+    @abc.abstractmethod
+    def __eq__(self, other):
+        """
+        Returns a boolean representing whether or not the two Specifier like
+        objects are equal.
+        """
+
+    @abc.abstractmethod
+    def __ne__(self, other):
+        """
+        Returns a boolean representing whether or not the two Specifier like
+        objects are not equal.
+        """
+
+    @abc.abstractproperty
+    def prereleases(self):
+        """
+        Returns whether or not pre-releases as a whole are allowed by this
+        specifier.
+        """
+
+    @prereleases.setter
+    def prereleases(self, value):
+        """
+        Sets whether or not pre-releases as a whole are allowed by this
+        specifier.
+        """
+
+    @abc.abstractmethod
+    def contains(self, item, prereleases=None):
+        """
+        Determines if the given item is contained within this specifier.
+        """
+
+    @abc.abstractmethod
+    def filter(self, iterable, prereleases=None):
+        """
+        Takes an iterable of items and filters them so that only items which
+        are contained within this specifier are allowed in it.
+        """
+
+
+class _IndividualSpecifier(BaseSpecifier):
+
+    _operators = {}
+
+    def __init__(self, spec="", prereleases=None):
+        match = self._regex.search(spec)
+        if not match:
+            raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec))
+
+        self._spec = (match.group("operator").strip(), match.group("version").strip())
+
+        # Store whether or not this Specifier should accept prereleases
+        self._prereleases = prereleases
+
+    def __repr__(self):
+        pre = (
+            ", prereleases={0!r}".format(self.prereleases)
+            if self._prereleases is not None
+            else ""
+        )
+
+        return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre)
+
+    def __str__(self):
+        return "{0}{1}".format(*self._spec)
+
+    def __hash__(self):
+        return hash(self._spec)
+
+    def __eq__(self, other):
+        if isinstance(other, string_types):
+            try:
+                other = self.__class__(other)
+            except InvalidSpecifier:
+                return NotImplemented
+        elif not isinstance(other, self.__class__):
+            return NotImplemented
+
+        return self._spec == other._spec
+
+    def __ne__(self, other):
+        if isinstance(other, string_types):
+            try:
+                other = self.__class__(other)
+            except InvalidSpecifier:
+                return NotImplemented
+        elif not isinstance(other, self.__class__):
+            return NotImplemented
+
+        return self._spec != other._spec
+
+    def _get_operator(self, op):
+        return getattr(self, "_compare_{0}".format(self._operators[op]))
+
+    def _coerce_version(self, version):
+        if not isinstance(version, (LegacyVersion, Version)):
+            version = parse(version)
+        return version
+
+    @property
+    def operator(self):
+        return self._spec[0]
+
+    @property
+    def version(self):
+        return self._spec[1]
+
+    @property
+    def prereleases(self):
+        return self._prereleases
+
+    @prereleases.setter
+    def prereleases(self, value):
+        self._prereleases = value
+
+    def __contains__(self, item):
+        return self.contains(item)
+
+    def contains(self, item, prereleases=None):
+        # Determine if prereleases are to be allowed or not.
+        if prereleases is None:
+            prereleases = self.prereleases
+
+        # Normalize item to a Version or LegacyVersion, this allows us to have
+        # a shortcut for ``"2.0" in Specifier(">=2")
+        item = self._coerce_version(item)
+
+        # Determine if we should be supporting prereleases in this specifier
+        # or not, if we do not support prereleases than we can short circuit
+        # logic if this version is a prereleases.
+        if item.is_prerelease and not prereleases:
+            return False
+
+        # Actually do the comparison to determine if this item is contained
+        # within this Specifier or not.
+        return self._get_operator(self.operator)(item, self.version)
+
+    def filter(self, iterable, prereleases=None):
+        yielded = False
+        found_prereleases = []
+
+        kw = {"prereleases": prereleases if prereleases is not None else True}
+
+        # Attempt to iterate over all the values in the iterable and if any of
+        # them match, yield them.
+        for version in iterable:
+            parsed_version = self._coerce_version(version)
+
+            if self.contains(parsed_version, **kw):
+                # If our version is a prerelease, and we were not set to allow
+                # prereleases, then we'll store it for later incase nothing
+                # else matches this specifier.
+                if parsed_version.is_prerelease and not (
+                    prereleases or self.prereleases
+                ):
+                    found_prereleases.append(version)
+                # Either this is not a prerelease, or we should have been
+                # accepting prereleases from the beginning.
+                else:
+                    yielded = True
+                    yield version
+
+        # Now that we've iterated over everything, determine if we've yielded
+        # any values, and if we have not and we have any prereleases stored up
+        # then we will go ahead and yield the prereleases.
+        if not yielded and found_prereleases:
+            for version in found_prereleases:
+                yield version
+
+
+class LegacySpecifier(_IndividualSpecifier):
+
+    _regex_str = r"""
+        (?P(==|!=|<=|>=|<|>))
+        \s*
+        (?P
+            [^,;\s)]* # Since this is a "legacy" specifier, and the version
+                      # string can be just about anything, we match everything
+                      # except for whitespace, a semi-colon for marker support,
+                      # a closing paren since versions can be enclosed in
+                      # them, and a comma since it's a version separator.
+        )
+        """
+
+    _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+    _operators = {
+        "==": "equal",
+        "!=": "not_equal",
+        "<=": "less_than_equal",
+        ">=": "greater_than_equal",
+        "<": "less_than",
+        ">": "greater_than",
+    }
+
+    def _coerce_version(self, version):
+        if not isinstance(version, LegacyVersion):
+            version = LegacyVersion(str(version))
+        return version
+
+    def _compare_equal(self, prospective, spec):
+        return prospective == self._coerce_version(spec)
+
+    def _compare_not_equal(self, prospective, spec):
+        return prospective != self._coerce_version(spec)
+
+    def _compare_less_than_equal(self, prospective, spec):
+        return prospective <= self._coerce_version(spec)
+
+    def _compare_greater_than_equal(self, prospective, spec):
+        return prospective >= self._coerce_version(spec)
+
+    def _compare_less_than(self, prospective, spec):
+        return prospective < self._coerce_version(spec)
+
+    def _compare_greater_than(self, prospective, spec):
+        return prospective > self._coerce_version(spec)
+
+
+def _require_version_compare(fn):
+    @functools.wraps(fn)
+    def wrapped(self, prospective, spec):
+        if not isinstance(prospective, Version):
+            return False
+        return fn(self, prospective, spec)
+
+    return wrapped
+
+
+class Specifier(_IndividualSpecifier):
+
+    _regex_str = r"""
+        (?P(~=|==|!=|<=|>=|<|>|===))
+        (?P
+            (?:
+                # The identity operators allow for an escape hatch that will
+                # do an exact string match of the version you wish to install.
+                # This will not be parsed by PEP 440 and we cannot determine
+                # any semantic meaning from it. This operator is discouraged
+                # but included entirely as an escape hatch.
+                (?<====)  # Only match for the identity operator
+                \s*
+                [^\s]*    # We just match everything, except for whitespace
+                          # since we are only testing for strict identity.
+            )
+            |
+            (?:
+                # The (non)equality operators allow for wild card and local
+                # versions to be specified so we have to define these two
+                # operators separately to enable that.
+                (?<===|!=)            # Only match for equals and not equals
+
+                \s*
+                v?
+                (?:[0-9]+!)?          # epoch
+                [0-9]+(?:\.[0-9]+)*   # release
+                (?:                   # pre release
+                    [-_\.]?
+                    (a|b|c|rc|alpha|beta|pre|preview)
+                    [-_\.]?
+                    [0-9]*
+                )?
+                (?:                   # post release
+                    (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*)
+                )?
+
+                # You cannot use a wild card and a dev or local version
+                # together so group them with a | and make them optional.
+                (?:
+                    (?:[-_\.]?dev[-_\.]?[0-9]*)?         # dev release
+                    (?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)? # local
+                    |
+                    \.\*  # Wild card syntax of .*
+                )?
+            )
+            |
+            (?:
+                # The compatible operator requires at least two digits in the
+                # release segment.
+                (?<=~=)               # Only match for the compatible operator
+
+                \s*
+                v?
+                (?:[0-9]+!)?          # epoch
+                [0-9]+(?:\.[0-9]+)+   # release  (We have a + instead of a *)
+                (?:                   # pre release
+                    [-_\.]?
+                    (a|b|c|rc|alpha|beta|pre|preview)
+                    [-_\.]?
+                    [0-9]*
+                )?
+                (?:                                   # post release
+                    (?:-[0-9]+)|(?:[-_\.]?(post|rev|r)[-_\.]?[0-9]*)
+                )?
+                (?:[-_\.]?dev[-_\.]?[0-9]*)?          # dev release
+            )
+            |
+            (?:
+                # All other operators only allow a sub set of what the
+                # (non)equality operators do. Specifically they do not allow
+                # local versions to be specified nor do they allow the prefix
+                # matching wild cards.
+                (?=": "greater_than_equal",
+        "<": "less_than",
+        ">": "greater_than",
+        "===": "arbitrary",
+    }
+
+    @_require_version_compare
+    def _compare_compatible(self, prospective, spec):
+        # Compatible releases have an equivalent combination of >= and ==. That
+        # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to
+        # implement this in terms of the other specifiers instead of
+        # implementing it ourselves. The only thing we need to do is construct
+        # the other specifiers.
+
+        # We want everything but the last item in the version, but we want to
+        # ignore post and dev releases and we want to treat the pre-release as
+        # it's own separate segment.
+        prefix = ".".join(
+            list(
+                itertools.takewhile(
+                    lambda x: (not x.startswith("post") and not x.startswith("dev")),
+                    _version_split(spec),
+                )
+            )[:-1]
+        )
+
+        # Add the prefix notation to the end of our string
+        prefix += ".*"
+
+        return self._get_operator(">=")(prospective, spec) and self._get_operator("==")(
+            prospective, prefix
+        )
+
+    @_require_version_compare
+    def _compare_equal(self, prospective, spec):
+        # We need special logic to handle prefix matching
+        if spec.endswith(".*"):
+            # In the case of prefix matching we want to ignore local segment.
+            prospective = Version(prospective.public)
+            # Split the spec out by dots, and pretend that there is an implicit
+            # dot in between a release segment and a pre-release segment.
+            spec = _version_split(spec[:-2])  # Remove the trailing .*
+
+            # Split the prospective version out by dots, and pretend that there
+            # is an implicit dot in between a release segment and a pre-release
+            # segment.
+            prospective = _version_split(str(prospective))
+
+            # Shorten the prospective version to be the same length as the spec
+            # so that we can determine if the specifier is a prefix of the
+            # prospective version or not.
+            prospective = prospective[: len(spec)]
+
+            # Pad out our two sides with zeros so that they both equal the same
+            # length.
+            spec, prospective = _pad_version(spec, prospective)
+        else:
+            # Convert our spec string into a Version
+            spec = Version(spec)
+
+            # If the specifier does not have a local segment, then we want to
+            # act as if the prospective version also does not have a local
+            # segment.
+            if not spec.local:
+                prospective = Version(prospective.public)
+
+        return prospective == spec
+
+    @_require_version_compare
+    def _compare_not_equal(self, prospective, spec):
+        return not self._compare_equal(prospective, spec)
+
+    @_require_version_compare
+    def _compare_less_than_equal(self, prospective, spec):
+        return prospective <= Version(spec)
+
+    @_require_version_compare
+    def _compare_greater_than_equal(self, prospective, spec):
+        return prospective >= Version(spec)
+
+    @_require_version_compare
+    def _compare_less_than(self, prospective, spec):
+        # Convert our spec to a Version instance, since we'll want to work with
+        # it as a version.
+        spec = Version(spec)
+
+        # Check to see if the prospective version is less than the spec
+        # version. If it's not we can short circuit and just return False now
+        # instead of doing extra unneeded work.
+        if not prospective < spec:
+            return False
+
+        # This special case is here so that, unless the specifier itself
+        # includes is a pre-release version, that we do not accept pre-release
+        # versions for the version mentioned in the specifier (e.g. <3.1 should
+        # not match 3.1.dev0, but should match 3.0.dev0).
+        if not spec.is_prerelease and prospective.is_prerelease:
+            if Version(prospective.base_version) == Version(spec.base_version):
+                return False
+
+        # If we've gotten to here, it means that prospective version is both
+        # less than the spec version *and* it's not a pre-release of the same
+        # version in the spec.
+        return True
+
+    @_require_version_compare
+    def _compare_greater_than(self, prospective, spec):
+        # Convert our spec to a Version instance, since we'll want to work with
+        # it as a version.
+        spec = Version(spec)
+
+        # Check to see if the prospective version is greater than the spec
+        # version. If it's not we can short circuit and just return False now
+        # instead of doing extra unneeded work.
+        if not prospective > spec:
+            return False
+
+        # This special case is here so that, unless the specifier itself
+        # includes is a post-release version, that we do not accept
+        # post-release versions for the version mentioned in the specifier
+        # (e.g. >3.1 should not match 3.0.post0, but should match 3.2.post0).
+        if not spec.is_postrelease and prospective.is_postrelease:
+            if Version(prospective.base_version) == Version(spec.base_version):
+                return False
+
+        # Ensure that we do not allow a local version of the version mentioned
+        # in the specifier, which is technically greater than, to match.
+        if prospective.local is not None:
+            if Version(prospective.base_version) == Version(spec.base_version):
+                return False
+
+        # If we've gotten to here, it means that prospective version is both
+        # greater than the spec version *and* it's not a pre-release of the
+        # same version in the spec.
+        return True
+
+    def _compare_arbitrary(self, prospective, spec):
+        return str(prospective).lower() == str(spec).lower()
+
+    @property
+    def prereleases(self):
+        # If there is an explicit prereleases set for this, then we'll just
+        # blindly use that.
+        if self._prereleases is not None:
+            return self._prereleases
+
+        # Look at all of our specifiers and determine if they are inclusive
+        # operators, and if they are if they are including an explicit
+        # prerelease.
+        operator, version = self._spec
+        if operator in ["==", ">=", "<=", "~=", "==="]:
+            # The == specifier can include a trailing .*, if it does we
+            # want to remove before parsing.
+            if operator == "==" and version.endswith(".*"):
+                version = version[:-2]
+
+            # Parse the version, and if it is a pre-release than this
+            # specifier allows pre-releases.
+            if parse(version).is_prerelease:
+                return True
+
+        return False
+
+    @prereleases.setter
+    def prereleases(self, value):
+        self._prereleases = value
+
+
+_prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$")
+
+
+def _version_split(version):
+    result = []
+    for item in version.split("."):
+        match = _prefix_regex.search(item)
+        if match:
+            result.extend(match.groups())
+        else:
+            result.append(item)
+    return result
+
+
+def _pad_version(left, right):
+    left_split, right_split = [], []
+
+    # Get the release segment of our versions
+    left_split.append(list(itertools.takewhile(lambda x: x.isdigit(), left)))
+    right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right)))
+
+    # Get the rest of our versions
+    left_split.append(left[len(left_split[0]) :])
+    right_split.append(right[len(right_split[0]) :])
+
+    # Insert our padding
+    left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0])))
+    right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0])))
+
+    return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split)))
+
+
+class SpecifierSet(BaseSpecifier):
+    def __init__(self, specifiers="", prereleases=None):
+        # Split on , to break each indidivual specifier into it's own item, and
+        # strip each item to remove leading/trailing whitespace.
+        specifiers = [s.strip() for s in specifiers.split(",") if s.strip()]
+
+        # Parsed each individual specifier, attempting first to make it a
+        # Specifier and falling back to a LegacySpecifier.
+        parsed = set()
+        for specifier in specifiers:
+            try:
+                parsed.add(Specifier(specifier))
+            except InvalidSpecifier:
+                parsed.add(LegacySpecifier(specifier))
+
+        # Turn our parsed specifiers into a frozen set and save them for later.
+        self._specs = frozenset(parsed)
+
+        # Store our prereleases value so we can use it later to determine if
+        # we accept prereleases or not.
+        self._prereleases = prereleases
+
+    def __repr__(self):
+        pre = (
+            ", prereleases={0!r}".format(self.prereleases)
+            if self._prereleases is not None
+            else ""
+        )
+
+        return "".format(str(self), pre)
+
+    def __str__(self):
+        return ",".join(sorted(str(s) for s in self._specs))
+
+    def __hash__(self):
+        return hash(self._specs)
+
+    def __and__(self, other):
+        if isinstance(other, string_types):
+            other = SpecifierSet(other)
+        elif not isinstance(other, SpecifierSet):
+            return NotImplemented
+
+        specifier = SpecifierSet()
+        specifier._specs = frozenset(self._specs | other._specs)
+
+        if self._prereleases is None and other._prereleases is not None:
+            specifier._prereleases = other._prereleases
+        elif self._prereleases is not None and other._prereleases is None:
+            specifier._prereleases = self._prereleases
+        elif self._prereleases == other._prereleases:
+            specifier._prereleases = self._prereleases
+        else:
+            raise ValueError(
+                "Cannot combine SpecifierSets with True and False prerelease "
+                "overrides."
+            )
+
+        return specifier
+
+    def __eq__(self, other):
+        if isinstance(other, string_types):
+            other = SpecifierSet(other)
+        elif isinstance(other, _IndividualSpecifier):
+            other = SpecifierSet(str(other))
+        elif not isinstance(other, SpecifierSet):
+            return NotImplemented
+
+        return self._specs == other._specs
+
+    def __ne__(self, other):
+        if isinstance(other, string_types):
+            other = SpecifierSet(other)
+        elif isinstance(other, _IndividualSpecifier):
+            other = SpecifierSet(str(other))
+        elif not isinstance(other, SpecifierSet):
+            return NotImplemented
+
+        return self._specs != other._specs
+
+    def __len__(self):
+        return len(self._specs)
+
+    def __iter__(self):
+        return iter(self._specs)
+
+    @property
+    def prereleases(self):
+        # If we have been given an explicit prerelease modifier, then we'll
+        # pass that through here.
+        if self._prereleases is not None:
+            return self._prereleases
+
+        # If we don't have any specifiers, and we don't have a forced value,
+        # then we'll just return None since we don't know if this should have
+        # pre-releases or not.
+        if not self._specs:
+            return None
+
+        # Otherwise we'll see if any of the given specifiers accept
+        # prereleases, if any of them do we'll return True, otherwise False.
+        return any(s.prereleases for s in self._specs)
+
+    @prereleases.setter
+    def prereleases(self, value):
+        self._prereleases = value
+
+    def __contains__(self, item):
+        return self.contains(item)
+
+    def contains(self, item, prereleases=None):
+        # Ensure that our item is a Version or LegacyVersion instance.
+        if not isinstance(item, (LegacyVersion, Version)):
+            item = parse(item)
+
+        # Determine if we're forcing a prerelease or not, if we're not forcing
+        # one for this particular filter call, then we'll use whatever the
+        # SpecifierSet thinks for whether or not we should support prereleases.
+        if prereleases is None:
+            prereleases = self.prereleases
+
+        # We can determine if we're going to allow pre-releases by looking to
+        # see if any of the underlying items supports them. If none of them do
+        # and this item is a pre-release then we do not allow it and we can
+        # short circuit that here.
+        # Note: This means that 1.0.dev1 would not be contained in something
+        #       like >=1.0.devabc however it would be in >=1.0.debabc,>0.0.dev0
+        if not prereleases and item.is_prerelease:
+            return False
+
+        # We simply dispatch to the underlying specs here to make sure that the
+        # given version is contained within all of them.
+        # Note: This use of all() here means that an empty set of specifiers
+        #       will always return True, this is an explicit design decision.
+        return all(s.contains(item, prereleases=prereleases) for s in self._specs)
+
+    def filter(self, iterable, prereleases=None):
+        # Determine if we're forcing a prerelease or not, if we're not forcing
+        # one for this particular filter call, then we'll use whatever the
+        # SpecifierSet thinks for whether or not we should support prereleases.
+        if prereleases is None:
+            prereleases = self.prereleases
+
+        # If we have any specifiers, then we want to wrap our iterable in the
+        # filter method for each one, this will act as a logical AND amongst
+        # each specifier.
+        if self._specs:
+            for spec in self._specs:
+                iterable = spec.filter(iterable, prereleases=bool(prereleases))
+            return iterable
+        # If we do not have any specifiers, then we need to have a rough filter
+        # which will filter out any pre-releases, unless there are no final
+        # releases, and which will filter out LegacyVersion in general.
+        else:
+            filtered = []
+            found_prereleases = []
+
+            for item in iterable:
+                # Ensure that we some kind of Version class for this item.
+                if not isinstance(item, (LegacyVersion, Version)):
+                    parsed_version = parse(item)
+                else:
+                    parsed_version = item
+
+                # Filter out any item which is parsed as a LegacyVersion
+                if isinstance(parsed_version, LegacyVersion):
+                    continue
+
+                # Store any item which is a pre-release for later unless we've
+                # already found a final version or we are accepting prereleases
+                if parsed_version.is_prerelease and not prereleases:
+                    if not filtered:
+                        found_prereleases.append(item)
+                else:
+                    filtered.append(item)
+
+            # If we've found no items except for pre-releases, then we'll go
+            # ahead and use the pre-releases
+            if not filtered and found_prereleases and prereleases is None:
+                return found_prereleases
+
+            return filtered
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/tags.py b/ubuntu/venv/setuptools/_vendor/packaging/tags.py
new file mode 100644
index 0000000..ec9942f
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/tags.py
@@ -0,0 +1,404 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+
+from __future__ import absolute_import
+
+import distutils.util
+
+try:
+    from importlib.machinery import EXTENSION_SUFFIXES
+except ImportError:  # pragma: no cover
+    import imp
+
+    EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()]
+    del imp
+import platform
+import re
+import sys
+import sysconfig
+import warnings
+
+
+INTERPRETER_SHORT_NAMES = {
+    "python": "py",  # Generic.
+    "cpython": "cp",
+    "pypy": "pp",
+    "ironpython": "ip",
+    "jython": "jy",
+}
+
+
+_32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32
+
+
+class Tag(object):
+
+    __slots__ = ["_interpreter", "_abi", "_platform"]
+
+    def __init__(self, interpreter, abi, platform):
+        self._interpreter = interpreter.lower()
+        self._abi = abi.lower()
+        self._platform = platform.lower()
+
+    @property
+    def interpreter(self):
+        return self._interpreter
+
+    @property
+    def abi(self):
+        return self._abi
+
+    @property
+    def platform(self):
+        return self._platform
+
+    def __eq__(self, other):
+        return (
+            (self.platform == other.platform)
+            and (self.abi == other.abi)
+            and (self.interpreter == other.interpreter)
+        )
+
+    def __hash__(self):
+        return hash((self._interpreter, self._abi, self._platform))
+
+    def __str__(self):
+        return "{}-{}-{}".format(self._interpreter, self._abi, self._platform)
+
+    def __repr__(self):
+        return "<{self} @ {self_id}>".format(self=self, self_id=id(self))
+
+
+def parse_tag(tag):
+    tags = set()
+    interpreters, abis, platforms = tag.split("-")
+    for interpreter in interpreters.split("."):
+        for abi in abis.split("."):
+            for platform_ in platforms.split("."):
+                tags.add(Tag(interpreter, abi, platform_))
+    return frozenset(tags)
+
+
+def _normalize_string(string):
+    return string.replace(".", "_").replace("-", "_")
+
+
+def _cpython_interpreter(py_version):
+    # TODO: Is using py_version_nodot for interpreter version critical?
+    return "cp{major}{minor}".format(major=py_version[0], minor=py_version[1])
+
+
+def _cpython_abis(py_version):
+    abis = []
+    version = "{}{}".format(*py_version[:2])
+    debug = pymalloc = ucs4 = ""
+    with_debug = sysconfig.get_config_var("Py_DEBUG")
+    has_refcount = hasattr(sys, "gettotalrefcount")
+    # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled
+    # extension modules is the best option.
+    # https://github.com/pypa/pip/issues/3383#issuecomment-173267692
+    has_ext = "_d.pyd" in EXTENSION_SUFFIXES
+    if with_debug or (with_debug is None and (has_refcount or has_ext)):
+        debug = "d"
+    if py_version < (3, 8):
+        with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC")
+        if with_pymalloc or with_pymalloc is None:
+            pymalloc = "m"
+        if py_version < (3, 3):
+            unicode_size = sysconfig.get_config_var("Py_UNICODE_SIZE")
+            if unicode_size == 4 or (
+                unicode_size is None and sys.maxunicode == 0x10FFFF
+            ):
+                ucs4 = "u"
+    elif debug:
+        # Debug builds can also load "normal" extension modules.
+        # We can also assume no UCS-4 or pymalloc requirement.
+        abis.append("cp{version}".format(version=version))
+    abis.insert(
+        0,
+        "cp{version}{debug}{pymalloc}{ucs4}".format(
+            version=version, debug=debug, pymalloc=pymalloc, ucs4=ucs4
+        ),
+    )
+    return abis
+
+
+def _cpython_tags(py_version, interpreter, abis, platforms):
+    for abi in abis:
+        for platform_ in platforms:
+            yield Tag(interpreter, abi, platform_)
+    for tag in (Tag(interpreter, "abi3", platform_) for platform_ in platforms):
+        yield tag
+    for tag in (Tag(interpreter, "none", platform_) for platform_ in platforms):
+        yield tag
+    # PEP 384 was first implemented in Python 3.2.
+    for minor_version in range(py_version[1] - 1, 1, -1):
+        for platform_ in platforms:
+            interpreter = "cp{major}{minor}".format(
+                major=py_version[0], minor=minor_version
+            )
+            yield Tag(interpreter, "abi3", platform_)
+
+
+def _pypy_interpreter():
+    return "pp{py_major}{pypy_major}{pypy_minor}".format(
+        py_major=sys.version_info[0],
+        pypy_major=sys.pypy_version_info.major,
+        pypy_minor=sys.pypy_version_info.minor,
+    )
+
+
+def _generic_abi():
+    abi = sysconfig.get_config_var("SOABI")
+    if abi:
+        return _normalize_string(abi)
+    else:
+        return "none"
+
+
+def _pypy_tags(py_version, interpreter, abi, platforms):
+    for tag in (Tag(interpreter, abi, platform) for platform in platforms):
+        yield tag
+    for tag in (Tag(interpreter, "none", platform) for platform in platforms):
+        yield tag
+
+
+def _generic_tags(interpreter, py_version, abi, platforms):
+    for tag in (Tag(interpreter, abi, platform) for platform in platforms):
+        yield tag
+    if abi != "none":
+        tags = (Tag(interpreter, "none", platform_) for platform_ in platforms)
+        for tag in tags:
+            yield tag
+
+
+def _py_interpreter_range(py_version):
+    """
+    Yield Python versions in descending order.
+
+    After the latest version, the major-only version will be yielded, and then
+    all following versions up to 'end'.
+    """
+    yield "py{major}{minor}".format(major=py_version[0], minor=py_version[1])
+    yield "py{major}".format(major=py_version[0])
+    for minor in range(py_version[1] - 1, -1, -1):
+        yield "py{major}{minor}".format(major=py_version[0], minor=minor)
+
+
+def _independent_tags(interpreter, py_version, platforms):
+    """
+    Return the sequence of tags that are consistent across implementations.
+
+    The tags consist of:
+    - py*-none-
+    - -none-any
+    - py*-none-any
+    """
+    for version in _py_interpreter_range(py_version):
+        for platform_ in platforms:
+            yield Tag(version, "none", platform_)
+    yield Tag(interpreter, "none", "any")
+    for version in _py_interpreter_range(py_version):
+        yield Tag(version, "none", "any")
+
+
+def _mac_arch(arch, is_32bit=_32_BIT_INTERPRETER):
+    if not is_32bit:
+        return arch
+
+    if arch.startswith("ppc"):
+        return "ppc"
+
+    return "i386"
+
+
+def _mac_binary_formats(version, cpu_arch):
+    formats = [cpu_arch]
+    if cpu_arch == "x86_64":
+        if version < (10, 4):
+            return []
+        formats.extend(["intel", "fat64", "fat32"])
+
+    elif cpu_arch == "i386":
+        if version < (10, 4):
+            return []
+        formats.extend(["intel", "fat32", "fat"])
+
+    elif cpu_arch == "ppc64":
+        # TODO: Need to care about 32-bit PPC for ppc64 through 10.2?
+        if version > (10, 5) or version < (10, 4):
+            return []
+        formats.append("fat64")
+
+    elif cpu_arch == "ppc":
+        if version > (10, 6):
+            return []
+        formats.extend(["fat32", "fat"])
+
+    formats.append("universal")
+    return formats
+
+
+def _mac_platforms(version=None, arch=None):
+    version_str, _, cpu_arch = platform.mac_ver()
+    if version is None:
+        version = tuple(map(int, version_str.split(".")[:2]))
+    if arch is None:
+        arch = _mac_arch(cpu_arch)
+    platforms = []
+    for minor_version in range(version[1], -1, -1):
+        compat_version = version[0], minor_version
+        binary_formats = _mac_binary_formats(compat_version, arch)
+        for binary_format in binary_formats:
+            platforms.append(
+                "macosx_{major}_{minor}_{binary_format}".format(
+                    major=compat_version[0],
+                    minor=compat_version[1],
+                    binary_format=binary_format,
+                )
+            )
+    return platforms
+
+
+# From PEP 513.
+def _is_manylinux_compatible(name, glibc_version):
+    # Check for presence of _manylinux module.
+    try:
+        import _manylinux
+
+        return bool(getattr(_manylinux, name + "_compatible"))
+    except (ImportError, AttributeError):
+        # Fall through to heuristic check below.
+        pass
+
+    return _have_compatible_glibc(*glibc_version)
+
+
+def _glibc_version_string():
+    # Returns glibc version string, or None if not using glibc.
+    import ctypes
+
+    # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen
+    # manpage says, "If filename is NULL, then the returned handle is for the
+    # main program". This way we can let the linker do the work to figure out
+    # which libc our process is actually using.
+    process_namespace = ctypes.CDLL(None)
+    try:
+        gnu_get_libc_version = process_namespace.gnu_get_libc_version
+    except AttributeError:
+        # Symbol doesn't exist -> therefore, we are not linked to
+        # glibc.
+        return None
+
+    # Call gnu_get_libc_version, which returns a string like "2.5"
+    gnu_get_libc_version.restype = ctypes.c_char_p
+    version_str = gnu_get_libc_version()
+    # py2 / py3 compatibility:
+    if not isinstance(version_str, str):
+        version_str = version_str.decode("ascii")
+
+    return version_str
+
+
+# Separated out from have_compatible_glibc for easier unit testing.
+def _check_glibc_version(version_str, required_major, minimum_minor):
+    # Parse string and check against requested version.
+    #
+    # We use a regexp instead of str.split because we want to discard any
+    # random junk that might come after the minor version -- this might happen
+    # in patched/forked versions of glibc (e.g. Linaro's version of glibc
+    # uses version strings like "2.20-2014.11"). See gh-3588.
+    m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str)
+    if not m:
+        warnings.warn(
+            "Expected glibc version with 2 components major.minor,"
+            " got: %s" % version_str,
+            RuntimeWarning,
+        )
+        return False
+    return (
+        int(m.group("major")) == required_major
+        and int(m.group("minor")) >= minimum_minor
+    )
+
+
+def _have_compatible_glibc(required_major, minimum_minor):
+    version_str = _glibc_version_string()
+    if version_str is None:
+        return False
+    return _check_glibc_version(version_str, required_major, minimum_minor)
+
+
+def _linux_platforms(is_32bit=_32_BIT_INTERPRETER):
+    linux = _normalize_string(distutils.util.get_platform())
+    if linux == "linux_x86_64" and is_32bit:
+        linux = "linux_i686"
+    manylinux_support = (
+        ("manylinux2014", (2, 17)),  # CentOS 7 w/ glibc 2.17 (PEP 599)
+        ("manylinux2010", (2, 12)),  # CentOS 6 w/ glibc 2.12 (PEP 571)
+        ("manylinux1", (2, 5)),  # CentOS 5 w/ glibc 2.5 (PEP 513)
+    )
+    manylinux_support_iter = iter(manylinux_support)
+    for name, glibc_version in manylinux_support_iter:
+        if _is_manylinux_compatible(name, glibc_version):
+            platforms = [linux.replace("linux", name)]
+            break
+    else:
+        platforms = []
+    # Support for a later manylinux implies support for an earlier version.
+    platforms += [linux.replace("linux", name) for name, _ in manylinux_support_iter]
+    platforms.append(linux)
+    return platforms
+
+
+def _generic_platforms():
+    platform = _normalize_string(distutils.util.get_platform())
+    return [platform]
+
+
+def _interpreter_name():
+    name = platform.python_implementation().lower()
+    return INTERPRETER_SHORT_NAMES.get(name) or name
+
+
+def _generic_interpreter(name, py_version):
+    version = sysconfig.get_config_var("py_version_nodot")
+    if not version:
+        version = "".join(map(str, py_version[:2]))
+    return "{name}{version}".format(name=name, version=version)
+
+
+def sys_tags():
+    """
+    Returns the sequence of tag triples for the running interpreter.
+
+    The order of the sequence corresponds to priority order for the
+    interpreter, from most to least important.
+    """
+    py_version = sys.version_info[:2]
+    interpreter_name = _interpreter_name()
+    if platform.system() == "Darwin":
+        platforms = _mac_platforms()
+    elif platform.system() == "Linux":
+        platforms = _linux_platforms()
+    else:
+        platforms = _generic_platforms()
+
+    if interpreter_name == "cp":
+        interpreter = _cpython_interpreter(py_version)
+        abis = _cpython_abis(py_version)
+        for tag in _cpython_tags(py_version, interpreter, abis, platforms):
+            yield tag
+    elif interpreter_name == "pp":
+        interpreter = _pypy_interpreter()
+        abi = _generic_abi()
+        for tag in _pypy_tags(py_version, interpreter, abi, platforms):
+            yield tag
+    else:
+        interpreter = _generic_interpreter(interpreter_name, py_version)
+        abi = _generic_abi()
+        for tag in _generic_tags(interpreter, py_version, abi, platforms):
+            yield tag
+    for tag in _independent_tags(interpreter, py_version, platforms):
+        yield tag
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/utils.py b/ubuntu/venv/setuptools/_vendor/packaging/utils.py
new file mode 100644
index 0000000..8841878
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/utils.py
@@ -0,0 +1,57 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import re
+
+from .version import InvalidVersion, Version
+
+
+_canonicalize_regex = re.compile(r"[-_.]+")
+
+
+def canonicalize_name(name):
+    # This is taken from PEP 503.
+    return _canonicalize_regex.sub("-", name).lower()
+
+
+def canonicalize_version(version):
+    """
+    This is very similar to Version.__str__, but has one subtle differences
+    with the way it handles the release segment.
+    """
+
+    try:
+        version = Version(version)
+    except InvalidVersion:
+        # Legacy versions cannot be normalized
+        return version
+
+    parts = []
+
+    # Epoch
+    if version.epoch != 0:
+        parts.append("{0}!".format(version.epoch))
+
+    # Release segment
+    # NB: This strips trailing '.0's to normalize
+    parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release)))
+
+    # Pre-release
+    if version.pre is not None:
+        parts.append("".join(str(x) for x in version.pre))
+
+    # Post-release
+    if version.post is not None:
+        parts.append(".post{0}".format(version.post))
+
+    # Development release
+    if version.dev is not None:
+        parts.append(".dev{0}".format(version.dev))
+
+    # Local version segment
+    if version.local is not None:
+        parts.append("+{0}".format(version.local))
+
+    return "".join(parts)
diff --git a/ubuntu/venv/setuptools/_vendor/packaging/version.py b/ubuntu/venv/setuptools/_vendor/packaging/version.py
new file mode 100644
index 0000000..95157a1
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/packaging/version.py
@@ -0,0 +1,420 @@
+# This file is dual licensed under the terms of the Apache License, Version
+# 2.0, and the BSD License. See the LICENSE file in the root of this repository
+# for complete details.
+from __future__ import absolute_import, division, print_function
+
+import collections
+import itertools
+import re
+
+from ._structures import Infinity
+
+
+__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"]
+
+
+_Version = collections.namedtuple(
+    "_Version", ["epoch", "release", "dev", "pre", "post", "local"]
+)
+
+
+def parse(version):
+    """
+    Parse the given version string and return either a :class:`Version` object
+    or a :class:`LegacyVersion` object depending on if the given version is
+    a valid PEP 440 version or a legacy version.
+    """
+    try:
+        return Version(version)
+    except InvalidVersion:
+        return LegacyVersion(version)
+
+
+class InvalidVersion(ValueError):
+    """
+    An invalid version was found, users should refer to PEP 440.
+    """
+
+
+class _BaseVersion(object):
+    def __hash__(self):
+        return hash(self._key)
+
+    def __lt__(self, other):
+        return self._compare(other, lambda s, o: s < o)
+
+    def __le__(self, other):
+        return self._compare(other, lambda s, o: s <= o)
+
+    def __eq__(self, other):
+        return self._compare(other, lambda s, o: s == o)
+
+    def __ge__(self, other):
+        return self._compare(other, lambda s, o: s >= o)
+
+    def __gt__(self, other):
+        return self._compare(other, lambda s, o: s > o)
+
+    def __ne__(self, other):
+        return self._compare(other, lambda s, o: s != o)
+
+    def _compare(self, other, method):
+        if not isinstance(other, _BaseVersion):
+            return NotImplemented
+
+        return method(self._key, other._key)
+
+
+class LegacyVersion(_BaseVersion):
+    def __init__(self, version):
+        self._version = str(version)
+        self._key = _legacy_cmpkey(self._version)
+
+    def __str__(self):
+        return self._version
+
+    def __repr__(self):
+        return "".format(repr(str(self)))
+
+    @property
+    def public(self):
+        return self._version
+
+    @property
+    def base_version(self):
+        return self._version
+
+    @property
+    def epoch(self):
+        return -1
+
+    @property
+    def release(self):
+        return None
+
+    @property
+    def pre(self):
+        return None
+
+    @property
+    def post(self):
+        return None
+
+    @property
+    def dev(self):
+        return None
+
+    @property
+    def local(self):
+        return None
+
+    @property
+    def is_prerelease(self):
+        return False
+
+    @property
+    def is_postrelease(self):
+        return False
+
+    @property
+    def is_devrelease(self):
+        return False
+
+
+_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE)
+
+_legacy_version_replacement_map = {
+    "pre": "c",
+    "preview": "c",
+    "-": "final-",
+    "rc": "c",
+    "dev": "@",
+}
+
+
+def _parse_version_parts(s):
+    for part in _legacy_version_component_re.split(s):
+        part = _legacy_version_replacement_map.get(part, part)
+
+        if not part or part == ".":
+            continue
+
+        if part[:1] in "0123456789":
+            # pad for numeric comparison
+            yield part.zfill(8)
+        else:
+            yield "*" + part
+
+    # ensure that alpha/beta/candidate are before final
+    yield "*final"
+
+
+def _legacy_cmpkey(version):
+    # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch
+    # greater than or equal to 0. This will effectively put the LegacyVersion,
+    # which uses the defacto standard originally implemented by setuptools,
+    # as before all PEP 440 versions.
+    epoch = -1
+
+    # This scheme is taken from pkg_resources.parse_version setuptools prior to
+    # it's adoption of the packaging library.
+    parts = []
+    for part in _parse_version_parts(version.lower()):
+        if part.startswith("*"):
+            # remove "-" before a prerelease tag
+            if part < "*final":
+                while parts and parts[-1] == "*final-":
+                    parts.pop()
+
+            # remove trailing zeros from each series of numeric parts
+            while parts and parts[-1] == "00000000":
+                parts.pop()
+
+        parts.append(part)
+    parts = tuple(parts)
+
+    return epoch, parts
+
+
+# Deliberately not anchored to the start and end of the string, to make it
+# easier for 3rd party code to reuse
+VERSION_PATTERN = r"""
+    v?
+    (?:
+        (?:(?P[0-9]+)!)?                           # epoch
+        (?P[0-9]+(?:\.[0-9]+)*)                  # release segment
+        (?P
                                          # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+
+class Version(_BaseVersion):
+
+    _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+    def __init__(self, version):
+        # Validate the version and parse it into pieces
+        match = self._regex.search(version)
+        if not match:
+            raise InvalidVersion("Invalid version: '{0}'".format(version))
+
+        # Store the parsed out pieces of the version
+        self._version = _Version(
+            epoch=int(match.group("epoch")) if match.group("epoch") else 0,
+            release=tuple(int(i) for i in match.group("release").split(".")),
+            pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")),
+            post=_parse_letter_version(
+                match.group("post_l"), match.group("post_n1") or match.group("post_n2")
+            ),
+            dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")),
+            local=_parse_local_version(match.group("local")),
+        )
+
+        # Generate a key which will be used for sorting
+        self._key = _cmpkey(
+            self._version.epoch,
+            self._version.release,
+            self._version.pre,
+            self._version.post,
+            self._version.dev,
+            self._version.local,
+        )
+
+    def __repr__(self):
+        return "".format(repr(str(self)))
+
+    def __str__(self):
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append("{0}!".format(self.epoch))
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        # Pre-release
+        if self.pre is not None:
+            parts.append("".join(str(x) for x in self.pre))
+
+        # Post-release
+        if self.post is not None:
+            parts.append(".post{0}".format(self.post))
+
+        # Development release
+        if self.dev is not None:
+            parts.append(".dev{0}".format(self.dev))
+
+        # Local version segment
+        if self.local is not None:
+            parts.append("+{0}".format(self.local))
+
+        return "".join(parts)
+
+    @property
+    def epoch(self):
+        return self._version.epoch
+
+    @property
+    def release(self):
+        return self._version.release
+
+    @property
+    def pre(self):
+        return self._version.pre
+
+    @property
+    def post(self):
+        return self._version.post[1] if self._version.post else None
+
+    @property
+    def dev(self):
+        return self._version.dev[1] if self._version.dev else None
+
+    @property
+    def local(self):
+        if self._version.local:
+            return ".".join(str(x) for x in self._version.local)
+        else:
+            return None
+
+    @property
+    def public(self):
+        return str(self).split("+", 1)[0]
+
+    @property
+    def base_version(self):
+        parts = []
+
+        # Epoch
+        if self.epoch != 0:
+            parts.append("{0}!".format(self.epoch))
+
+        # Release segment
+        parts.append(".".join(str(x) for x in self.release))
+
+        return "".join(parts)
+
+    @property
+    def is_prerelease(self):
+        return self.dev is not None or self.pre is not None
+
+    @property
+    def is_postrelease(self):
+        return self.post is not None
+
+    @property
+    def is_devrelease(self):
+        return self.dev is not None
+
+
+def _parse_letter_version(letter, number):
+    if letter:
+        # We consider there to be an implicit 0 in a pre-release if there is
+        # not a numeral associated with it.
+        if number is None:
+            number = 0
+
+        # We normalize any letters to their lower case form
+        letter = letter.lower()
+
+        # We consider some words to be alternate spellings of other words and
+        # in those cases we want to normalize the spellings to our preferred
+        # spelling.
+        if letter == "alpha":
+            letter = "a"
+        elif letter == "beta":
+            letter = "b"
+        elif letter in ["c", "pre", "preview"]:
+            letter = "rc"
+        elif letter in ["rev", "r"]:
+            letter = "post"
+
+        return letter, int(number)
+    if not letter and number:
+        # We assume if we are given a number, but we are not given a letter
+        # then this is using the implicit post release syntax (e.g. 1.0-1)
+        letter = "post"
+
+        return letter, int(number)
+
+
+_local_version_separators = re.compile(r"[\._-]")
+
+
+def _parse_local_version(local):
+    """
+    Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve").
+    """
+    if local is not None:
+        return tuple(
+            part.lower() if not part.isdigit() else int(part)
+            for part in _local_version_separators.split(local)
+        )
+
+
+def _cmpkey(epoch, release, pre, post, dev, local):
+    # When we compare a release version, we want to compare it with all of the
+    # trailing zeros removed. So we'll use a reverse the list, drop all the now
+    # leading zeros until we come to something non zero, then take the rest
+    # re-reverse it back into the correct order and make it a tuple and use
+    # that for our sorting key.
+    release = tuple(
+        reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+    )
+
+    # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0.
+    # We'll do this by abusing the pre segment, but we _only_ want to do this
+    # if there is not a pre or a post segment. If we have one of those then
+    # the normal sorting rules will handle this case correctly.
+    if pre is None and post is None and dev is not None:
+        pre = -Infinity
+    # Versions without a pre-release (except as noted above) should sort after
+    # those with one.
+    elif pre is None:
+        pre = Infinity
+
+    # Versions without a post segment should sort before those with one.
+    if post is None:
+        post = -Infinity
+
+    # Versions without a development segment should sort after those with one.
+    if dev is None:
+        dev = Infinity
+
+    if local is None:
+        # Versions without a local segment should sort before those with one.
+        local = -Infinity
+    else:
+        # Versions with a local segment need that segment parsed to implement
+        # the sorting rules in PEP440.
+        # - Alpha numeric segments sort before numeric segments
+        # - Alpha numeric segments sort lexicographically
+        # - Numeric segments sort numerically
+        # - Shorter versions sort before longer versions when the prefixes
+        #   match exactly
+        local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local)
+
+    return epoch, release, pre, post, dev, local
diff --git a/ubuntu/venv/setuptools/_vendor/pyparsing.py b/ubuntu/venv/setuptools/_vendor/pyparsing.py
new file mode 100644
index 0000000..cf75e1e
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/pyparsing.py
@@ -0,0 +1,5742 @@
+# module pyparsing.py
+#
+# Copyright (c) 2003-2018  Paul T. McGuire
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+__doc__ = \
+"""
+pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
+
+The pyparsing module is an alternative approach to creating and executing simple grammars,
+vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
+don't need to learn a new syntax for defining grammars or matching expressions - the parsing module
+provides a library of classes that you use to construct the grammar directly in Python.
+
+Here is a program to parse "Hello, World!" (or any greeting of the form 
+C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements 
+(L{'+'} operator gives L{And} expressions, strings are auto-converted to
+L{Literal} expressions)::
+
+    from pyparsing import Word, alphas
+
+    # define grammar of a greeting
+    greet = Word(alphas) + "," + Word(alphas) + "!"
+
+    hello = "Hello, World!"
+    print (hello, "->", greet.parseString(hello))
+
+The program outputs the following::
+
+    Hello, World! -> ['Hello', ',', 'World', '!']
+
+The Python representation of the grammar is quite readable, owing to the self-explanatory
+class names, and the use of '+', '|' and '^' operators.
+
+The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an
+object with named attributes.
+
+The pyparsing module handles some of the problems that are typically vexing when writing text parsers:
+ - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
+ - quoted strings
+ - embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
+"""
+
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
+__author__ = "Paul McGuire "
+
+import string
+from weakref import ref as wkref
+import copy
+import sys
+import warnings
+import re
+import sre_constants
+import collections
+import pprint
+import traceback
+import types
+from datetime import datetime
+
+try:
+    from _thread import RLock
+except ImportError:
+    from threading import RLock
+
+try:
+    # Python 3
+    from collections.abc import Iterable
+    from collections.abc import MutableMapping
+except ImportError:
+    # Python 2.7
+    from collections import Iterable
+    from collections import MutableMapping
+
+try:
+    from collections import OrderedDict as _OrderedDict
+except ImportError:
+    try:
+        from ordereddict import OrderedDict as _OrderedDict
+    except ImportError:
+        _OrderedDict = None
+
+#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
+
+__all__ = [
+'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty',
+'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal',
+'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or',
+'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException',
+'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException',
+'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', 
+'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
+'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
+'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
+'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno',
+'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
+'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
+'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 
+'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
+'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass',
+'CloseMatch', 'tokenMap', 'pyparsing_common',
+]
+
+system_version = tuple(sys.version_info)[:3]
+PY_3 = system_version[0] == 3
+if PY_3:
+    _MAX_INT = sys.maxsize
+    basestring = str
+    unichr = chr
+    _ustr = str
+
+    # build list of single arg builtins, that can be used as parse actions
+    singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
+else:
+    _MAX_INT = sys.maxint
+    range = xrange
+
+    def _ustr(obj):
+        """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
+           str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
+           then < returns the unicode object | encodes it with the default encoding | ... >.
+        """
+        if isinstance(obj,unicode):
+            return obj
+
+        try:
+            # If this works, then _ustr(obj) has the same behaviour as str(obj), so
+            # it won't break any existing code.
+            return str(obj)
+
+        except UnicodeEncodeError:
+            # Else encode it
+            ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
+            xmlcharref = Regex(r'&#\d+;')
+            xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
+            return xmlcharref.transformString(ret)
+
+    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+    singleArgBuiltins = []
+    import __builtin__
+    for fname in "sum len sorted reversed list tuple set any all min max".split():
+        try:
+            singleArgBuiltins.append(getattr(__builtin__,fname))
+        except AttributeError:
+            continue
+            
+_generatorType = type((y for y in range(1)))
+ 
+def _xml_escape(data):
+    """Escape &, <, >, ", ', etc. in a string of data."""
+
+    # ampersand must be replaced first
+    from_symbols = '&><"\''
+    to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
+    for from_,to_ in zip(from_symbols, to_symbols):
+        data = data.replace(from_, to_)
+    return data
+
+class _Constants(object):
+    pass
+
+alphas     = string.ascii_uppercase + string.ascii_lowercase
+nums       = "0123456789"
+hexnums    = nums + "ABCDEFabcdef"
+alphanums  = alphas + nums
+_bslash    = chr(92)
+printables = "".join(c for c in string.printable if c not in string.whitespace)
+
+class ParseBaseException(Exception):
+    """base exception class for all parsing runtime exceptions"""
+    # Performance tuning: we construct a *lot* of these, so keep this
+    # constructor as small and fast as possible
+    def __init__( self, pstr, loc=0, msg=None, elem=None ):
+        self.loc = loc
+        if msg is None:
+            self.msg = pstr
+            self.pstr = ""
+        else:
+            self.msg = msg
+            self.pstr = pstr
+        self.parserElement = elem
+        self.args = (pstr, loc, msg)
+
+    @classmethod
+    def _from_exception(cls, pe):
+        """
+        internal factory method to simplify creating one type of ParseException 
+        from another - avoids having __init__ signature conflicts among subclasses
+        """
+        return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement)
+
+    def __getattr__( self, aname ):
+        """supported attributes by name are:
+            - lineno - returns the line number of the exception text
+            - col - returns the column number of the exception text
+            - line - returns the line containing the exception text
+        """
+        if( aname == "lineno" ):
+            return lineno( self.loc, self.pstr )
+        elif( aname in ("col", "column") ):
+            return col( self.loc, self.pstr )
+        elif( aname == "line" ):
+            return line( self.loc, self.pstr )
+        else:
+            raise AttributeError(aname)
+
+    def __str__( self ):
+        return "%s (at char %d), (line:%d, col:%d)" % \
+                ( self.msg, self.loc, self.lineno, self.column )
+    def __repr__( self ):
+        return _ustr(self)
+    def markInputline( self, markerString = ">!<" ):
+        """Extracts the exception line from the input string, and marks
+           the location of the exception with a special symbol.
+        """
+        line_str = self.line
+        line_column = self.column - 1
+        if markerString:
+            line_str = "".join((line_str[:line_column],
+                                markerString, line_str[line_column:]))
+        return line_str.strip()
+    def __dir__(self):
+        return "lineno col line".split() + dir(type(self))
+
+class ParseException(ParseBaseException):
+    """
+    Exception thrown when parse expressions don't match class;
+    supported attributes by name are:
+     - lineno - returns the line number of the exception text
+     - col - returns the column number of the exception text
+     - line - returns the line containing the exception text
+        
+    Example::
+        try:
+            Word(nums).setName("integer").parseString("ABC")
+        except ParseException as pe:
+            print(pe)
+            print("column: {}".format(pe.col))
+            
+    prints::
+       Expected integer (at char 0), (line:1, col:1)
+        column: 1
+    """
+    pass
+
+class ParseFatalException(ParseBaseException):
+    """user-throwable exception thrown when inconsistent parse content
+       is found; stops all parsing immediately"""
+    pass
+
+class ParseSyntaxException(ParseFatalException):
+    """just like L{ParseFatalException}, but thrown internally when an
+       L{ErrorStop} ('-' operator) indicates that parsing is to stop 
+       immediately because an unbacktrackable syntax error has been found"""
+    pass
+
+#~ class ReparseException(ParseBaseException):
+    #~ """Experimental class - parse actions can raise this exception to cause
+       #~ pyparsing to reparse the input string:
+        #~ - with a modified input string, and/or
+        #~ - with a modified start location
+       #~ Set the values of the ReparseException in the constructor, and raise the
+       #~ exception in a parse action to cause pyparsing to use the new string/location.
+       #~ Setting the values as None causes no change to be made.
+       #~ """
+    #~ def __init_( self, newstring, restartLoc ):
+        #~ self.newParseText = newstring
+        #~ self.reparseLoc = restartLoc
+
+class RecursiveGrammarException(Exception):
+    """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive"""
+    def __init__( self, parseElementList ):
+        self.parseElementTrace = parseElementList
+
+    def __str__( self ):
+        return "RecursiveGrammarException: %s" % self.parseElementTrace
+
+class _ParseResultsWithOffset(object):
+    def __init__(self,p1,p2):
+        self.tup = (p1,p2)
+    def __getitem__(self,i):
+        return self.tup[i]
+    def __repr__(self):
+        return repr(self.tup[0])
+    def setOffset(self,i):
+        self.tup = (self.tup[0],i)
+
+class ParseResults(object):
+    """
+    Structured parse results, to provide multiple means of access to the parsed data:
+       - as a list (C{len(results)})
+       - by list index (C{results[0], results[1]}, etc.)
+       - by attribute (C{results.} - see L{ParserElement.setResultsName})
+
+    Example::
+        integer = Word(nums)
+        date_str = (integer.setResultsName("year") + '/' 
+                        + integer.setResultsName("month") + '/' 
+                        + integer.setResultsName("day"))
+        # equivalent form:
+        # date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+
+        # parseString returns a ParseResults object
+        result = date_str.parseString("1999/12/31")
+
+        def test(s, fn=repr):
+            print("%s -> %s" % (s, fn(eval(s))))
+        test("list(result)")
+        test("result[0]")
+        test("result['month']")
+        test("result.day")
+        test("'month' in result")
+        test("'minutes' in result")
+        test("result.dump()", str)
+    prints::
+        list(result) -> ['1999', '/', '12', '/', '31']
+        result[0] -> '1999'
+        result['month'] -> '12'
+        result.day -> '31'
+        'month' in result -> True
+        'minutes' in result -> False
+        result.dump() -> ['1999', '/', '12', '/', '31']
+        - day: 31
+        - month: 12
+        - year: 1999
+    """
+    def __new__(cls, toklist=None, name=None, asList=True, modal=True ):
+        if isinstance(toklist, cls):
+            return toklist
+        retobj = object.__new__(cls)
+        retobj.__doinit = True
+        return retobj
+
+    # Performance tuning: we construct a *lot* of these, so keep this
+    # constructor as small and fast as possible
+    def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ):
+        if self.__doinit:
+            self.__doinit = False
+            self.__name = None
+            self.__parent = None
+            self.__accumNames = {}
+            self.__asList = asList
+            self.__modal = modal
+            if toklist is None:
+                toklist = []
+            if isinstance(toklist, list):
+                self.__toklist = toklist[:]
+            elif isinstance(toklist, _generatorType):
+                self.__toklist = list(toklist)
+            else:
+                self.__toklist = [toklist]
+            self.__tokdict = dict()
+
+        if name is not None and name:
+            if not modal:
+                self.__accumNames[name] = 0
+            if isinstance(name,int):
+                name = _ustr(name) # will always return a str, but use _ustr for consistency
+            self.__name = name
+            if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])):
+                if isinstance(toklist,basestring):
+                    toklist = [ toklist ]
+                if asList:
+                    if isinstance(toklist,ParseResults):
+                        self[name] = _ParseResultsWithOffset(toklist.copy(),0)
+                    else:
+                        self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0)
+                    self[name].__name = name
+                else:
+                    try:
+                        self[name] = toklist[0]
+                    except (KeyError,TypeError,IndexError):
+                        self[name] = toklist
+
+    def __getitem__( self, i ):
+        if isinstance( i, (int,slice) ):
+            return self.__toklist[i]
+        else:
+            if i not in self.__accumNames:
+                return self.__tokdict[i][-1][0]
+            else:
+                return ParseResults([ v[0] for v in self.__tokdict[i] ])
+
+    def __setitem__( self, k, v, isinstance=isinstance ):
+        if isinstance(v,_ParseResultsWithOffset):
+            self.__tokdict[k] = self.__tokdict.get(k,list()) + [v]
+            sub = v[0]
+        elif isinstance(k,(int,slice)):
+            self.__toklist[k] = v
+            sub = v
+        else:
+            self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)]
+            sub = v
+        if isinstance(sub,ParseResults):
+            sub.__parent = wkref(self)
+
+    def __delitem__( self, i ):
+        if isinstance(i,(int,slice)):
+            mylen = len( self.__toklist )
+            del self.__toklist[i]
+
+            # convert int to slice
+            if isinstance(i, int):
+                if i < 0:
+                    i += mylen
+                i = slice(i, i+1)
+            # get removed indices
+            removed = list(range(*i.indices(mylen)))
+            removed.reverse()
+            # fixup indices in token dictionary
+            for name,occurrences in self.__tokdict.items():
+                for j in removed:
+                    for k, (value, position) in enumerate(occurrences):
+                        occurrences[k] = _ParseResultsWithOffset(value, position - (position > j))
+        else:
+            del self.__tokdict[i]
+
+    def __contains__( self, k ):
+        return k in self.__tokdict
+
+    def __len__( self ): return len( self.__toklist )
+    def __bool__(self): return ( not not self.__toklist )
+    __nonzero__ = __bool__
+    def __iter__( self ): return iter( self.__toklist )
+    def __reversed__( self ): return iter( self.__toklist[::-1] )
+    def _iterkeys( self ):
+        if hasattr(self.__tokdict, "iterkeys"):
+            return self.__tokdict.iterkeys()
+        else:
+            return iter(self.__tokdict)
+
+    def _itervalues( self ):
+        return (self[k] for k in self._iterkeys())
+            
+    def _iteritems( self ):
+        return ((k, self[k]) for k in self._iterkeys())
+
+    if PY_3:
+        keys = _iterkeys       
+        """Returns an iterator of all named result keys (Python 3.x only)."""
+
+        values = _itervalues
+        """Returns an iterator of all named result values (Python 3.x only)."""
+
+        items = _iteritems
+        """Returns an iterator of all named result key-value tuples (Python 3.x only)."""
+
+    else:
+        iterkeys = _iterkeys
+        """Returns an iterator of all named result keys (Python 2.x only)."""
+
+        itervalues = _itervalues
+        """Returns an iterator of all named result values (Python 2.x only)."""
+
+        iteritems = _iteritems
+        """Returns an iterator of all named result key-value tuples (Python 2.x only)."""
+
+        def keys( self ):
+            """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.iterkeys())
+
+        def values( self ):
+            """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.itervalues())
+                
+        def items( self ):
+            """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x)."""
+            return list(self.iteritems())
+
+    def haskeys( self ):
+        """Since keys() returns an iterator, this method is helpful in bypassing
+           code that looks for the existence of any defined results names."""
+        return bool(self.__tokdict)
+        
+    def pop( self, *args, **kwargs):
+        """
+        Removes and returns item at specified index (default=C{last}).
+        Supports both C{list} and C{dict} semantics for C{pop()}. If passed no
+        argument or an integer argument, it will use C{list} semantics
+        and pop tokens from the list of parsed tokens. If passed a 
+        non-integer argument (most likely a string), it will use C{dict}
+        semantics and pop the corresponding value from any defined 
+        results names. A second default return value argument is 
+        supported, just as in C{dict.pop()}.
+
+        Example::
+            def remove_first(tokens):
+                tokens.pop(0)
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+            print(OneOrMore(Word(nums)).addParseAction(remove_first).parseString("0 123 321")) # -> ['123', '321']
+
+            label = Word(alphas)
+            patt = label("LABEL") + OneOrMore(Word(nums))
+            print(patt.parseString("AAB 123 321").dump())
+
+            # Use pop() in a parse action to remove named result (note that corresponding value is not
+            # removed from list form of results)
+            def remove_LABEL(tokens):
+                tokens.pop("LABEL")
+                return tokens
+            patt.addParseAction(remove_LABEL)
+            print(patt.parseString("AAB 123 321").dump())
+        prints::
+            ['AAB', '123', '321']
+            - LABEL: AAB
+
+            ['AAB', '123', '321']
+        """
+        if not args:
+            args = [-1]
+        for k,v in kwargs.items():
+            if k == 'default':
+                args = (args[0], v)
+            else:
+                raise TypeError("pop() got an unexpected keyword argument '%s'" % k)
+        if (isinstance(args[0], int) or 
+                        len(args) == 1 or 
+                        args[0] in self):
+            index = args[0]
+            ret = self[index]
+            del self[index]
+            return ret
+        else:
+            defaultvalue = args[1]
+            return defaultvalue
+
+    def get(self, key, defaultValue=None):
+        """
+        Returns named result matching the given key, or if there is no
+        such name, then returns the given C{defaultValue} or C{None} if no
+        C{defaultValue} is specified.
+
+        Similar to C{dict.get()}.
+        
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            result = date_str.parseString("1999/12/31")
+            print(result.get("year")) # -> '1999'
+            print(result.get("hour", "not specified")) # -> 'not specified'
+            print(result.get("hour")) # -> None
+        """
+        if key in self:
+            return self[key]
+        else:
+            return defaultValue
+
+    def insert( self, index, insStr ):
+        """
+        Inserts new element at location index in the list of parsed tokens.
+        
+        Similar to C{list.insert()}.
+
+        Example::
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+
+            # use a parse action to insert the parse location in the front of the parsed results
+            def insert_locn(locn, tokens):
+                tokens.insert(0, locn)
+            print(OneOrMore(Word(nums)).addParseAction(insert_locn).parseString("0 123 321")) # -> [0, '0', '123', '321']
+        """
+        self.__toklist.insert(index, insStr)
+        # fixup indices in token dictionary
+        for name,occurrences in self.__tokdict.items():
+            for k, (value, position) in enumerate(occurrences):
+                occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
+
+    def append( self, item ):
+        """
+        Add single element to end of ParseResults list of elements.
+
+        Example::
+            print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321']
+            
+            # use a parse action to compute the sum of the parsed integers, and add it to the end
+            def append_sum(tokens):
+                tokens.append(sum(map(int, tokens)))
+            print(OneOrMore(Word(nums)).addParseAction(append_sum).parseString("0 123 321")) # -> ['0', '123', '321', 444]
+        """
+        self.__toklist.append(item)
+
+    def extend( self, itemseq ):
+        """
+        Add sequence of elements to end of ParseResults list of elements.
+
+        Example::
+            patt = OneOrMore(Word(alphas))
+            
+            # use a parse action to append the reverse of the matched strings, to make a palindrome
+            def make_palindrome(tokens):
+                tokens.extend(reversed([t[::-1] for t in tokens]))
+                return ''.join(tokens)
+            print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl'
+        """
+        if isinstance(itemseq, ParseResults):
+            self += itemseq
+        else:
+            self.__toklist.extend(itemseq)
+
+    def clear( self ):
+        """
+        Clear all elements and results names.
+        """
+        del self.__toklist[:]
+        self.__tokdict.clear()
+
+    def __getattr__( self, name ):
+        try:
+            return self[name]
+        except KeyError:
+            return ""
+            
+        if name in self.__tokdict:
+            if name not in self.__accumNames:
+                return self.__tokdict[name][-1][0]
+            else:
+                return ParseResults([ v[0] for v in self.__tokdict[name] ])
+        else:
+            return ""
+
+    def __add__( self, other ):
+        ret = self.copy()
+        ret += other
+        return ret
+
+    def __iadd__( self, other ):
+        if other.__tokdict:
+            offset = len(self.__toklist)
+            addoffset = lambda a: offset if a<0 else a+offset
+            otheritems = other.__tokdict.items()
+            otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) )
+                                for (k,vlist) in otheritems for v in vlist]
+            for k,v in otherdictitems:
+                self[k] = v
+                if isinstance(v[0],ParseResults):
+                    v[0].__parent = wkref(self)
+            
+        self.__toklist += other.__toklist
+        self.__accumNames.update( other.__accumNames )
+        return self
+
+    def __radd__(self, other):
+        if isinstance(other,int) and other == 0:
+            # useful for merging many ParseResults using sum() builtin
+            return self.copy()
+        else:
+            # this may raise a TypeError - so be it
+            return other + self
+        
+    def __repr__( self ):
+        return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
+
+    def __str__( self ):
+        return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']'
+
+    def _asStringList( self, sep='' ):
+        out = []
+        for item in self.__toklist:
+            if out and sep:
+                out.append(sep)
+            if isinstance( item, ParseResults ):
+                out += item._asStringList()
+            else:
+                out.append( _ustr(item) )
+        return out
+
+    def asList( self ):
+        """
+        Returns the parse results as a nested list of matching tokens, all converted to strings.
+
+        Example::
+            patt = OneOrMore(Word(alphas))
+            result = patt.parseString("sldkj lsdkj sldkj")
+            # even though the result prints in string-like form, it is actually a pyparsing ParseResults
+            print(type(result), result) # ->  ['sldkj', 'lsdkj', 'sldkj']
+            
+            # Use asList() to create an actual list
+            result_list = result.asList()
+            print(type(result_list), result_list) # ->  ['sldkj', 'lsdkj', 'sldkj']
+        """
+        return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist]
+
+    def asDict( self ):
+        """
+        Returns the named parse results as a nested dictionary.
+
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+            
+            result = date_str.parseString('12/31/1999')
+            print(type(result), repr(result)) # ->  (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]})
+            
+            result_dict = result.asDict()
+            print(type(result_dict), repr(result_dict)) # ->  {'day': '1999', 'year': '12', 'month': '31'}
+
+            # even though a ParseResults supports dict-like access, sometime you just need to have a dict
+            import json
+            print(json.dumps(result)) # -> Exception: TypeError: ... is not JSON serializable
+            print(json.dumps(result.asDict())) # -> {"month": "31", "day": "1999", "year": "12"}
+        """
+        if PY_3:
+            item_fn = self.items
+        else:
+            item_fn = self.iteritems
+            
+        def toItem(obj):
+            if isinstance(obj, ParseResults):
+                if obj.haskeys():
+                    return obj.asDict()
+                else:
+                    return [toItem(v) for v in obj]
+            else:
+                return obj
+                
+        return dict((k,toItem(v)) for k,v in item_fn())
+
+    def copy( self ):
+        """
+        Returns a new copy of a C{ParseResults} object.
+        """
+        ret = ParseResults( self.__toklist )
+        ret.__tokdict = self.__tokdict.copy()
+        ret.__parent = self.__parent
+        ret.__accumNames.update( self.__accumNames )
+        ret.__name = self.__name
+        return ret
+
+    def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ):
+        """
+        (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names.
+        """
+        nl = "\n"
+        out = []
+        namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+                                                            for v in vlist)
+        nextLevelIndent = indent + "  "
+
+        # collapse out indents if formatting is not desired
+        if not formatted:
+            indent = ""
+            nextLevelIndent = ""
+            nl = ""
+
+        selfTag = None
+        if doctag is not None:
+            selfTag = doctag
+        else:
+            if self.__name:
+                selfTag = self.__name
+
+        if not selfTag:
+            if namedItemsOnly:
+                return ""
+            else:
+                selfTag = "ITEM"
+
+        out += [ nl, indent, "<", selfTag, ">" ]
+
+        for i,res in enumerate(self.__toklist):
+            if isinstance(res,ParseResults):
+                if i in namedItems:
+                    out += [ res.asXML(namedItems[i],
+                                        namedItemsOnly and doctag is None,
+                                        nextLevelIndent,
+                                        formatted)]
+                else:
+                    out += [ res.asXML(None,
+                                        namedItemsOnly and doctag is None,
+                                        nextLevelIndent,
+                                        formatted)]
+            else:
+                # individual token, see if there is a name for it
+                resTag = None
+                if i in namedItems:
+                    resTag = namedItems[i]
+                if not resTag:
+                    if namedItemsOnly:
+                        continue
+                    else:
+                        resTag = "ITEM"
+                xmlBodyText = _xml_escape(_ustr(res))
+                out += [ nl, nextLevelIndent, "<", resTag, ">",
+                                                xmlBodyText,
+                                                "" ]
+
+        out += [ nl, indent, "" ]
+        return "".join(out)
+
+    def __lookup(self,sub):
+        for k,vlist in self.__tokdict.items():
+            for v,loc in vlist:
+                if sub is v:
+                    return k
+        return None
+
+    def getName(self):
+        r"""
+        Returns the results name for this token expression. Useful when several 
+        different expressions might match at a particular location.
+
+        Example::
+            integer = Word(nums)
+            ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d")
+            house_number_expr = Suppress('#') + Word(nums, alphanums)
+            user_data = (Group(house_number_expr)("house_number") 
+                        | Group(ssn_expr)("ssn")
+                        | Group(integer)("age"))
+            user_info = OneOrMore(user_data)
+            
+            result = user_info.parseString("22 111-22-3333 #221B")
+            for item in result:
+                print(item.getName(), ':', item[0])
+        prints::
+            age : 22
+            ssn : 111-22-3333
+            house_number : 221B
+        """
+        if self.__name:
+            return self.__name
+        elif self.__parent:
+            par = self.__parent()
+            if par:
+                return par.__lookup(self)
+            else:
+                return None
+        elif (len(self) == 1 and
+               len(self.__tokdict) == 1 and
+               next(iter(self.__tokdict.values()))[0][1] in (0,-1)):
+            return next(iter(self.__tokdict.keys()))
+        else:
+            return None
+
+    def dump(self, indent='', depth=0, full=True):
+        """
+        Diagnostic method for listing out the contents of a C{ParseResults}.
+        Accepts an optional C{indent} argument so that this string can be embedded
+        in a nested display of other data.
+
+        Example::
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+            
+            result = date_str.parseString('12/31/1999')
+            print(result.dump())
+        prints::
+            ['12', '/', '31', '/', '1999']
+            - day: 1999
+            - month: 31
+            - year: 12
+        """
+        out = []
+        NL = '\n'
+        out.append( indent+_ustr(self.asList()) )
+        if full:
+            if self.haskeys():
+                items = sorted((str(k), v) for k,v in self.items())
+                for k,v in items:
+                    if out:
+                        out.append(NL)
+                    out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
+                    if isinstance(v,ParseResults):
+                        if v:
+                            out.append( v.dump(indent,depth+1) )
+                        else:
+                            out.append(_ustr(v))
+                    else:
+                        out.append(repr(v))
+            elif any(isinstance(vv,ParseResults) for vv in self):
+                v = self
+                for i,vv in enumerate(v):
+                    if isinstance(vv,ParseResults):
+                        out.append("\n%s%s[%d]:\n%s%s%s" % (indent,('  '*(depth)),i,indent,('  '*(depth+1)),vv.dump(indent,depth+1) ))
+                    else:
+                        out.append("\n%s%s[%d]:\n%s%s%s" % (indent,('  '*(depth)),i,indent,('  '*(depth+1)),_ustr(vv)))
+            
+        return "".join(out)
+
+    def pprint(self, *args, **kwargs):
+        """
+        Pretty-printer for parsed results as a list, using the C{pprint} module.
+        Accepts additional positional or keyword args as defined for the 
+        C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})
+
+        Example::
+            ident = Word(alphas, alphanums)
+            num = Word(nums)
+            func = Forward()
+            term = ident | num | Group('(' + func + ')')
+            func <<= ident + Group(Optional(delimitedList(term)))
+            result = func.parseString("fna a,b,(fnb c,d,200),100")
+            result.pprint(width=40)
+        prints::
+            ['fna',
+             ['a',
+              'b',
+              ['(', 'fnb', ['c', 'd', '200'], ')'],
+              '100']]
+        """
+        pprint.pprint(self.asList(), *args, **kwargs)
+
+    # add support for pickle protocol
+    def __getstate__(self):
+        return ( self.__toklist,
+                 ( self.__tokdict.copy(),
+                   self.__parent is not None and self.__parent() or None,
+                   self.__accumNames,
+                   self.__name ) )
+
+    def __setstate__(self,state):
+        self.__toklist = state[0]
+        (self.__tokdict,
+         par,
+         inAccumNames,
+         self.__name) = state[1]
+        self.__accumNames = {}
+        self.__accumNames.update(inAccumNames)
+        if par is not None:
+            self.__parent = wkref(par)
+        else:
+            self.__parent = None
+
+    def __getnewargs__(self):
+        return self.__toklist, self.__name, self.__asList, self.__modal
+
+    def __dir__(self):
+        return (dir(type(self)) + list(self.keys()))
+
+MutableMapping.register(ParseResults)
+
+def col (loc,strg):
+    """Returns current column within a string, counting newlines as line separators.
+   The first column is number 1.
+
+   Note: the default parsing behavior is to expand tabs in the input string
+   before starting the parsing process.  See L{I{ParserElement.parseString}} for more information
+   on parsing strings containing C{}s, and suggested methods to maintain a
+   consistent view of the parsed string, the parse location, and line and column
+   positions within the parsed string.
+   """
+    s = strg
+    return 1 if 0} for more information
+   on parsing strings containing C{}s, and suggested methods to maintain a
+   consistent view of the parsed string, the parse location, and line and column
+   positions within the parsed string.
+   """
+    return strg.count("\n",0,loc) + 1
+
+def line( loc, strg ):
+    """Returns the line of text containing loc within a string, counting newlines as line separators.
+       """
+    lastCR = strg.rfind("\n", 0, loc)
+    nextCR = strg.find("\n", loc)
+    if nextCR >= 0:
+        return strg[lastCR+1:nextCR]
+    else:
+        return strg[lastCR+1:]
+
+def _defaultStartDebugAction( instring, loc, expr ):
+    print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
+
+def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
+    print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
+
+def _defaultExceptionDebugAction( instring, loc, expr, exc ):
+    print ("Exception raised:" + _ustr(exc))
+
+def nullDebugAction(*args):
+    """'Do-nothing' debug action, to suppress debugging output during parsing."""
+    pass
+
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+    #~ if func in singleArgBuiltins:
+        #~ return lambda s,l,t: func(t)
+    #~ limit = 0
+    #~ foundArity = False
+    #~ def wrapper(*args):
+        #~ nonlocal limit,foundArity
+        #~ while 1:
+            #~ try:
+                #~ ret = func(*args[limit:])
+                #~ foundArity = True
+                #~ return ret
+            #~ except TypeError:
+                #~ if limit == maxargs or foundArity:
+                    #~ raise
+                #~ limit += 1
+                #~ continue
+    #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
+'decorator to trim function calls to match the arity of the target'
+def _trim_arity(func, maxargs=2):
+    if func in singleArgBuiltins:
+        return lambda s,l,t: func(t)
+    limit = [0]
+    foundArity = [False]
+    
+    # traceback return data structure changed in Py3.5 - normalize back to plain tuples
+    if system_version[:2] >= (3,5):
+        def extract_stack(limit=0):
+            # special handling for Python 3.5.0 - extra deep call stack by 1
+            offset = -3 if system_version == (3,5,0) else -2
+            frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
+            return [frame_summary[:2]]
+        def extract_tb(tb, limit=0):
+            frames = traceback.extract_tb(tb, limit=limit)
+            frame_summary = frames[-1]
+            return [frame_summary[:2]]
+    else:
+        extract_stack = traceback.extract_stack
+        extract_tb = traceback.extract_tb
+    
+    # synthesize what would be returned by traceback.extract_stack at the call to 
+    # user's parse action 'func', so that we don't incur call penalty at parse time
+    
+    LINE_DIFF = 6
+    # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND 
+    # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
+    this_line = extract_stack(limit=2)[-1]
+    pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF)
+
+    def wrapper(*args):
+        while 1:
+            try:
+                ret = func(*args[limit[0]:])
+                foundArity[0] = True
+                return ret
+            except TypeError:
+                # re-raise TypeErrors if they did not come from our arity testing
+                if foundArity[0]:
+                    raise
+                else:
+                    try:
+                        tb = sys.exc_info()[-1]
+                        if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth:
+                            raise
+                    finally:
+                        del tb
+
+                if limit[0] <= maxargs:
+                    limit[0] += 1
+                    continue
+                raise
+
+    # copy func name to wrapper for sensible debug output
+    func_name = ""
+    try:
+        func_name = getattr(func, '__name__', 
+                            getattr(func, '__class__').__name__)
+    except Exception:
+        func_name = str(func)
+    wrapper.__name__ = func_name
+
+    return wrapper
+
+class ParserElement(object):
+    """Abstract base level parser element class."""
+    DEFAULT_WHITE_CHARS = " \n\t\r"
+    verbose_stacktrace = False
+
+    @staticmethod
+    def setDefaultWhitespaceChars( chars ):
+        r"""
+        Overrides the default whitespace chars
+
+        Example::
+            # default whitespace chars are space,  and newline
+            OneOrMore(Word(alphas)).parseString("abc def\nghi jkl")  # -> ['abc', 'def', 'ghi', 'jkl']
+            
+            # change to just treat newline as significant
+            ParserElement.setDefaultWhitespaceChars(" \t")
+            OneOrMore(Word(alphas)).parseString("abc def\nghi jkl")  # -> ['abc', 'def']
+        """
+        ParserElement.DEFAULT_WHITE_CHARS = chars
+
+    @staticmethod
+    def inlineLiteralsUsing(cls):
+        """
+        Set class to be used for inclusion of string literals into a parser.
+        
+        Example::
+            # default literal class used is Literal
+            integer = Word(nums)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '/', '12', '/', '31']
+
+
+            # change to Suppress
+            ParserElement.inlineLiteralsUsing(Suppress)
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")           
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '12', '31']
+        """
+        ParserElement._literalStringClass = cls
+
+    def __init__( self, savelist=False ):
+        self.parseAction = list()
+        self.failAction = None
+        #~ self.name = ""  # don't define self.name, let subclasses try/except upcall
+        self.strRepr = None
+        self.resultsName = None
+        self.saveAsList = savelist
+        self.skipWhitespace = True
+        self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+        self.copyDefaultWhiteChars = True
+        self.mayReturnEmpty = False # used when checking for left-recursion
+        self.keepTabs = False
+        self.ignoreExprs = list()
+        self.debug = False
+        self.streamlined = False
+        self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index
+        self.errmsg = ""
+        self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all)
+        self.debugActions = ( None, None, None ) #custom debug actions
+        self.re = None
+        self.callPreparse = True # used to avoid redundant calls to preParse
+        self.callDuringTry = False
+
+    def copy( self ):
+        """
+        Make a copy of this C{ParserElement}.  Useful for defining different parse actions
+        for the same parsing pattern, using copies of the original parse element.
+        
+        Example::
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K")
+            integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+            
+            print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M"))
+        prints::
+            [5120, 100, 655360, 268435456]
+        Equivalent form of C{expr.copy()} is just C{expr()}::
+            integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M")
+        """
+        cpy = copy.copy( self )
+        cpy.parseAction = self.parseAction[:]
+        cpy.ignoreExprs = self.ignoreExprs[:]
+        if self.copyDefaultWhiteChars:
+            cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS
+        return cpy
+
+    def setName( self, name ):
+        """
+        Define name for this expression, makes debugging and exception messages clearer.
+        
+        Example::
+            Word(nums).parseString("ABC")  # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1)
+            Word(nums).setName("integer").parseString("ABC")  # -> Exception: Expected integer (at char 0), (line:1, col:1)
+        """
+        self.name = name
+        self.errmsg = "Expected " + self.name
+        if hasattr(self,"exception"):
+            self.exception.msg = self.errmsg
+        return self
+
+    def setResultsName( self, name, listAllMatches=False ):
+        """
+        Define name for referencing matching tokens as a nested attribute
+        of the returned parse results.
+        NOTE: this returns a *copy* of the original C{ParserElement} object;
+        this is so that the client can define a basic element, such as an
+        integer, and reference it in multiple places with different names.
+
+        You can also set results names using the abbreviated syntax,
+        C{expr("name")} in place of C{expr.setResultsName("name")} - 
+        see L{I{__call__}<__call__>}.
+
+        Example::
+            date_str = (integer.setResultsName("year") + '/' 
+                        + integer.setResultsName("month") + '/' 
+                        + integer.setResultsName("day"))
+
+            # equivalent form:
+            date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
+        """
+        newself = self.copy()
+        if name.endswith("*"):
+            name = name[:-1]
+            listAllMatches=True
+        newself.resultsName = name
+        newself.modalResults = not listAllMatches
+        return newself
+
+    def setBreak(self,breakFlag = True):
+        """Method to invoke the Python pdb debugger when this element is
+           about to be parsed. Set C{breakFlag} to True to enable, False to
+           disable.
+        """
+        if breakFlag:
+            _parseMethod = self._parse
+            def breaker(instring, loc, doActions=True, callPreParse=True):
+                import pdb
+                pdb.set_trace()
+                return _parseMethod( instring, loc, doActions, callPreParse )
+            breaker._originalParseMethod = _parseMethod
+            self._parse = breaker
+        else:
+            if hasattr(self._parse,"_originalParseMethod"):
+                self._parse = self._parse._originalParseMethod
+        return self
+
+    def setParseAction( self, *fns, **kwargs ):
+        """
+        Define one or more actions to perform when successfully matching parse element definition.
+        Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)},
+        C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
+         - s   = the original string being parsed (see note below)
+         - loc = the location of the matching substring
+         - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
+        If the functions in fns modify the tokens, they can return them as the return
+        value from fn, and the modified list of tokens will replace the original.
+        Otherwise, fn does not need to return any value.
+
+        Optional keyword arguments:
+         - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing
+
+        Note: the default parsing behavior is to expand tabs in the input string
+        before starting the parsing process.  See L{I{parseString}} for more information
+        on parsing strings containing C{}s, and suggested methods to maintain a
+        consistent view of the parsed string, the parse location, and line and column
+        positions within the parsed string.
+        
+        Example::
+            integer = Word(nums)
+            date_str = integer + '/' + integer + '/' + integer
+
+            date_str.parseString("1999/12/31")  # -> ['1999', '/', '12', '/', '31']
+
+            # use parse action to convert to ints at parse time
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            date_str = integer + '/' + integer + '/' + integer
+
+            # note that integer fields are now ints, not strings
+            date_str.parseString("1999/12/31")  # -> [1999, '/', 12, '/', 31]
+        """
+        self.parseAction = list(map(_trim_arity, list(fns)))
+        self.callDuringTry = kwargs.get("callDuringTry", False)
+        return self
+
+    def addParseAction( self, *fns, **kwargs ):
+        """
+        Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}.
+        
+        See examples in L{I{copy}}.
+        """
+        self.parseAction += list(map(_trim_arity, list(fns)))
+        self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+        return self
+
+    def addCondition(self, *fns, **kwargs):
+        """Add a boolean predicate function to expression's list of parse actions. See 
+        L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, 
+        functions passed to C{addCondition} need to return boolean success/fail of the condition.
+
+        Optional keyword arguments:
+         - message = define a custom message to be used in the raised exception
+         - fatal   = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException
+         
+        Example::
+            integer = Word(nums).setParseAction(lambda toks: int(toks[0]))
+            year_int = integer.copy()
+            year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later")
+            date_str = year_int + '/' + integer + '/' + integer
+
+            result = date_str.parseString("1999/12/31")  # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1)
+        """
+        msg = kwargs.get("message", "failed user-defined condition")
+        exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException
+        for fn in fns:
+            def pa(s,l,t):
+                if not bool(_trim_arity(fn)(s,l,t)):
+                    raise exc_type(s,l,msg)
+            self.parseAction.append(pa)
+        self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False)
+        return self
+
+    def setFailAction( self, fn ):
+        """Define action to perform if parsing fails at this expression.
+           Fail acton fn is a callable function that takes the arguments
+           C{fn(s,loc,expr,err)} where:
+            - s = string being parsed
+            - loc = location where expression match was attempted and failed
+            - expr = the parse expression that failed
+            - err = the exception thrown
+           The function returns no value.  It may throw C{L{ParseFatalException}}
+           if it is desired to stop parsing immediately."""
+        self.failAction = fn
+        return self
+
+    def _skipIgnorables( self, instring, loc ):
+        exprsFound = True
+        while exprsFound:
+            exprsFound = False
+            for e in self.ignoreExprs:
+                try:
+                    while 1:
+                        loc,dummy = e._parse( instring, loc )
+                        exprsFound = True
+                except ParseException:
+                    pass
+        return loc
+
+    def preParse( self, instring, loc ):
+        if self.ignoreExprs:
+            loc = self._skipIgnorables( instring, loc )
+
+        if self.skipWhitespace:
+            wt = self.whiteChars
+            instrlen = len(instring)
+            while loc < instrlen and instring[loc] in wt:
+                loc += 1
+
+        return loc
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        return loc, []
+
+    def postParse( self, instring, loc, tokenlist ):
+        return tokenlist
+
+    #~ @profile
+    def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ):
+        debugging = ( self.debug ) #and doActions )
+
+        if debugging or self.failAction:
+            #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+            if (self.debugActions[0] ):
+                self.debugActions[0]( instring, loc, self )
+            if callPreParse and self.callPreparse:
+                preloc = self.preParse( instring, loc )
+            else:
+                preloc = loc
+            tokensStart = preloc
+            try:
+                try:
+                    loc,tokens = self.parseImpl( instring, preloc, doActions )
+                except IndexError:
+                    raise ParseException( instring, len(instring), self.errmsg, self )
+            except ParseBaseException as err:
+                #~ print ("Exception raised:", err)
+                if self.debugActions[2]:
+                    self.debugActions[2]( instring, tokensStart, self, err )
+                if self.failAction:
+                    self.failAction( instring, tokensStart, self, err )
+                raise
+        else:
+            if callPreParse and self.callPreparse:
+                preloc = self.preParse( instring, loc )
+            else:
+                preloc = loc
+            tokensStart = preloc
+            if self.mayIndexError or preloc >= len(instring):
+                try:
+                    loc,tokens = self.parseImpl( instring, preloc, doActions )
+                except IndexError:
+                    raise ParseException( instring, len(instring), self.errmsg, self )
+            else:
+                loc,tokens = self.parseImpl( instring, preloc, doActions )
+
+        tokens = self.postParse( instring, loc, tokens )
+
+        retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults )
+        if self.parseAction and (doActions or self.callDuringTry):
+            if debugging:
+                try:
+                    for fn in self.parseAction:
+                        tokens = fn( instring, tokensStart, retTokens )
+                        if tokens is not None:
+                            retTokens = ParseResults( tokens,
+                                                      self.resultsName,
+                                                      asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+                                                      modal=self.modalResults )
+                except ParseBaseException as err:
+                    #~ print "Exception raised in user parse action:", err
+                    if (self.debugActions[2] ):
+                        self.debugActions[2]( instring, tokensStart, self, err )
+                    raise
+            else:
+                for fn in self.parseAction:
+                    tokens = fn( instring, tokensStart, retTokens )
+                    if tokens is not None:
+                        retTokens = ParseResults( tokens,
+                                                  self.resultsName,
+                                                  asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
+                                                  modal=self.modalResults )
+        if debugging:
+            #~ print ("Matched",self,"->",retTokens.asList())
+            if (self.debugActions[1] ):
+                self.debugActions[1]( instring, tokensStart, loc, self, retTokens )
+
+        return loc, retTokens
+
+    def tryParse( self, instring, loc ):
+        try:
+            return self._parse( instring, loc, doActions=False )[0]
+        except ParseFatalException:
+            raise ParseException( instring, loc, self.errmsg, self)
+    
+    def canParseNext(self, instring, loc):
+        try:
+            self.tryParse(instring, loc)
+        except (ParseException, IndexError):
+            return False
+        else:
+            return True
+
+    class _UnboundedCache(object):
+        def __init__(self):
+            cache = {}
+            self.not_in_cache = not_in_cache = object()
+
+            def get(self, key):
+                return cache.get(key, not_in_cache)
+
+            def set(self, key, value):
+                cache[key] = value
+
+            def clear(self):
+                cache.clear()
+                
+            def cache_len(self):
+                return len(cache)
+
+            self.get = types.MethodType(get, self)
+            self.set = types.MethodType(set, self)
+            self.clear = types.MethodType(clear, self)
+            self.__len__ = types.MethodType(cache_len, self)
+
+    if _OrderedDict is not None:
+        class _FifoCache(object):
+            def __init__(self, size):
+                self.not_in_cache = not_in_cache = object()
+
+                cache = _OrderedDict()
+
+                def get(self, key):
+                    return cache.get(key, not_in_cache)
+
+                def set(self, key, value):
+                    cache[key] = value
+                    while len(cache) > size:
+                        try:
+                            cache.popitem(False)
+                        except KeyError:
+                            pass
+
+                def clear(self):
+                    cache.clear()
+
+                def cache_len(self):
+                    return len(cache)
+
+                self.get = types.MethodType(get, self)
+                self.set = types.MethodType(set, self)
+                self.clear = types.MethodType(clear, self)
+                self.__len__ = types.MethodType(cache_len, self)
+
+    else:
+        class _FifoCache(object):
+            def __init__(self, size):
+                self.not_in_cache = not_in_cache = object()
+
+                cache = {}
+                key_fifo = collections.deque([], size)
+
+                def get(self, key):
+                    return cache.get(key, not_in_cache)
+
+                def set(self, key, value):
+                    cache[key] = value
+                    while len(key_fifo) > size:
+                        cache.pop(key_fifo.popleft(), None)
+                    key_fifo.append(key)
+
+                def clear(self):
+                    cache.clear()
+                    key_fifo.clear()
+
+                def cache_len(self):
+                    return len(cache)
+
+                self.get = types.MethodType(get, self)
+                self.set = types.MethodType(set, self)
+                self.clear = types.MethodType(clear, self)
+                self.__len__ = types.MethodType(cache_len, self)
+
+    # argument cache for optimizing repeated calls when backtracking through recursive expressions
+    packrat_cache = {} # this is set later by enabledPackrat(); this is here so that resetCache() doesn't fail
+    packrat_cache_lock = RLock()
+    packrat_cache_stats = [0, 0]
+
+    # this method gets repeatedly called during backtracking with the same arguments -
+    # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
+    def _parseCache( self, instring, loc, doActions=True, callPreParse=True ):
+        HIT, MISS = 0, 1
+        lookup = (self, instring, loc, callPreParse, doActions)
+        with ParserElement.packrat_cache_lock:
+            cache = ParserElement.packrat_cache
+            value = cache.get(lookup)
+            if value is cache.not_in_cache:
+                ParserElement.packrat_cache_stats[MISS] += 1
+                try:
+                    value = self._parseNoCache(instring, loc, doActions, callPreParse)
+                except ParseBaseException as pe:
+                    # cache a copy of the exception, without the traceback
+                    cache.set(lookup, pe.__class__(*pe.args))
+                    raise
+                else:
+                    cache.set(lookup, (value[0], value[1].copy()))
+                    return value
+            else:
+                ParserElement.packrat_cache_stats[HIT] += 1
+                if isinstance(value, Exception):
+                    raise value
+                return (value[0], value[1].copy())
+
+    _parse = _parseNoCache
+
+    @staticmethod
+    def resetCache():
+        ParserElement.packrat_cache.clear()
+        ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats)
+
+    _packratEnabled = False
+    @staticmethod
+    def enablePackrat(cache_size_limit=128):
+        """Enables "packrat" parsing, which adds memoizing to the parsing logic.
+           Repeated parse attempts at the same string location (which happens
+           often in many complex grammars) can immediately return a cached value,
+           instead of re-executing parsing/validating code.  Memoizing is done of
+           both valid results and parsing exceptions.
+           
+           Parameters:
+            - cache_size_limit - (default=C{128}) - if an integer value is provided
+              will limit the size of the packrat cache; if None is passed, then
+              the cache size will be unbounded; if 0 is passed, the cache will
+              be effectively disabled.
+            
+           This speedup may break existing programs that use parse actions that
+           have side-effects.  For this reason, packrat parsing is disabled when
+           you first import pyparsing.  To activate the packrat feature, your
+           program must call the class method C{ParserElement.enablePackrat()}.  If
+           your program uses C{psyco} to "compile as you go", you must call
+           C{enablePackrat} before calling C{psyco.full()}.  If you do not do this,
+           Python will crash.  For best results, call C{enablePackrat()} immediately
+           after importing pyparsing.
+           
+           Example::
+               import pyparsing
+               pyparsing.ParserElement.enablePackrat()
+        """
+        if not ParserElement._packratEnabled:
+            ParserElement._packratEnabled = True
+            if cache_size_limit is None:
+                ParserElement.packrat_cache = ParserElement._UnboundedCache()
+            else:
+                ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit)
+            ParserElement._parse = ParserElement._parseCache
+
+    def parseString( self, instring, parseAll=False ):
+        """
+        Execute the parse expression with the given string.
+        This is the main interface to the client code, once the complete
+        expression has been built.
+
+        If you want the grammar to require that the entire input string be
+        successfully parsed, then set C{parseAll} to True (equivalent to ending
+        the grammar with C{L{StringEnd()}}).
+
+        Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
+        in order to report proper column numbers in parse actions.
+        If the input string contains tabs and
+        the grammar uses parse actions that use the C{loc} argument to index into the
+        string being parsed, you can ensure you have a consistent view of the input
+        string by:
+         - calling C{parseWithTabs} on your grammar before calling C{parseString}
+           (see L{I{parseWithTabs}})
+         - define your parse action using the full C{(s,loc,toks)} signature, and
+           reference the input string using the parse action's C{s} argument
+         - explictly expand the tabs in your input string before calling
+           C{parseString}
+        
+        Example::
+            Word('a').parseString('aaaaabaaa')  # -> ['aaaaa']
+            Word('a').parseString('aaaaabaaa', parseAll=True)  # -> Exception: Expected end of text
+        """
+        ParserElement.resetCache()
+        if not self.streamlined:
+            self.streamline()
+            #~ self.saveAsList = True
+        for e in self.ignoreExprs:
+            e.streamline()
+        if not self.keepTabs:
+            instring = instring.expandtabs()
+        try:
+            loc, tokens = self._parse( instring, 0 )
+            if parseAll:
+                loc = self.preParse( instring, loc )
+                se = Empty() + StringEnd()
+                se._parse( instring, loc )
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+        else:
+            return tokens
+
+    def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ):
+        """
+        Scan the input string for expression matches.  Each match will return the
+        matching tokens, start location, and end location.  May be called with optional
+        C{maxMatches} argument, to clip scanning after 'n' matches are found.  If
+        C{overlap} is specified, then overlapping matches will be reported.
+
+        Note that the start and end locations are reported relative to the string
+        being parsed.  See L{I{parseString}} for more information on parsing
+        strings with embedded tabs.
+
+        Example::
+            source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
+            print(source)
+            for tokens,start,end in Word(alphas).scanString(source):
+                print(' '*start + '^'*(end-start))
+                print(' '*start + tokens[0])
+        
+        prints::
+        
+            sldjf123lsdjjkf345sldkjf879lkjsfd987
+            ^^^^^
+            sldjf
+                    ^^^^^^^
+                    lsdjjkf
+                              ^^^^^^
+                              sldkjf
+                                       ^^^^^^
+                                       lkjsfd
+        """
+        if not self.streamlined:
+            self.streamline()
+        for e in self.ignoreExprs:
+            e.streamline()
+
+        if not self.keepTabs:
+            instring = _ustr(instring).expandtabs()
+        instrlen = len(instring)
+        loc = 0
+        preparseFn = self.preParse
+        parseFn = self._parse
+        ParserElement.resetCache()
+        matches = 0
+        try:
+            while loc <= instrlen and matches < maxMatches:
+                try:
+                    preloc = preparseFn( instring, loc )
+                    nextLoc,tokens = parseFn( instring, preloc, callPreParse=False )
+                except ParseException:
+                    loc = preloc+1
+                else:
+                    if nextLoc > loc:
+                        matches += 1
+                        yield tokens, preloc, nextLoc
+                        if overlap:
+                            nextloc = preparseFn( instring, loc )
+                            if nextloc > loc:
+                                loc = nextLoc
+                            else:
+                                loc += 1
+                        else:
+                            loc = nextLoc
+                    else:
+                        loc = preloc+1
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def transformString( self, instring ):
+        """
+        Extension to C{L{scanString}}, to modify matching text with modified tokens that may
+        be returned from a parse action.  To use C{transformString}, define a grammar and
+        attach a parse action to it that modifies the returned token list.
+        Invoking C{transformString()} on a target string will then scan for matches,
+        and replace the matched text patterns according to the logic in the parse
+        action.  C{transformString()} returns the resulting transformed string.
+        
+        Example::
+            wd = Word(alphas)
+            wd.setParseAction(lambda toks: toks[0].title())
+            
+            print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york."))
+        Prints::
+            Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York.
+        """
+        out = []
+        lastE = 0
+        # force preservation of s, to minimize unwanted transformation of string, and to
+        # keep string locs straight between transformString and scanString
+        self.keepTabs = True
+        try:
+            for t,s,e in self.scanString( instring ):
+                out.append( instring[lastE:s] )
+                if t:
+                    if isinstance(t,ParseResults):
+                        out += t.asList()
+                    elif isinstance(t,list):
+                        out += t
+                    else:
+                        out.append(t)
+                lastE = e
+            out.append(instring[lastE:])
+            out = [o for o in out if o]
+            return "".join(map(_ustr,_flatten(out)))
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def searchString( self, instring, maxMatches=_MAX_INT ):
+        """
+        Another extension to C{L{scanString}}, simplifying the access to the tokens found
+        to match the given parse expression.  May be called with optional
+        C{maxMatches} argument, to clip searching after 'n' matches are found.
+        
+        Example::
+            # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters
+            cap_word = Word(alphas.upper(), alphas.lower())
+            
+            print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))
+
+            # the sum() builtin can be used to merge results into a single ParseResults object
+            print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")))
+        prints::
+            [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
+            ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
+        """
+        try:
+            return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False):
+        """
+        Generator method to split a string using the given expression as a separator.
+        May be called with optional C{maxsplit} argument, to limit the number of splits;
+        and the optional C{includeSeparators} argument (default=C{False}), if the separating
+        matching text should be included in the split results.
+        
+        Example::        
+            punc = oneOf(list(".,;:/-!?"))
+            print(list(punc.split("This, this?, this sentence, is badly punctuated!")))
+        prints::
+            ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
+        """
+        splits = 0
+        last = 0
+        for t,s,e in self.scanString(instring, maxMatches=maxsplit):
+            yield instring[last:s]
+            if includeSeparators:
+                yield t[0]
+            last = e
+        yield instring[last:]
+
+    def __add__(self, other ):
+        """
+        Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement
+        converts them to L{Literal}s by default.
+        
+        Example::
+            greet = Word(alphas) + "," + Word(alphas) + "!"
+            hello = "Hello, World!"
+            print (hello, "->", greet.parseString(hello))
+        Prints::
+            Hello, World! -> ['Hello', ',', 'World', '!']
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return And( [ self, other ] )
+
+    def __radd__(self, other ):
+        """
+        Implementation of + operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other + self
+
+    def __sub__(self, other):
+        """
+        Implementation of - operator, returns C{L{And}} with error stop
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return self + And._ErrorStop() + other
+
+    def __rsub__(self, other ):
+        """
+        Implementation of - operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other - self
+
+    def __mul__(self,other):
+        """
+        Implementation of * operator, allows use of C{expr * 3} in place of
+        C{expr + expr + expr}.  Expressions may also me multiplied by a 2-integer
+        tuple, similar to C{{min,max}} multipliers in regular expressions.  Tuples
+        may also include C{None} as in:
+         - C{expr*(n,None)} or C{expr*(n,)} is equivalent
+              to C{expr*n + L{ZeroOrMore}(expr)}
+              (read as "at least n instances of C{expr}")
+         - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
+              (read as "0 to n instances of C{expr}")
+         - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+         - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
+
+        Note that C{expr*(None,n)} does not raise an exception if
+        more than n exprs exist in the input stream; that is,
+        C{expr*(None,n)} does not enforce a maximum number of expr
+        occurrences.  If this behavior is desired, then write
+        C{expr*(None,n) + ~expr}
+        """
+        if isinstance(other,int):
+            minElements, optElements = other,0
+        elif isinstance(other,tuple):
+            other = (other + (None, None))[:2]
+            if other[0] is None:
+                other = (0, other[1])
+            if isinstance(other[0],int) and other[1] is None:
+                if other[0] == 0:
+                    return ZeroOrMore(self)
+                if other[0] == 1:
+                    return OneOrMore(self)
+                else:
+                    return self*other[0] + ZeroOrMore(self)
+            elif isinstance(other[0],int) and isinstance(other[1],int):
+                minElements, optElements = other
+                optElements -= minElements
+            else:
+                raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1]))
+        else:
+            raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other))
+
+        if minElements < 0:
+            raise ValueError("cannot multiply ParserElement by negative value")
+        if optElements < 0:
+            raise ValueError("second tuple value must be greater or equal to first tuple value")
+        if minElements == optElements == 0:
+            raise ValueError("cannot multiply ParserElement by 0 or (0,0)")
+
+        if (optElements):
+            def makeOptionalList(n):
+                if n>1:
+                    return Optional(self + makeOptionalList(n-1))
+                else:
+                    return Optional(self)
+            if minElements:
+                if minElements == 1:
+                    ret = self + makeOptionalList(optElements)
+                else:
+                    ret = And([self]*minElements) + makeOptionalList(optElements)
+            else:
+                ret = makeOptionalList(optElements)
+        else:
+            if minElements == 1:
+                ret = self
+            else:
+                ret = And([self]*minElements)
+        return ret
+
+    def __rmul__(self, other):
+        return self.__mul__(other)
+
+    def __or__(self, other ):
+        """
+        Implementation of | operator - returns C{L{MatchFirst}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return MatchFirst( [ self, other ] )
+
+    def __ror__(self, other ):
+        """
+        Implementation of | operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other | self
+
+    def __xor__(self, other ):
+        """
+        Implementation of ^ operator - returns C{L{Or}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return Or( [ self, other ] )
+
+    def __rxor__(self, other ):
+        """
+        Implementation of ^ operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other ^ self
+
+    def __and__(self, other ):
+        """
+        Implementation of & operator - returns C{L{Each}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return Each( [ self, other ] )
+
+    def __rand__(self, other ):
+        """
+        Implementation of & operator when left operand is not a C{L{ParserElement}}
+        """
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        if not isinstance( other, ParserElement ):
+            warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
+                    SyntaxWarning, stacklevel=2)
+            return None
+        return other & self
+
+    def __invert__( self ):
+        """
+        Implementation of ~ operator - returns C{L{NotAny}}
+        """
+        return NotAny( self )
+
+    def __call__(self, name=None):
+        """
+        Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}.
+        
+        If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
+        passed as C{True}.
+           
+        If C{name} is omitted, same as calling C{L{copy}}.
+
+        Example::
+            # these are equivalent
+            userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
+            userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")             
+        """
+        if name is not None:
+            return self.setResultsName(name)
+        else:
+            return self.copy()
+
+    def suppress( self ):
+        """
+        Suppresses the output of this C{ParserElement}; useful to keep punctuation from
+        cluttering up returned output.
+        """
+        return Suppress( self )
+
+    def leaveWhitespace( self ):
+        """
+        Disables the skipping of whitespace before matching the characters in the
+        C{ParserElement}'s defined pattern.  This is normally only used internally by
+        the pyparsing module, but may be needed in some whitespace-sensitive grammars.
+        """
+        self.skipWhitespace = False
+        return self
+
+    def setWhitespaceChars( self, chars ):
+        """
+        Overrides the default whitespace chars
+        """
+        self.skipWhitespace = True
+        self.whiteChars = chars
+        self.copyDefaultWhiteChars = False
+        return self
+
+    def parseWithTabs( self ):
+        """
+        Overrides default behavior to expand C{}s to spaces before parsing the input string.
+        Must be called before C{parseString} when the input grammar contains elements that
+        match C{} characters.
+        """
+        self.keepTabs = True
+        return self
+
+    def ignore( self, other ):
+        """
+        Define expression to be ignored (e.g., comments) while doing pattern
+        matching; may be called repeatedly, to define multiple comment or other
+        ignorable patterns.
+        
+        Example::
+            patt = OneOrMore(Word(alphas))
+            patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj']
+            
+            patt.ignore(cStyleComment)
+            patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd']
+        """
+        if isinstance(other, basestring):
+            other = Suppress(other)
+
+        if isinstance( other, Suppress ):
+            if other not in self.ignoreExprs:
+                self.ignoreExprs.append(other)
+        else:
+            self.ignoreExprs.append( Suppress( other.copy() ) )
+        return self
+
+    def setDebugActions( self, startAction, successAction, exceptionAction ):
+        """
+        Enable display of debugging messages while doing pattern matching.
+        """
+        self.debugActions = (startAction or _defaultStartDebugAction,
+                             successAction or _defaultSuccessDebugAction,
+                             exceptionAction or _defaultExceptionDebugAction)
+        self.debug = True
+        return self
+
+    def setDebug( self, flag=True ):
+        """
+        Enable display of debugging messages while doing pattern matching.
+        Set C{flag} to True to enable, False to disable.
+
+        Example::
+            wd = Word(alphas).setName("alphaword")
+            integer = Word(nums).setName("numword")
+            term = wd | integer
+            
+            # turn on debugging for wd
+            wd.setDebug()
+
+            OneOrMore(term).parseString("abc 123 xyz 890")
+        
+        prints::
+            Match alphaword at loc 0(1,1)
+            Matched alphaword -> ['abc']
+            Match alphaword at loc 3(1,4)
+            Exception raised:Expected alphaword (at char 4), (line:1, col:5)
+            Match alphaword at loc 7(1,8)
+            Matched alphaword -> ['xyz']
+            Match alphaword at loc 11(1,12)
+            Exception raised:Expected alphaword (at char 12), (line:1, col:13)
+            Match alphaword at loc 15(1,16)
+            Exception raised:Expected alphaword (at char 15), (line:1, col:16)
+
+        The output shown is that produced by the default debug actions - custom debug actions can be
+        specified using L{setDebugActions}. Prior to attempting
+        to match the C{wd} expression, the debugging message C{"Match  at loc (,)"}
+        is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"}
+        message is shown. Also note the use of L{setName} to assign a human-readable name to the expression,
+        which makes debugging and exception messages easier to understand - for instance, the default
+        name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}.
+        """
+        if flag:
+            self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction )
+        else:
+            self.debug = False
+        return self
+
+    def __str__( self ):
+        return self.name
+
+    def __repr__( self ):
+        return _ustr(self)
+
+    def streamline( self ):
+        self.streamlined = True
+        self.strRepr = None
+        return self
+
+    def checkRecursion( self, parseElementList ):
+        pass
+
+    def validate( self, validateTrace=[] ):
+        """
+        Check defined expressions for valid structure, check for infinite recursive definitions.
+        """
+        self.checkRecursion( [] )
+
+    def parseFile( self, file_or_filename, parseAll=False ):
+        """
+        Execute the parse expression on the given file or filename.
+        If a filename is specified (instead of a file object),
+        the entire file is opened, read, and closed before parsing.
+        """
+        try:
+            file_contents = file_or_filename.read()
+        except AttributeError:
+            with open(file_or_filename, "r") as f:
+                file_contents = f.read()
+        try:
+            return self.parseString(file_contents, parseAll)
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
+
+    def __eq__(self,other):
+        if isinstance(other, ParserElement):
+            return self is other or vars(self) == vars(other)
+        elif isinstance(other, basestring):
+            return self.matches(other)
+        else:
+            return super(ParserElement,self)==other
+
+    def __ne__(self,other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(id(self))
+
+    def __req__(self,other):
+        return self == other
+
+    def __rne__(self,other):
+        return not (self == other)
+
+    def matches(self, testString, parseAll=True):
+        """
+        Method for quick testing of a parser against a test string. Good for simple 
+        inline microtests of sub expressions while building up larger parser.
+           
+        Parameters:
+         - testString - to test against this expression for a match
+         - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests
+            
+        Example::
+            expr = Word(nums)
+            assert expr.matches("100")
+        """
+        try:
+            self.parseString(_ustr(testString), parseAll=parseAll)
+            return True
+        except ParseBaseException:
+            return False
+                
+    def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False):
+        """
+        Execute the parse expression on a series of test strings, showing each
+        test, the parsed results or where the parse failed. Quick and easy way to
+        run a parse expression against a list of sample strings.
+           
+        Parameters:
+         - tests - a list of separate test strings, or a multiline string of test strings
+         - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests           
+         - comment - (default=C{'#'}) - expression for indicating embedded comments in the test 
+              string; pass None to disable comment filtering
+         - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline;
+              if False, only dump nested list
+         - printResults - (default=C{True}) prints test output to stdout
+         - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing
+
+        Returns: a (success, results) tuple, where success indicates that all tests succeeded
+        (or failed if C{failureTests} is True), and the results contain a list of lines of each 
+        test's output
+        
+        Example::
+            number_expr = pyparsing_common.number.copy()
+
+            result = number_expr.runTests('''
+                # unsigned integer
+                100
+                # negative integer
+                -100
+                # float with scientific notation
+                6.02e23
+                # integer with scientific notation
+                1e-12
+                ''')
+            print("Success" if result[0] else "Failed!")
+
+            result = number_expr.runTests('''
+                # stray character
+                100Z
+                # missing leading digit before '.'
+                -.100
+                # too many '.'
+                3.14.159
+                ''', failureTests=True)
+            print("Success" if result[0] else "Failed!")
+        prints::
+            # unsigned integer
+            100
+            [100]
+
+            # negative integer
+            -100
+            [-100]
+
+            # float with scientific notation
+            6.02e23
+            [6.02e+23]
+
+            # integer with scientific notation
+            1e-12
+            [1e-12]
+
+            Success
+            
+            # stray character
+            100Z
+               ^
+            FAIL: Expected end of text (at char 3), (line:1, col:4)
+
+            # missing leading digit before '.'
+            -.100
+            ^
+            FAIL: Expected {real number with scientific notation | real number | signed integer} (at char 0), (line:1, col:1)
+
+            # too many '.'
+            3.14.159
+                ^
+            FAIL: Expected end of text (at char 4), (line:1, col:5)
+
+            Success
+
+        Each test string must be on a single line. If you want to test a string that spans multiple
+        lines, create a test like this::
+
+            expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines")
+        
+        (Note that this is a raw string literal, you must include the leading 'r'.)
+        """
+        if isinstance(tests, basestring):
+            tests = list(map(str.strip, tests.rstrip().splitlines()))
+        if isinstance(comment, basestring):
+            comment = Literal(comment)
+        allResults = []
+        comments = []
+        success = True
+        for t in tests:
+            if comment is not None and comment.matches(t, False) or comments and not t:
+                comments.append(t)
+                continue
+            if not t:
+                continue
+            out = ['\n'.join(comments), t]
+            comments = []
+            try:
+                t = t.replace(r'\n','\n')
+                result = self.parseString(t, parseAll=parseAll)
+                out.append(result.dump(full=fullDump))
+                success = success and not failureTests
+            except ParseBaseException as pe:
+                fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else ""
+                if '\n' in t:
+                    out.append(line(pe.loc, t))
+                    out.append(' '*(col(pe.loc,t)-1) + '^' + fatal)
+                else:
+                    out.append(' '*pe.loc + '^' + fatal)
+                out.append("FAIL: " + str(pe))
+                success = success and failureTests
+                result = pe
+            except Exception as exc:
+                out.append("FAIL-EXCEPTION: " + str(exc))
+                success = success and failureTests
+                result = exc
+
+            if printResults:
+                if fullDump:
+                    out.append('')
+                print('\n'.join(out))
+
+            allResults.append((t, result))
+        
+        return success, allResults
+
+        
+class Token(ParserElement):
+    """
+    Abstract C{ParserElement} subclass, for defining atomic matching patterns.
+    """
+    def __init__( self ):
+        super(Token,self).__init__( savelist=False )
+
+
+class Empty(Token):
+    """
+    An empty token, will always match.
+    """
+    def __init__( self ):
+        super(Empty,self).__init__()
+        self.name = "Empty"
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+
+
+class NoMatch(Token):
+    """
+    A token that will never match.
+    """
+    def __init__( self ):
+        super(NoMatch,self).__init__()
+        self.name = "NoMatch"
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+        self.errmsg = "Unmatchable token"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Literal(Token):
+    """
+    Token to exactly match a specified string.
+    
+    Example::
+        Literal('blah').parseString('blah')  # -> ['blah']
+        Literal('blah').parseString('blahfooblah')  # -> ['blah']
+        Literal('blah').parseString('bla')  # -> Exception: Expected "blah"
+    
+    For case-insensitive matching, use L{CaselessLiteral}.
+    
+    For keyword matching (force word break before and after the matched string),
+    use L{Keyword} or L{CaselessKeyword}.
+    """
+    def __init__( self, matchString ):
+        super(Literal,self).__init__()
+        self.match = matchString
+        self.matchLen = len(matchString)
+        try:
+            self.firstMatchChar = matchString[0]
+        except IndexError:
+            warnings.warn("null string passed to Literal; use Empty() instead",
+                            SyntaxWarning, stacklevel=2)
+            self.__class__ = Empty
+        self.name = '"%s"' % _ustr(self.match)
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = False
+        self.mayIndexError = False
+
+    # Performance tuning: this routine gets called a *lot*
+    # if this is a single character match string  and the first character matches,
+    # short-circuit as quickly as possible, and avoid calling startswith
+    #~ @profile
+    def parseImpl( self, instring, loc, doActions=True ):
+        if (instring[loc] == self.firstMatchChar and
+            (self.matchLen==1 or instring.startswith(self.match,loc)) ):
+            return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+_L = Literal
+ParserElement._literalStringClass = Literal
+
+class Keyword(Token):
+    """
+    Token to exactly match a specified string as a keyword, that is, it must be
+    immediately followed by a non-keyword character.  Compare with C{L{Literal}}:
+     - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}.
+     - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
+    Accepts two optional constructor arguments in addition to the keyword string:
+     - C{identChars} is a string of characters that would be valid identifier characters,
+          defaulting to all alphanumerics + "_" and "$"
+     - C{caseless} allows case-insensitive matching, default is C{False}.
+       
+    Example::
+        Keyword("start").parseString("start")  # -> ['start']
+        Keyword("start").parseString("starting")  # -> Exception
+
+    For case-insensitive matching, use L{CaselessKeyword}.
+    """
+    DEFAULT_KEYWORD_CHARS = alphanums+"_$"
+
+    def __init__( self, matchString, identChars=None, caseless=False ):
+        super(Keyword,self).__init__()
+        if identChars is None:
+            identChars = Keyword.DEFAULT_KEYWORD_CHARS
+        self.match = matchString
+        self.matchLen = len(matchString)
+        try:
+            self.firstMatchChar = matchString[0]
+        except IndexError:
+            warnings.warn("null string passed to Keyword; use Empty() instead",
+                            SyntaxWarning, stacklevel=2)
+        self.name = '"%s"' % self.match
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = False
+        self.mayIndexError = False
+        self.caseless = caseless
+        if caseless:
+            self.caselessmatch = matchString.upper()
+            identChars = identChars.upper()
+        self.identChars = set(identChars)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.caseless:
+            if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and
+                 (loc == 0 or instring[loc-1].upper() not in self.identChars) ):
+                return loc+self.matchLen, self.match
+        else:
+            if (instring[loc] == self.firstMatchChar and
+                (self.matchLen==1 or instring.startswith(self.match,loc)) and
+                (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
+                (loc == 0 or instring[loc-1] not in self.identChars) ):
+                return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+
+    def copy(self):
+        c = super(Keyword,self).copy()
+        c.identChars = Keyword.DEFAULT_KEYWORD_CHARS
+        return c
+
+    @staticmethod
+    def setDefaultKeywordChars( chars ):
+        """Overrides the default Keyword chars
+        """
+        Keyword.DEFAULT_KEYWORD_CHARS = chars
+
+class CaselessLiteral(Literal):
+    """
+    Token to match a specified string, ignoring case of letters.
+    Note: the matched results will always be in the case of the given
+    match string, NOT the case of the input text.
+
+    Example::
+        OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD']
+        
+    (Contrast with example for L{CaselessKeyword}.)
+    """
+    def __init__( self, matchString ):
+        super(CaselessLiteral,self).__init__( matchString.upper() )
+        # Preserve the defining literal.
+        self.returnString = matchString
+        self.name = "'%s'" % self.returnString
+        self.errmsg = "Expected " + self.name
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if instring[ loc:loc+self.matchLen ].upper() == self.match:
+            return loc+self.matchLen, self.returnString
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class CaselessKeyword(Keyword):
+    """
+    Caseless version of L{Keyword}.
+
+    Example::
+        OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD']
+        
+    (Contrast with example for L{CaselessLiteral}.)
+    """
+    def __init__( self, matchString, identChars=None ):
+        super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True )
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
+             (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
+            return loc+self.matchLen, self.match
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class CloseMatch(Token):
+    """
+    A variation on L{Literal} which matches "close" matches, that is, 
+    strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters:
+     - C{match_string} - string to be matched
+     - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match
+    
+    The results from a successful parse will contain the matched text from the input string and the following named results:
+     - C{mismatches} - a list of the positions within the match_string where mismatches were found
+     - C{original} - the original match_string used to compare against the input string
+    
+    If C{mismatches} is an empty list, then the match was an exact match.
+    
+    Example::
+        patt = CloseMatch("ATCATCGAATGGA")
+        patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']})
+        patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1)
+
+        # exact match
+        patt.parseString("ATCATCGAATGGA") # -> (['ATCATCGAATGGA'], {'mismatches': [[]], 'original': ['ATCATCGAATGGA']})
+
+        # close match allowing up to 2 mismatches
+        patt = CloseMatch("ATCATCGAATGGA", maxMismatches=2)
+        patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']})
+    """
+    def __init__(self, match_string, maxMismatches=1):
+        super(CloseMatch,self).__init__()
+        self.name = match_string
+        self.match_string = match_string
+        self.maxMismatches = maxMismatches
+        self.errmsg = "Expected %r (with up to %d mismatches)" % (self.match_string, self.maxMismatches)
+        self.mayIndexError = False
+        self.mayReturnEmpty = False
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        start = loc
+        instrlen = len(instring)
+        maxloc = start + len(self.match_string)
+
+        if maxloc <= instrlen:
+            match_string = self.match_string
+            match_stringloc = 0
+            mismatches = []
+            maxMismatches = self.maxMismatches
+
+            for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)):
+                src,mat = s_m
+                if src != mat:
+                    mismatches.append(match_stringloc)
+                    if len(mismatches) > maxMismatches:
+                        break
+            else:
+                loc = match_stringloc + 1
+                results = ParseResults([instring[start:loc]])
+                results['original'] = self.match_string
+                results['mismatches'] = mismatches
+                return loc, results
+
+        raise ParseException(instring, loc, self.errmsg, self)
+
+
+class Word(Token):
+    """
+    Token for matching words composed of allowed character sets.
+    Defined with string containing all allowed initial characters,
+    an optional string containing allowed body characters (if omitted,
+    defaults to the initial character set), and an optional minimum,
+    maximum, and/or exact length.  The default value for C{min} is 1 (a
+    minimum value < 1 is not valid); the default values for C{max} and C{exact}
+    are 0, meaning no maximum or exact length restriction. An optional
+    C{excludeChars} parameter can list characters that might be found in 
+    the input C{bodyChars} string; useful to define a word of all printables
+    except for one or two characters, for instance.
+    
+    L{srange} is useful for defining custom character set strings for defining 
+    C{Word} expressions, using range notation from regular expression character sets.
+    
+    A common mistake is to use C{Word} to match a specific literal string, as in 
+    C{Word("Address")}. Remember that C{Word} uses the string argument to define
+    I{sets} of matchable characters. This expression would match "Add", "AAA",
+    "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'.
+    To match an exact literal string, use L{Literal} or L{Keyword}.
+
+    pyparsing includes helper strings for building Words:
+     - L{alphas}
+     - L{nums}
+     - L{alphanums}
+     - L{hexnums}
+     - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.)
+     - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.)
+     - L{printables} (any non-whitespace character)
+
+    Example::
+        # a word composed of digits
+        integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9"))
+        
+        # a word with a leading capital, and zero or more lowercase
+        capital_word = Word(alphas.upper(), alphas.lower())
+
+        # hostnames are alphanumeric, with leading alpha, and '-'
+        hostname = Word(alphas, alphanums+'-')
+        
+        # roman numeral (not a strict parser, accepts invalid mix of characters)
+        roman = Word("IVXLCDM")
+        
+        # any string of non-whitespace characters, except for ','
+        csv_value = Word(printables, excludeChars=",")
+    """
+    def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
+        super(Word,self).__init__()
+        if excludeChars:
+            initChars = ''.join(c for c in initChars if c not in excludeChars)
+            if bodyChars:
+                bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
+        self.initCharsOrig = initChars
+        self.initChars = set(initChars)
+        if bodyChars :
+            self.bodyCharsOrig = bodyChars
+            self.bodyChars = set(bodyChars)
+        else:
+            self.bodyCharsOrig = initChars
+            self.bodyChars = set(initChars)
+
+        self.maxSpecified = max > 0
+
+        if min < 1:
+            raise ValueError("cannot specify a minimum length < 1; use Optional(Word()) if zero-length word is permitted")
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.asKeyword = asKeyword
+
+        if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0):
+            if self.bodyCharsOrig == self.initCharsOrig:
+                self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig)
+            elif len(self.initCharsOrig) == 1:
+                self.reString = "%s[%s]*" % \
+                                      (re.escape(self.initCharsOrig),
+                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
+            else:
+                self.reString = "[%s][%s]*" % \
+                                      (_escapeRegexRangeChars(self.initCharsOrig),
+                                      _escapeRegexRangeChars(self.bodyCharsOrig),)
+            if self.asKeyword:
+                self.reString = r"\b"+self.reString+r"\b"
+            try:
+                self.re = re.compile( self.reString )
+            except Exception:
+                self.re = None
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.re:
+            result = self.re.match(instring,loc)
+            if not result:
+                raise ParseException(instring, loc, self.errmsg, self)
+
+            loc = result.end()
+            return loc, result.group()
+
+        if not(instring[ loc ] in self.initChars):
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        start = loc
+        loc += 1
+        instrlen = len(instring)
+        bodychars = self.bodyChars
+        maxloc = start + self.maxLen
+        maxloc = min( maxloc, instrlen )
+        while loc < maxloc and instring[loc] in bodychars:
+            loc += 1
+
+        throwException = False
+        if loc - start < self.minLen:
+            throwException = True
+        if self.maxSpecified and loc < instrlen and instring[loc] in bodychars:
+            throwException = True
+        if self.asKeyword:
+            if (start>0 and instring[start-1] in bodychars) or (loc4:
+                    return s[:4]+"..."
+                else:
+                    return s
+
+            if ( self.initCharsOrig != self.bodyCharsOrig ):
+                self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) )
+            else:
+                self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig)
+
+        return self.strRepr
+
+
+class Regex(Token):
+    r"""
+    Token for matching strings that match a given regular expression.
+    Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module.
+    If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as 
+    named parse results.
+
+    Example::
+        realnum = Regex(r"[+-]?\d+\.\d*")
+        date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)')
+        # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
+        roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
+    """
+    compiledREtype = type(re.compile("[A-Z]"))
+    def __init__( self, pattern, flags=0):
+        """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags."""
+        super(Regex,self).__init__()
+
+        if isinstance(pattern, basestring):
+            if not pattern:
+                warnings.warn("null string passed to Regex; use Empty() instead",
+                        SyntaxWarning, stacklevel=2)
+
+            self.pattern = pattern
+            self.flags = flags
+
+            try:
+                self.re = re.compile(self.pattern, self.flags)
+                self.reString = self.pattern
+            except sre_constants.error:
+                warnings.warn("invalid pattern (%s) passed to Regex" % pattern,
+                    SyntaxWarning, stacklevel=2)
+                raise
+
+        elif isinstance(pattern, Regex.compiledREtype):
+            self.re = pattern
+            self.pattern = \
+            self.reString = str(pattern)
+            self.flags = flags
+            
+        else:
+            raise ValueError("Regex may only be constructed with a string or a compiled RE object")
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        result = self.re.match(instring,loc)
+        if not result:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        loc = result.end()
+        d = result.groupdict()
+        ret = ParseResults(result.group())
+        if d:
+            for k in d:
+                ret[k] = d[k]
+        return loc,ret
+
+    def __str__( self ):
+        try:
+            return super(Regex,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            self.strRepr = "Re:(%s)" % repr(self.pattern)
+
+        return self.strRepr
+
+
+class QuotedString(Token):
+    r"""
+    Token for matching strings that are delimited by quoting characters.
+    
+    Defined with the following parameters:
+        - quoteChar - string of one or more characters defining the quote delimiting string
+        - escChar - character to escape quotes, typically backslash (default=C{None})
+        - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None})
+        - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+        - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+        - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
+        - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True})
+
+    Example::
+        qs = QuotedString('"')
+        print(qs.searchString('lsjdf "This is the quote" sldjf'))
+        complex_qs = QuotedString('{{', endQuoteChar='}}')
+        print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf'))
+        sql_qs = QuotedString('"', escQuote='""')
+        print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
+    prints::
+        [['This is the quote']]
+        [['This is the "quote"']]
+        [['This is the quote with "embedded" quotes']]
+    """
+    def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True):
+        super(QuotedString,self).__init__()
+
+        # remove white space from quote chars - wont work anyway
+        quoteChar = quoteChar.strip()
+        if not quoteChar:
+            warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+            raise SyntaxError()
+
+        if endQuoteChar is None:
+            endQuoteChar = quoteChar
+        else:
+            endQuoteChar = endQuoteChar.strip()
+            if not endQuoteChar:
+                warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2)
+                raise SyntaxError()
+
+        self.quoteChar = quoteChar
+        self.quoteCharLen = len(quoteChar)
+        self.firstQuoteChar = quoteChar[0]
+        self.endQuoteChar = endQuoteChar
+        self.endQuoteCharLen = len(endQuoteChar)
+        self.escChar = escChar
+        self.escQuote = escQuote
+        self.unquoteResults = unquoteResults
+        self.convertWhitespaceEscapes = convertWhitespaceEscapes
+
+        if multiline:
+            self.flags = re.MULTILINE | re.DOTALL
+            self.pattern = r'%s(?:[^%s%s]' % \
+                ( re.escape(self.quoteChar),
+                  _escapeRegexRangeChars(self.endQuoteChar[0]),
+                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+        else:
+            self.flags = 0
+            self.pattern = r'%s(?:[^%s\n\r%s]' % \
+                ( re.escape(self.quoteChar),
+                  _escapeRegexRangeChars(self.endQuoteChar[0]),
+                  (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
+        if len(self.endQuoteChar) > 1:
+            self.pattern += (
+                '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+                                               _escapeRegexRangeChars(self.endQuoteChar[i]))
+                                    for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
+                )
+        if escQuote:
+            self.pattern += (r'|(?:%s)' % re.escape(escQuote))
+        if escChar:
+            self.pattern += (r'|(?:%s.)' % re.escape(escChar))
+            self.escCharReplacePattern = re.escape(self.escChar)+"(.)"
+        self.pattern += (r')*%s' % re.escape(self.endQuoteChar))
+
+        try:
+            self.re = re.compile(self.pattern, self.flags)
+            self.reString = self.pattern
+        except sre_constants.error:
+            warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern,
+                SyntaxWarning, stacklevel=2)
+            raise
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayIndexError = False
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
+        if not result:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        loc = result.end()
+        ret = result.group()
+
+        if self.unquoteResults:
+
+            # strip off quotes
+            ret = ret[self.quoteCharLen:-self.endQuoteCharLen]
+
+            if isinstance(ret,basestring):
+                # replace escaped whitespace
+                if '\\' in ret and self.convertWhitespaceEscapes:
+                    ws_map = {
+                        r'\t' : '\t',
+                        r'\n' : '\n',
+                        r'\f' : '\f',
+                        r'\r' : '\r',
+                    }
+                    for wslit,wschar in ws_map.items():
+                        ret = ret.replace(wslit, wschar)
+
+                # replace escaped characters
+                if self.escChar:
+                    ret = re.sub(self.escCharReplacePattern, r"\g<1>", ret)
+
+                # replace escaped quotes
+                if self.escQuote:
+                    ret = ret.replace(self.escQuote, self.endQuoteChar)
+
+        return loc, ret
+
+    def __str__( self ):
+        try:
+            return super(QuotedString,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            self.strRepr = "quoted string, starting with %s ending with %s" % (self.quoteChar, self.endQuoteChar)
+
+        return self.strRepr
+
+
+class CharsNotIn(Token):
+    """
+    Token for matching words composed of characters I{not} in a given set (will
+    include whitespace in matched characters if not listed in the provided exclusion set - see example).
+    Defined with string containing all disallowed characters, and an optional
+    minimum, maximum, and/or exact length.  The default value for C{min} is 1 (a
+    minimum value < 1 is not valid); the default values for C{max} and C{exact}
+    are 0, meaning no maximum or exact length restriction.
+
+    Example::
+        # define a comma-separated-value as anything that is not a ','
+        csv_value = CharsNotIn(',')
+        print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213"))
+    prints::
+        ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
+    """
+    def __init__( self, notChars, min=1, max=0, exact=0 ):
+        super(CharsNotIn,self).__init__()
+        self.skipWhitespace = False
+        self.notChars = notChars
+
+        if min < 1:
+            raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted")
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+        self.name = _ustr(self)
+        self.errmsg = "Expected " + self.name
+        self.mayReturnEmpty = ( self.minLen == 0 )
+        self.mayIndexError = False
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if instring[loc] in self.notChars:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        start = loc
+        loc += 1
+        notchars = self.notChars
+        maxlen = min( start+self.maxLen, len(instring) )
+        while loc < maxlen and \
+              (instring[loc] not in notchars):
+            loc += 1
+
+        if loc - start < self.minLen:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        return loc, instring[start:loc]
+
+    def __str__( self ):
+        try:
+            return super(CharsNotIn, self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None:
+            if len(self.notChars) > 4:
+                self.strRepr = "!W:(%s...)" % self.notChars[:4]
+            else:
+                self.strRepr = "!W:(%s)" % self.notChars
+
+        return self.strRepr
+
+class White(Token):
+    """
+    Special matching class for matching whitespace.  Normally, whitespace is ignored
+    by pyparsing grammars.  This class is included when some whitespace structures
+    are significant.  Define with a string containing the whitespace characters to be
+    matched; default is C{" \\t\\r\\n"}.  Also takes optional C{min}, C{max}, and C{exact} arguments,
+    as defined for the C{L{Word}} class.
+    """
+    whiteStrs = {
+        " " : "",
+        "\t": "",
+        "\n": "",
+        "\r": "",
+        "\f": "",
+        }
+    def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
+        super(White,self).__init__()
+        self.matchWhite = ws
+        self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
+        #~ self.leaveWhitespace()
+        self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
+        self.mayReturnEmpty = True
+        self.errmsg = "Expected " + self.name
+
+        self.minLen = min
+
+        if max > 0:
+            self.maxLen = max
+        else:
+            self.maxLen = _MAX_INT
+
+        if exact > 0:
+            self.maxLen = exact
+            self.minLen = exact
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if not(instring[ loc ] in self.matchWhite):
+            raise ParseException(instring, loc, self.errmsg, self)
+        start = loc
+        loc += 1
+        maxloc = start + self.maxLen
+        maxloc = min( maxloc, len(instring) )
+        while loc < maxloc and instring[loc] in self.matchWhite:
+            loc += 1
+
+        if loc - start < self.minLen:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        return loc, instring[start:loc]
+
+
+class _PositionToken(Token):
+    def __init__( self ):
+        super(_PositionToken,self).__init__()
+        self.name=self.__class__.__name__
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+
+class GoToColumn(_PositionToken):
+    """
+    Token to advance to a specific column of input text; useful for tabular report scraping.
+    """
+    def __init__( self, colno ):
+        super(GoToColumn,self).__init__()
+        self.col = colno
+
+    def preParse( self, instring, loc ):
+        if col(loc,instring) != self.col:
+            instrlen = len(instring)
+            if self.ignoreExprs:
+                loc = self._skipIgnorables( instring, loc )
+            while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col :
+                loc += 1
+        return loc
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        thiscol = col( loc, instring )
+        if thiscol > self.col:
+            raise ParseException( instring, loc, "Text not in expected column", self )
+        newloc = loc + self.col - thiscol
+        ret = instring[ loc: newloc ]
+        return newloc, ret
+
+
+class LineStart(_PositionToken):
+    """
+    Matches if current position is at the beginning of a line within the parse string
+    
+    Example::
+    
+        test = '''\
+        AAA this line
+        AAA and this line
+          AAA but not this one
+        B AAA and definitely not this one
+        '''
+
+        for t in (LineStart() + 'AAA' + restOfLine).searchString(test):
+            print(t)
+    
+    Prints::
+        ['AAA', ' this line']
+        ['AAA', ' and this line']    
+
+    """
+    def __init__( self ):
+        super(LineStart,self).__init__()
+        self.errmsg = "Expected start of line"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if col(loc, instring) == 1:
+            return loc, []
+        raise ParseException(instring, loc, self.errmsg, self)
+
+class LineEnd(_PositionToken):
+    """
+    Matches if current position is at the end of a line within the parse string
+    """
+    def __init__( self ):
+        super(LineEnd,self).__init__()
+        self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") )
+        self.errmsg = "Expected end of line"
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if loc len(instring):
+            return loc, []
+        else:
+            raise ParseException(instring, loc, self.errmsg, self)
+
+class WordStart(_PositionToken):
+    """
+    Matches if the current position is at the beginning of a Word, and
+    is not preceded by any character in a given set of C{wordChars}
+    (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+    use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of
+    the string being parsed, or at the beginning of a line.
+    """
+    def __init__(self, wordChars = printables):
+        super(WordStart,self).__init__()
+        self.wordChars = set(wordChars)
+        self.errmsg = "Not at the start of a word"
+
+    def parseImpl(self, instring, loc, doActions=True ):
+        if loc != 0:
+            if (instring[loc-1] in self.wordChars or
+                instring[loc] not in self.wordChars):
+                raise ParseException(instring, loc, self.errmsg, self)
+        return loc, []
+
+class WordEnd(_PositionToken):
+    """
+    Matches if the current position is at the end of a Word, and
+    is not followed by any character in a given set of C{wordChars}
+    (default=C{printables}). To emulate the C{\b} behavior of regular expressions,
+    use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of
+    the string being parsed, or at the end of a line.
+    """
+    def __init__(self, wordChars = printables):
+        super(WordEnd,self).__init__()
+        self.wordChars = set(wordChars)
+        self.skipWhitespace = False
+        self.errmsg = "Not at the end of a word"
+
+    def parseImpl(self, instring, loc, doActions=True ):
+        instrlen = len(instring)
+        if instrlen>0 and loc maxExcLoc:
+                    maxException = err
+                    maxExcLoc = err.loc
+            except IndexError:
+                if len(instring) > maxExcLoc:
+                    maxException = ParseException(instring,len(instring),e.errmsg,self)
+                    maxExcLoc = len(instring)
+            else:
+                # save match among all matches, to retry longest to shortest
+                matches.append((loc2, e))
+
+        if matches:
+            matches.sort(key=lambda x: -x[0])
+            for _,e in matches:
+                try:
+                    return e._parse( instring, loc, doActions )
+                except ParseException as err:
+                    err.__traceback__ = None
+                    if err.loc > maxExcLoc:
+                        maxException = err
+                        maxExcLoc = err.loc
+
+        if maxException is not None:
+            maxException.msg = self.errmsg
+            raise maxException
+        else:
+            raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+
+    def __ixor__(self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        return self.append( other ) #Or( [ self, other ] )
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class MatchFirst(ParseExpression):
+    """
+    Requires that at least one C{ParseExpression} is found.
+    If two expressions match, the first one listed is the one that will match.
+    May be constructed using the C{'|'} operator.
+
+    Example::
+        # construct MatchFirst using '|' operator
+        
+        # watch the order of expressions to match
+        number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
+        print(number.searchString("123 3.1416 789")) #  Fail! -> [['123'], ['3'], ['1416'], ['789']]
+
+        # put more selective expression first
+        number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
+        print(number.searchString("123 3.1416 789")) #  Better -> [['123'], ['3.1416'], ['789']]
+    """
+    def __init__( self, exprs, savelist = False ):
+        super(MatchFirst,self).__init__(exprs, savelist)
+        if self.exprs:
+            self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+        else:
+            self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        maxExcLoc = -1
+        maxException = None
+        for e in self.exprs:
+            try:
+                ret = e._parse( instring, loc, doActions )
+                return ret
+            except ParseException as err:
+                if err.loc > maxExcLoc:
+                    maxException = err
+                    maxExcLoc = err.loc
+            except IndexError:
+                if len(instring) > maxExcLoc:
+                    maxException = ParseException(instring,len(instring),e.errmsg,self)
+                    maxExcLoc = len(instring)
+
+        # only got here if no expression matched, raise exception for match that made it the furthest
+        else:
+            if maxException is not None:
+                maxException.msg = self.errmsg
+                raise maxException
+            else:
+                raise ParseException(instring, loc, "no defined alternatives to match", self)
+
+    def __ior__(self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass( other )
+        return self.append( other ) #MatchFirst( [ self, other ] )
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class Each(ParseExpression):
+    """
+    Requires all given C{ParseExpression}s to be found, but in any order.
+    Expressions may be separated by whitespace.
+    May be constructed using the C{'&'} operator.
+
+    Example::
+        color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
+        shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
+        integer = Word(nums)
+        shape_attr = "shape:" + shape_type("shape")
+        posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
+        color_attr = "color:" + color("color")
+        size_attr = "size:" + integer("size")
+
+        # use Each (using operator '&') to accept attributes in any order 
+        # (shape and posn are required, color and size are optional)
+        shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr)
+
+        shape_spec.runTests('''
+            shape: SQUARE color: BLACK posn: 100, 120
+            shape: CIRCLE size: 50 color: BLUE posn: 50,80
+            color:GREEN size:20 shape:TRIANGLE posn:20,40
+            '''
+            )
+    prints::
+        shape: SQUARE color: BLACK posn: 100, 120
+        ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
+        - color: BLACK
+        - posn: ['100', ',', '120']
+          - x: 100
+          - y: 120
+        - shape: SQUARE
+
+
+        shape: CIRCLE size: 50 color: BLUE posn: 50,80
+        ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE', 'posn:', ['50', ',', '80']]
+        - color: BLUE
+        - posn: ['50', ',', '80']
+          - x: 50
+          - y: 80
+        - shape: CIRCLE
+        - size: 50
+
+
+        color: GREEN size: 20 shape: TRIANGLE posn: 20,40
+        ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE', 'posn:', ['20', ',', '40']]
+        - color: GREEN
+        - posn: ['20', ',', '40']
+          - x: 20
+          - y: 40
+        - shape: TRIANGLE
+        - size: 20
+    """
+    def __init__( self, exprs, savelist = True ):
+        super(Each,self).__init__(exprs, savelist)
+        self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
+        self.skipWhitespace = True
+        self.initExprGroups = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.initExprGroups:
+            self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional))
+            opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ]
+            opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)]
+            self.optionals = opt1 + opt2
+            self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ]
+            self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ]
+            self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ]
+            self.required += self.multirequired
+            self.initExprGroups = False
+        tmpLoc = loc
+        tmpReqd = self.required[:]
+        tmpOpt  = self.optionals[:]
+        matchOrder = []
+
+        keepMatching = True
+        while keepMatching:
+            tmpExprs = tmpReqd + tmpOpt + self.multioptionals + self.multirequired
+            failed = []
+            for e in tmpExprs:
+                try:
+                    tmpLoc = e.tryParse( instring, tmpLoc )
+                except ParseException:
+                    failed.append(e)
+                else:
+                    matchOrder.append(self.opt1map.get(id(e),e))
+                    if e in tmpReqd:
+                        tmpReqd.remove(e)
+                    elif e in tmpOpt:
+                        tmpOpt.remove(e)
+            if len(failed) == len(tmpExprs):
+                keepMatching = False
+
+        if tmpReqd:
+            missing = ", ".join(_ustr(e) for e in tmpReqd)
+            raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
+
+        # add any unmatched Optionals, in case they have default values defined
+        matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt]
+
+        resultlist = []
+        for e in matchOrder:
+            loc,results = e._parse(instring,loc,doActions)
+            resultlist.append(results)
+
+        finalResults = sum(resultlist, ParseResults([]))
+        return loc, finalResults
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
+
+        return self.strRepr
+
+    def checkRecursion( self, parseElementList ):
+        subRecCheckList = parseElementList[:] + [ self ]
+        for e in self.exprs:
+            e.checkRecursion( subRecCheckList )
+
+
+class ParseElementEnhance(ParserElement):
+    """
+    Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens.
+    """
+    def __init__( self, expr, savelist=False ):
+        super(ParseElementEnhance,self).__init__(savelist)
+        if isinstance( expr, basestring ):
+            if issubclass(ParserElement._literalStringClass, Token):
+                expr = ParserElement._literalStringClass(expr)
+            else:
+                expr = ParserElement._literalStringClass(Literal(expr))
+        self.expr = expr
+        self.strRepr = None
+        if expr is not None:
+            self.mayIndexError = expr.mayIndexError
+            self.mayReturnEmpty = expr.mayReturnEmpty
+            self.setWhitespaceChars( expr.whiteChars )
+            self.skipWhitespace = expr.skipWhitespace
+            self.saveAsList = expr.saveAsList
+            self.callPreparse = expr.callPreparse
+            self.ignoreExprs.extend(expr.ignoreExprs)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.expr is not None:
+            return self.expr._parse( instring, loc, doActions, callPreParse=False )
+        else:
+            raise ParseException("",loc,self.errmsg,self)
+
+    def leaveWhitespace( self ):
+        self.skipWhitespace = False
+        self.expr = self.expr.copy()
+        if self.expr is not None:
+            self.expr.leaveWhitespace()
+        return self
+
+    def ignore( self, other ):
+        if isinstance( other, Suppress ):
+            if other not in self.ignoreExprs:
+                super( ParseElementEnhance, self).ignore( other )
+                if self.expr is not None:
+                    self.expr.ignore( self.ignoreExprs[-1] )
+        else:
+            super( ParseElementEnhance, self).ignore( other )
+            if self.expr is not None:
+                self.expr.ignore( self.ignoreExprs[-1] )
+        return self
+
+    def streamline( self ):
+        super(ParseElementEnhance,self).streamline()
+        if self.expr is not None:
+            self.expr.streamline()
+        return self
+
+    def checkRecursion( self, parseElementList ):
+        if self in parseElementList:
+            raise RecursiveGrammarException( parseElementList+[self] )
+        subRecCheckList = parseElementList[:] + [ self ]
+        if self.expr is not None:
+            self.expr.checkRecursion( subRecCheckList )
+
+    def validate( self, validateTrace=[] ):
+        tmp = validateTrace[:]+[self]
+        if self.expr is not None:
+            self.expr.validate(tmp)
+        self.checkRecursion( [] )
+
+    def __str__( self ):
+        try:
+            return super(ParseElementEnhance,self).__str__()
+        except Exception:
+            pass
+
+        if self.strRepr is None and self.expr is not None:
+            self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) )
+        return self.strRepr
+
+
+class FollowedBy(ParseElementEnhance):
+    """
+    Lookahead matching of the given parse expression.  C{FollowedBy}
+    does I{not} advance the parsing position within the input string, it only
+    verifies that the specified parse expression matches at the current
+    position.  C{FollowedBy} always returns a null token list.
+
+    Example::
+        # use FollowedBy to match a label only if it is followed by a ':'
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        
+        OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint()
+    prints::
+        [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
+    """
+    def __init__( self, expr ):
+        super(FollowedBy,self).__init__(expr)
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        self.expr.tryParse( instring, loc )
+        return loc, []
+
+
+class NotAny(ParseElementEnhance):
+    """
+    Lookahead to disallow matching with the given parse expression.  C{NotAny}
+    does I{not} advance the parsing position within the input string, it only
+    verifies that the specified parse expression does I{not} match at the current
+    position.  Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny}
+    always returns a null token list.  May be constructed using the '~' operator.
+
+    Example::
+        
+    """
+    def __init__( self, expr ):
+        super(NotAny,self).__init__(expr)
+        #~ self.leaveWhitespace()
+        self.skipWhitespace = False  # do NOT use self.leaveWhitespace(), don't want to propagate to exprs
+        self.mayReturnEmpty = True
+        self.errmsg = "Found unwanted token, "+_ustr(self.expr)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        if self.expr.canParseNext(instring, loc):
+            raise ParseException(instring, loc, self.errmsg, self)
+        return loc, []
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "~{" + _ustr(self.expr) + "}"
+
+        return self.strRepr
+
+class _MultipleMatch(ParseElementEnhance):
+    def __init__( self, expr, stopOn=None):
+        super(_MultipleMatch, self).__init__(expr)
+        self.saveAsList = True
+        ender = stopOn
+        if isinstance(ender, basestring):
+            ender = ParserElement._literalStringClass(ender)
+        self.not_ender = ~ender if ender is not None else None
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        self_expr_parse = self.expr._parse
+        self_skip_ignorables = self._skipIgnorables
+        check_ender = self.not_ender is not None
+        if check_ender:
+            try_not_ender = self.not_ender.tryParse
+        
+        # must be at least one (but first see if we are the stopOn sentinel;
+        # if so, fail)
+        if check_ender:
+            try_not_ender(instring, loc)
+        loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False )
+        try:
+            hasIgnoreExprs = (not not self.ignoreExprs)
+            while 1:
+                if check_ender:
+                    try_not_ender(instring, loc)
+                if hasIgnoreExprs:
+                    preloc = self_skip_ignorables( instring, loc )
+                else:
+                    preloc = loc
+                loc, tmptokens = self_expr_parse( instring, preloc, doActions )
+                if tmptokens or tmptokens.haskeys():
+                    tokens += tmptokens
+        except (ParseException,IndexError):
+            pass
+
+        return loc, tokens
+        
+class OneOrMore(_MultipleMatch):
+    """
+    Repetition of one or more of the given expression.
+    
+    Parameters:
+     - expr - expression that must match one or more times
+     - stopOn - (default=C{None}) - expression for a terminating sentinel
+          (only required if the sentinel would ordinarily match the repetition 
+          expression)          
+
+    Example::
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+        text = "shape: SQUARE posn: upper left color: BLACK"
+        OneOrMore(attr_expr).parseString(text).pprint()  # Fail! read 'color' as data instead of next label -> [['shape', 'SQUARE color']]
+
+        # use stopOn attribute for OneOrMore to avoid reading label string as part of the data
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
+        
+        # could also be written as
+        (attr_expr * (1,)).parseString(text).pprint()
+    """
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "{" + _ustr(self.expr) + "}..."
+
+        return self.strRepr
+
+class ZeroOrMore(_MultipleMatch):
+    """
+    Optional repetition of zero or more of the given expression.
+    
+    Parameters:
+     - expr - expression that must match zero or more times
+     - stopOn - (default=C{None}) - expression for a terminating sentinel
+          (only required if the sentinel would ordinarily match the repetition 
+          expression)          
+
+    Example: similar to L{OneOrMore}
+    """
+    def __init__( self, expr, stopOn=None):
+        super(ZeroOrMore,self).__init__(expr, stopOn=stopOn)
+        self.mayReturnEmpty = True
+        
+    def parseImpl( self, instring, loc, doActions=True ):
+        try:
+            return super(ZeroOrMore, self).parseImpl(instring, loc, doActions)
+        except (ParseException,IndexError):
+            return loc, []
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "[" + _ustr(self.expr) + "]..."
+
+        return self.strRepr
+
+class _NullToken(object):
+    def __bool__(self):
+        return False
+    __nonzero__ = __bool__
+    def __str__(self):
+        return ""
+
+_optionalNotMatched = _NullToken()
+class Optional(ParseElementEnhance):
+    """
+    Optional matching of the given expression.
+
+    Parameters:
+     - expr - expression that must match zero or more times
+     - default (optional) - value to be returned if the optional expression is not found.
+
+    Example::
+        # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
+        zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4)))
+        zip.runTests('''
+            # traditional ZIP code
+            12345
+            
+            # ZIP+4 form
+            12101-0001
+            
+            # invalid ZIP
+            98765-
+            ''')
+    prints::
+        # traditional ZIP code
+        12345
+        ['12345']
+
+        # ZIP+4 form
+        12101-0001
+        ['12101-0001']
+
+        # invalid ZIP
+        98765-
+             ^
+        FAIL: Expected end of text (at char 5), (line:1, col:6)
+    """
+    def __init__( self, expr, default=_optionalNotMatched ):
+        super(Optional,self).__init__( expr, savelist=False )
+        self.saveAsList = self.expr.saveAsList
+        self.defaultValue = default
+        self.mayReturnEmpty = True
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        try:
+            loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False )
+        except (ParseException,IndexError):
+            if self.defaultValue is not _optionalNotMatched:
+                if self.expr.resultsName:
+                    tokens = ParseResults([ self.defaultValue ])
+                    tokens[self.expr.resultsName] = self.defaultValue
+                else:
+                    tokens = [ self.defaultValue ]
+            else:
+                tokens = []
+        return loc, tokens
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+
+        if self.strRepr is None:
+            self.strRepr = "[" + _ustr(self.expr) + "]"
+
+        return self.strRepr
+
+class SkipTo(ParseElementEnhance):
+    """
+    Token for skipping over all undefined text until the matched expression is found.
+
+    Parameters:
+     - expr - target expression marking the end of the data to be skipped
+     - include - (default=C{False}) if True, the target expression is also parsed 
+          (the skipped text and target expression are returned as a 2-element list).
+     - ignore - (default=C{None}) used to define grammars (typically quoted strings and 
+          comments) that might contain false matches to the target expression
+     - failOn - (default=C{None}) define expressions that are not allowed to be 
+          included in the skipped test; if found before the target expression is found, 
+          the SkipTo is not a match
+
+    Example::
+        report = '''
+            Outstanding Issues Report - 1 Jan 2000
+
+               # | Severity | Description                               |  Days Open
+            -----+----------+-------------------------------------------+-----------
+             101 | Critical | Intermittent system crash                 |          6
+              94 | Cosmetic | Spelling error on Login ('log|n')         |         14
+              79 | Minor    | System slow when running too many reports |         47
+            '''
+        integer = Word(nums)
+        SEP = Suppress('|')
+        # use SkipTo to simply match everything up until the next SEP
+        # - ignore quoted strings, so that a '|' character inside a quoted string does not match
+        # - parse action will call token.strip() for each matched token, i.e., the description body
+        string_data = SkipTo(SEP, ignore=quotedString)
+        string_data.setParseAction(tokenMap(str.strip))
+        ticket_expr = (integer("issue_num") + SEP 
+                      + string_data("sev") + SEP 
+                      + string_data("desc") + SEP 
+                      + integer("days_open"))
+        
+        for tkt in ticket_expr.searchString(report):
+            print tkt.dump()
+    prints::
+        ['101', 'Critical', 'Intermittent system crash', '6']
+        - days_open: 6
+        - desc: Intermittent system crash
+        - issue_num: 101
+        - sev: Critical
+        ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
+        - days_open: 14
+        - desc: Spelling error on Login ('log|n')
+        - issue_num: 94
+        - sev: Cosmetic
+        ['79', 'Minor', 'System slow when running too many reports', '47']
+        - days_open: 47
+        - desc: System slow when running too many reports
+        - issue_num: 79
+        - sev: Minor
+    """
+    def __init__( self, other, include=False, ignore=None, failOn=None ):
+        super( SkipTo, self ).__init__( other )
+        self.ignoreExpr = ignore
+        self.mayReturnEmpty = True
+        self.mayIndexError = False
+        self.includeMatch = include
+        self.asList = False
+        if isinstance(failOn, basestring):
+            self.failOn = ParserElement._literalStringClass(failOn)
+        else:
+            self.failOn = failOn
+        self.errmsg = "No match found for "+_ustr(self.expr)
+
+    def parseImpl( self, instring, loc, doActions=True ):
+        startloc = loc
+        instrlen = len(instring)
+        expr = self.expr
+        expr_parse = self.expr._parse
+        self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None
+        self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None
+        
+        tmploc = loc
+        while tmploc <= instrlen:
+            if self_failOn_canParseNext is not None:
+                # break if failOn expression matches
+                if self_failOn_canParseNext(instring, tmploc):
+                    break
+                    
+            if self_ignoreExpr_tryParse is not None:
+                # advance past ignore expressions
+                while 1:
+                    try:
+                        tmploc = self_ignoreExpr_tryParse(instring, tmploc)
+                    except ParseBaseException:
+                        break
+            
+            try:
+                expr_parse(instring, tmploc, doActions=False, callPreParse=False)
+            except (ParseException, IndexError):
+                # no match, advance loc in string
+                tmploc += 1
+            else:
+                # matched skipto expr, done
+                break
+
+        else:
+            # ran off the end of the input string without matching skipto expr, fail
+            raise ParseException(instring, loc, self.errmsg, self)
+
+        # build up return values
+        loc = tmploc
+        skiptext = instring[startloc:loc]
+        skipresult = ParseResults(skiptext)
+        
+        if self.includeMatch:
+            loc, mat = expr_parse(instring,loc,doActions,callPreParse=False)
+            skipresult += mat
+
+        return loc, skipresult
+
+class Forward(ParseElementEnhance):
+    """
+    Forward declaration of an expression to be defined later -
+    used for recursive grammars, such as algebraic infix notation.
+    When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator.
+
+    Note: take care when assigning to C{Forward} not to overlook precedence of operators.
+    Specifically, '|' has a lower precedence than '<<', so that::
+        fwdExpr << a | b | c
+    will actually be evaluated as::
+        (fwdExpr << a) | b | c
+    thereby leaving b and c out as parseable alternatives.  It is recommended that you
+    explicitly group the values inserted into the C{Forward}::
+        fwdExpr << (a | b | c)
+    Converting to use the '<<=' operator instead will avoid this problem.
+
+    See L{ParseResults.pprint} for an example of a recursive parser created using
+    C{Forward}.
+    """
+    def __init__( self, other=None ):
+        super(Forward,self).__init__( other, savelist=False )
+
+    def __lshift__( self, other ):
+        if isinstance( other, basestring ):
+            other = ParserElement._literalStringClass(other)
+        self.expr = other
+        self.strRepr = None
+        self.mayIndexError = self.expr.mayIndexError
+        self.mayReturnEmpty = self.expr.mayReturnEmpty
+        self.setWhitespaceChars( self.expr.whiteChars )
+        self.skipWhitespace = self.expr.skipWhitespace
+        self.saveAsList = self.expr.saveAsList
+        self.ignoreExprs.extend(self.expr.ignoreExprs)
+        return self
+        
+    def __ilshift__(self, other):
+        return self << other
+    
+    def leaveWhitespace( self ):
+        self.skipWhitespace = False
+        return self
+
+    def streamline( self ):
+        if not self.streamlined:
+            self.streamlined = True
+            if self.expr is not None:
+                self.expr.streamline()
+        return self
+
+    def validate( self, validateTrace=[] ):
+        if self not in validateTrace:
+            tmp = validateTrace[:]+[self]
+            if self.expr is not None:
+                self.expr.validate(tmp)
+        self.checkRecursion([])
+
+    def __str__( self ):
+        if hasattr(self,"name"):
+            return self.name
+        return self.__class__.__name__ + ": ..."
+
+        # stubbed out for now - creates awful memory and perf issues
+        self._revertClass = self.__class__
+        self.__class__ = _ForwardNoRecurse
+        try:
+            if self.expr is not None:
+                retString = _ustr(self.expr)
+            else:
+                retString = "None"
+        finally:
+            self.__class__ = self._revertClass
+        return self.__class__.__name__ + ": " + retString
+
+    def copy(self):
+        if self.expr is not None:
+            return super(Forward,self).copy()
+        else:
+            ret = Forward()
+            ret <<= self
+            return ret
+
+class _ForwardNoRecurse(Forward):
+    def __str__( self ):
+        return "..."
+
+class TokenConverter(ParseElementEnhance):
+    """
+    Abstract subclass of C{ParseExpression}, for converting parsed results.
+    """
+    def __init__( self, expr, savelist=False ):
+        super(TokenConverter,self).__init__( expr )#, savelist )
+        self.saveAsList = False
+
+class Combine(TokenConverter):
+    """
+    Converter to concatenate all matching tokens to a single string.
+    By default, the matching patterns must also be contiguous in the input string;
+    this can be disabled by specifying C{'adjacent=False'} in the constructor.
+
+    Example::
+        real = Word(nums) + '.' + Word(nums)
+        print(real.parseString('3.1416')) # -> ['3', '.', '1416']
+        # will also erroneously match the following
+        print(real.parseString('3. 1416')) # -> ['3', '.', '1416']
+
+        real = Combine(Word(nums) + '.' + Word(nums))
+        print(real.parseString('3.1416')) # -> ['3.1416']
+        # no match when there are internal spaces
+        print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...)
+    """
+    def __init__( self, expr, joinString="", adjacent=True ):
+        super(Combine,self).__init__( expr )
+        # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
+        if adjacent:
+            self.leaveWhitespace()
+        self.adjacent = adjacent
+        self.skipWhitespace = True
+        self.joinString = joinString
+        self.callPreparse = True
+
+    def ignore( self, other ):
+        if self.adjacent:
+            ParserElement.ignore(self, other)
+        else:
+            super( Combine, self).ignore( other )
+        return self
+
+    def postParse( self, instring, loc, tokenlist ):
+        retToks = tokenlist.copy()
+        del retToks[:]
+        retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
+
+        if self.resultsName and retToks.haskeys():
+            return [ retToks ]
+        else:
+            return retToks
+
+class Group(TokenConverter):
+    """
+    Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions.
+
+    Example::
+        ident = Word(alphas)
+        num = Word(nums)
+        term = ident | num
+        func = ident + Optional(delimitedList(term))
+        print(func.parseString("fn a,b,100"))  # -> ['fn', 'a', 'b', '100']
+
+        func = ident + Group(Optional(delimitedList(term)))
+        print(func.parseString("fn a,b,100"))  # -> ['fn', ['a', 'b', '100']]
+    """
+    def __init__( self, expr ):
+        super(Group,self).__init__( expr )
+        self.saveAsList = True
+
+    def postParse( self, instring, loc, tokenlist ):
+        return [ tokenlist ]
+
+class Dict(TokenConverter):
+    """
+    Converter to return a repetitive expression as a list, but also as a dictionary.
+    Each element can also be referenced using the first token in the expression as its key.
+    Useful for tabular report scraping when the first column can be used as a item key.
+
+    Example::
+        data_word = Word(alphas)
+        label = data_word + FollowedBy(':')
+        attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join))
+
+        text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+        attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        
+        # print attributes as plain groups
+        print(OneOrMore(attr_expr).parseString(text).dump())
+        
+        # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names
+        result = Dict(OneOrMore(Group(attr_expr))).parseString(text)
+        print(result.dump())
+        
+        # access named fields as dict entries, or output as dict
+        print(result['shape'])        
+        print(result.asDict())
+    prints::
+        ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
+
+        [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+        - color: light blue
+        - posn: upper left
+        - shape: SQUARE
+        - texture: burlap
+        SQUARE
+        {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'}
+    See more examples at L{ParseResults} of accessing fields by results name.
+    """
+    def __init__( self, expr ):
+        super(Dict,self).__init__( expr )
+        self.saveAsList = True
+
+    def postParse( self, instring, loc, tokenlist ):
+        for i,tok in enumerate(tokenlist):
+            if len(tok) == 0:
+                continue
+            ikey = tok[0]
+            if isinstance(ikey,int):
+                ikey = _ustr(tok[0]).strip()
+            if len(tok)==1:
+                tokenlist[ikey] = _ParseResultsWithOffset("",i)
+            elif len(tok)==2 and not isinstance(tok[1],ParseResults):
+                tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i)
+            else:
+                dictvalue = tok.copy() #ParseResults(i)
+                del dictvalue[0]
+                if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
+                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
+                else:
+                    tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
+
+        if self.resultsName:
+            return [ tokenlist ]
+        else:
+            return tokenlist
+
+
+class Suppress(TokenConverter):
+    """
+    Converter for ignoring the results of a parsed expression.
+
+    Example::
+        source = "a, b, c,d"
+        wd = Word(alphas)
+        wd_list1 = wd + ZeroOrMore(',' + wd)
+        print(wd_list1.parseString(source))
+
+        # often, delimiters that are useful during parsing are just in the
+        # way afterward - use Suppress to keep them out of the parsed output
+        wd_list2 = wd + ZeroOrMore(Suppress(',') + wd)
+        print(wd_list2.parseString(source))
+    prints::
+        ['a', ',', 'b', ',', 'c', ',', 'd']
+        ['a', 'b', 'c', 'd']
+    (See also L{delimitedList}.)
+    """
+    def postParse( self, instring, loc, tokenlist ):
+        return []
+
+    def suppress( self ):
+        return self
+
+
+class OnlyOnce(object):
+    """
+    Wrapper for parse actions, to ensure they are only called once.
+    """
+    def __init__(self, methodCall):
+        self.callable = _trim_arity(methodCall)
+        self.called = False
+    def __call__(self,s,l,t):
+        if not self.called:
+            results = self.callable(s,l,t)
+            self.called = True
+            return results
+        raise ParseException(s,l,"")
+    def reset(self):
+        self.called = False
+
+def traceParseAction(f):
+    """
+    Decorator for debugging parse actions. 
+    
+    When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".}
+    When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised.
+
+    Example::
+        wd = Word(alphas)
+
+        @traceParseAction
+        def remove_duplicate_chars(tokens):
+            return ''.join(sorted(set(''.join(tokens))))
+
+        wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
+        print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
+    prints::
+        >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
+        <3:
+            thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc
+        sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) )
+        try:
+            ret = f(*paArgs)
+        except Exception as exc:
+            sys.stderr.write( "< ['aa', 'bb', 'cc']
+        delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE']
+    """
+    dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..."
+    if combine:
+        return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName)
+    else:
+        return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName)
+
+def countedArray( expr, intExpr=None ):
+    """
+    Helper to define a counted list of expressions.
+    This helper defines a pattern of the form::
+        integer expr expr expr...
+    where the leading integer tells how many expr expressions follow.
+    The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed.
+    
+    If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value.
+
+    Example::
+        countedArray(Word(alphas)).parseString('2 ab cd ef')  # -> ['ab', 'cd']
+
+        # in this parser, the leading integer value is given in binary,
+        # '10' indicating that 2 values are in the array
+        binaryConstant = Word('01').setParseAction(lambda t: int(t[0], 2))
+        countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef')  # -> ['ab', 'cd']
+    """
+    arrayExpr = Forward()
+    def countFieldParseAction(s,l,t):
+        n = t[0]
+        arrayExpr << (n and Group(And([expr]*n)) or Group(empty))
+        return []
+    if intExpr is None:
+        intExpr = Word(nums).setParseAction(lambda t:int(t[0]))
+    else:
+        intExpr = intExpr.copy()
+    intExpr.setName("arrayLen")
+    intExpr.addParseAction(countFieldParseAction, callDuringTry=True)
+    return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...')
+
+def _flatten(L):
+    ret = []
+    for i in L:
+        if isinstance(i,list):
+            ret.extend(_flatten(i))
+        else:
+            ret.append(i)
+    return ret
+
+def matchPreviousLiteral(expr):
+    """
+    Helper to define an expression that is indirectly defined from
+    the tokens matched in a previous expression, that is, it looks
+    for a 'repeat' of a previous expression.  For example::
+        first = Word(nums)
+        second = matchPreviousLiteral(first)
+        matchExpr = first + ":" + second
+    will match C{"1:1"}, but not C{"1:2"}.  Because this matches a
+    previous literal, will also match the leading C{"1:1"} in C{"1:10"}.
+    If this is not desired, use C{matchPreviousExpr}.
+    Do I{not} use with packrat parsing enabled.
+    """
+    rep = Forward()
+    def copyTokenToRepeater(s,l,t):
+        if t:
+            if len(t) == 1:
+                rep << t[0]
+            else:
+                # flatten t tokens
+                tflat = _flatten(t.asList())
+                rep << And(Literal(tt) for tt in tflat)
+        else:
+            rep << Empty()
+    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+    rep.setName('(prev) ' + _ustr(expr))
+    return rep
+
+def matchPreviousExpr(expr):
+    """
+    Helper to define an expression that is indirectly defined from
+    the tokens matched in a previous expression, that is, it looks
+    for a 'repeat' of a previous expression.  For example::
+        first = Word(nums)
+        second = matchPreviousExpr(first)
+        matchExpr = first + ":" + second
+    will match C{"1:1"}, but not C{"1:2"}.  Because this matches by
+    expressions, will I{not} match the leading C{"1:1"} in C{"1:10"};
+    the expressions are evaluated first, and then compared, so
+    C{"1"} is compared with C{"10"}.
+    Do I{not} use with packrat parsing enabled.
+    """
+    rep = Forward()
+    e2 = expr.copy()
+    rep <<= e2
+    def copyTokenToRepeater(s,l,t):
+        matchTokens = _flatten(t.asList())
+        def mustMatchTheseTokens(s,l,t):
+            theseTokens = _flatten(t.asList())
+            if  theseTokens != matchTokens:
+                raise ParseException("",0,"")
+        rep.setParseAction( mustMatchTheseTokens, callDuringTry=True )
+    expr.addParseAction(copyTokenToRepeater, callDuringTry=True)
+    rep.setName('(prev) ' + _ustr(expr))
+    return rep
+
+def _escapeRegexRangeChars(s):
+    #~  escape these chars: ^-]
+    for c in r"\^-]":
+        s = s.replace(c,_bslash+c)
+    s = s.replace("\n",r"\n")
+    s = s.replace("\t",r"\t")
+    return _ustr(s)
+
+def oneOf( strs, caseless=False, useRegex=True ):
+    """
+    Helper to quickly define a set of alternative Literals, and makes sure to do
+    longest-first testing when there is a conflict, regardless of the input order,
+    but returns a C{L{MatchFirst}} for best performance.
+
+    Parameters:
+     - strs - a string of space-delimited literals, or a collection of string literals
+     - caseless - (default=C{False}) - treat all literals as caseless
+     - useRegex - (default=C{True}) - as an optimization, will generate a Regex
+          object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or
+          if creating a C{Regex} raises an exception)
+
+    Example::
+        comp_oper = oneOf("< = > <= >= !=")
+        var = Word(alphas)
+        number = Word(nums)
+        term = var | number
+        comparison_expr = term + comp_oper + term
+        print(comparison_expr.searchString("B = 12  AA=23 B<=AA AA>12"))
+    prints::
+        [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']]
+    """
+    if caseless:
+        isequal = ( lambda a,b: a.upper() == b.upper() )
+        masks = ( lambda a,b: b.upper().startswith(a.upper()) )
+        parseElementClass = CaselessLiteral
+    else:
+        isequal = ( lambda a,b: a == b )
+        masks = ( lambda a,b: b.startswith(a) )
+        parseElementClass = Literal
+
+    symbols = []
+    if isinstance(strs,basestring):
+        symbols = strs.split()
+    elif isinstance(strs, Iterable):
+        symbols = list(strs)
+    else:
+        warnings.warn("Invalid argument to oneOf, expected string or iterable",
+                SyntaxWarning, stacklevel=2)
+    if not symbols:
+        return NoMatch()
+
+    i = 0
+    while i < len(symbols)-1:
+        cur = symbols[i]
+        for j,other in enumerate(symbols[i+1:]):
+            if ( isequal(other, cur) ):
+                del symbols[i+j+1]
+                break
+            elif ( masks(cur, other) ):
+                del symbols[i+j+1]
+                symbols.insert(i,other)
+                cur = other
+                break
+        else:
+            i += 1
+
+    if not caseless and useRegex:
+        #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
+        try:
+            if len(symbols)==len("".join(symbols)):
+                return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols))
+            else:
+                return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols))
+        except Exception:
+            warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
+                    SyntaxWarning, stacklevel=2)
+
+
+    # last resort, just use MatchFirst
+    return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols))
+
+def dictOf( key, value ):
+    """
+    Helper to easily and clearly define a dictionary by specifying the respective patterns
+    for the key and value.  Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
+    in the proper order.  The key pattern can include delimiting markers or punctuation,
+    as long as they are suppressed, thereby leaving the significant key text.  The value
+    pattern can include named results, so that the C{Dict} results can include named token
+    fields.
+
+    Example::
+        text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
+        attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join))
+        print(OneOrMore(attr_expr).parseString(text).dump())
+        
+        attr_label = label
+        attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)
+
+        # similar to Dict, but simpler call format
+        result = dictOf(attr_label, attr_value).parseString(text)
+        print(result.dump())
+        print(result['shape'])
+        print(result.shape)  # object attribute access works too
+        print(result.asDict())
+    prints::
+        [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
+        - color: light blue
+        - posn: upper left
+        - shape: SQUARE
+        - texture: burlap
+        SQUARE
+        SQUARE
+        {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'}
+    """
+    return Dict( ZeroOrMore( Group ( key + value ) ) )
+
+def originalTextFor(expr, asString=True):
+    """
+    Helper to return the original, untokenized text for a given expression.  Useful to
+    restore the parsed fields of an HTML start tag into the raw tag text itself, or to
+    revert separate tokens with intervening whitespace back to the original matching
+    input text. By default, returns astring containing the original parsed text.  
+       
+    If the optional C{asString} argument is passed as C{False}, then the return value is a 
+    C{L{ParseResults}} containing any results names that were originally matched, and a 
+    single token containing the original matched text from the input string.  So if 
+    the expression passed to C{L{originalTextFor}} contains expressions with defined
+    results names, you must set C{asString} to C{False} if you want to preserve those
+    results name values.
+
+    Example::
+        src = "this is test  bold text  normal text "
+        for tag in ("b","i"):
+            opener,closer = makeHTMLTags(tag)
+            patt = originalTextFor(opener + SkipTo(closer) + closer)
+            print(patt.searchString(src)[0])
+    prints::
+        [' bold text ']
+        ['text']
+    """
+    locMarker = Empty().setParseAction(lambda s,loc,t: loc)
+    endlocMarker = locMarker.copy()
+    endlocMarker.callPreparse = False
+    matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end")
+    if asString:
+        extractText = lambda s,l,t: s[t._original_start:t._original_end]
+    else:
+        def extractText(s,l,t):
+            t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]]
+    matchExpr.setParseAction(extractText)
+    matchExpr.ignoreExprs = expr.ignoreExprs
+    return matchExpr
+
+def ungroup(expr): 
+    """
+    Helper to undo pyparsing's default grouping of And expressions, even
+    if all but one are non-empty.
+    """
+    return TokenConverter(expr).setParseAction(lambda t:t[0])
+
+def locatedExpr(expr):
+    """
+    Helper to decorate a returned token with its starting and ending locations in the input string.
+    This helper adds the following results names:
+     - locn_start = location where matched expression begins
+     - locn_end = location where matched expression ends
+     - value = the actual parsed results
+
+    Be careful if the input text contains C{} characters, you may want to call
+    C{L{ParserElement.parseWithTabs}}
+
+    Example::
+        wd = Word(alphas)
+        for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"):
+            print(match)
+    prints::
+        [[0, 'ljsdf', 5]]
+        [[8, 'lksdjjf', 15]]
+        [[18, 'lkkjj', 23]]
+    """
+    locator = Empty().setParseAction(lambda s,l,t: l)
+    return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
+# convenience constants for positional expressions
+empty       = Empty().setName("empty")
+lineStart   = LineStart().setName("lineStart")
+lineEnd     = LineEnd().setName("lineEnd")
+stringStart = StringStart().setName("stringStart")
+stringEnd   = StringEnd().setName("stringEnd")
+
+_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
+_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
+_charRange = Group(_singleChar + Suppress("-") + _singleChar)
+_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
+
+def srange(s):
+    r"""
+    Helper to easily define string ranges for use in Word construction.  Borrows
+    syntax from regexp '[]' string range definitions::
+        srange("[0-9]")   -> "0123456789"
+        srange("[a-z]")   -> "abcdefghijklmnopqrstuvwxyz"
+        srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
+    The input string must be enclosed in []'s, and the returned string is the expanded
+    character set joined into a single string.
+    The values enclosed in the []'s may be:
+     - a single character
+     - an escaped character with a leading backslash (such as C{\-} or C{\]})
+     - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) 
+         (C{\0x##} is also supported for backwards compatibility) 
+     - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character)
+     - a range of any of the above, separated by a dash (C{'a-z'}, etc.)
+     - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.)
+    """
+    _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
+    try:
+        return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
+    except Exception:
+        return ""
+
+def matchOnlyAtCol(n):
+    """
+    Helper method for defining parse actions that require matching at a specific
+    column in the input text.
+    """
+    def verifyCol(strg,locn,toks):
+        if col(locn,strg) != n:
+            raise ParseException(strg,locn,"matched token not at column %d" % n)
+    return verifyCol
+
+def replaceWith(replStr):
+    """
+    Helper method for common parse actions that simply return a literal value.  Especially
+    useful when used with C{L{transformString}()}.
+
+    Example::
+        num = Word(nums).setParseAction(lambda toks: int(toks[0]))
+        na = oneOf("N/A NA").setParseAction(replaceWith(math.nan))
+        term = na | num
+        
+        OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234]
+    """
+    return lambda s,l,t: [replStr]
+
+def removeQuotes(s,l,t):
+    """
+    Helper parse action for removing quotation marks from parsed quoted strings.
+
+    Example::
+        # by default, quotation marks are included in parsed results
+        quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"]
+
+        # use removeQuotes to strip quotation marks from parsed results
+        quotedString.setParseAction(removeQuotes)
+        quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["Now is the Winter of our Discontent"]
+    """
+    return t[0][1:-1]
+
+def tokenMap(func, *args):
+    """
+    Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional 
+    args are passed, they are forwarded to the given function as additional arguments after
+    the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the
+    parsed data to an integer using base 16.
+
+    Example (compare the last to example in L{ParserElement.transformString}::
+        hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16))
+        hex_ints.runTests('''
+            00 11 22 aa FF 0a 0d 1a
+            ''')
+        
+        upperword = Word(alphas).setParseAction(tokenMap(str.upper))
+        OneOrMore(upperword).runTests('''
+            my kingdom for a horse
+            ''')
+
+        wd = Word(alphas).setParseAction(tokenMap(str.title))
+        OneOrMore(wd).setParseAction(' '.join).runTests('''
+            now is the winter of our discontent made glorious summer by this sun of york
+            ''')
+    prints::
+        00 11 22 aa FF 0a 0d 1a
+        [0, 17, 34, 170, 255, 10, 13, 26]
+
+        my kingdom for a horse
+        ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
+
+        now is the winter of our discontent made glorious summer by this sun of york
+        ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
+    """
+    def pa(s,l,t):
+        return [func(tokn, *args) for tokn in t]
+
+    try:
+        func_name = getattr(func, '__name__', 
+                            getattr(func, '__class__').__name__)
+    except Exception:
+        func_name = str(func)
+    pa.__name__ = func_name
+
+    return pa
+
+upcaseTokens = tokenMap(lambda t: _ustr(t).upper())
+"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}"""
+
+downcaseTokens = tokenMap(lambda t: _ustr(t).lower())
+"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}"""
+    
+def _makeTags(tagStr, xml):
+    """Internal helper to construct opening and closing tag expressions, given a tag name"""
+    if isinstance(tagStr,basestring):
+        resname = tagStr
+        tagStr = Keyword(tagStr, caseless=not xml)
+    else:
+        resname = tagStr.name
+
+    tagAttrName = Word(alphas,alphanums+"_-:")
+    if (xml):
+        tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes )
+        openTag = Suppress("<") + tagStr("tag") + \
+                Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
+                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+    else:
+        printablesLessRAbrack = "".join(c for c in printables if c not in ">")
+        tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
+        openTag = Suppress("<") + tagStr("tag") + \
+                Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
+                Optional( Suppress("=") + tagAttrValue ) ))) + \
+                Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
+    closeTag = Combine(_L("")
+
+    openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname)
+    closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("" % resname)
+    openTag.tag = resname
+    closeTag.tag = resname
+    return openTag, closeTag
+
+def makeHTMLTags(tagStr):
+    """
+    Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches
+    tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values.
+
+    Example::
+        text = 'More info at the pyparsing wiki page'
+        # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple
+        a,a_end = makeHTMLTags("A")
+        link_expr = a + SkipTo(a_end)("link_text") + a_end
+        
+        for link in link_expr.searchString(text):
+            # attributes in the  tag (like "href" shown here) are also accessible as named results
+            print(link.link_text, '->', link.href)
+    prints::
+        pyparsing -> http://pyparsing.wikispaces.com
+    """
+    return _makeTags( tagStr, False )
+
+def makeXMLTags(tagStr):
+    """
+    Helper to construct opening and closing tag expressions for XML, given a tag name. Matches
+    tags only in the given upper/lower case.
+
+    Example: similar to L{makeHTMLTags}
+    """
+    return _makeTags( tagStr, True )
+
+def withAttribute(*args,**attrDict):
+    """
+    Helper to create a validating parse action to be used with start tags created
+    with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
+    with a required attribute value, to avoid false matches on common tags such as
+    C{} or C{
}.
+
+    Call C{withAttribute} with a series of attribute names and values. Specify the list
+    of filter attributes names and values as:
+     - keyword arguments, as in C{(align="right")}, or
+     - as an explicit dict with C{**} operator, when an attribute name is also a Python
+          reserved word, as in C{**{"class":"Customer", "align":"right"}}
+     - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") )
+    For attribute names with a namespace prefix, you must use the second form.  Attribute
+    names are matched insensitive to upper/lower case.
+       
+    If just testing for C{class} (with or without a namespace), use C{L{withClass}}.
+
+    To verify that the attribute exists, but without specifying a value, pass
+    C{withAttribute.ANY_VALUE} as the value.
+
+    Example::
+        html = '''
+            

+            Some text
+            
1 4 0 1 0

+            
1,3 2,3 1,1

+            
this has no type

+            

+                
+        '''
+        div,div_end = makeHTMLTags("div")
+
+        # only match div tag having a type attribute with value "grid"
+        div_grid = div().setParseAction(withAttribute(type="grid"))
+        grid_expr = div_grid + SkipTo(div | div_end)("body")
+        for grid_header in grid_expr.searchString(html):
+            print(grid_header.body)
+        
+        # construct a match with any div tag having a type attribute, regardless of the value
+        div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE))
+        div_expr = div_any_type + SkipTo(div | div_end)("body")
+        for div_header in div_expr.searchString(html):
+            print(div_header.body)
+    prints::
+        1 4 0 1 0
+
+        1 4 0 1 0
+        1,3 2,3 1,1
+    """
+    if args:
+        attrs = args[:]
+    else:
+        attrs = attrDict.items()
+    attrs = [(k,v) for k,v in attrs]
+    def pa(s,l,tokens):
+        for attrName,attrValue in attrs:
+            if attrName not in tokens:
+                raise ParseException(s,l,"no matching attribute " + attrName)
+            if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue:
+                raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" %
+                                            (attrName, tokens[attrName], attrValue))
+    return pa
+withAttribute.ANY_VALUE = object()
+
+def withClass(classname, namespace=''):
+    """
+    Simplified version of C{L{withAttribute}} when matching on a div class - made
+    difficult because C{class} is a reserved word in Python.
+
+    Example::
+        html = '''
+            

+            Some text
+            
1 4 0 1 0

+            
1,3 2,3 1,1

+            
this <div> has no class

+            

+                
+        '''
+        div,div_end = makeHTMLTags("div")
+        div_grid = div().setParseAction(withClass("grid"))
+        
+        grid_expr = div_grid + SkipTo(div | div_end)("body")
+        for grid_header in grid_expr.searchString(html):
+            print(grid_header.body)
+        
+        div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE))
+        div_expr = div_any_type + SkipTo(div | div_end)("body")
+        for div_header in div_expr.searchString(html):
+            print(div_header.body)
+    prints::
+        1 4 0 1 0
+
+        1 4 0 1 0
+        1,3 2,3 1,1
+    """
+    classattr = "%s:class" % namespace if namespace else "class"
+    return withAttribute(**{classattr : classname})        
+
+opAssoc = _Constants()
+opAssoc.LEFT = object()
+opAssoc.RIGHT = object()
+
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
+    """
+    Helper method for constructing grammars of expressions made up of
+    operators working in a precedence hierarchy.  Operators may be unary or
+    binary, left- or right-associative.  Parse actions can also be attached
+    to operator expressions. The generated parser will also recognize the use 
+    of parentheses to override operator precedences (see example below).
+    
+    Note: if you define a deep operator list, you may see performance issues
+    when using infixNotation. See L{ParserElement.enablePackrat} for a
+    mechanism to potentially improve your parser performance.
+
+    Parameters:
+     - baseExpr - expression representing the most basic element for the nested
+     - opList - list of tuples, one for each operator precedence level in the
+      expression grammar; each tuple is of the form
+      (opExpr, numTerms, rightLeftAssoc, parseAction), where:
+       - opExpr is the pyparsing expression for the operator;
+          may also be a string, which will be converted to a Literal;
+          if numTerms is 3, opExpr is a tuple of two expressions, for the
+          two operators separating the 3 terms
+       - numTerms is the number of terms for this operator (must
+          be 1, 2, or 3)
+       - rightLeftAssoc is the indicator whether the operator is
+          right or left associative, using the pyparsing-defined
+          constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
+       - parseAction is the parse action to be associated with
+          expressions matching this operator expression (the
+          parse action tuple member may be omitted); if the parse action
+          is passed a tuple or list of functions, this is equivalent to
+          calling C{setParseAction(*fn)} (L{ParserElement.setParseAction})
+     - lpar - expression for matching left-parentheses (default=C{Suppress('(')})
+     - rpar - expression for matching right-parentheses (default=C{Suppress(')')})
+
+    Example::
+        # simple example of four-function arithmetic with ints and variable names
+        integer = pyparsing_common.signed_integer
+        varname = pyparsing_common.identifier 
+        
+        arith_expr = infixNotation(integer | varname,
+            [
+            ('-', 1, opAssoc.RIGHT),
+            (oneOf('* /'), 2, opAssoc.LEFT),
+            (oneOf('+ -'), 2, opAssoc.LEFT),
+            ])
+        
+        arith_expr.runTests('''
+            5+3*6
+            (5+3)*6
+            -2--11
+            ''', fullDump=False)
+    prints::
+        5+3*6
+        [[5, '+', [3, '*', 6]]]
+
+        (5+3)*6
+        [[[5, '+', 3], '*', 6]]
+
+        -2--11
+        [[['-', 2], '-', ['-', 11]]]
+    """
+    ret = Forward()
+    lastExpr = baseExpr | ( lpar + ret + rpar )
+    for i,operDef in enumerate(opList):
+        opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
+        termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr
+        if arity == 3:
+            if opExpr is None or len(opExpr) != 2:
+                raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions")
+            opExpr1, opExpr2 = opExpr
+        thisExpr = Forward().setName(termName)
+        if rightLeftAssoc == opAssoc.LEFT:
+            if arity == 1:
+                matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) )
+            elif arity == 2:
+                if opExpr is not None:
+                    matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) )
+                else:
+                    matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) )
+            elif arity == 3:
+                matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \
+                            Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr )
+            else:
+                raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+        elif rightLeftAssoc == opAssoc.RIGHT:
+            if arity == 1:
+                # try to avoid LR with this extra test
+                if not isinstance(opExpr, Optional):
+                    opExpr = Optional(opExpr)
+                matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr )
+            elif arity == 2:
+                if opExpr is not None:
+                    matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) )
+                else:
+                    matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) )
+            elif arity == 3:
+                matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \
+                            Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr )
+            else:
+                raise ValueError("operator must be unary (1), binary (2), or ternary (3)")
+        else:
+            raise ValueError("operator must indicate right or left associativity")
+        if pa:
+            if isinstance(pa, (tuple, list)):
+                matchExpr.setParseAction(*pa)
+            else:
+                matchExpr.setParseAction(pa)
+        thisExpr <<= ( matchExpr.setName(termName) | lastExpr )
+        lastExpr = thisExpr
+    ret <<= lastExpr
+    return ret
+
+operatorPrecedence = infixNotation
+"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release."""
+
+dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes")
+sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes")
+quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'|
+                       Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes")
+unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal")
+
+def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()):
+    """
+    Helper method for defining nested lists enclosed in opening and closing
+    delimiters ("(" and ")" are the default).
+
+    Parameters:
+     - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression
+     - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression
+     - content - expression for items within the nested lists (default=C{None})
+     - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString})
+
+    If an expression is not provided for the content argument, the nested
+    expression will capture all whitespace-delimited content between delimiters
+    as a list of separate values.
+
+    Use the C{ignoreExpr} argument to define expressions that may contain
+    opening or closing characters that should not be treated as opening
+    or closing characters for nesting, such as quotedString or a comment
+    expression.  Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}.
+    The default is L{quotedString}, but if no expressions are to be ignored,
+    then pass C{None} for this argument.
+
+    Example::
+        data_type = oneOf("void int short long char float double")
+        decl_data_type = Combine(data_type + Optional(Word('*')))
+        ident = Word(alphas+'_', alphanums+'_')
+        number = pyparsing_common.number
+        arg = Group(decl_data_type + ident)
+        LPAR,RPAR = map(Suppress, "()")
+
+        code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment))
+
+        c_function = (decl_data_type("type") 
+                      + ident("name")
+                      + LPAR + Optional(delimitedList(arg), [])("args") + RPAR 
+                      + code_body("body"))
+        c_function.ignore(cStyleComment)
+        
+        source_code = '''
+            int is_odd(int x) { 
+                return (x%2); 
+            }
+                
+            int dec_to_hex(char hchar) { 
+                if (hchar >= '0' && hchar <= '9') { 
+                    return (ord(hchar)-ord('0')); 
+                } else { 
+                    return (10+ord(hchar)-ord('A'));
+                } 
+            }
+        '''
+        for func in c_function.searchString(source_code):
+            print("%(name)s (%(type)s) args: %(args)s" % func)
+
+    prints::
+        is_odd (int) args: [['int', 'x']]
+        dec_to_hex (int) args: [['char', 'hchar']]
+    """
+    if opener == closer:
+        raise ValueError("opening and closing strings cannot be the same")
+    if content is None:
+        if isinstance(opener,basestring) and isinstance(closer,basestring):
+            if len(opener) == 1 and len(closer)==1:
+                if ignoreExpr is not None:
+                    content = (Combine(OneOrMore(~ignoreExpr +
+                                    CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+                else:
+                    content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS
+                                ).setParseAction(lambda t:t[0].strip()))
+            else:
+                if ignoreExpr is not None:
+                    content = (Combine(OneOrMore(~ignoreExpr + 
+                                    ~Literal(opener) + ~Literal(closer) +
+                                    CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+                else:
+                    content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) +
+                                    CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1))
+                                ).setParseAction(lambda t:t[0].strip()))
+        else:
+            raise ValueError("opening and closing arguments must be strings if no content expression is given")
+    ret = Forward()
+    if ignoreExpr is not None:
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+    else:
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
+    ret.setName('nested %s%s expression' % (opener,closer))
+    return ret
+
+def indentedBlock(blockStatementExpr, indentStack, indent=True):
+    """
+    Helper method for defining space-delimited indentation blocks, such as
+    those used to define block statements in Python source code.
+
+    Parameters:
+     - blockStatementExpr - expression defining syntax of statement that
+            is repeated within the indented block
+     - indentStack - list created by caller to manage indentation stack
+            (multiple statementWithIndentedBlock expressions within a single grammar
+            should share a common indentStack)
+     - indent - boolean indicating whether block must be indented beyond the
+            the current level; set to False for block of left-most statements
+            (default=C{True})
+
+    A valid block must contain at least one C{blockStatement}.
+
+    Example::
+        data = '''
+        def A(z):
+          A1
+          B = 100
+          G = A2
+          A2
+          A3
+        B
+        def BB(a,b,c):
+          BB1
+          def BBA():
+            bba1
+            bba2
+            bba3
+        C
+        D
+        def spam(x,y):
+             def eggs(z):
+                 pass
+        '''
+
+
+        indentStack = [1]
+        stmt = Forward()
+
+        identifier = Word(alphas, alphanums)
+        funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":")
+        func_body = indentedBlock(stmt, indentStack)
+        funcDef = Group( funcDecl + func_body )
+
+        rvalue = Forward()
+        funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")")
+        rvalue << (funcCall | identifier | Word(nums))
+        assignment = Group(identifier + "=" + rvalue)
+        stmt << ( funcDef | assignment | identifier )
+
+        module_body = OneOrMore(stmt)
+
+        parseTree = module_body.parseString(data)
+        parseTree.pprint()
+    prints::
+        [['def',
+          'A',
+          ['(', 'z', ')'],
+          ':',
+          [['A1'], [['B', '=', '100']], [['G', '=', 'A2']], ['A2'], ['A3']]],
+         'B',
+         ['def',
+          'BB',
+          ['(', 'a', 'b', 'c', ')'],
+          ':',
+          [['BB1'], [['def', 'BBA', ['(', ')'], ':', [['bba1'], ['bba2'], ['bba3']]]]]],
+         'C',
+         'D',
+         ['def',
+          'spam',
+          ['(', 'x', 'y', ')'],
+          ':',
+          [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] 
+    """
+    def checkPeerIndent(s,l,t):
+        if l >= len(s): return
+        curCol = col(l,s)
+        if curCol != indentStack[-1]:
+            if curCol > indentStack[-1]:
+                raise ParseFatalException(s,l,"illegal nesting")
+            raise ParseException(s,l,"not a peer entry")
+
+    def checkSubIndent(s,l,t):
+        curCol = col(l,s)
+        if curCol > indentStack[-1]:
+            indentStack.append( curCol )
+        else:
+            raise ParseException(s,l,"not a subentry")
+
+    def checkUnindent(s,l,t):
+        if l >= len(s): return
+        curCol = col(l,s)
+        if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]):
+            raise ParseException(s,l,"not an unindent")
+        indentStack.pop()
+
+    NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress())
+    INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT')
+    PEER   = Empty().setParseAction(checkPeerIndent).setName('')
+    UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT')
+    if indent:
+        smExpr = Group( Optional(NL) +
+            #~ FollowedBy(blockStatementExpr) +
+            INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT)
+    else:
+        smExpr = Group( Optional(NL) +
+            (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) )
+    blockStatementExpr.ignore(_bslash + LineEnd())
+    return smExpr.setName('indented block')
+
+alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
+punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
+
+anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag'))
+_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\''))
+commonHTMLEntity = Regex('&(?P' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity")
+def replaceHTMLEntity(t):
+    """Helper parser action to replace common HTML entities with their special characters"""
+    return _htmlEntityMap.get(t.entity)
+
+# it's easy to get these comment structures wrong - they're very common, so may as well make them available
+cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment")
+"Comment of the form C{/* ... */}"
+
+htmlComment = Regex(r"").setName("HTML comment")
+"Comment of the form C{}"
+
+restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line")
+dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment")
+"Comment of the form C{// ... (to end of line)}"
+
+cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment")
+"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}"
+
+javaStyleComment = cppStyleComment
+"Same as C{L{cppStyleComment}}"
+
+pythonStyleComment = Regex(r"#.*").setName("Python style comment")
+"Comment of the form C{# ... (to end of line)}"
+
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
+                                  Optional( Word(" \t") +
+                                            ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
+commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
+"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas.
+   This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}."""
+
+# some other useful expressions - using lower-case class name since we are really using this as a namespace
+class pyparsing_common:
+    """
+    Here are some common low-level expressions that may be useful in jump-starting parser development:
+     - numeric forms (L{integers}, L{reals}, L{scientific notation})
+     - common L{programming identifiers}
+     - network addresses (L{MAC}, L{IPv4}, L{IPv6})
+     - ISO8601 L{dates} and L{datetime}
+     - L{UUID}
+     - L{comma-separated list}
+    Parse actions:
+     - C{L{convertToInteger}}
+     - C{L{convertToFloat}}
+     - C{L{convertToDate}}
+     - C{L{convertToDatetime}}
+     - C{L{stripHTMLTags}}
+     - C{L{upcaseTokens}}
+     - C{L{downcaseTokens}}
+
+    Example::
+        pyparsing_common.number.runTests('''
+            # any int or real number, returned as the appropriate type
+            100
+            -100
+            +100
+            3.14159
+            6.02e23
+            1e-12
+            ''')
+
+        pyparsing_common.fnumber.runTests('''
+            # any int or real number, returned as float
+            100
+            -100
+            +100
+            3.14159
+            6.02e23
+            1e-12
+            ''')
+
+        pyparsing_common.hex_integer.runTests('''
+            # hex numbers
+            100
+            FF
+            ''')
+
+        pyparsing_common.fraction.runTests('''
+            # fractions
+            1/2
+            -3/4
+            ''')
+
+        pyparsing_common.mixed_integer.runTests('''
+            # mixed fractions
+            1
+            1/2
+            -3/4
+            1-3/4
+            ''')
+
+        import uuid
+        pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+        pyparsing_common.uuid.runTests('''
+            # uuid
+            12345678-1234-5678-1234-567812345678
+            ''')
+    prints::
+        # any int or real number, returned as the appropriate type
+        100
+        [100]
+
+        -100
+        [-100]
+
+        +100
+        [100]
+
+        3.14159
+        [3.14159]
+
+        6.02e23
+        [6.02e+23]
+
+        1e-12
+        [1e-12]
+
+        # any int or real number, returned as float
+        100
+        [100.0]
+
+        -100
+        [-100.0]
+
+        +100
+        [100.0]
+
+        3.14159
+        [3.14159]
+
+        6.02e23
+        [6.02e+23]
+
+        1e-12
+        [1e-12]
+
+        # hex numbers
+        100
+        [256]
+
+        FF
+        [255]
+
+        # fractions
+        1/2
+        [0.5]
+
+        -3/4
+        [-0.75]
+
+        # mixed fractions
+        1
+        [1]
+
+        1/2
+        [0.5]
+
+        -3/4
+        [-0.75]
+
+        1-3/4
+        [1.75]
+
+        # uuid
+        12345678-1234-5678-1234-567812345678
+        [UUID('12345678-1234-5678-1234-567812345678')]
+    """
+
+    convertToInteger = tokenMap(int)
+    """
+    Parse action for converting parsed integers to Python int
+    """
+
+    convertToFloat = tokenMap(float)
+    """
+    Parse action for converting parsed numbers to Python float
+    """
+
+    integer = Word(nums).setName("integer").setParseAction(convertToInteger)
+    """expression that parses an unsigned integer, returns an int"""
+
+    hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16))
+    """expression that parses a hexadecimal integer, returns an int"""
+
+    signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger)
+    """expression that parses an integer with optional leading sign, returns an int"""
+
+    fraction = (signed_integer().setParseAction(convertToFloat) + '/' + signed_integer().setParseAction(convertToFloat)).setName("fraction")
+    """fractional expression of an integer divided by an integer, returns a float"""
+    fraction.addParseAction(lambda t: t[0]/t[-1])
+
+    mixed_integer = (fraction | signed_integer + Optional(Optional('-').suppress() + fraction)).setName("fraction or mixed integer-fraction")
+    """mixed integer of the form 'integer - fraction', with optional leading integer, returns float"""
+    mixed_integer.addParseAction(sum)
+
+    real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat)
+    """expression that parses a floating point number and returns a float"""
+
+    sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat)
+    """expression that parses a floating point number with optional scientific notation and returns a float"""
+
+    # streamlining this expression makes the docs nicer-looking
+    number = (sci_real | real | signed_integer).streamline()
+    """any numeric expression, returns the corresponding Python type"""
+
+    fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat)
+    """any int or real number, returned as float"""
+    
+    identifier = Word(alphas+'_', alphanums+'_').setName("identifier")
+    """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')"""
+    
+    ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address")
+    "IPv4 address (C{0.0.0.0 - 255.255.255.255})"
+
+    _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer")
+    _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address")
+    _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address")
+    _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8)
+    _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address")
+    ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address")
+    "IPv6 address (long, short, or mixed form)"
+    
+    mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address")
+    "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)"
+
+    @staticmethod
+    def convertToDate(fmt="%Y-%m-%d"):
+        """
+        Helper to create a parse action for converting parsed date string to Python datetime.date
+
+        Params -
+         - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"})
+
+        Example::
+            date_expr = pyparsing_common.iso8601_date.copy()
+            date_expr.setParseAction(pyparsing_common.convertToDate())
+            print(date_expr.parseString("1999-12-31"))
+        prints::
+            [datetime.date(1999, 12, 31)]
+        """
+        def cvt_fn(s,l,t):
+            try:
+                return datetime.strptime(t[0], fmt).date()
+            except ValueError as ve:
+                raise ParseException(s, l, str(ve))
+        return cvt_fn
+
+    @staticmethod
+    def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"):
+        """
+        Helper to create a parse action for converting parsed datetime string to Python datetime.datetime
+
+        Params -
+         - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"})
+
+        Example::
+            dt_expr = pyparsing_common.iso8601_datetime.copy()
+            dt_expr.setParseAction(pyparsing_common.convertToDatetime())
+            print(dt_expr.parseString("1999-12-31T23:59:59.999"))
+        prints::
+            [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)]
+        """
+        def cvt_fn(s,l,t):
+            try:
+                return datetime.strptime(t[0], fmt)
+            except ValueError as ve:
+                raise ParseException(s, l, str(ve))
+        return cvt_fn
+
+    iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date")
+    "ISO8601 date (C{yyyy-mm-dd})"
+
+    iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime")
+    "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}"
+
+    uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID")
+    "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})"
+
+    _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress()
+    @staticmethod
+    def stripHTMLTags(s, l, tokens):
+        """
+        Parse action to remove HTML tags from web page HTML source
+
+        Example::
+            # strip HTML links from normal text 
+            text = 'More info at the pyparsing wiki page'
+            td,td_end = makeHTMLTags("TD")
+            table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end
+            
+            print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page'
+        """
+        return pyparsing_common._html_stripper.transformString(tokens[0])
+
+    _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') 
+                                        + Optional( White(" \t") ) ) ).streamline().setName("commaItem")
+    comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list")
+    """Predefined expression of 1 or more printable words or quoted strings, separated by commas."""
+
+    upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper()))
+    """Parse action to convert tokens to upper case."""
+
+    downcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).lower()))
+    """Parse action to convert tokens to lower case."""
+
+
+if __name__ == "__main__":
+
+    selectToken    = CaselessLiteral("select")
+    fromToken      = CaselessLiteral("from")
+
+    ident          = Word(alphas, alphanums + "_$")
+
+    columnName     = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+    columnNameList = Group(delimitedList(columnName)).setName("columns")
+    columnSpec     = ('*' | columnNameList)
+
+    tableName      = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens)
+    tableNameList  = Group(delimitedList(tableName)).setName("tables")
+    
+    simpleSQL      = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables")
+
+    # demo runTests method, including embedded comments in test string
+    simpleSQL.runTests("""
+        # '*' as column list and dotted table name
+        select * from SYS.XYZZY
+
+        # caseless match on "SELECT", and casts back to "select"
+        SELECT * from XYZZY, ABC
+
+        # list of column names, and mixed case SELECT keyword
+        Select AA,BB,CC from Sys.dual
+
+        # multiple tables
+        Select A, B, C from Sys.dual, Table2
+
+        # invalid SELECT keyword - should fail
+        Xelect A, B, C from Sys.dual
+
+        # incomplete command - should fail
+        Select
+
+        # invalid column name - should fail
+        Select ^^^ frox Sys.dual
+
+        """)
+
+    pyparsing_common.number.runTests("""
+        100
+        -100
+        +100
+        3.14159
+        6.02e23
+        1e-12
+        """)
+
+    # any int or real number, returned as float
+    pyparsing_common.fnumber.runTests("""
+        100
+        -100
+        +100
+        3.14159
+        6.02e23
+        1e-12
+        """)
+
+    pyparsing_common.hex_integer.runTests("""
+        100
+        FF
+        """)
+
+    import uuid
+    pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID))
+    pyparsing_common.uuid.runTests("""
+        12345678-1234-5678-1234-567812345678
+        """)
diff --git a/ubuntu/venv/setuptools/_vendor/six.py b/ubuntu/venv/setuptools/_vendor/six.py
new file mode 100644
index 0000000..190c023
--- /dev/null
+++ b/ubuntu/venv/setuptools/_vendor/six.py
@@ -0,0 +1,868 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2015 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+import functools
+import itertools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson "
+__version__ = "1.10.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)  # Invokes __set__.
+        try:
+            # This is a bit ugly, but it avoids running this again by
+            # removing this descriptor.
+            delattr(obj.__class__, self.name)
+        except AttributeError:
+            pass
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("URLopener", "urllib", "urllib.request"),
+    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    def create_unbound_method(func, cls):
+        return func
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+
+    viewkeys = operator.methodcaller("keys")
+
+    viewvalues = operator.methodcaller("values")
+
+    viewitems = operator.methodcaller("items")
+else:
+    def iterkeys(d, **kw):
+        return d.iterkeys(**kw)
+
+    def itervalues(d, **kw):
+        return d.itervalues(**kw)
+
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+    def iterlists(d, **kw):
+        return d.iterlists(**kw)
+
+    viewkeys = operator.methodcaller("viewkeys")
+
+    viewvalues = operator.methodcaller("viewvalues")
+
+    viewitems = operator.methodcaller("viewitems")
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+
+    def u(s):
+        return s
+    unichr = chr
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+    _assertCountEqual = "assertCountEqual"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+
+    def byte2int(bs):
+        return ord(bs[0])
+
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    iterbytes = functools.partial(itertools.imap, ord)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+    _assertCountEqual = "assertItemsEqual"
+    _assertRaisesRegex = "assertRaisesRegexp"
+    _assertRegex = "assertRegexpMatches"
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+def assertCountEqual(self, *args, **kwargs):
+    return getattr(self, _assertCountEqual)(*args, **kwargs)
+
+
+def assertRaisesRegex(self, *args, **kwargs):
+    return getattr(self, _assertRaisesRegex)(*args, **kwargs)
+
+
+def assertRegex(self, *args, **kwargs):
+    return getattr(self, _assertRegex)(*args, **kwargs)
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+    def reraise(tp, value, tb=None):
+        if value is None:
+            value = tp()
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+if sys.version_info[:2] == (3, 2):
+    exec_("""def raise_from(value, from_value):
+    if from_value is None:
+        raise value
+    raise value from from_value
+""")
+elif sys.version_info[:2] > (3, 2):
+    exec_("""def raise_from(value, from_value):
+    raise value from from_value
+""")
+else:
+    def raise_from(value, from_value):
+        raise value
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+if sys.version_info[:2] < (3, 3):
+    _print = print_
+
+    def print_(*args, **kwargs):
+        fp = kwargs.get("file", sys.stdout)
+        flush = kwargs.pop("flush", False)
+        _print(*args, **kwargs)
+        if flush and fp is not None:
+            fp.flush()
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        def wrapper(f):
+            f = functools.wraps(wrapped, assigned, updated)(f)
+            f.__wrapped__ = wrapped
+            return f
+        return wrapper
+else:
+    wraps = functools.wraps
+
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(meta):
+
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+
+def python_2_unicode_compatible(klass):
+    """
+    A decorator that defines __unicode__ and __str__ methods under Python 2.
+    Under Python 3 it does nothing.
+
+    To support Python 2 and 3 with a single code base, define a __str__ method
+    returning text and apply this decorator to the class.
+    """
+    if PY2:
+        if '__str__' not in klass.__dict__:
+            raise ValueError("@python_2_unicode_compatible cannot be applied "
+                             "to %s because it doesn't define __str__()." %
+                             klass.__name__)
+        klass.__unicode__ = klass.__str__
+        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
+    return klass
+
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+                importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)
diff --git a/ubuntu/venv/setuptools/archive_util.py b/ubuntu/venv/setuptools/archive_util.py
new file mode 100644
index 0000000..8143604
--- /dev/null
+++ b/ubuntu/venv/setuptools/archive_util.py
@@ -0,0 +1,173 @@
+"""Utilities for extracting common archive formats"""
+
+import zipfile
+import tarfile
+import os
+import shutil
+import posixpath
+import contextlib
+from distutils.errors import DistutilsError
+
+from pkg_resources import ensure_directory
+
+__all__ = [
+    "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter",
+    "UnrecognizedFormat", "extraction_drivers", "unpack_directory",
+]
+
+
+class UnrecognizedFormat(DistutilsError):
+    """Couldn't recognize the archive type"""
+
+
+def default_filter(src, dst):
+    """The default progress/filter callback; returns True for all files"""
+    return dst
+
+
+def unpack_archive(filename, extract_dir, progress_filter=default_filter,
+        drivers=None):
+    """Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat``
+
+    `progress_filter` is a function taking two arguments: a source path
+    internal to the archive ('/'-separated), and a filesystem path where it
+    will be extracted.  The callback must return the desired extract path
+    (which may be the same as the one passed in), or else ``None`` to skip
+    that file or directory.  The callback can thus be used to report on the
+    progress of the extraction, as well as to filter the items extracted or
+    alter their extraction paths.
+
+    `drivers`, if supplied, must be a non-empty sequence of functions with the
+    same signature as this function (minus the `drivers` argument), that raise
+    ``UnrecognizedFormat`` if they do not support extracting the designated
+    archive type.  The `drivers` are tried in sequence until one is found that
+    does not raise an error, or until all are exhausted (in which case
+    ``UnrecognizedFormat`` is raised).  If you do not supply a sequence of
+    drivers, the module's ``extraction_drivers`` constant will be used, which
+    means that ``unpack_zipfile`` and ``unpack_tarfile`` will be tried, in that
+    order.
+    """
+    for driver in drivers or extraction_drivers:
+        try:
+            driver(filename, extract_dir, progress_filter)
+        except UnrecognizedFormat:
+            continue
+        else:
+            return
+    else:
+        raise UnrecognizedFormat(
+            "Not a recognized archive type: %s" % filename
+        )
+
+
+def unpack_directory(filename, extract_dir, progress_filter=default_filter):
+    """"Unpack" a directory, using the same interface as for archives
+
+    Raises ``UnrecognizedFormat`` if `filename` is not a directory
+    """
+    if not os.path.isdir(filename):
+        raise UnrecognizedFormat("%s is not a directory" % filename)
+
+    paths = {
+        filename: ('', extract_dir),
+    }
+    for base, dirs, files in os.walk(filename):
+        src, dst = paths[base]
+        for d in dirs:
+            paths[os.path.join(base, d)] = src + d + '/', os.path.join(dst, d)
+        for f in files:
+            target = os.path.join(dst, f)
+            target = progress_filter(src + f, target)
+            if not target:
+                # skip non-files
+                continue
+            ensure_directory(target)
+            f = os.path.join(base, f)
+            shutil.copyfile(f, target)
+            shutil.copystat(f, target)
+
+
+def unpack_zipfile(filename, extract_dir, progress_filter=default_filter):
+    """Unpack zip `filename` to `extract_dir`
+
+    Raises ``UnrecognizedFormat`` if `filename` is not a zipfile (as determined
+    by ``zipfile.is_zipfile()``).  See ``unpack_archive()`` for an explanation
+    of the `progress_filter` argument.
+    """
+
+    if not zipfile.is_zipfile(filename):
+        raise UnrecognizedFormat("%s is not a zip file" % (filename,))
+
+    with zipfile.ZipFile(filename) as z:
+        for info in z.infolist():
+            name = info.filename
+
+            # don't extract absolute paths or ones with .. in them
+            if name.startswith('/') or '..' in name.split('/'):
+                continue
+
+            target = os.path.join(extract_dir, *name.split('/'))
+            target = progress_filter(name, target)
+            if not target:
+                continue
+            if name.endswith('/'):
+                # directory
+                ensure_directory(target)
+            else:
+                # file
+                ensure_directory(target)
+                data = z.read(info.filename)
+                with open(target, 'wb') as f:
+                    f.write(data)
+            unix_attributes = info.external_attr >> 16
+            if unix_attributes:
+                os.chmod(target, unix_attributes)
+
+
+def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):
+    """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir`
+
+    Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined
+    by ``tarfile.open()``).  See ``unpack_archive()`` for an explanation
+    of the `progress_filter` argument.
+    """
+    try:
+        tarobj = tarfile.open(filename)
+    except tarfile.TarError:
+        raise UnrecognizedFormat(
+            "%s is not a compressed or uncompressed tar file" % (filename,)
+        )
+    with contextlib.closing(tarobj):
+        # don't do any chowning!
+        tarobj.chown = lambda *args: None
+        for member in tarobj:
+            name = member.name
+            # don't extract absolute paths or ones with .. in them
+            if not name.startswith('/') and '..' not in name.split('/'):
+                prelim_dst = os.path.join(extract_dir, *name.split('/'))
+
+                # resolve any links and to extract the link targets as normal
+                # files
+                while member is not None and (member.islnk() or member.issym()):
+                    linkpath = member.linkname
+                    if member.issym():
+                        base = posixpath.dirname(member.name)
+                        linkpath = posixpath.join(base, linkpath)
+                        linkpath = posixpath.normpath(linkpath)
+                    member = tarobj._getmember(linkpath)
+
+                if member is not None and (member.isfile() or member.isdir()):
+                    final_dst = progress_filter(name, prelim_dst)
+                    if final_dst:
+                        if final_dst.endswith(os.sep):
+                            final_dst = final_dst[:-1]
+                        try:
+                            # XXX Ugh
+                            tarobj._extract_member(member, final_dst)
+                        except tarfile.ExtractError:
+                            # chown/chmod/mkfifo/mknode/makedev failed
+                            pass
+        return True
+
+
+extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile
diff --git a/ubuntu/venv/setuptools/build_meta.py b/ubuntu/venv/setuptools/build_meta.py
new file mode 100644
index 0000000..10c4b52
--- /dev/null
+++ b/ubuntu/venv/setuptools/build_meta.py
@@ -0,0 +1,257 @@
+"""A PEP 517 interface to setuptools
+
+Previously, when a user or a command line tool (let's call it a "frontend")
+needed to make a request of setuptools to take a certain action, for
+example, generating a list of installation requirements, the frontend would
+would call "setup.py egg_info" or "setup.py bdist_wheel" on the command line.
+
+PEP 517 defines a different method of interfacing with setuptools. Rather
+than calling "setup.py" directly, the frontend should:
+
+  1. Set the current directory to the directory with a setup.py file
+  2. Import this module into a safe python interpreter (one in which
+     setuptools can potentially set global variables or crash hard).
+  3. Call one of the functions defined in PEP 517.
+
+What each function does is defined in PEP 517. However, here is a "casual"
+definition of the functions (this definition should not be relied on for
+bug reports or API stability):
+
+  - `build_wheel`: build a wheel in the folder and return the basename
+  - `get_requires_for_build_wheel`: get the `setup_requires` to build
+  - `prepare_metadata_for_build_wheel`: get the `install_requires`
+  - `build_sdist`: build an sdist in the folder and return the basename
+  - `get_requires_for_build_sdist`: get the `setup_requires` to build
+
+Again, this is not a formal definition! Just a "taste" of the module.
+"""
+
+import io
+import os
+import sys
+import tokenize
+import shutil
+import contextlib
+
+import setuptools
+import distutils
+from setuptools.py31compat import TemporaryDirectory
+
+from pkg_resources import parse_requirements
+from pkg_resources.py31compat import makedirs
+
+__all__ = ['get_requires_for_build_sdist',
+           'get_requires_for_build_wheel',
+           'prepare_metadata_for_build_wheel',
+           'build_wheel',
+           'build_sdist',
+           '__legacy__',
+           'SetupRequirementsError']
+
+class SetupRequirementsError(BaseException):
+    def __init__(self, specifiers):
+        self.specifiers = specifiers
+
+
+class Distribution(setuptools.dist.Distribution):
+    def fetch_build_eggs(self, specifiers):
+        specifier_list = list(map(str, parse_requirements(specifiers)))
+
+        raise SetupRequirementsError(specifier_list)
+
+    @classmethod
+    @contextlib.contextmanager
+    def patch(cls):
+        """
+        Replace
+        distutils.dist.Distribution with this class
+        for the duration of this context.
+        """
+        orig = distutils.core.Distribution
+        distutils.core.Distribution = cls
+        try:
+            yield
+        finally:
+            distutils.core.Distribution = orig
+
+
+def _to_str(s):
+    """
+    Convert a filename to a string (on Python 2, explicitly
+    a byte string, not Unicode) as distutils checks for the
+    exact type str.
+    """
+    if sys.version_info[0] == 2 and not isinstance(s, str):
+        # Assume it's Unicode, as that's what the PEP says
+        # should be provided.
+        return s.encode(sys.getfilesystemencoding())
+    return s
+
+
+def _get_immediate_subdirectories(a_dir):
+    return [name for name in os.listdir(a_dir)
+            if os.path.isdir(os.path.join(a_dir, name))]
+
+
+def _file_with_extension(directory, extension):
+    matching = (
+        f for f in os.listdir(directory)
+        if f.endswith(extension)
+    )
+    file, = matching
+    return file
+
+
+def _open_setup_script(setup_script):
+    if not os.path.exists(setup_script):
+        # Supply a default setup.py
+        return io.StringIO(u"from setuptools import setup; setup()")
+
+    return getattr(tokenize, 'open', open)(setup_script)
+
+
+class _BuildMetaBackend(object):
+
+    def _fix_config(self, config_settings):
+        config_settings = config_settings or {}
+        config_settings.setdefault('--global-option', [])
+        return config_settings
+
+    def _get_build_requires(self, config_settings, requirements):
+        config_settings = self._fix_config(config_settings)
+
+        sys.argv = sys.argv[:1] + ['egg_info'] + \
+            config_settings["--global-option"]
+        try:
+            with Distribution.patch():
+                self.run_setup()
+        except SetupRequirementsError as e:
+            requirements += e.specifiers
+
+        return requirements
+
+    def run_setup(self, setup_script='setup.py'):
+        # Note that we can reuse our build directory between calls
+        # Correctness comes first, then optimization later
+        __file__ = setup_script
+        __name__ = '__main__'
+
+        with _open_setup_script(__file__) as f:
+            code = f.read().replace(r'\r\n', r'\n')
+
+        exec(compile(code, __file__, 'exec'), locals())
+
+    def get_requires_for_build_wheel(self, config_settings=None):
+        config_settings = self._fix_config(config_settings)
+        return self._get_build_requires(config_settings, requirements=['wheel'])
+
+    def get_requires_for_build_sdist(self, config_settings=None):
+        config_settings = self._fix_config(config_settings)
+        return self._get_build_requires(config_settings, requirements=[])
+
+    def prepare_metadata_for_build_wheel(self, metadata_directory,
+                                         config_settings=None):
+        sys.argv = sys.argv[:1] + ['dist_info', '--egg-base',
+                                   _to_str(metadata_directory)]
+        self.run_setup()
+
+        dist_info_directory = metadata_directory
+        while True:
+            dist_infos = [f for f in os.listdir(dist_info_directory)
+                          if f.endswith('.dist-info')]
+
+            if (len(dist_infos) == 0 and
+                len(_get_immediate_subdirectories(dist_info_directory)) == 1):
+
+                dist_info_directory = os.path.join(
+                    dist_info_directory, os.listdir(dist_info_directory)[0])
+                continue
+
+            assert len(dist_infos) == 1
+            break
+
+        # PEP 517 requires that the .dist-info directory be placed in the
+        # metadata_directory. To comply, we MUST copy the directory to the root
+        if dist_info_directory != metadata_directory:
+            shutil.move(
+                os.path.join(dist_info_directory, dist_infos[0]),
+                metadata_directory)
+            shutil.rmtree(dist_info_directory, ignore_errors=True)
+
+        return dist_infos[0]
+
+    def _build_with_temp_dir(self, setup_command, result_extension,
+                             result_directory, config_settings):
+        config_settings = self._fix_config(config_settings)
+        result_directory = os.path.abspath(result_directory)
+
+        # Build in a temporary directory, then copy to the target.
+        makedirs(result_directory, exist_ok=True)
+        with TemporaryDirectory(dir=result_directory) as tmp_dist_dir:
+            sys.argv = (sys.argv[:1] + setup_command +
+                        ['--dist-dir', tmp_dist_dir] +
+                        config_settings["--global-option"])
+            self.run_setup()
+
+            result_basename = _file_with_extension(tmp_dist_dir, result_extension)
+            result_path = os.path.join(result_directory, result_basename)
+            if os.path.exists(result_path):
+                # os.rename will fail overwriting on non-Unix.
+                os.remove(result_path)
+            os.rename(os.path.join(tmp_dist_dir, result_basename), result_path)
+
+        return result_basename
+
+
+    def build_wheel(self, wheel_directory, config_settings=None,
+                    metadata_directory=None):
+        return self._build_with_temp_dir(['bdist_wheel'], '.whl',
+                                         wheel_directory, config_settings)
+
+    def build_sdist(self, sdist_directory, config_settings=None):
+        return self._build_with_temp_dir(['sdist', '--formats', 'gztar'],
+                                         '.tar.gz', sdist_directory,
+                                         config_settings)
+
+
+class _BuildMetaLegacyBackend(_BuildMetaBackend):
+    """Compatibility backend for setuptools
+
+    This is a version of setuptools.build_meta that endeavors to maintain backwards
+    compatibility with pre-PEP 517 modes of invocation. It exists as a temporary
+    bridge between the old packaging mechanism and the new packaging mechanism,
+    and will eventually be removed.
+    """
+    def run_setup(self, setup_script='setup.py'):
+        # In order to maintain compatibility with scripts assuming that
+        # the setup.py script is in a directory on the PYTHONPATH, inject
+        # '' into sys.path. (pypa/setuptools#1642)
+        sys_path = list(sys.path)           # Save the original path
+
+        script_dir = os.path.dirname(os.path.abspath(setup_script))
+        if script_dir not in sys.path:
+            sys.path.insert(0, script_dir)
+
+        try:
+            super(_BuildMetaLegacyBackend,
+                  self).run_setup(setup_script=setup_script)
+        finally:
+            # While PEP 517 frontends should be calling each hook in a fresh
+            # subprocess according to the standard (and thus it should not be
+            # strictly necessary to restore the old sys.path), we'll restore
+            # the original path so that the path manipulation does not persist
+            # within the hook after run_setup is called.
+            sys.path[:] = sys_path
+
+# The primary backend
+_BACKEND = _BuildMetaBackend()
+
+get_requires_for_build_wheel = _BACKEND.get_requires_for_build_wheel
+get_requires_for_build_sdist = _BACKEND.get_requires_for_build_sdist
+prepare_metadata_for_build_wheel = _BACKEND.prepare_metadata_for_build_wheel
+build_wheel = _BACKEND.build_wheel
+build_sdist = _BACKEND.build_sdist
+
+
+# The legacy backend
+__legacy__ = _BuildMetaLegacyBackend()
diff --git a/ubuntu/venv/setuptools/command/__init__.py b/ubuntu/venv/setuptools/command/__init__.py
new file mode 100644
index 0000000..743f558
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/__init__.py
@@ -0,0 +1,17 @@
+__all__ = [
+    'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop',
+    'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts',
+    'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts',
+    'bdist_wininst', 'upload_docs', 'build_clib', 'dist_info',
+]
+
+from distutils.command.bdist import bdist
+import sys
+
+from setuptools.command import install_scripts
+
+if 'egg' not in bdist.format_commands:
+    bdist.format_command['egg'] = ('bdist_egg', "Python .egg file")
+    bdist.format_commands.append('egg')
+
+del bdist, sys
diff --git a/ubuntu/venv/setuptools/command/alias.py b/ubuntu/venv/setuptools/command/alias.py
new file mode 100644
index 0000000..4532b1c
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/alias.py
@@ -0,0 +1,80 @@
+from distutils.errors import DistutilsOptionError
+
+from setuptools.extern.six.moves import map
+
+from setuptools.command.setopt import edit_config, option_base, config_file
+
+
+def shquote(arg):
+    """Quote an argument for later parsing by shlex.split()"""
+    for c in '"', "'", "\\", "#":
+        if c in arg:
+            return repr(arg)
+    if arg.split() != [arg]:
+        return repr(arg)
+    return arg
+
+
+class alias(option_base):
+    """Define a shortcut that invokes one or more commands"""
+
+    description = "define a shortcut to invoke one or more commands"
+    command_consumes_arguments = True
+
+    user_options = [
+        ('remove', 'r', 'remove (unset) the alias'),
+    ] + option_base.user_options
+
+    boolean_options = option_base.boolean_options + ['remove']
+
+    def initialize_options(self):
+        option_base.initialize_options(self)
+        self.args = None
+        self.remove = None
+
+    def finalize_options(self):
+        option_base.finalize_options(self)
+        if self.remove and len(self.args) != 1:
+            raise DistutilsOptionError(
+                "Must specify exactly one argument (the alias name) when "
+                "using --remove"
+            )
+
+    def run(self):
+        aliases = self.distribution.get_option_dict('aliases')
+
+        if not self.args:
+            print("Command Aliases")
+            print("---------------")
+            for alias in aliases:
+                print("setup.py alias", format_alias(alias, aliases))
+            return
+
+        elif len(self.args) == 1:
+            alias, = self.args
+            if self.remove:
+                command = None
+            elif alias in aliases:
+                print("setup.py alias", format_alias(alias, aliases))
+                return
+            else:
+                print("No alias definition found for %r" % alias)
+                return
+        else:
+            alias = self.args[0]
+            command = ' '.join(map(shquote, self.args[1:]))
+
+        edit_config(self.filename, {'aliases': {alias: command}}, self.dry_run)
+
+
+def format_alias(name, aliases):
+    source, command = aliases[name]
+    if source == config_file('global'):
+        source = '--global-config '
+    elif source == config_file('user'):
+        source = '--user-config '
+    elif source == config_file('local'):
+        source = ''
+    else:
+        source = '--filename=%r' % source
+    return source + name + ' ' + command
diff --git a/ubuntu/venv/setuptools/command/bdist_egg.py b/ubuntu/venv/setuptools/command/bdist_egg.py
new file mode 100644
index 0000000..98470f1
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/bdist_egg.py
@@ -0,0 +1,502 @@
+"""setuptools.command.bdist_egg
+
+Build .egg distributions"""
+
+from distutils.errors import DistutilsSetupError
+from distutils.dir_util import remove_tree, mkpath
+from distutils import log
+from types import CodeType
+import sys
+import os
+import re
+import textwrap
+import marshal
+
+from setuptools.extern import six
+
+from pkg_resources import get_build_platform, Distribution, ensure_directory
+from pkg_resources import EntryPoint
+from setuptools.extension import Library
+from setuptools import Command
+
+try:
+    # Python 2.7 or >=3.2
+    from sysconfig import get_path, get_python_version
+
+    def _get_purelib():
+        return get_path("purelib")
+except ImportError:
+    from distutils.sysconfig import get_python_lib, get_python_version
+
+    def _get_purelib():
+        return get_python_lib(False)
+
+
+def strip_module(filename):
+    if '.' in filename:
+        filename = os.path.splitext(filename)[0]
+    if filename.endswith('module'):
+        filename = filename[:-6]
+    return filename
+
+
+def sorted_walk(dir):
+    """Do os.walk in a reproducible way,
+    independent of indeterministic filesystem readdir order
+    """
+    for base, dirs, files in os.walk(dir):
+        dirs.sort()
+        files.sort()
+        yield base, dirs, files
+
+
+def write_stub(resource, pyfile):
+    _stub_template = textwrap.dedent("""
+        def __bootstrap__():
+            global __bootstrap__, __loader__, __file__
+            import sys, pkg_resources, imp
+            __file__ = pkg_resources.resource_filename(__name__, %r)
+            __loader__ = None; del __bootstrap__, __loader__
+            imp.load_dynamic(__name__,__file__)
+        __bootstrap__()
+        """).lstrip()
+    with open(pyfile, 'w') as f:
+        f.write(_stub_template % resource)
+
+
+class bdist_egg(Command):
+    description = "create an \"egg\" distribution"
+
+    user_options = [
+        ('bdist-dir=', 'b',
+         "temporary directory for creating the distribution"),
+        ('plat-name=', 'p', "platform name to embed in generated filenames "
+                            "(default: %s)" % get_build_platform()),
+        ('exclude-source-files', None,
+         "remove all .py files from the generated egg"),
+        ('keep-temp', 'k',
+         "keep the pseudo-installation tree around after " +
+         "creating the distribution archive"),
+        ('dist-dir=', 'd',
+         "directory to put final built distributions in"),
+        ('skip-build', None,
+         "skip rebuilding everything (for testing/debugging)"),
+    ]
+
+    boolean_options = [
+        'keep-temp', 'skip-build', 'exclude-source-files'
+    ]
+
+    def initialize_options(self):
+        self.bdist_dir = None
+        self.plat_name = None
+        self.keep_temp = 0
+        self.dist_dir = None
+        self.skip_build = 0
+        self.egg_output = None
+        self.exclude_source_files = None
+
+    def finalize_options(self):
+        ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info")
+        self.egg_info = ei_cmd.egg_info
+
+        if self.bdist_dir is None:
+            bdist_base = self.get_finalized_command('bdist').bdist_base
+            self.bdist_dir = os.path.join(bdist_base, 'egg')
+
+        if self.plat_name is None:
+            self.plat_name = get_build_platform()
+
+        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))
+
+        if self.egg_output is None:
+
+            # Compute filename of the output egg
+            basename = Distribution(
+                None, None, ei_cmd.egg_name, ei_cmd.egg_version,
+                get_python_version(),
+                self.distribution.has_ext_modules() and self.plat_name
+            ).egg_name()
+
+            self.egg_output = os.path.join(self.dist_dir, basename + '.egg')
+
+    def do_install_data(self):
+        # Hack for packages that install data to install's --install-lib
+        self.get_finalized_command('install').install_lib = self.bdist_dir
+
+        site_packages = os.path.normcase(os.path.realpath(_get_purelib()))
+        old, self.distribution.data_files = self.distribution.data_files, []
+
+        for item in old:
+            if isinstance(item, tuple) and len(item) == 2:
+                if os.path.isabs(item[0]):
+                    realpath = os.path.realpath(item[0])
+                    normalized = os.path.normcase(realpath)
+                    if normalized == site_packages or normalized.startswith(
+                        site_packages + os.sep
+                    ):
+                        item = realpath[len(site_packages) + 1:], item[1]
+                        # XXX else: raise ???
+            self.distribution.data_files.append(item)
+
+        try:
+            log.info("installing package data to %s", self.bdist_dir)
+            self.call_command('install_data', force=0, root=None)
+        finally:
+            self.distribution.data_files = old
+
+    def get_outputs(self):
+        return [self.egg_output]
+
+    def call_command(self, cmdname, **kw):
+        """Invoke reinitialized command `cmdname` with keyword args"""
+        for dirname in INSTALL_DIRECTORY_ATTRS:
+            kw.setdefault(dirname, self.bdist_dir)
+        kw.setdefault('skip_build', self.skip_build)
+        kw.setdefault('dry_run', self.dry_run)
+        cmd = self.reinitialize_command(cmdname, **kw)
+        self.run_command(cmdname)
+        return cmd
+
+    def run(self):
+        # Generate metadata first
+        self.run_command("egg_info")
+        # We run install_lib before install_data, because some data hacks
+        # pull their data path from the install_lib command.
+        log.info("installing library code to %s", self.bdist_dir)
+        instcmd = self.get_finalized_command('install')
+        old_root = instcmd.root
+        instcmd.root = None
+        if self.distribution.has_c_libraries() and not self.skip_build:
+            self.run_command('build_clib')
+        cmd = self.call_command('install_lib', warn_dir=0)
+        instcmd.root = old_root
+
+        all_outputs, ext_outputs = self.get_ext_outputs()
+        self.stubs = []
+        to_compile = []
+        for (p, ext_name) in enumerate(ext_outputs):
+            filename, ext = os.path.splitext(ext_name)
+            pyfile = os.path.join(self.bdist_dir, strip_module(filename) +
+                                  '.py')
+            self.stubs.append(pyfile)
+            log.info("creating stub loader for %s", ext_name)
+            if not self.dry_run:
+                write_stub(os.path.basename(ext_name), pyfile)
+            to_compile.append(pyfile)
+            ext_outputs[p] = ext_name.replace(os.sep, '/')
+
+        if to_compile:
+            cmd.byte_compile(to_compile)
+        if self.distribution.data_files:
+            self.do_install_data()
+
+        # Make the EGG-INFO directory
+        archive_root = self.bdist_dir
+        egg_info = os.path.join(archive_root, 'EGG-INFO')
+        self.mkpath(egg_info)
+        if self.distribution.scripts:
+            script_dir = os.path.join(egg_info, 'scripts')
+            log.info("installing scripts to %s", script_dir)
+            self.call_command('install_scripts', install_dir=script_dir,
+                              no_ep=1)
+
+        self.copy_metadata_to(egg_info)
+        native_libs = os.path.join(egg_info, "native_libs.txt")
+        if all_outputs:
+            log.info("writing %s", native_libs)
+            if not self.dry_run:
+                ensure_directory(native_libs)
+                libs_file = open(native_libs, 'wt')
+                libs_file.write('\n'.join(all_outputs))
+                libs_file.write('\n')
+                libs_file.close()
+        elif os.path.isfile(native_libs):
+            log.info("removing %s", native_libs)
+            if not self.dry_run:
+                os.unlink(native_libs)
+
+        write_safety_flag(
+            os.path.join(archive_root, 'EGG-INFO'), self.zip_safe()
+        )
+
+        if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):
+            log.warn(
+                "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"
+                "Use the install_requires/extras_require setup() args instead."
+            )
+
+        if self.exclude_source_files:
+            self.zap_pyfiles()
+
+        # Make the archive
+        make_zipfile(self.egg_output, archive_root, verbose=self.verbose,
+                     dry_run=self.dry_run, mode=self.gen_header())
+        if not self.keep_temp:
+            remove_tree(self.bdist_dir, dry_run=self.dry_run)
+
+        # Add to 'Distribution.dist_files' so that the "upload" command works
+        getattr(self.distribution, 'dist_files', []).append(
+            ('bdist_egg', get_python_version(), self.egg_output))
+
+    def zap_pyfiles(self):
+        log.info("Removing .py files from temporary directory")
+        for base, dirs, files in walk_egg(self.bdist_dir):
+            for name in files:
+                path = os.path.join(base, name)
+
+                if name.endswith('.py'):
+                    log.debug("Deleting %s", path)
+                    os.unlink(path)
+
+                if base.endswith('__pycache__'):
+                    path_old = path
+
+                    pattern = r'(?P.+)\.(?P[^.]+)\.pyc'
+                    m = re.match(pattern, name)
+                    path_new = os.path.join(
+                        base, os.pardir, m.group('name') + '.pyc')
+                    log.info(
+                        "Renaming file from [%s] to [%s]"
+                        % (path_old, path_new))
+                    try:
+                        os.remove(path_new)
+                    except OSError:
+                        pass
+                    os.rename(path_old, path_new)
+
+    def zip_safe(self):
+        safe = getattr(self.distribution, 'zip_safe', None)
+        if safe is not None:
+            return safe
+        log.warn("zip_safe flag not set; analyzing archive contents...")
+        return analyze_egg(self.bdist_dir, self.stubs)
+
+    def gen_header(self):
+        epm = EntryPoint.parse_map(self.distribution.entry_points or '')
+        ep = epm.get('setuptools.installation', {}).get('eggsecutable')
+        if ep is None:
+            return 'w'  # not an eggsecutable, do it the usual way.
+
+        if not ep.attrs or ep.extras:
+            raise DistutilsSetupError(
+                "eggsecutable entry point (%r) cannot have 'extras' "
+                "or refer to a module" % (ep,)
+            )
+
+        pyver = '{}.{}'.format(*sys.version_info)
+        pkg = ep.module_name
+        full = '.'.join(ep.attrs)
+        base = ep.attrs[0]
+        basename = os.path.basename(self.egg_output)
+
+        header = (
+            "#!/bin/sh\n"
+            'if [ `basename $0` = "%(basename)s" ]\n'
+            'then exec python%(pyver)s -c "'
+            "import sys, os; sys.path.insert(0, os.path.abspath('$0')); "
+            "from %(pkg)s import %(base)s; sys.exit(%(full)s())"
+            '" "$@"\n'
+            'else\n'
+            '  echo $0 is not the correct name for this egg file.\n'
+            '  echo Please rename it back to %(basename)s and try again.\n'
+            '  exec false\n'
+            'fi\n'
+        ) % locals()
+
+        if not self.dry_run:
+            mkpath(os.path.dirname(self.egg_output), dry_run=self.dry_run)
+            f = open(self.egg_output, 'w')
+            f.write(header)
+            f.close()
+        return 'a'
+
+    def copy_metadata_to(self, target_dir):
+        "Copy metadata (egg info) to the target_dir"
+        # normalize the path (so that a forward-slash in egg_info will
+        # match using startswith below)
+        norm_egg_info = os.path.normpath(self.egg_info)
+        prefix = os.path.join(norm_egg_info, '')
+        for path in self.ei_cmd.filelist.files:
+            if path.startswith(prefix):
+                target = os.path.join(target_dir, path[len(prefix):])
+                ensure_directory(target)
+                self.copy_file(path, target)
+
+    def get_ext_outputs(self):
+        """Get a list of relative paths to C extensions in the output distro"""
+
+        all_outputs = []
+        ext_outputs = []
+
+        paths = {self.bdist_dir: ''}
+        for base, dirs, files in sorted_walk(self.bdist_dir):
+            for filename in files:
+                if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS:
+                    all_outputs.append(paths[base] + filename)
+            for filename in dirs:
+                paths[os.path.join(base, filename)] = (paths[base] +
+                                                       filename + '/')
+
+        if self.distribution.has_ext_modules():
+            build_cmd = self.get_finalized_command('build_ext')
+            for ext in build_cmd.extensions:
+                if isinstance(ext, Library):
+                    continue
+                fullname = build_cmd.get_ext_fullname(ext.name)
+                filename = build_cmd.get_ext_filename(fullname)
+                if not os.path.basename(filename).startswith('dl-'):
+                    if os.path.exists(os.path.join(self.bdist_dir, filename)):
+                        ext_outputs.append(filename)
+
+        return all_outputs, ext_outputs
+
+
+NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split())
+
+
+def walk_egg(egg_dir):
+    """Walk an unpacked egg's contents, skipping the metadata directory"""
+    walker = sorted_walk(egg_dir)
+    base, dirs, files = next(walker)
+    if 'EGG-INFO' in dirs:
+        dirs.remove('EGG-INFO')
+    yield base, dirs, files
+    for bdf in walker:
+        yield bdf
+
+
+def analyze_egg(egg_dir, stubs):
+    # check for existing flag in EGG-INFO
+    for flag, fn in safety_flags.items():
+        if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)):
+            return flag
+    if not can_scan():
+        return False
+    safe = True
+    for base, dirs, files in walk_egg(egg_dir):
+        for name in files:
+            if name.endswith('.py') or name.endswith('.pyw'):
+                continue
+            elif name.endswith('.pyc') or name.endswith('.pyo'):
+                # always scan, even if we already know we're not safe
+                safe = scan_module(egg_dir, base, name, stubs) and safe
+    return safe
+
+
+def write_safety_flag(egg_dir, safe):
+    # Write or remove zip safety flag file(s)
+    for flag, fn in safety_flags.items():
+        fn = os.path.join(egg_dir, fn)
+        if os.path.exists(fn):
+            if safe is None or bool(safe) != flag:
+                os.unlink(fn)
+        elif safe is not None and bool(safe) == flag:
+            f = open(fn, 'wt')
+            f.write('\n')
+            f.close()
+
+
+safety_flags = {
+    True: 'zip-safe',
+    False: 'not-zip-safe',
+}
+
+
+def scan_module(egg_dir, base, name, stubs):
+    """Check whether module possibly uses unsafe-for-zipfile stuff"""
+
+    filename = os.path.join(base, name)
+    if filename[:-1] in stubs:
+        return True  # Extension module
+    pkg = base[len(egg_dir) + 1:].replace(os.sep, '.')
+    module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0]
+    if six.PY2:
+        skip = 8  # skip magic & date
+    elif sys.version_info < (3, 7):
+        skip = 12  # skip magic & date & file size
+    else:
+        skip = 16  # skip magic & reserved? & date & file size
+    f = open(filename, 'rb')
+    f.read(skip)
+    code = marshal.load(f)
+    f.close()
+    safe = True
+    symbols = dict.fromkeys(iter_symbols(code))
+    for bad in ['__file__', '__path__']:
+        if bad in symbols:
+            log.warn("%s: module references %s", module, bad)
+            safe = False
+    if 'inspect' in symbols:
+        for bad in [
+            'getsource', 'getabsfile', 'getsourcefile', 'getfile'
+            'getsourcelines', 'findsource', 'getcomments', 'getframeinfo',
+            'getinnerframes', 'getouterframes', 'stack', 'trace'
+        ]:
+            if bad in symbols:
+                log.warn("%s: module MAY be using inspect.%s", module, bad)
+                safe = False
+    return safe
+
+
+def iter_symbols(code):
+    """Yield names and strings used by `code` and its nested code objects"""
+    for name in code.co_names:
+        yield name
+    for const in code.co_consts:
+        if isinstance(const, six.string_types):
+            yield const
+        elif isinstance(const, CodeType):
+            for name in iter_symbols(const):
+                yield name
+
+
+def can_scan():
+    if not sys.platform.startswith('java') and sys.platform != 'cli':
+        # CPython, PyPy, etc.
+        return True
+    log.warn("Unable to analyze compiled code on this platform.")
+    log.warn("Please ask the author to include a 'zip_safe'"
+             " setting (either True or False) in the package's setup.py")
+
+
+# Attribute names of options for commands that might need to be convinced to
+# install to the egg build directory
+
+INSTALL_DIRECTORY_ATTRS = [
+    'install_lib', 'install_dir', 'install_data', 'install_base'
+]
+
+
+def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True,
+                 mode='w'):
+    """Create a zip file from all the files under 'base_dir'.  The output
+    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"
+    Python module (if available) or the InfoZIP "zip" utility (if installed
+    and found on the default search path).  If neither tool is available,
+    raises DistutilsExecError.  Returns the name of the output zip file.
+    """
+    import zipfile
+
+    mkpath(os.path.dirname(zip_filename), dry_run=dry_run)
+    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)
+
+    def visit(z, dirname, names):
+        for name in names:
+            path = os.path.normpath(os.path.join(dirname, name))
+            if os.path.isfile(path):
+                p = path[len(base_dir) + 1:]
+                if not dry_run:
+                    z.write(path, p)
+                log.debug("adding '%s'", p)
+
+    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
+    if not dry_run:
+        z = zipfile.ZipFile(zip_filename, mode, compression=compression)
+        for dirname, dirs, files in sorted_walk(base_dir):
+            visit(z, dirname, files)
+        z.close()
+    else:
+        for dirname, dirs, files in sorted_walk(base_dir):
+            visit(None, dirname, files)
+    return zip_filename
diff --git a/ubuntu/venv/setuptools/command/bdist_rpm.py b/ubuntu/venv/setuptools/command/bdist_rpm.py
new file mode 100644
index 0000000..7073092
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/bdist_rpm.py
@@ -0,0 +1,43 @@
+import distutils.command.bdist_rpm as orig
+
+
+class bdist_rpm(orig.bdist_rpm):
+    """
+    Override the default bdist_rpm behavior to do the following:
+
+    1. Run egg_info to ensure the name and version are properly calculated.
+    2. Always run 'install' using --single-version-externally-managed to
+       disable eggs in RPM distributions.
+    3. Replace dash with underscore in the version numbers for better RPM
+       compatibility.
+    """
+
+    def run(self):
+        # ensure distro name is up-to-date
+        self.run_command('egg_info')
+
+        orig.bdist_rpm.run(self)
+
+    def _make_spec_file(self):
+        version = self.distribution.get_version()
+        rpmversion = version.replace('-', '_')
+        spec = orig.bdist_rpm._make_spec_file(self)
+        line23 = '%define version ' + version
+        line24 = '%define version ' + rpmversion
+        spec = [
+            line.replace(
+                "Source0: %{name}-%{version}.tar",
+                "Source0: %{name}-%{unmangled_version}.tar"
+            ).replace(
+                "setup.py install ",
+                "setup.py install --single-version-externally-managed "
+            ).replace(
+                "%setup",
+                "%setup -n %{name}-%{unmangled_version}"
+            ).replace(line23, line24)
+            for line in spec
+        ]
+        insert_loc = spec.index(line24) + 1
+        unmangled_version = "%define unmangled_version " + version
+        spec.insert(insert_loc, unmangled_version)
+        return spec
diff --git a/ubuntu/venv/setuptools/command/bdist_wininst.py b/ubuntu/venv/setuptools/command/bdist_wininst.py
new file mode 100644
index 0000000..073de97
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/bdist_wininst.py
@@ -0,0 +1,21 @@
+import distutils.command.bdist_wininst as orig
+
+
+class bdist_wininst(orig.bdist_wininst):
+    def reinitialize_command(self, command, reinit_subcommands=0):
+        """
+        Supplement reinitialize_command to work around
+        http://bugs.python.org/issue20819
+        """
+        cmd = self.distribution.reinitialize_command(
+            command, reinit_subcommands)
+        if command in ('install', 'install_lib'):
+            cmd.install_lib = None
+        return cmd
+
+    def run(self):
+        self._is_running = True
+        try:
+            orig.bdist_wininst.run(self)
+        finally:
+            self._is_running = False
diff --git a/ubuntu/venv/setuptools/command/build_clib.py b/ubuntu/venv/setuptools/command/build_clib.py
new file mode 100644
index 0000000..09caff6
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/build_clib.py
@@ -0,0 +1,98 @@
+import distutils.command.build_clib as orig
+from distutils.errors import DistutilsSetupError
+from distutils import log
+from setuptools.dep_util import newer_pairwise_group
+
+
+class build_clib(orig.build_clib):
+    """
+    Override the default build_clib behaviour to do the following:
+
+    1. Implement a rudimentary timestamp-based dependency system
+       so 'compile()' doesn't run every time.
+    2. Add more keys to the 'build_info' dictionary:
+        * obj_deps - specify dependencies for each object compiled.
+                     this should be a dictionary mapping a key
+                     with the source filename to a list of
+                     dependencies. Use an empty string for global
+                     dependencies.
+        * cflags   - specify a list of additional flags to pass to
+                     the compiler.
+    """
+
+    def build_libraries(self, libraries):
+        for (lib_name, build_info) in libraries:
+            sources = build_info.get('sources')
+            if sources is None or not isinstance(sources, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'sources' must be present and must be "
+                       "a list of source filenames" % lib_name)
+            sources = list(sources)
+
+            log.info("building '%s' library", lib_name)
+
+            # Make sure everything is the correct type.
+            # obj_deps should be a dictionary of keys as sources
+            # and a list/tuple of files that are its dependencies.
+            obj_deps = build_info.get('obj_deps', dict())
+            if not isinstance(obj_deps, dict):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'obj_deps' must be a dictionary of "
+                       "type 'source: list'" % lib_name)
+            dependencies = []
+
+            # Get the global dependencies that are specified by the '' key.
+            # These will go into every source's dependency list.
+            global_deps = obj_deps.get('', list())
+            if not isinstance(global_deps, (list, tuple)):
+                raise DistutilsSetupError(
+                       "in 'libraries' option (library '%s'), "
+                       "'obj_deps' must be a dictionary of "
+                       "type 'source: list'" % lib_name)
+
+            # Build the list to be used by newer_pairwise_group
+            # each source will be auto-added to its dependencies.
+            for source in sources:
+                src_deps = [source]
+                src_deps.extend(global_deps)
+                extra_deps = obj_deps.get(source, list())
+                if not isinstance(extra_deps, (list, tuple)):
+                    raise DistutilsSetupError(
+                           "in 'libraries' option (library '%s'), "
+                           "'obj_deps' must be a dictionary of "
+                           "type 'source: list'" % lib_name)
+                src_deps.extend(extra_deps)
+                dependencies.append(src_deps)
+
+            expected_objects = self.compiler.object_filenames(
+                    sources,
+                    output_dir=self.build_temp
+                    )
+
+            if newer_pairwise_group(dependencies, expected_objects) != ([], []):
+                # First, compile the source code to object files in the library
+                # directory.  (This should probably change to putting object
+                # files in a temporary build directory.)
+                macros = build_info.get('macros')
+                include_dirs = build_info.get('include_dirs')
+                cflags = build_info.get('cflags')
+                objects = self.compiler.compile(
+                        sources,
+                        output_dir=self.build_temp,
+                        macros=macros,
+                        include_dirs=include_dirs,
+                        extra_postargs=cflags,
+                        debug=self.debug
+                        )
+
+            # Now "link" the object files together into a static library.
+            # (On Unix at least, this isn't really linking -- it just
+            # builds an archive.  Whatever.)
+            self.compiler.create_static_lib(
+                    expected_objects,
+                    lib_name,
+                    output_dir=self.build_clib,
+                    debug=self.debug
+                    )
diff --git a/ubuntu/venv/setuptools/command/build_ext.py b/ubuntu/venv/setuptools/command/build_ext.py
new file mode 100644
index 0000000..daa8e4f
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/build_ext.py
@@ -0,0 +1,327 @@
+import os
+import sys
+import itertools
+from distutils.command.build_ext import build_ext as _du_build_ext
+from distutils.file_util import copy_file
+from distutils.ccompiler import new_compiler
+from distutils.sysconfig import customize_compiler, get_config_var
+from distutils.errors import DistutilsError
+from distutils import log
+
+from setuptools.extension import Library
+from setuptools.extern import six
+
+if six.PY2:
+    import imp
+
+    EXTENSION_SUFFIXES = [s for s, _, tp in imp.get_suffixes() if tp == imp.C_EXTENSION]
+else:
+    from importlib.machinery import EXTENSION_SUFFIXES
+
+try:
+    # Attempt to use Cython for building extensions, if available
+    from Cython.Distutils.build_ext import build_ext as _build_ext
+    # Additionally, assert that the compiler module will load
+    # also. Ref #1229.
+    __import__('Cython.Compiler.Main')
+except ImportError:
+    _build_ext = _du_build_ext
+
+# make sure _config_vars is initialized
+get_config_var("LDSHARED")
+from distutils.sysconfig import _config_vars as _CONFIG_VARS
+
+
+def _customize_compiler_for_shlib(compiler):
+    if sys.platform == "darwin":
+        # building .dylib requires additional compiler flags on OSX; here we
+        # temporarily substitute the pyconfig.h variables so that distutils'
+        # 'customize_compiler' uses them before we build the shared libraries.
+        tmp = _CONFIG_VARS.copy()
+        try:
+            # XXX Help!  I don't have any idea whether these are right...
+            _CONFIG_VARS['LDSHARED'] = (
+                "gcc -Wl,-x -dynamiclib -undefined dynamic_lookup")
+            _CONFIG_VARS['CCSHARED'] = " -dynamiclib"
+            _CONFIG_VARS['SO'] = ".dylib"
+            customize_compiler(compiler)
+        finally:
+            _CONFIG_VARS.clear()
+            _CONFIG_VARS.update(tmp)
+    else:
+        customize_compiler(compiler)
+
+
+have_rtld = False
+use_stubs = False
+libtype = 'shared'
+
+if sys.platform == "darwin":
+    use_stubs = True
+elif os.name != 'nt':
+    try:
+        import dl
+        use_stubs = have_rtld = hasattr(dl, 'RTLD_NOW')
+    except ImportError:
+        pass
+
+if_dl = lambda s: s if have_rtld else ''
+
+
+def get_abi3_suffix():
+    """Return the file extension for an abi3-compliant Extension()"""
+    for suffix in EXTENSION_SUFFIXES:
+        if '.abi3' in suffix:  # Unix
+            return suffix
+        elif suffix == '.pyd':  # Windows
+            return suffix
+
+
+class build_ext(_build_ext):
+    def run(self):
+        """Build extensions in build directory, then copy if --inplace"""
+        old_inplace, self.inplace = self.inplace, 0
+        _build_ext.run(self)
+        self.inplace = old_inplace
+        if old_inplace:
+            self.copy_extensions_to_source()
+
+    def copy_extensions_to_source(self):
+        build_py = self.get_finalized_command('build_py')
+        for ext in self.extensions:
+            fullname = self.get_ext_fullname(ext.name)
+            filename = self.get_ext_filename(fullname)
+            modpath = fullname.split('.')
+            package = '.'.join(modpath[:-1])
+            package_dir = build_py.get_package_dir(package)
+            dest_filename = os.path.join(package_dir,
+                                         os.path.basename(filename))
+            src_filename = os.path.join(self.build_lib, filename)
+
+            # Always copy, even if source is older than destination, to ensure
+            # that the right extensions for the current Python/platform are
+            # used.
+            copy_file(
+                src_filename, dest_filename, verbose=self.verbose,
+                dry_run=self.dry_run
+            )
+            if ext._needs_stub:
+                self.write_stub(package_dir or os.curdir, ext, True)
+
+    def get_ext_filename(self, fullname):
+        filename = _build_ext.get_ext_filename(self, fullname)
+        if fullname in self.ext_map:
+            ext = self.ext_map[fullname]
+            use_abi3 = (
+                six.PY3
+                and getattr(ext, 'py_limited_api')
+                and get_abi3_suffix()
+            )
+            if use_abi3:
+                so_ext = get_config_var('EXT_SUFFIX')
+                filename = filename[:-len(so_ext)]
+                filename = filename + get_abi3_suffix()
+            if isinstance(ext, Library):
+                fn, ext = os.path.splitext(filename)
+                return self.shlib_compiler.library_filename(fn, libtype)
+            elif use_stubs and ext._links_to_dynamic:
+                d, fn = os.path.split(filename)
+                return os.path.join(d, 'dl-' + fn)
+        return filename
+
+    def initialize_options(self):
+        _build_ext.initialize_options(self)
+        self.shlib_compiler = None
+        self.shlibs = []
+        self.ext_map = {}
+
+    def finalize_options(self):
+        _build_ext.finalize_options(self)
+        self.extensions = self.extensions or []
+        self.check_extensions_list(self.extensions)
+        self.shlibs = [ext for ext in self.extensions
+                       if isinstance(ext, Library)]
+        if self.shlibs:
+            self.setup_shlib_compiler()
+        for ext in self.extensions:
+            ext._full_name = self.get_ext_fullname(ext.name)
+        for ext in self.extensions:
+            fullname = ext._full_name
+            self.ext_map[fullname] = ext
+
+            # distutils 3.1 will also ask for module names
+            # XXX what to do with conflicts?
+            self.ext_map[fullname.split('.')[-1]] = ext
+
+            ltd = self.shlibs and self.links_to_dynamic(ext) or False
+            ns = ltd and use_stubs and not isinstance(ext, Library)
+            ext._links_to_dynamic = ltd
+            ext._needs_stub = ns
+            filename = ext._file_name = self.get_ext_filename(fullname)
+            libdir = os.path.dirname(os.path.join(self.build_lib, filename))
+            if ltd and libdir not in ext.library_dirs:
+                ext.library_dirs.append(libdir)
+            if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs:
+                ext.runtime_library_dirs.append(os.curdir)
+
+    def setup_shlib_compiler(self):
+        compiler = self.shlib_compiler = new_compiler(
+            compiler=self.compiler, dry_run=self.dry_run, force=self.force
+        )
+        _customize_compiler_for_shlib(compiler)
+
+        if self.include_dirs is not None:
+            compiler.set_include_dirs(self.include_dirs)
+        if self.define is not None:
+            # 'define' option is a list of (name,value) tuples
+            for (name, value) in self.define:
+                compiler.define_macro(name, value)
+        if self.undef is not None:
+            for macro in self.undef:
+                compiler.undefine_macro(macro)
+        if self.libraries is not None:
+            compiler.set_libraries(self.libraries)
+        if self.library_dirs is not None:
+            compiler.set_library_dirs(self.library_dirs)
+        if self.rpath is not None:
+            compiler.set_runtime_library_dirs(self.rpath)
+        if self.link_objects is not None:
+            compiler.set_link_objects(self.link_objects)
+
+        # hack so distutils' build_extension() builds a library instead
+        compiler.link_shared_object = link_shared_object.__get__(compiler)
+
+    def get_export_symbols(self, ext):
+        if isinstance(ext, Library):
+            return ext.export_symbols
+        return _build_ext.get_export_symbols(self, ext)
+
+    def build_extension(self, ext):
+        ext._convert_pyx_sources_to_lang()
+        _compiler = self.compiler
+        try:
+            if isinstance(ext, Library):
+                self.compiler = self.shlib_compiler
+            _build_ext.build_extension(self, ext)
+            if ext._needs_stub:
+                cmd = self.get_finalized_command('build_py').build_lib
+                self.write_stub(cmd, ext)
+        finally:
+            self.compiler = _compiler
+
+    def links_to_dynamic(self, ext):
+        """Return true if 'ext' links to a dynamic lib in the same package"""
+        # XXX this should check to ensure the lib is actually being built
+        # XXX as dynamic, and not just using a locally-found version or a
+        # XXX static-compiled version
+        libnames = dict.fromkeys([lib._full_name for lib in self.shlibs])
+        pkg = '.'.join(ext._full_name.split('.')[:-1] + [''])
+        return any(pkg + libname in libnames for libname in ext.libraries)
+
+    def get_outputs(self):
+        return _build_ext.get_outputs(self) + self.__get_stubs_outputs()
+
+    def __get_stubs_outputs(self):
+        # assemble the base name for each extension that needs a stub
+        ns_ext_bases = (
+            os.path.join(self.build_lib, *ext._full_name.split('.'))
+            for ext in self.extensions
+            if ext._needs_stub
+        )
+        # pair each base with the extension
+        pairs = itertools.product(ns_ext_bases, self.__get_output_extensions())
+        return list(base + fnext for base, fnext in pairs)
+
+    def __get_output_extensions(self):
+        yield '.py'
+        yield '.pyc'
+        if self.get_finalized_command('build_py').optimize:
+            yield '.pyo'
+
+    def write_stub(self, output_dir, ext, compile=False):
+        log.info("writing stub loader for %s to %s", ext._full_name,
+                 output_dir)
+        stub_file = (os.path.join(output_dir, *ext._full_name.split('.')) +
+                     '.py')
+        if compile and os.path.exists(stub_file):
+            raise DistutilsError(stub_file + " already exists! Please delete.")
+        if not self.dry_run:
+            f = open(stub_file, 'w')
+            f.write(
+                '\n'.join([
+                    "def __bootstrap__():",
+                    "   global __bootstrap__, __file__, __loader__",
+                    "   import sys, os, pkg_resources, imp" + if_dl(", dl"),
+                    "   __file__ = pkg_resources.resource_filename"
+                    "(__name__,%r)"
+                    % os.path.basename(ext._file_name),
+                    "   del __bootstrap__",
+                    "   if '__loader__' in globals():",
+                    "       del __loader__",
+                    if_dl("   old_flags = sys.getdlopenflags()"),
+                    "   old_dir = os.getcwd()",
+                    "   try:",
+                    "     os.chdir(os.path.dirname(__file__))",
+                    if_dl("     sys.setdlopenflags(dl.RTLD_NOW)"),
+                    "     imp.load_dynamic(__name__,__file__)",
+                    "   finally:",
+                    if_dl("     sys.setdlopenflags(old_flags)"),
+                    "     os.chdir(old_dir)",
+                    "__bootstrap__()",
+                    ""  # terminal \n
+                ])
+            )
+            f.close()
+        if compile:
+            from distutils.util import byte_compile
+
+            byte_compile([stub_file], optimize=0,
+                         force=True, dry_run=self.dry_run)
+            optimize = self.get_finalized_command('install_lib').optimize
+            if optimize > 0:
+                byte_compile([stub_file], optimize=optimize,
+                             force=True, dry_run=self.dry_run)
+            if os.path.exists(stub_file) and not self.dry_run:
+                os.unlink(stub_file)
+
+
+if use_stubs or os.name == 'nt':
+    # Build shared libraries
+    #
+    def link_shared_object(
+            self, objects, output_libname, output_dir=None, libraries=None,
+            library_dirs=None, runtime_library_dirs=None, export_symbols=None,
+            debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
+            target_lang=None):
+        self.link(
+            self.SHARED_LIBRARY, objects, output_libname,
+            output_dir, libraries, library_dirs, runtime_library_dirs,
+            export_symbols, debug, extra_preargs, extra_postargs,
+            build_temp, target_lang
+        )
+else:
+    # Build static libraries everywhere else
+    libtype = 'static'
+
+    def link_shared_object(
+            self, objects, output_libname, output_dir=None, libraries=None,
+            library_dirs=None, runtime_library_dirs=None, export_symbols=None,
+            debug=0, extra_preargs=None, extra_postargs=None, build_temp=None,
+            target_lang=None):
+        # XXX we need to either disallow these attrs on Library instances,
+        # or warn/abort here if set, or something...
+        # libraries=None, library_dirs=None, runtime_library_dirs=None,
+        # export_symbols=None, extra_preargs=None, extra_postargs=None,
+        # build_temp=None
+
+        assert output_dir is None  # distutils build_ext doesn't pass this
+        output_dir, filename = os.path.split(output_libname)
+        basename, ext = os.path.splitext(filename)
+        if self.library_filename("x").startswith('lib'):
+            # strip 'lib' prefix; this is kludgy if some platform uses
+            # a different prefix
+            basename = basename[3:]
+
+        self.create_static_lib(
+            objects, basename, output_dir, debug, target_lang
+        )
diff --git a/ubuntu/venv/setuptools/command/build_py.py b/ubuntu/venv/setuptools/command/build_py.py
new file mode 100644
index 0000000..b0314fd
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/build_py.py
@@ -0,0 +1,270 @@
+from glob import glob
+from distutils.util import convert_path
+import distutils.command.build_py as orig
+import os
+import fnmatch
+import textwrap
+import io
+import distutils.errors
+import itertools
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import map, filter, filterfalse
+
+try:
+    from setuptools.lib2to3_ex import Mixin2to3
+except ImportError:
+
+    class Mixin2to3:
+        def run_2to3(self, files, doctests=True):
+            "do nothing"
+
+
+class build_py(orig.build_py, Mixin2to3):
+    """Enhanced 'build_py' command that includes data files with packages
+
+    The data files are specified via a 'package_data' argument to 'setup()'.
+    See 'setuptools.dist.Distribution' for more details.
+
+    Also, this version of the 'build_py' command allows you to specify both
+    'py_modules' and 'packages' in the same setup operation.
+    """
+
+    def finalize_options(self):
+        orig.build_py.finalize_options(self)
+        self.package_data = self.distribution.package_data
+        self.exclude_package_data = (self.distribution.exclude_package_data or
+                                     {})
+        if 'data_files' in self.__dict__:
+            del self.__dict__['data_files']
+        self.__updated_files = []
+        self.__doctests_2to3 = []
+
+    def run(self):
+        """Build modules, packages, and copy data files to build directory"""
+        if not self.py_modules and not self.packages:
+            return
+
+        if self.py_modules:
+            self.build_modules()
+
+        if self.packages:
+            self.build_packages()
+            self.build_package_data()
+
+        self.run_2to3(self.__updated_files, False)
+        self.run_2to3(self.__updated_files, True)
+        self.run_2to3(self.__doctests_2to3, True)
+
+        # Only compile actual .py files, using our base class' idea of what our
+        # output files are.
+        self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
+
+    def __getattr__(self, attr):
+        "lazily compute data files"
+        if attr == 'data_files':
+            self.data_files = self._get_data_files()
+            return self.data_files
+        return orig.build_py.__getattr__(self, attr)
+
+    def build_module(self, module, module_file, package):
+        if six.PY2 and isinstance(package, six.string_types):
+            # avoid errors on Python 2 when unicode is passed (#190)
+            package = package.split('.')
+        outfile, copied = orig.build_py.build_module(self, module, module_file,
+                                                     package)
+        if copied:
+            self.__updated_files.append(outfile)
+        return outfile, copied
+
+    def _get_data_files(self):
+        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
+        self.analyze_manifest()
+        return list(map(self._get_pkg_data_files, self.packages or ()))
+
+    def _get_pkg_data_files(self, package):
+        # Locate package source directory
+        src_dir = self.get_package_dir(package)
+
+        # Compute package build directory
+        build_dir = os.path.join(*([self.build_lib] + package.split('.')))
+
+        # Strip directory from globbed filenames
+        filenames = [
+            os.path.relpath(file, src_dir)
+            for file in self.find_data_files(package, src_dir)
+        ]
+        return package, src_dir, build_dir, filenames
+
+    def find_data_files(self, package, src_dir):
+        """Return filenames for package's data files in 'src_dir'"""
+        patterns = self._get_platform_patterns(
+            self.package_data,
+            package,
+            src_dir,
+        )
+        globs_expanded = map(glob, patterns)
+        # flatten the expanded globs into an iterable of matches
+        globs_matches = itertools.chain.from_iterable(globs_expanded)
+        glob_files = filter(os.path.isfile, globs_matches)
+        files = itertools.chain(
+            self.manifest_files.get(package, []),
+            glob_files,
+        )
+        return self.exclude_data_files(package, src_dir, files)
+
+    def build_package_data(self):
+        """Copy data files into build directory"""
+        for package, src_dir, build_dir, filenames in self.data_files:
+            for filename in filenames:
+                target = os.path.join(build_dir, filename)
+                self.mkpath(os.path.dirname(target))
+                srcfile = os.path.join(src_dir, filename)
+                outf, copied = self.copy_file(srcfile, target)
+                srcfile = os.path.abspath(srcfile)
+                if (copied and
+                        srcfile in self.distribution.convert_2to3_doctests):
+                    self.__doctests_2to3.append(outf)
+
+    def analyze_manifest(self):
+        self.manifest_files = mf = {}
+        if not self.distribution.include_package_data:
+            return
+        src_dirs = {}
+        for package in self.packages or ():
+            # Locate package source directory
+            src_dirs[assert_relative(self.get_package_dir(package))] = package
+
+        self.run_command('egg_info')
+        ei_cmd = self.get_finalized_command('egg_info')
+        for path in ei_cmd.filelist.files:
+            d, f = os.path.split(assert_relative(path))
+            prev = None
+            oldf = f
+            while d and d != prev and d not in src_dirs:
+                prev = d
+                d, df = os.path.split(d)
+                f = os.path.join(df, f)
+            if d in src_dirs:
+                if path.endswith('.py') and f == oldf:
+                    continue  # it's a module, not data
+                mf.setdefault(src_dirs[d], []).append(path)
+
+    def get_data_files(self):
+        pass  # Lazily compute data files in _get_data_files() function.
+
+    def check_package(self, package, package_dir):
+        """Check namespace packages' __init__ for declare_namespace"""
+        try:
+            return self.packages_checked[package]
+        except KeyError:
+            pass
+
+        init_py = orig.build_py.check_package(self, package, package_dir)
+        self.packages_checked[package] = init_py
+
+        if not init_py or not self.distribution.namespace_packages:
+            return init_py
+
+        for pkg in self.distribution.namespace_packages:
+            if pkg == package or pkg.startswith(package + '.'):
+                break
+        else:
+            return init_py
+
+        with io.open(init_py, 'rb') as f:
+            contents = f.read()
+        if b'declare_namespace' not in contents:
+            raise distutils.errors.DistutilsError(
+                "Namespace package problem: %s is a namespace package, but "
+                "its\n__init__.py does not call declare_namespace()! Please "
+                'fix it.\n(See the setuptools manual under '
+                '"Namespace Packages" for details.)\n"' % (package,)
+            )
+        return init_py
+
+    def initialize_options(self):
+        self.packages_checked = {}
+        orig.build_py.initialize_options(self)
+
+    def get_package_dir(self, package):
+        res = orig.build_py.get_package_dir(self, package)
+        if self.distribution.src_root is not None:
+            return os.path.join(self.distribution.src_root, res)
+        return res
+
+    def exclude_data_files(self, package, src_dir, files):
+        """Filter filenames for package's data files in 'src_dir'"""
+        files = list(files)
+        patterns = self._get_platform_patterns(
+            self.exclude_package_data,
+            package,
+            src_dir,
+        )
+        match_groups = (
+            fnmatch.filter(files, pattern)
+            for pattern in patterns
+        )
+        # flatten the groups of matches into an iterable of matches
+        matches = itertools.chain.from_iterable(match_groups)
+        bad = set(matches)
+        keepers = (
+            fn
+            for fn in files
+            if fn not in bad
+        )
+        # ditch dupes
+        return list(_unique_everseen(keepers))
+
+    @staticmethod
+    def _get_platform_patterns(spec, package, src_dir):
+        """
+        yield platform-specific path patterns (suitable for glob
+        or fn_match) from a glob-based spec (such as
+        self.package_data or self.exclude_package_data)
+        matching package in src_dir.
+        """
+        raw_patterns = itertools.chain(
+            spec.get('', []),
+            spec.get(package, []),
+        )
+        return (
+            # Each pattern has to be converted to a platform-specific path
+            os.path.join(src_dir, convert_path(pattern))
+            for pattern in raw_patterns
+        )
+
+
+# from Python docs
+def _unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
+
+
+def assert_relative(path):
+    if not os.path.isabs(path):
+        return path
+    from distutils.errors import DistutilsSetupError
+
+    msg = textwrap.dedent("""
+        Error: setup script specifies an absolute path:
+
+            %s
+
+        setup() arguments must *always* be /-separated paths relative to the
+        setup.py directory, *never* absolute paths.
+        """).lstrip() % path
+    raise DistutilsSetupError(msg)
diff --git a/ubuntu/venv/setuptools/command/develop.py b/ubuntu/venv/setuptools/command/develop.py
new file mode 100644
index 0000000..009e4f9
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/develop.py
@@ -0,0 +1,221 @@
+from distutils.util import convert_path
+from distutils import log
+from distutils.errors import DistutilsError, DistutilsOptionError
+import os
+import glob
+import io
+
+from setuptools.extern import six
+
+import pkg_resources
+from setuptools.command.easy_install import easy_install
+from setuptools import namespaces
+import setuptools
+
+__metaclass__ = type
+
+
+class develop(namespaces.DevelopInstaller, easy_install):
+    """Set up package for development"""
+
+    description = "install package in 'development mode'"
+
+    user_options = easy_install.user_options + [
+        ("uninstall", "u", "Uninstall this source package"),
+        ("egg-path=", None, "Set the path to be used in the .egg-link file"),
+    ]
+
+    boolean_options = easy_install.boolean_options + ['uninstall']
+
+    command_consumes_arguments = False  # override base
+
+    def run(self):
+        if self.uninstall:
+            self.multi_version = True
+            self.uninstall_link()
+            self.uninstall_namespaces()
+        else:
+            self.install_for_development()
+        self.warn_deprecated_options()
+
+    def initialize_options(self):
+        self.uninstall = None
+        self.egg_path = None
+        easy_install.initialize_options(self)
+        self.setup_path = None
+        self.always_copy_from = '.'  # always copy eggs installed in curdir
+
+    def finalize_options(self):
+        ei = self.get_finalized_command("egg_info")
+        if ei.broken_egg_info:
+            template = "Please rename %r to %r before using 'develop'"
+            args = ei.egg_info, ei.broken_egg_info
+            raise DistutilsError(template % args)
+        self.args = [ei.egg_name]
+
+        easy_install.finalize_options(self)
+        self.expand_basedirs()
+        self.expand_dirs()
+        # pick up setup-dir .egg files only: no .egg-info
+        self.package_index.scan(glob.glob('*.egg'))
+
+        egg_link_fn = ei.egg_name + '.egg-link'
+        self.egg_link = os.path.join(self.install_dir, egg_link_fn)
+        self.egg_base = ei.egg_base
+        if self.egg_path is None:
+            self.egg_path = os.path.abspath(ei.egg_base)
+
+        target = pkg_resources.normalize_path(self.egg_base)
+        egg_path = pkg_resources.normalize_path(
+            os.path.join(self.install_dir, self.egg_path))
+        if egg_path != target:
+            raise DistutilsOptionError(
+                "--egg-path must be a relative path from the install"
+                " directory to " + target
+            )
+
+        # Make a distribution for the package's source
+        self.dist = pkg_resources.Distribution(
+            target,
+            pkg_resources.PathMetadata(target, os.path.abspath(ei.egg_info)),
+            project_name=ei.egg_name
+        )
+
+        self.setup_path = self._resolve_setup_path(
+            self.egg_base,
+            self.install_dir,
+            self.egg_path,
+        )
+
+    @staticmethod
+    def _resolve_setup_path(egg_base, install_dir, egg_path):
+        """
+        Generate a path from egg_base back to '.' where the
+        setup script resides and ensure that path points to the
+        setup path from $install_dir/$egg_path.
+        """
+        path_to_setup = egg_base.replace(os.sep, '/').rstrip('/')
+        if path_to_setup != os.curdir:
+            path_to_setup = '../' * (path_to_setup.count('/') + 1)
+        resolved = pkg_resources.normalize_path(
+            os.path.join(install_dir, egg_path, path_to_setup)
+        )
+        if resolved != pkg_resources.normalize_path(os.curdir):
+            raise DistutilsOptionError(
+                "Can't get a consistent path to setup script from"
+                " installation directory", resolved,
+                pkg_resources.normalize_path(os.curdir))
+        return path_to_setup
+
+    def install_for_development(self):
+        if six.PY3 and getattr(self.distribution, 'use_2to3', False):
+            # If we run 2to3 we can not do this inplace:
+
+            # Ensure metadata is up-to-date
+            self.reinitialize_command('build_py', inplace=0)
+            self.run_command('build_py')
+            bpy_cmd = self.get_finalized_command("build_py")
+            build_path = pkg_resources.normalize_path(bpy_cmd.build_lib)
+
+            # Build extensions
+            self.reinitialize_command('egg_info', egg_base=build_path)
+            self.run_command('egg_info')
+
+            self.reinitialize_command('build_ext', inplace=0)
+            self.run_command('build_ext')
+
+            # Fixup egg-link and easy-install.pth
+            ei_cmd = self.get_finalized_command("egg_info")
+            self.egg_path = build_path
+            self.dist.location = build_path
+            # XXX
+            self.dist._provider = pkg_resources.PathMetadata(
+                build_path, ei_cmd.egg_info)
+        else:
+            # Without 2to3 inplace works fine:
+            self.run_command('egg_info')
+
+            # Build extensions in-place
+            self.reinitialize_command('build_ext', inplace=1)
+            self.run_command('build_ext')
+
+        self.install_site_py()  # ensure that target dir is site-safe
+        if setuptools.bootstrap_install_from:
+            self.easy_install(setuptools.bootstrap_install_from)
+            setuptools.bootstrap_install_from = None
+
+        self.install_namespaces()
+
+        # create an .egg-link in the installation dir, pointing to our egg
+        log.info("Creating %s (link to %s)", self.egg_link, self.egg_base)
+        if not self.dry_run:
+            with open(self.egg_link, "w") as f:
+                f.write(self.egg_path + "\n" + self.setup_path)
+        # postprocess the installed distro, fixing up .pth, installing scripts,
+        # and handling requirements
+        self.process_distribution(None, self.dist, not self.no_deps)
+
+    def uninstall_link(self):
+        if os.path.exists(self.egg_link):
+            log.info("Removing %s (link to %s)", self.egg_link, self.egg_base)
+            egg_link_file = open(self.egg_link)
+            contents = [line.rstrip() for line in egg_link_file]
+            egg_link_file.close()
+            if contents not in ([self.egg_path],
+                                [self.egg_path, self.setup_path]):
+                log.warn("Link points to %s: uninstall aborted", contents)
+                return
+            if not self.dry_run:
+                os.unlink(self.egg_link)
+        if not self.dry_run:
+            self.update_pth(self.dist)  # remove any .pth link to us
+        if self.distribution.scripts:
+            # XXX should also check for entry point scripts!
+            log.warn("Note: you must uninstall or replace scripts manually!")
+
+    def install_egg_scripts(self, dist):
+        if dist is not self.dist:
+            # Installing a dependency, so fall back to normal behavior
+            return easy_install.install_egg_scripts(self, dist)
+
+        # create wrapper scripts in the script dir, pointing to dist.scripts
+
+        # new-style...
+        self.install_wrapper_scripts(dist)
+
+        # ...and old-style
+        for script_name in self.distribution.scripts or []:
+            script_path = os.path.abspath(convert_path(script_name))
+            script_name = os.path.basename(script_path)
+            with io.open(script_path) as strm:
+                script_text = strm.read()
+            self.install_script(dist, script_name, script_text, script_path)
+
+    def install_wrapper_scripts(self, dist):
+        dist = VersionlessRequirement(dist)
+        return easy_install.install_wrapper_scripts(self, dist)
+
+
+class VersionlessRequirement:
+    """
+    Adapt a pkg_resources.Distribution to simply return the project
+    name as the 'requirement' so that scripts will work across
+    multiple versions.
+
+    >>> from pkg_resources import Distribution
+    >>> dist = Distribution(project_name='foo', version='1.0')
+    >>> str(dist.as_requirement())
+    'foo==1.0'
+    >>> adapted_dist = VersionlessRequirement(dist)
+    >>> str(adapted_dist.as_requirement())
+    'foo'
+    """
+
+    def __init__(self, dist):
+        self.__dist = dist
+
+    def __getattr__(self, name):
+        return getattr(self.__dist, name)
+
+    def as_requirement(self):
+        return self.project_name
diff --git a/ubuntu/venv/setuptools/command/dist_info.py b/ubuntu/venv/setuptools/command/dist_info.py
new file mode 100644
index 0000000..c45258f
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/dist_info.py
@@ -0,0 +1,36 @@
+"""
+Create a dist_info directory
+As defined in the wheel specification
+"""
+
+import os
+
+from distutils.core import Command
+from distutils import log
+
+
+class dist_info(Command):
+
+    description = 'create a .dist-info directory'
+
+    user_options = [
+        ('egg-base=', 'e', "directory containing .egg-info directories"
+                           " (default: top of the source tree)"),
+    ]
+
+    def initialize_options(self):
+        self.egg_base = None
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        egg_info = self.get_finalized_command('egg_info')
+        egg_info.egg_base = self.egg_base
+        egg_info.finalize_options()
+        egg_info.run()
+        dist_info_dir = egg_info.egg_info[:-len('.egg-info')] + '.dist-info'
+        log.info("creating '{}'".format(os.path.abspath(dist_info_dir)))
+
+        bdist_wheel = self.get_finalized_command('bdist_wheel')
+        bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir)
diff --git a/ubuntu/venv/setuptools/command/easy_install.py b/ubuntu/venv/setuptools/command/easy_install.py
new file mode 100644
index 0000000..1f6839c
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/easy_install.py
@@ -0,0 +1,2402 @@
+#!/usr/bin/env python
+"""
+Easy Install
+------------
+
+A tool for doing automatic download/extract/build of distutils-based Python
+packages.  For detailed documentation, see the accompanying EasyInstall.txt
+file, or visit the `EasyInstall home page`__.
+
+__ https://setuptools.readthedocs.io/en/latest/easy_install.html
+
+"""
+
+from glob import glob
+from distutils.util import get_platform
+from distutils.util import convert_path, subst_vars
+from distutils.errors import (
+    DistutilsArgError, DistutilsOptionError,
+    DistutilsError, DistutilsPlatformError,
+)
+from distutils.command.install import INSTALL_SCHEMES, SCHEME_KEYS
+from distutils import log, dir_util
+from distutils.command.build_scripts import first_line_re
+from distutils.spawn import find_executable
+import sys
+import os
+import zipimport
+import shutil
+import tempfile
+import zipfile
+import re
+import stat
+import random
+import textwrap
+import warnings
+import site
+import struct
+import contextlib
+import subprocess
+import shlex
+import io
+
+
+from sysconfig import get_config_vars, get_path
+
+from setuptools import SetuptoolsDeprecationWarning
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import configparser, map
+
+from setuptools import Command
+from setuptools.sandbox import run_setup
+from setuptools.py27compat import rmtree_safe
+from setuptools.command import setopt
+from setuptools.archive_util import unpack_archive
+from setuptools.package_index import (
+    PackageIndex, parse_requirement_arg, URL_SCHEME,
+)
+from setuptools.command import bdist_egg, egg_info
+from setuptools.wheel import Wheel
+from pkg_resources import (
+    yield_lines, normalize_path, resource_string, ensure_directory,
+    get_distribution, find_distributions, Environment, Requirement,
+    Distribution, PathMetadata, EggMetadata, WorkingSet, DistributionNotFound,
+    VersionConflict, DEVELOP_DIST,
+)
+import pkg_resources.py31compat
+
+__metaclass__ = type
+
+# Turn on PEP440Warnings
+warnings.filterwarnings("default", category=pkg_resources.PEP440Warning)
+
+__all__ = [
+    'samefile', 'easy_install', 'PthDistributions', 'extract_wininst_cfg',
+    'main', 'get_exe_prefixes',
+]
+
+
+def is_64bit():
+    return struct.calcsize("P") == 8
+
+
+def samefile(p1, p2):
+    """
+    Determine if two paths reference the same file.
+
+    Augments os.path.samefile to work on Windows and
+    suppresses errors if the path doesn't exist.
+    """
+    both_exist = os.path.exists(p1) and os.path.exists(p2)
+    use_samefile = hasattr(os.path, 'samefile') and both_exist
+    if use_samefile:
+        return os.path.samefile(p1, p2)
+    norm_p1 = os.path.normpath(os.path.normcase(p1))
+    norm_p2 = os.path.normpath(os.path.normcase(p2))
+    return norm_p1 == norm_p2
+
+
+if six.PY2:
+
+    def _to_bytes(s):
+        return s
+
+    def isascii(s):
+        try:
+            six.text_type(s, 'ascii')
+            return True
+        except UnicodeError:
+            return False
+else:
+
+    def _to_bytes(s):
+        return s.encode('utf8')
+
+    def isascii(s):
+        try:
+            s.encode('ascii')
+            return True
+        except UnicodeError:
+            return False
+
+
+_one_liner = lambda text: textwrap.dedent(text).strip().replace('\n', '; ')
+
+
+class easy_install(Command):
+    """Manage a download/build/install process"""
+    description = "Find/get/install Python packages"
+    command_consumes_arguments = True
+
+    user_options = [
+        ('prefix=', None, "installation prefix"),
+        ("zip-ok", "z", "install package as a zipfile"),
+        ("multi-version", "m", "make apps have to require() a version"),
+        ("upgrade", "U", "force upgrade (searches PyPI for latest versions)"),
+        ("install-dir=", "d", "install package to DIR"),
+        ("script-dir=", "s", "install scripts to DIR"),
+        ("exclude-scripts", "x", "Don't install scripts"),
+        ("always-copy", "a", "Copy all needed packages to install dir"),
+        ("index-url=", "i", "base URL of Python Package Index"),
+        ("find-links=", "f", "additional URL(s) to search for packages"),
+        ("build-directory=", "b",
+         "download/extract/build in DIR; keep the results"),
+        ('optimize=', 'O',
+         "also compile with optimization: -O1 for \"python -O\", "
+         "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
+        ('record=', None,
+         "filename in which to record list of installed files"),
+        ('always-unzip', 'Z', "don't install as a zipfile, no matter what"),
+        ('site-dirs=', 'S', "list of directories where .pth files work"),
+        ('editable', 'e', "Install specified packages in editable form"),
+        ('no-deps', 'N', "don't install dependencies"),
+        ('allow-hosts=', 'H', "pattern(s) that hostnames must match"),
+        ('local-snapshots-ok', 'l',
+         "allow building eggs from local checkouts"),
+        ('version', None, "print version information and exit"),
+        ('install-layout=', None, "installation layout to choose (known values: deb)"),
+        ('force-installation-into-system-dir', '0', "force installation into /usr"),
+        ('no-find-links', None,
+         "Don't load find-links defined in packages being installed")
+    ]
+    boolean_options = [
+        'zip-ok', 'multi-version', 'exclude-scripts', 'upgrade', 'always-copy',
+        'editable',
+        'no-deps', 'local-snapshots-ok', 'version', 'force-installation-into-system-dir'
+    ]
+
+    if site.ENABLE_USER_SITE:
+        help_msg = "install in user site-package '%s'" % site.USER_SITE
+        user_options.append(('user', None, help_msg))
+        boolean_options.append('user')
+
+    negative_opt = {'always-unzip': 'zip-ok'}
+    create_index = PackageIndex
+
+    def initialize_options(self):
+        # the --user option seems to be an opt-in one,
+        # so the default should be False.
+        self.user = 0
+        self.zip_ok = self.local_snapshots_ok = None
+        self.install_dir = self.script_dir = self.exclude_scripts = None
+        self.index_url = None
+        self.find_links = None
+        self.build_directory = None
+        self.args = None
+        self.optimize = self.record = None
+        self.upgrade = self.always_copy = self.multi_version = None
+        self.editable = self.no_deps = self.allow_hosts = None
+        self.root = self.prefix = self.no_report = None
+        self.version = None
+        self.install_purelib = None  # for pure module distributions
+        self.install_platlib = None  # non-pure (dists w/ extensions)
+        self.install_headers = None  # for C/C++ headers
+        self.install_lib = None  # set to either purelib or platlib
+        self.install_scripts = None
+        self.install_data = None
+        self.install_base = None
+        self.install_platbase = None
+        if site.ENABLE_USER_SITE:
+            self.install_userbase = site.USER_BASE
+            self.install_usersite = site.USER_SITE
+        else:
+            self.install_userbase = None
+            self.install_usersite = None
+        self.no_find_links = None
+
+        # Options not specifiable via command line
+        self.package_index = None
+        self.pth_file = self.always_copy_from = None
+        self.site_dirs = None
+        self.installed_projects = {}
+        self.sitepy_installed = False
+        # enable custom installation, known values: deb
+        self.install_layout = None
+        self.force_installation_into_system_dir = None
+        self.multiarch = None
+
+        # Always read easy_install options, even if we are subclassed, or have
+        # an independent instance created.  This ensures that defaults will
+        # always come from the standard configuration file(s)' "easy_install"
+        # section, even if this is a "develop" or "install" command, or some
+        # other embedding.
+        self._dry_run = None
+        self.verbose = self.distribution.verbose
+        self.distribution._set_command_options(
+            self, self.distribution.get_option_dict('easy_install')
+        )
+
+    def delete_blockers(self, blockers):
+        extant_blockers = (
+            filename for filename in blockers
+            if os.path.exists(filename) or os.path.islink(filename)
+        )
+        list(map(self._delete_path, extant_blockers))
+
+    def _delete_path(self, path):
+        log.info("Deleting %s", path)
+        if self.dry_run:
+            return
+
+        is_tree = os.path.isdir(path) and not os.path.islink(path)
+        remover = rmtree if is_tree else os.unlink
+        remover(path)
+
+    @staticmethod
+    def _render_version():
+        """
+        Render the Setuptools version and installation details, then exit.
+        """
+        ver = '{}.{}'.format(*sys.version_info)
+        dist = get_distribution('setuptools')
+        tmpl = 'setuptools {dist.version} from {dist.location} (Python {ver})'
+        print(tmpl.format(**locals()))
+        raise SystemExit()
+
+    def finalize_options(self):
+        self.version and self._render_version()
+
+        py_version = sys.version.split()[0]
+        prefix, exec_prefix = get_config_vars('prefix', 'exec_prefix')
+
+        self.config_vars = {
+            'dist_name': self.distribution.get_name(),
+            'dist_version': self.distribution.get_version(),
+            'dist_fullname': self.distribution.get_fullname(),
+            'py_version': py_version,
+            'py_version_short': py_version[0:3],
+            'py_version_nodot': py_version[0] + py_version[2],
+            'sys_prefix': prefix,
+            'prefix': prefix,
+            'sys_exec_prefix': exec_prefix,
+            'exec_prefix': exec_prefix,
+            # Only python 3.2+ has abiflags
+            'abiflags': getattr(sys, 'abiflags', ''),
+        }
+
+        if site.ENABLE_USER_SITE:
+            self.config_vars['userbase'] = self.install_userbase
+            self.config_vars['usersite'] = self.install_usersite
+
+        self._fix_install_dir_for_user_site()
+
+        self.expand_basedirs()
+        self.expand_dirs()
+
+        if self.install_layout:
+            if not self.install_layout.lower() in ['deb']:
+                raise DistutilsOptionError("unknown value for --install-layout")
+            self.install_layout = self.install_layout.lower()
+
+            import sysconfig
+            if sys.version_info[:2] >= (3, 3):
+                self.multiarch = sysconfig.get_config_var('MULTIARCH')
+
+        self._expand(
+            'install_dir', 'script_dir', 'build_directory',
+            'site_dirs',
+        )
+        # If a non-default installation directory was specified, default the
+        # script directory to match it.
+        if self.script_dir is None:
+            self.script_dir = self.install_dir
+
+        if self.no_find_links is None:
+            self.no_find_links = False
+
+        # Let install_dir get set by install_lib command, which in turn
+        # gets its info from the install command, and takes into account
+        # --prefix and --home and all that other crud.
+        self.set_undefined_options(
+            'install_lib', ('install_dir', 'install_dir')
+        )
+        # Likewise, set default script_dir from 'install_scripts.install_dir'
+        self.set_undefined_options(
+            'install_scripts', ('install_dir', 'script_dir')
+        )
+
+        if self.user and self.install_purelib:
+            self.install_dir = self.install_purelib
+            self.script_dir = self.install_scripts
+
+        if self.prefix == '/usr' and not self.force_installation_into_system_dir:
+            raise DistutilsOptionError("""installation into /usr
+
+Trying to install into the system managed parts of the file system. Please
+consider to install to another location, or use the option
+--force-installation-into-system-dir to overwrite this warning.
+""")
+
+        # default --record from the install command
+        self.set_undefined_options('install', ('record', 'record'))
+        # Should this be moved to the if statement below? It's not used
+        # elsewhere
+        normpath = map(normalize_path, sys.path)
+        self.all_site_dirs = get_site_dirs()
+        if self.site_dirs is not None:
+            site_dirs = [
+                os.path.expanduser(s.strip()) for s in
+                self.site_dirs.split(',')
+            ]
+            for d in site_dirs:
+                if not os.path.isdir(d):
+                    log.warn("%s (in --site-dirs) does not exist", d)
+                elif normalize_path(d) not in normpath:
+                    raise DistutilsOptionError(
+                        d + " (in --site-dirs) is not on sys.path"
+                    )
+                else:
+                    self.all_site_dirs.append(normalize_path(d))
+        if not self.editable:
+            self.check_site_dir()
+        self.index_url = self.index_url or "https://pypi.org/simple/"
+        self.shadow_path = self.all_site_dirs[:]
+        for path_item in self.install_dir, normalize_path(self.script_dir):
+            if path_item not in self.shadow_path:
+                self.shadow_path.insert(0, path_item)
+
+        if self.allow_hosts is not None:
+            hosts = [s.strip() for s in self.allow_hosts.split(',')]
+        else:
+            hosts = ['*']
+        if self.package_index is None:
+            self.package_index = self.create_index(
+                self.index_url, search_path=self.shadow_path, hosts=hosts,
+            )
+        self.local_index = Environment(self.shadow_path + sys.path)
+
+        if self.find_links is not None:
+            if isinstance(self.find_links, six.string_types):
+                self.find_links = self.find_links.split()
+        else:
+            self.find_links = []
+        if self.local_snapshots_ok:
+            self.package_index.scan_egg_links(self.shadow_path + sys.path)
+        if not self.no_find_links:
+            self.package_index.add_find_links(self.find_links)
+        self.set_undefined_options('install_lib', ('optimize', 'optimize'))
+        if not isinstance(self.optimize, int):
+            try:
+                self.optimize = int(self.optimize)
+                if not (0 <= self.optimize <= 2):
+                    raise ValueError
+            except ValueError:
+                raise DistutilsOptionError("--optimize must be 0, 1, or 2")
+
+        if self.editable and not self.build_directory:
+            raise DistutilsArgError(
+                "Must specify a build directory (-b) when using --editable"
+            )
+        if not self.args:
+            raise DistutilsArgError(
+                "No urls, filenames, or requirements specified (see --help)")
+
+        self.outputs = []
+
+    def _fix_install_dir_for_user_site(self):
+        """
+        Fix the install_dir if "--user" was used.
+        """
+        if not self.user or not site.ENABLE_USER_SITE:
+            return
+
+        self.create_home_path()
+        if self.install_userbase is None:
+            msg = "User base directory is not specified"
+            raise DistutilsPlatformError(msg)
+        self.install_base = self.install_platbase = self.install_userbase
+        scheme_name = os.name.replace('posix', 'unix') + '_user'
+        self.select_scheme(scheme_name)
+
+    def _expand_attrs(self, attrs):
+        for attr in attrs:
+            val = getattr(self, attr)
+            if val is not None:
+                if os.name == 'posix' or os.name == 'nt':
+                    val = os.path.expanduser(val)
+                val = subst_vars(val, self.config_vars)
+                setattr(self, attr, val)
+
+    def expand_basedirs(self):
+        """Calls `os.path.expanduser` on install_base, install_platbase and
+        root."""
+        self._expand_attrs(['install_base', 'install_platbase', 'root'])
+
+    def expand_dirs(self):
+        """Calls `os.path.expanduser` on install dirs."""
+        dirs = [
+            'install_purelib',
+            'install_platlib',
+            'install_lib',
+            'install_headers',
+            'install_scripts',
+            'install_data',
+        ]
+        self._expand_attrs(dirs)
+
+    def run(self, show_deprecation=True):
+        if show_deprecation:
+            self.announce(
+                "WARNING: The easy_install command is deprecated "
+                "and will be removed in a future version."
+                , log.WARN,
+            )
+        if self.verbose != self.distribution.verbose:
+            log.set_verbosity(self.verbose)
+        try:
+            for spec in self.args:
+                self.easy_install(spec, not self.no_deps)
+            if self.record:
+                outputs = list(sorted(self.outputs))
+                if self.root:  # strip any package prefix
+                    root_len = len(self.root)
+                    for counter in range(len(outputs)):
+                        outputs[counter] = outputs[counter][root_len:]
+                from distutils import file_util
+
+                self.execute(
+                    file_util.write_file, (self.record, outputs),
+                    "writing list of installed files to '%s'" %
+                    self.record
+                )
+            self.warn_deprecated_options()
+        finally:
+            log.set_verbosity(self.distribution.verbose)
+
+    def pseudo_tempname(self):
+        """Return a pseudo-tempname base in the install directory.
+        This code is intentionally naive; if a malicious party can write to
+        the target directory you're already in deep doodoo.
+        """
+        try:
+            pid = os.getpid()
+        except Exception:
+            pid = random.randint(0, sys.maxsize)
+        return os.path.join(self.install_dir, "test-easy-install-%s" % pid)
+
+    def warn_deprecated_options(self):
+        pass
+
+    def check_site_dir(self):
+        """Verify that self.install_dir is .pth-capable dir, if needed"""
+
+        instdir = normalize_path(self.install_dir)
+        pth_file = os.path.join(instdir, 'easy-install.pth')
+
+        # Is it a configured, PYTHONPATH, implicit, or explicit site dir?
+        is_site_dir = instdir in self.all_site_dirs
+
+        if not is_site_dir and not self.multi_version:
+            # No?  Then directly test whether it does .pth file processing
+            is_site_dir = self.check_pth_processing()
+        else:
+            # make sure we can write to target dir
+            testfile = self.pseudo_tempname() + '.write-test'
+            test_exists = os.path.exists(testfile)
+            try:
+                if test_exists:
+                    os.unlink(testfile)
+                open(testfile, 'w').close()
+                os.unlink(testfile)
+            except (OSError, IOError):
+                self.cant_write_to_target()
+
+        if not is_site_dir and not self.multi_version:
+            # Can't install non-multi to non-site dir
+            raise DistutilsError(self.no_default_version_msg())
+
+        if is_site_dir:
+            if self.pth_file is None:
+                self.pth_file = PthDistributions(pth_file, self.all_site_dirs)
+        else:
+            self.pth_file = None
+
+        if instdir not in map(normalize_path, _pythonpath()):
+            # only PYTHONPATH dirs need a site.py, so pretend it's there
+            self.sitepy_installed = True
+        elif self.multi_version and not os.path.exists(pth_file):
+            self.sitepy_installed = True  # don't need site.py in this case
+            self.pth_file = None  # and don't create a .pth file
+        self.install_dir = instdir
+
+    __cant_write_msg = textwrap.dedent("""
+        can't create or remove files in install directory
+
+        The following error occurred while trying to add or remove files in the
+        installation directory:
+
+            %s
+
+        The installation directory you specified (via --install-dir, --prefix, or
+        the distutils default setting) was:
+
+            %s
+        """).lstrip()
+
+    __not_exists_id = textwrap.dedent("""
+        This directory does not currently exist.  Please create it and try again, or
+        choose a different installation directory (using the -d or --install-dir
+        option).
+        """).lstrip()
+
+    __access_msg = textwrap.dedent("""
+        Perhaps your account does not have write access to this directory?  If the
+        installation directory is a system-owned directory, you may need to sign in
+        as the administrator or "root" account.  If you do not have administrative
+        access to this machine, you may wish to choose a different installation
+        directory, preferably one that is listed in your PYTHONPATH environment
+        variable.
+
+        For information on other options, you may wish to consult the
+        documentation at:
+
+          https://setuptools.readthedocs.io/en/latest/easy_install.html
+
+        Please make the appropriate changes for your system and try again.
+        """).lstrip()
+
+    def cant_write_to_target(self):
+        msg = self.__cant_write_msg % (sys.exc_info()[1], self.install_dir,)
+
+        if not os.path.exists(self.install_dir):
+            msg += '\n' + self.__not_exists_id
+        else:
+            msg += '\n' + self.__access_msg
+        raise DistutilsError(msg)
+
+    def check_pth_processing(self):
+        """Empirically verify whether .pth files are supported in inst. dir"""
+        instdir = self.install_dir
+        log.info("Checking .pth file support in %s", instdir)
+        pth_file = self.pseudo_tempname() + ".pth"
+        ok_file = pth_file + '.ok'
+        ok_exists = os.path.exists(ok_file)
+        tmpl = _one_liner("""
+            import os
+            f = open({ok_file!r}, 'w')
+            f.write('OK')
+            f.close()
+            """) + '\n'
+        try:
+            if ok_exists:
+                os.unlink(ok_file)
+            dirname = os.path.dirname(ok_file)
+            pkg_resources.py31compat.makedirs(dirname, exist_ok=True)
+            f = open(pth_file, 'w')
+        except (OSError, IOError):
+            self.cant_write_to_target()
+        else:
+            try:
+                f.write(tmpl.format(**locals()))
+                f.close()
+                f = None
+                executable = sys.executable
+                if os.name == 'nt':
+                    dirname, basename = os.path.split(executable)
+                    alt = os.path.join(dirname, 'pythonw.exe')
+                    use_alt = (
+                        basename.lower() == 'python.exe' and
+                        os.path.exists(alt)
+                    )
+                    if use_alt:
+                        # use pythonw.exe to avoid opening a console window
+                        executable = alt
+
+                from distutils.spawn import spawn
+
+                spawn([executable, '-E', '-c', 'pass'], 0)
+
+                if os.path.exists(ok_file):
+                    log.info(
+                        "TEST PASSED: %s appears to support .pth files",
+                        instdir
+                    )
+                    return True
+            finally:
+                if f:
+                    f.close()
+                if os.path.exists(ok_file):
+                    os.unlink(ok_file)
+                if os.path.exists(pth_file):
+                    os.unlink(pth_file)
+        if not self.multi_version:
+            log.warn("TEST FAILED: %s does NOT support .pth files", instdir)
+        return False
+
+    def install_egg_scripts(self, dist):
+        """Write all the scripts for `dist`, unless scripts are excluded"""
+        if not self.exclude_scripts and dist.metadata_isdir('scripts'):
+            for script_name in dist.metadata_listdir('scripts'):
+                if dist.metadata_isdir('scripts/' + script_name):
+                    # The "script" is a directory, likely a Python 3
+                    # __pycache__ directory, so skip it.
+                    continue
+                self.install_script(
+                    dist, script_name,
+                    dist.get_metadata('scripts/' + script_name)
+                )
+        self.install_wrapper_scripts(dist)
+
+    def add_output(self, path):
+        if os.path.isdir(path):
+            for base, dirs, files in os.walk(path):
+                for filename in files:
+                    self.outputs.append(os.path.join(base, filename))
+        else:
+            self.outputs.append(path)
+
+    def not_editable(self, spec):
+        if self.editable:
+            raise DistutilsArgError(
+                "Invalid argument %r: you can't use filenames or URLs "
+                "with --editable (except via the --find-links option)."
+                % (spec,)
+            )
+
+    def check_editable(self, spec):
+        if not self.editable:
+            return
+
+        if os.path.exists(os.path.join(self.build_directory, spec.key)):
+            raise DistutilsArgError(
+                "%r already exists in %s; can't do a checkout there" %
+                (spec.key, self.build_directory)
+            )
+
+    @contextlib.contextmanager
+    def _tmpdir(self):
+        tmpdir = tempfile.mkdtemp(prefix=u"easy_install-")
+        try:
+            # cast to str as workaround for #709 and #710 and #712
+            yield str(tmpdir)
+        finally:
+            os.path.exists(tmpdir) and rmtree(rmtree_safe(tmpdir))
+
+    def easy_install(self, spec, deps=False):
+        if not self.editable:
+            self.install_site_py()
+
+        with self._tmpdir() as tmpdir:
+            if not isinstance(spec, Requirement):
+                if URL_SCHEME(spec):
+                    # It's a url, download it to tmpdir and process
+                    self.not_editable(spec)
+                    dl = self.package_index.download(spec, tmpdir)
+                    return self.install_item(None, dl, tmpdir, deps, True)
+
+                elif os.path.exists(spec):
+                    # Existing file or directory, just process it directly
+                    self.not_editable(spec)
+                    return self.install_item(None, spec, tmpdir, deps, True)
+                else:
+                    spec = parse_requirement_arg(spec)
+
+            self.check_editable(spec)
+            dist = self.package_index.fetch_distribution(
+                spec, tmpdir, self.upgrade, self.editable,
+                not self.always_copy, self.local_index
+            )
+            if dist is None:
+                msg = "Could not find suitable distribution for %r" % spec
+                if self.always_copy:
+                    msg += " (--always-copy skips system and development eggs)"
+                raise DistutilsError(msg)
+            elif dist.precedence == DEVELOP_DIST:
+                # .egg-info dists don't need installing, just process deps
+                self.process_distribution(spec, dist, deps, "Using")
+                return dist
+            else:
+                return self.install_item(spec, dist.location, tmpdir, deps)
+
+    def install_item(self, spec, download, tmpdir, deps, install_needed=False):
+
+        # Installation is also needed if file in tmpdir or is not an egg
+        install_needed = install_needed or self.always_copy
+        install_needed = install_needed or os.path.dirname(download) == tmpdir
+        install_needed = install_needed or not download.endswith('.egg')
+        install_needed = install_needed or (
+            self.always_copy_from is not None and
+            os.path.dirname(normalize_path(download)) ==
+            normalize_path(self.always_copy_from)
+        )
+
+        if spec and not install_needed:
+            # at this point, we know it's a local .egg, we just don't know if
+            # it's already installed.
+            for dist in self.local_index[spec.project_name]:
+                if dist.location == download:
+                    break
+            else:
+                install_needed = True  # it's not in the local index
+
+        log.info("Processing %s", os.path.basename(download))
+
+        if install_needed:
+            dists = self.install_eggs(spec, download, tmpdir)
+            for dist in dists:
+                self.process_distribution(spec, dist, deps)
+        else:
+            dists = [self.egg_distribution(download)]
+            self.process_distribution(spec, dists[0], deps, "Using")
+
+        if spec is not None:
+            for dist in dists:
+                if dist in spec:
+                    return dist
+
+    def select_scheme(self, name):
+        """Sets the install directories by applying the install schemes."""
+        # it's the caller's problem if they supply a bad name!
+        scheme = INSTALL_SCHEMES[name]
+        for key in SCHEME_KEYS:
+            attrname = 'install_' + key
+            if getattr(self, attrname) is None:
+                setattr(self, attrname, scheme[key])
+
+    def process_distribution(self, requirement, dist, deps=True, *info):
+        self.update_pth(dist)
+        self.package_index.add(dist)
+        if dist in self.local_index[dist.key]:
+            self.local_index.remove(dist)
+        self.local_index.add(dist)
+        self.install_egg_scripts(dist)
+        self.installed_projects[dist.key] = dist
+        log.info(self.installation_report(requirement, dist, *info))
+        if (dist.has_metadata('dependency_links.txt') and
+                not self.no_find_links):
+            self.package_index.add_find_links(
+                dist.get_metadata_lines('dependency_links.txt')
+            )
+        if not deps and not self.always_copy:
+            return
+        elif requirement is not None and dist.key != requirement.key:
+            log.warn("Skipping dependencies for %s", dist)
+            return  # XXX this is not the distribution we were looking for
+        elif requirement is None or dist not in requirement:
+            # if we wound up with a different version, resolve what we've got
+            distreq = dist.as_requirement()
+            requirement = Requirement(str(distreq))
+        log.info("Processing dependencies for %s", requirement)
+        try:
+            distros = WorkingSet([]).resolve(
+                [requirement], self.local_index, self.easy_install
+            )
+        except DistributionNotFound as e:
+            raise DistutilsError(str(e))
+        except VersionConflict as e:
+            raise DistutilsError(e.report())
+        if self.always_copy or self.always_copy_from:
+            # Force all the relevant distros to be copied or activated
+            for dist in distros:
+                if dist.key not in self.installed_projects:
+                    self.easy_install(dist.as_requirement())
+        log.info("Finished processing dependencies for %s", requirement)
+
+    def should_unzip(self, dist):
+        if self.zip_ok is not None:
+            return not self.zip_ok
+        if dist.has_metadata('not-zip-safe'):
+            return True
+        if not dist.has_metadata('zip-safe'):
+            return True
+        return False
+
+    def maybe_move(self, spec, dist_filename, setup_base):
+        dst = os.path.join(self.build_directory, spec.key)
+        if os.path.exists(dst):
+            msg = (
+                "%r already exists in %s; build directory %s will not be kept"
+            )
+            log.warn(msg, spec.key, self.build_directory, setup_base)
+            return setup_base
+        if os.path.isdir(dist_filename):
+            setup_base = dist_filename
+        else:
+            if os.path.dirname(dist_filename) == setup_base:
+                os.unlink(dist_filename)  # get it out of the tmp dir
+            contents = os.listdir(setup_base)
+            if len(contents) == 1:
+                dist_filename = os.path.join(setup_base, contents[0])
+                if os.path.isdir(dist_filename):
+                    # if the only thing there is a directory, move it instead
+                    setup_base = dist_filename
+        ensure_directory(dst)
+        shutil.move(setup_base, dst)
+        return dst
+
+    def install_wrapper_scripts(self, dist):
+        if self.exclude_scripts:
+            return
+        for args in ScriptWriter.best().get_args(dist):
+            self.write_script(*args)
+
+    def install_script(self, dist, script_name, script_text, dev_path=None):
+        """Generate a legacy script wrapper and install it"""
+        spec = str(dist.as_requirement())
+        is_script = is_python_script(script_text, script_name)
+
+        if is_script:
+            body = self._load_template(dev_path) % locals()
+            script_text = ScriptWriter.get_header(script_text) + body
+        self.write_script(script_name, _to_bytes(script_text), 'b')
+
+    @staticmethod
+    def _load_template(dev_path):
+        """
+        There are a couple of template scripts in the package. This
+        function loads one of them and prepares it for use.
+        """
+        # See https://github.com/pypa/setuptools/issues/134 for info
+        # on script file naming and downstream issues with SVR4
+        name = 'script.tmpl'
+        if dev_path:
+            name = name.replace('.tmpl', ' (dev).tmpl')
+
+        raw_bytes = resource_string('setuptools', name)
+        return raw_bytes.decode('utf-8')
+
+    def write_script(self, script_name, contents, mode="t", blockers=()):
+        """Write an executable file to the scripts directory"""
+        self.delete_blockers(  # clean up old .py/.pyw w/o a script
+            [os.path.join(self.script_dir, x) for x in blockers]
+        )
+        log.info("Installing %s script to %s", script_name, self.script_dir)
+        target = os.path.join(self.script_dir, script_name)
+        self.add_output(target)
+
+        if self.dry_run:
+            return
+
+        mask = current_umask()
+        ensure_directory(target)
+        if os.path.exists(target):
+            os.unlink(target)
+        with open(target, "w" + mode) as f:
+            f.write(contents)
+        chmod(target, 0o777 - mask)
+
+    def install_eggs(self, spec, dist_filename, tmpdir):
+        # .egg dirs or files are already built, so just return them
+        if dist_filename.lower().endswith('.egg'):
+            return [self.install_egg(dist_filename, tmpdir)]
+        elif dist_filename.lower().endswith('.exe'):
+            return [self.install_exe(dist_filename, tmpdir)]
+        elif dist_filename.lower().endswith('.whl'):
+            return [self.install_wheel(dist_filename, tmpdir)]
+
+        # Anything else, try to extract and build
+        setup_base = tmpdir
+        if os.path.isfile(dist_filename) and not dist_filename.endswith('.py'):
+            unpack_archive(dist_filename, tmpdir, self.unpack_progress)
+        elif os.path.isdir(dist_filename):
+            setup_base = os.path.abspath(dist_filename)
+
+        if (setup_base.startswith(tmpdir)  # something we downloaded
+                and self.build_directory and spec is not None):
+            setup_base = self.maybe_move(spec, dist_filename, setup_base)
+
+        # Find the setup.py file
+        setup_script = os.path.join(setup_base, 'setup.py')
+
+        if not os.path.exists(setup_script):
+            setups = glob(os.path.join(setup_base, '*', 'setup.py'))
+            if not setups:
+                raise DistutilsError(
+                    "Couldn't find a setup script in %s" %
+                    os.path.abspath(dist_filename)
+                )
+            if len(setups) > 1:
+                raise DistutilsError(
+                    "Multiple setup scripts in %s" %
+                    os.path.abspath(dist_filename)
+                )
+            setup_script = setups[0]
+
+        # Now run it, and return the result
+        if self.editable:
+            log.info(self.report_editable(spec, setup_script))
+            return []
+        else:
+            return self.build_and_install(setup_script, setup_base)
+
+    def egg_distribution(self, egg_path):
+        if os.path.isdir(egg_path):
+            metadata = PathMetadata(egg_path, os.path.join(egg_path,
+                                                           'EGG-INFO'))
+        else:
+            metadata = EggMetadata(zipimport.zipimporter(egg_path))
+        return Distribution.from_filename(egg_path, metadata=metadata)
+
+    def install_egg(self, egg_path, tmpdir):
+        destination = os.path.join(
+            self.install_dir,
+            os.path.basename(egg_path),
+        )
+        destination = os.path.abspath(destination)
+        if not self.dry_run:
+            ensure_directory(destination)
+
+        dist = self.egg_distribution(egg_path)
+        if not samefile(egg_path, destination):
+            if os.path.isdir(destination) and not os.path.islink(destination):
+                dir_util.remove_tree(destination, dry_run=self.dry_run)
+            elif os.path.exists(destination):
+                self.execute(
+                    os.unlink,
+                    (destination,),
+                    "Removing " + destination,
+                )
+            try:
+                new_dist_is_zipped = False
+                if os.path.isdir(egg_path):
+                    if egg_path.startswith(tmpdir):
+                        f, m = shutil.move, "Moving"
+                    else:
+                        f, m = shutil.copytree, "Copying"
+                elif self.should_unzip(dist):
+                    self.mkpath(destination)
+                    f, m = self.unpack_and_compile, "Extracting"
+                else:
+                    new_dist_is_zipped = True
+                    if egg_path.startswith(tmpdir):
+                        f, m = shutil.move, "Moving"
+                    else:
+                        f, m = shutil.copy2, "Copying"
+                self.execute(
+                    f,
+                    (egg_path, destination),
+                    (m + " %s to %s") % (
+                        os.path.basename(egg_path),
+                        os.path.dirname(destination)
+                    ),
+                )
+                update_dist_caches(
+                    destination,
+                    fix_zipimporter_caches=new_dist_is_zipped,
+                )
+            except Exception:
+                update_dist_caches(destination, fix_zipimporter_caches=False)
+                raise
+
+        self.add_output(destination)
+        return self.egg_distribution(destination)
+
+    def install_exe(self, dist_filename, tmpdir):
+        # See if it's valid, get data
+        cfg = extract_wininst_cfg(dist_filename)
+        if cfg is None:
+            raise DistutilsError(
+                "%s is not a valid distutils Windows .exe" % dist_filename
+            )
+        # Create a dummy distribution object until we build the real distro
+        dist = Distribution(
+            None,
+            project_name=cfg.get('metadata', 'name'),
+            version=cfg.get('metadata', 'version'), platform=get_platform(),
+        )
+
+        # Convert the .exe to an unpacked egg
+        egg_path = os.path.join(tmpdir, dist.egg_name() + '.egg')
+        dist.location = egg_path
+        egg_tmp = egg_path + '.tmp'
+        _egg_info = os.path.join(egg_tmp, 'EGG-INFO')
+        pkg_inf = os.path.join(_egg_info, 'PKG-INFO')
+        ensure_directory(pkg_inf)  # make sure EGG-INFO dir exists
+        dist._provider = PathMetadata(egg_tmp, _egg_info)  # XXX
+        self.exe_to_egg(dist_filename, egg_tmp)
+
+        # Write EGG-INFO/PKG-INFO
+        if not os.path.exists(pkg_inf):
+            f = open(pkg_inf, 'w')
+            f.write('Metadata-Version: 1.0\n')
+            for k, v in cfg.items('metadata'):
+                if k != 'target_version':
+                    f.write('%s: %s\n' % (k.replace('_', '-').title(), v))
+            f.close()
+        script_dir = os.path.join(_egg_info, 'scripts')
+        # delete entry-point scripts to avoid duping
+        self.delete_blockers([
+            os.path.join(script_dir, args[0])
+            for args in ScriptWriter.get_args(dist)
+        ])
+        # Build .egg file from tmpdir
+        bdist_egg.make_zipfile(
+            egg_path, egg_tmp, verbose=self.verbose, dry_run=self.dry_run,
+        )
+        # install the .egg
+        return self.install_egg(egg_path, tmpdir)
+
+    def exe_to_egg(self, dist_filename, egg_tmp):
+        """Extract a bdist_wininst to the directories an egg would use"""
+        # Check for .pth file and set up prefix translations
+        prefixes = get_exe_prefixes(dist_filename)
+        to_compile = []
+        native_libs = []
+        top_level = {}
+
+        def process(src, dst):
+            s = src.lower()
+            for old, new in prefixes:
+                if s.startswith(old):
+                    src = new + src[len(old):]
+                    parts = src.split('/')
+                    dst = os.path.join(egg_tmp, *parts)
+                    dl = dst.lower()
+                    if dl.endswith('.pyd') or dl.endswith('.dll'):
+                        parts[-1] = bdist_egg.strip_module(parts[-1])
+                        top_level[os.path.splitext(parts[0])[0]] = 1
+                        native_libs.append(src)
+                    elif dl.endswith('.py') and old != 'SCRIPTS/':
+                        top_level[os.path.splitext(parts[0])[0]] = 1
+                        to_compile.append(dst)
+                    return dst
+            if not src.endswith('.pth'):
+                log.warn("WARNING: can't process %s", src)
+            return None
+
+        # extract, tracking .pyd/.dll->native_libs and .py -> to_compile
+        unpack_archive(dist_filename, egg_tmp, process)
+        stubs = []
+        for res in native_libs:
+            if res.lower().endswith('.pyd'):  # create stubs for .pyd's
+                parts = res.split('/')
+                resource = parts[-1]
+                parts[-1] = bdist_egg.strip_module(parts[-1]) + '.py'
+                pyfile = os.path.join(egg_tmp, *parts)
+                to_compile.append(pyfile)
+                stubs.append(pyfile)
+                bdist_egg.write_stub(resource, pyfile)
+        self.byte_compile(to_compile)  # compile .py's
+        bdist_egg.write_safety_flag(
+            os.path.join(egg_tmp, 'EGG-INFO'),
+            bdist_egg.analyze_egg(egg_tmp, stubs))  # write zip-safety flag
+
+        for name in 'top_level', 'native_libs':
+            if locals()[name]:
+                txt = os.path.join(egg_tmp, 'EGG-INFO', name + '.txt')
+                if not os.path.exists(txt):
+                    f = open(txt, 'w')
+                    f.write('\n'.join(locals()[name]) + '\n')
+                    f.close()
+
+    def install_wheel(self, wheel_path, tmpdir):
+        wheel = Wheel(wheel_path)
+        assert wheel.is_compatible()
+        destination = os.path.join(self.install_dir, wheel.egg_name())
+        destination = os.path.abspath(destination)
+        if not self.dry_run:
+            ensure_directory(destination)
+        if os.path.isdir(destination) and not os.path.islink(destination):
+            dir_util.remove_tree(destination, dry_run=self.dry_run)
+        elif os.path.exists(destination):
+            self.execute(
+                os.unlink,
+                (destination,),
+                "Removing " + destination,
+            )
+        try:
+            self.execute(
+                wheel.install_as_egg,
+                (destination,),
+                ("Installing %s to %s") % (
+                    os.path.basename(wheel_path),
+                    os.path.dirname(destination)
+                ),
+            )
+        finally:
+            update_dist_caches(destination, fix_zipimporter_caches=False)
+        self.add_output(destination)
+        return self.egg_distribution(destination)
+
+    __mv_warning = textwrap.dedent("""
+        Because this distribution was installed --multi-version, before you can
+        import modules from this package in an application, you will need to
+        'import pkg_resources' and then use a 'require()' call similar to one of
+        these examples, in order to select the desired version:
+
+            pkg_resources.require("%(name)s")  # latest installed version
+            pkg_resources.require("%(name)s==%(version)s")  # this exact version
+            pkg_resources.require("%(name)s>=%(version)s")  # this version or higher
+        """).lstrip()
+
+    __id_warning = textwrap.dedent("""
+        Note also that the installation directory must be on sys.path at runtime for
+        this to work.  (e.g. by being the application's script directory, by being on
+        PYTHONPATH, or by being added to sys.path by your code.)
+        """)
+
+    def installation_report(self, req, dist, what="Installed"):
+        """Helpful installation message for display to package users"""
+        msg = "\n%(what)s %(eggloc)s%(extras)s"
+        if self.multi_version and not self.no_report:
+            msg += '\n' + self.__mv_warning
+            if self.install_dir not in map(normalize_path, sys.path):
+                msg += '\n' + self.__id_warning
+
+        eggloc = dist.location
+        name = dist.project_name
+        version = dist.version
+        extras = ''  # TODO: self.report_extras(req, dist)
+        return msg % locals()
+
+    __editable_msg = textwrap.dedent("""
+        Extracted editable version of %(spec)s to %(dirname)s
+
+        If it uses setuptools in its setup script, you can activate it in
+        "development" mode by going to that directory and running::
+
+            %(python)s setup.py develop
+
+        See the setuptools documentation for the "develop" command for more info.
+        """).lstrip()
+
+    def report_editable(self, spec, setup_script):
+        dirname = os.path.dirname(setup_script)
+        python = sys.executable
+        return '\n' + self.__editable_msg % locals()
+
+    def run_setup(self, setup_script, setup_base, args):
+        sys.modules.setdefault('distutils.command.bdist_egg', bdist_egg)
+        sys.modules.setdefault('distutils.command.egg_info', egg_info)
+
+        args = list(args)
+        if self.verbose > 2:
+            v = 'v' * (self.verbose - 1)
+            args.insert(0, '-' + v)
+        elif self.verbose < 2:
+            args.insert(0, '-q')
+        if self.dry_run:
+            args.insert(0, '-n')
+        log.info(
+            "Running %s %s", setup_script[len(setup_base) + 1:], ' '.join(args)
+        )
+        try:
+            run_setup(setup_script, args)
+        except SystemExit as v:
+            raise DistutilsError("Setup script exited with %s" % (v.args[0],))
+
+    def build_and_install(self, setup_script, setup_base):
+        args = ['bdist_egg', '--dist-dir']
+
+        dist_dir = tempfile.mkdtemp(
+            prefix='egg-dist-tmp-', dir=os.path.dirname(setup_script)
+        )
+        try:
+            self._set_fetcher_options(os.path.dirname(setup_script))
+            args.append(dist_dir)
+
+            self.run_setup(setup_script, setup_base, args)
+            all_eggs = Environment([dist_dir])
+            eggs = []
+            for key in all_eggs:
+                for dist in all_eggs[key]:
+                    eggs.append(self.install_egg(dist.location, setup_base))
+            if not eggs and not self.dry_run:
+                log.warn("No eggs found in %s (setup script problem?)",
+                         dist_dir)
+            return eggs
+        finally:
+            rmtree(dist_dir)
+            log.set_verbosity(self.verbose)  # restore our log verbosity
+
+    def _set_fetcher_options(self, base):
+        """
+        When easy_install is about to run bdist_egg on a source dist, that
+        source dist might have 'setup_requires' directives, requiring
+        additional fetching. Ensure the fetcher options given to easy_install
+        are available to that command as well.
+        """
+        # find the fetch options from easy_install and write them out
+        # to the setup.cfg file.
+        ei_opts = self.distribution.get_option_dict('easy_install').copy()
+        fetch_directives = (
+            'find_links', 'site_dirs', 'index_url', 'optimize', 'allow_hosts',
+        )
+        fetch_options = {}
+        for key, val in ei_opts.items():
+            if key not in fetch_directives:
+                continue
+            fetch_options[key.replace('_', '-')] = val[1]
+        # create a settings dictionary suitable for `edit_config`
+        settings = dict(easy_install=fetch_options)
+        cfg_filename = os.path.join(base, 'setup.cfg')
+        setopt.edit_config(cfg_filename, settings)
+
+    def update_pth(self, dist):
+        if self.pth_file is None:
+            return
+
+        for d in self.pth_file[dist.key]:  # drop old entries
+            if self.multi_version or d.location != dist.location:
+                log.info("Removing %s from easy-install.pth file", d)
+                self.pth_file.remove(d)
+                if d.location in self.shadow_path:
+                    self.shadow_path.remove(d.location)
+
+        if not self.multi_version:
+            if dist.location in self.pth_file.paths:
+                log.info(
+                    "%s is already the active version in easy-install.pth",
+                    dist,
+                )
+            else:
+                log.info("Adding %s to easy-install.pth file", dist)
+                self.pth_file.add(dist)  # add new entry
+                if dist.location not in self.shadow_path:
+                    self.shadow_path.append(dist.location)
+
+        if not self.dry_run:
+
+            self.pth_file.save()
+
+            if dist.key == 'setuptools':
+                # Ensure that setuptools itself never becomes unavailable!
+                # XXX should this check for latest version?
+                filename = os.path.join(self.install_dir, 'setuptools.pth')
+                if os.path.islink(filename):
+                    os.unlink(filename)
+                f = open(filename, 'wt')
+                f.write(self.pth_file.make_relative(dist.location) + '\n')
+                f.close()
+
+    def unpack_progress(self, src, dst):
+        # Progress filter for unpacking
+        log.debug("Unpacking %s to %s", src, dst)
+        return dst  # only unpack-and-compile skips files for dry run
+
+    def unpack_and_compile(self, egg_path, destination):
+        to_compile = []
+        to_chmod = []
+
+        def pf(src, dst):
+            if dst.endswith('.py') and not src.startswith('EGG-INFO/'):
+                to_compile.append(dst)
+            elif dst.endswith('.dll') or dst.endswith('.so'):
+                to_chmod.append(dst)
+            self.unpack_progress(src, dst)
+            return not self.dry_run and dst or None
+
+        unpack_archive(egg_path, destination, pf)
+        self.byte_compile(to_compile)
+        if not self.dry_run:
+            for f in to_chmod:
+                mode = ((os.stat(f)[stat.ST_MODE]) | 0o555) & 0o7755
+                chmod(f, mode)
+
+    def byte_compile(self, to_compile):
+        if sys.dont_write_bytecode:
+            return
+
+        from distutils.util import byte_compile
+
+        try:
+            # try to make the byte compile messages quieter
+            log.set_verbosity(self.verbose - 1)
+
+            byte_compile(to_compile, optimize=0, force=1, dry_run=self.dry_run)
+            if self.optimize:
+                byte_compile(
+                    to_compile, optimize=self.optimize, force=1,
+                    dry_run=self.dry_run,
+                )
+        finally:
+            log.set_verbosity(self.verbose)  # restore original verbosity
+
+    __no_default_msg = textwrap.dedent("""
+        bad install directory or PYTHONPATH
+
+        You are attempting to install a package to a directory that is not
+        on PYTHONPATH and which Python does not read ".pth" files from.  The
+        installation directory you specified (via --install-dir, --prefix, or
+        the distutils default setting) was:
+
+            %s
+
+        and your PYTHONPATH environment variable currently contains:
+
+            %r
+
+        Here are some of your options for correcting the problem:
+
+        * You can choose a different installation directory, i.e., one that is
+          on PYTHONPATH or supports .pth files
+
+        * You can add the installation directory to the PYTHONPATH environment
+          variable.  (It must then also be on PYTHONPATH whenever you run
+          Python and want to use the package(s) you are installing.)
+
+        * You can set up the installation directory to support ".pth" files by
+          using one of the approaches described here:
+
+          https://setuptools.readthedocs.io/en/latest/easy_install.html#custom-installation-locations
+
+
+        Please make the appropriate changes for your system and try again.""").lstrip()
+
+    def no_default_version_msg(self):
+        template = self.__no_default_msg
+        return template % (self.install_dir, os.environ.get('PYTHONPATH', ''))
+
+    def install_site_py(self):
+        """Make sure there's a site.py in the target dir, if needed"""
+
+        if self.sitepy_installed:
+            return  # already did it, or don't need to
+
+        sitepy = os.path.join(self.install_dir, "site.py")
+        source = resource_string("setuptools", "site-patch.py")
+        source = source.decode('utf-8')
+        current = ""
+
+        if os.path.exists(sitepy):
+            log.debug("Checking existing site.py in %s", self.install_dir)
+            with io.open(sitepy) as strm:
+                current = strm.read()
+
+            if not current.startswith('def __boot():'):
+                raise DistutilsError(
+                    "%s is not a setuptools-generated site.py; please"
+                    " remove it." % sitepy
+                )
+
+        if current != source:
+            log.info("Creating %s", sitepy)
+            if not self.dry_run:
+                ensure_directory(sitepy)
+                with io.open(sitepy, 'w', encoding='utf-8') as strm:
+                    strm.write(source)
+            self.byte_compile([sitepy])
+
+        self.sitepy_installed = True
+
+    def create_home_path(self):
+        """Create directories under ~."""
+        if not self.user:
+            return
+        home = convert_path(os.path.expanduser("~"))
+        for name, path in six.iteritems(self.config_vars):
+            if path.startswith(home) and not os.path.isdir(path):
+                self.debug_print("os.makedirs('%s', 0o700)" % path)
+                os.makedirs(path, 0o700)
+
+    if sys.version[:3] in ('2.3', '2.4', '2.5') or 'real_prefix' in sys.__dict__:
+        sitedir_name = 'site-packages'
+    else:
+        sitedir_name = 'dist-packages'
+
+    INSTALL_SCHEMES = dict(
+        posix=dict(
+            install_dir='$base/lib/python$py_version_short/site-packages',
+            script_dir='$base/bin',
+        ),
+        unix_local = dict(
+            install_dir = '$base/local/lib/python$py_version_short/%s' % sitedir_name,
+            script_dir  = '$base/local/bin',
+        ),
+        posix_local = dict(
+            install_dir = '$base/local/lib/python$py_version_short/%s' % sitedir_name,
+            script_dir  = '$base/local/bin',
+        ),
+        deb_system = dict(
+            install_dir = '$base/lib/python3/%s' % sitedir_name,
+            script_dir  = '$base/bin',
+        ),
+    )
+
+    DEFAULT_SCHEME = dict(
+        install_dir='$base/Lib/site-packages',
+        script_dir='$base/Scripts',
+    )
+
+    def _expand(self, *attrs):
+        config_vars = self.get_finalized_command('install').config_vars
+
+        if self.prefix or self.install_layout:
+            if self.install_layout and self.install_layout in ['deb']:
+                    scheme_name = "deb_system"
+                    self.prefix = '/usr'
+            elif self.prefix or 'real_prefix' in sys.__dict__:
+                scheme_name = os.name
+            else:
+                scheme_name = "posix_local"
+            # Set default install_dir/scripts from --prefix
+            config_vars = config_vars.copy()
+            config_vars['base'] = self.prefix
+            scheme = self.INSTALL_SCHEMES.get(scheme_name,self.DEFAULT_SCHEME)
+            for attr, val in scheme.items():
+                if getattr(self, attr, None) is None:
+                    setattr(self, attr, val)
+
+        from distutils.util import subst_vars
+
+        for attr in attrs:
+            val = getattr(self, attr)
+            if val is not None:
+                val = subst_vars(val, config_vars)
+                if os.name == 'posix':
+                    val = os.path.expanduser(val)
+                setattr(self, attr, val)
+
+
+def _pythonpath():
+    items = os.environ.get('PYTHONPATH', '').split(os.pathsep)
+    return filter(None, items)
+
+
+def get_site_dirs():
+    """
+    Return a list of 'site' dirs
+    """
+
+    sitedirs = []
+
+    # start with PYTHONPATH
+    sitedirs.extend(_pythonpath())
+
+    prefixes = [sys.prefix]
+    if sys.exec_prefix != sys.prefix:
+        prefixes.append(sys.exec_prefix)
+    for prefix in prefixes:
+        if prefix:
+            if sys.platform in ('os2emx', 'riscos'):
+                sitedirs.append(os.path.join(prefix, "Lib", "site-packages"))
+            elif os.sep == '/':
+                sitedirs.extend([
+                    os.path.join(
+                        prefix,
+                        "local/lib",
+                        "python" + sys.version[:3],
+                        "dist-packages",
+                    ),
+                    os.path.join(
+                        prefix,
+                        "lib",
+                        "python{}.{}".format(*sys.version_info),
+                        "dist-packages",
+                    ),
+                    os.path.join(prefix, "lib", "site-python"),
+                ])
+            else:
+                sitedirs.extend([
+                    prefix,
+                    os.path.join(prefix, "lib", "site-packages"),
+                ])
+            if sys.platform == 'darwin':
+                # for framework builds *only* we add the standard Apple
+                # locations. Currently only per-user, but /Library and
+                # /Network/Library could be added too
+                if 'Python.framework' in prefix:
+                    home = os.environ.get('HOME')
+                    if home:
+                        home_sp = os.path.join(
+                            home,
+                            'Library',
+                            'Python',
+                            '{}.{}'.format(*sys.version_info),
+                            'site-packages',
+                        )
+                        sitedirs.append(home_sp)
+    lib_paths = get_path('purelib'), get_path('platlib')
+    for site_lib in lib_paths:
+        if site_lib not in sitedirs:
+            sitedirs.append(site_lib)
+
+    if site.ENABLE_USER_SITE:
+        sitedirs.append(site.USER_SITE)
+
+    try:
+        sitedirs.extend(site.getsitepackages())
+    except AttributeError:
+        pass
+
+    sitedirs = list(map(normalize_path, sitedirs))
+
+    return sitedirs
+
+
+def expand_paths(inputs):
+    """Yield sys.path directories that might contain "old-style" packages"""
+
+    seen = {}
+
+    for dirname in inputs:
+        dirname = normalize_path(dirname)
+        if dirname in seen:
+            continue
+
+        seen[dirname] = 1
+        if not os.path.isdir(dirname):
+            continue
+
+        files = os.listdir(dirname)
+        yield dirname, files
+
+        for name in files:
+            if not name.endswith('.pth'):
+                # We only care about the .pth files
+                continue
+            if name in ('easy-install.pth', 'setuptools.pth'):
+                # Ignore .pth files that we control
+                continue
+
+            # Read the .pth file
+            f = open(os.path.join(dirname, name))
+            lines = list(yield_lines(f))
+            f.close()
+
+            # Yield existing non-dupe, non-import directory lines from it
+            for line in lines:
+                if not line.startswith("import"):
+                    line = normalize_path(line.rstrip())
+                    if line not in seen:
+                        seen[line] = 1
+                        if not os.path.isdir(line):
+                            continue
+                        yield line, os.listdir(line)
+
+
+def extract_wininst_cfg(dist_filename):
+    """Extract configuration data from a bdist_wininst .exe
+
+    Returns a configparser.RawConfigParser, or None
+    """
+    f = open(dist_filename, 'rb')
+    try:
+        endrec = zipfile._EndRecData(f)
+        if endrec is None:
+            return None
+
+        prepended = (endrec[9] - endrec[5]) - endrec[6]
+        if prepended < 12:  # no wininst data here
+            return None
+        f.seek(prepended - 12)
+
+        tag, cfglen, bmlen = struct.unpack("egg path translations for a given .exe file"""
+
+    prefixes = [
+        ('PURELIB/', ''),
+        ('PLATLIB/pywin32_system32', ''),
+        ('PLATLIB/', ''),
+        ('SCRIPTS/', 'EGG-INFO/scripts/'),
+        ('DATA/lib/site-packages', ''),
+    ]
+    z = zipfile.ZipFile(exe_filename)
+    try:
+        for info in z.infolist():
+            name = info.filename
+            parts = name.split('/')
+            if len(parts) == 3 and parts[2] == 'PKG-INFO':
+                if parts[1].endswith('.egg-info'):
+                    prefixes.insert(0, ('/'.join(parts[:2]), 'EGG-INFO/'))
+                    break
+            if len(parts) != 2 or not name.endswith('.pth'):
+                continue
+            if name.endswith('-nspkg.pth'):
+                continue
+            if parts[0].upper() in ('PURELIB', 'PLATLIB'):
+                contents = z.read(name)
+                if six.PY3:
+                    contents = contents.decode()
+                for pth in yield_lines(contents):
+                    pth = pth.strip().replace('\\', '/')
+                    if not pth.startswith('import'):
+                        prefixes.append((('%s/%s/' % (parts[0], pth)), ''))
+    finally:
+        z.close()
+    prefixes = [(x.lower(), y) for x, y in prefixes]
+    prefixes.sort()
+    prefixes.reverse()
+    return prefixes
+
+
+class PthDistributions(Environment):
+    """A .pth file with Distribution paths in it"""
+
+    dirty = False
+
+    def __init__(self, filename, sitedirs=()):
+        self.filename = filename
+        self.sitedirs = list(map(normalize_path, sitedirs))
+        self.basedir = normalize_path(os.path.dirname(self.filename))
+        self._load()
+        Environment.__init__(self, [], None, None)
+        for path in yield_lines(self.paths):
+            list(map(self.add, find_distributions(path, True)))
+
+    def _load(self):
+        self.paths = []
+        saw_import = False
+        seen = dict.fromkeys(self.sitedirs)
+        if os.path.isfile(self.filename):
+            f = open(self.filename, 'rt')
+            for line in f:
+                if line.startswith('import'):
+                    saw_import = True
+                    continue
+                path = line.rstrip()
+                self.paths.append(path)
+                if not path.strip() or path.strip().startswith('#'):
+                    continue
+                # skip non-existent paths, in case somebody deleted a package
+                # manually, and duplicate paths as well
+                path = self.paths[-1] = normalize_path(
+                    os.path.join(self.basedir, path)
+                )
+                if not os.path.exists(path) or path in seen:
+                    self.paths.pop()  # skip it
+                    self.dirty = True  # we cleaned up, so we're dirty now :)
+                    continue
+                seen[path] = 1
+            f.close()
+
+        if self.paths and not saw_import:
+            self.dirty = True  # ensure anything we touch has import wrappers
+        while self.paths and not self.paths[-1].strip():
+            self.paths.pop()
+
+    def save(self):
+        """Write changed .pth file back to disk"""
+        if not self.dirty:
+            return
+
+        rel_paths = list(map(self.make_relative, self.paths))
+        if rel_paths:
+            log.debug("Saving %s", self.filename)
+            lines = self._wrap_lines(rel_paths)
+            data = '\n'.join(lines) + '\n'
+
+            if os.path.islink(self.filename):
+                os.unlink(self.filename)
+            with open(self.filename, 'wt') as f:
+                f.write(data)
+
+        elif os.path.exists(self.filename):
+            log.debug("Deleting empty %s", self.filename)
+            os.unlink(self.filename)
+
+        self.dirty = False
+
+    @staticmethod
+    def _wrap_lines(lines):
+        return lines
+
+    def add(self, dist):
+        """Add `dist` to the distribution map"""
+        new_path = (
+            dist.location not in self.paths and (
+                dist.location not in self.sitedirs or
+                # account for '.' being in PYTHONPATH
+                dist.location == os.getcwd()
+            )
+        )
+        if new_path:
+            self.paths.append(dist.location)
+            self.dirty = True
+        Environment.add(self, dist)
+
+    def remove(self, dist):
+        """Remove `dist` from the distribution map"""
+        while dist.location in self.paths:
+            self.paths.remove(dist.location)
+            self.dirty = True
+        Environment.remove(self, dist)
+
+    def make_relative(self, path):
+        npath, last = os.path.split(normalize_path(path))
+        baselen = len(self.basedir)
+        parts = [last]
+        sep = os.altsep == '/' and '/' or os.sep
+        while len(npath) >= baselen:
+            if npath == self.basedir:
+                parts.append(os.curdir)
+                parts.reverse()
+                return sep.join(parts)
+            npath, last = os.path.split(npath)
+            parts.append(last)
+        else:
+            return path
+
+
+class RewritePthDistributions(PthDistributions):
+    @classmethod
+    def _wrap_lines(cls, lines):
+        yield cls.prelude
+        for line in lines:
+            yield line
+        yield cls.postlude
+
+    prelude = _one_liner("""
+        import sys
+        sys.__plen = len(sys.path)
+        """)
+    postlude = _one_liner("""
+        import sys
+        new = sys.path[sys.__plen:]
+        del sys.path[sys.__plen:]
+        p = getattr(sys, '__egginsert', 0)
+        sys.path[p:p] = new
+        sys.__egginsert = p + len(new)
+        """)
+
+
+if os.environ.get('SETUPTOOLS_SYS_PATH_TECHNIQUE', 'raw') == 'rewrite':
+    PthDistributions = RewritePthDistributions
+
+
+def _first_line_re():
+    """
+    Return a regular expression based on first_line_re suitable for matching
+    strings.
+    """
+    if isinstance(first_line_re.pattern, str):
+        return first_line_re
+
+    # first_line_re in Python >=3.1.4 and >=3.2.1 is a bytes pattern.
+    return re.compile(first_line_re.pattern.decode())
+
+
+def auto_chmod(func, arg, exc):
+    if func in [os.unlink, os.remove] and os.name == 'nt':
+        chmod(arg, stat.S_IWRITE)
+        return func(arg)
+    et, ev, _ = sys.exc_info()
+    six.reraise(et, (ev[0], ev[1] + (" %s %s" % (func, arg))))
+
+
+def update_dist_caches(dist_path, fix_zipimporter_caches):
+    """
+    Fix any globally cached `dist_path` related data
+
+    `dist_path` should be a path of a newly installed egg distribution (zipped
+    or unzipped).
+
+    sys.path_importer_cache contains finder objects that have been cached when
+    importing data from the original distribution. Any such finders need to be
+    cleared since the replacement distribution might be packaged differently,
+    e.g. a zipped egg distribution might get replaced with an unzipped egg
+    folder or vice versa. Having the old finders cached may then cause Python
+    to attempt loading modules from the replacement distribution using an
+    incorrect loader.
+
+    zipimport.zipimporter objects are Python loaders charged with importing
+    data packaged inside zip archives. If stale loaders referencing the
+    original distribution, are left behind, they can fail to load modules from
+    the replacement distribution. E.g. if an old zipimport.zipimporter instance
+    is used to load data from a new zipped egg archive, it may cause the
+    operation to attempt to locate the requested data in the wrong location -
+    one indicated by the original distribution's zip archive directory
+    information. Such an operation may then fail outright, e.g. report having
+    read a 'bad local file header', or even worse, it may fail silently &
+    return invalid data.
+
+    zipimport._zip_directory_cache contains cached zip archive directory
+    information for all existing zipimport.zipimporter instances and all such
+    instances connected to the same archive share the same cached directory
+    information.
+
+    If asked, and the underlying Python implementation allows it, we can fix
+    all existing zipimport.zipimporter instances instead of having to track
+    them down and remove them one by one, by updating their shared cached zip
+    archive directory information. This, of course, assumes that the
+    replacement distribution is packaged as a zipped egg.
+
+    If not asked to fix existing zipimport.zipimporter instances, we still do
+    our best to clear any remaining zipimport.zipimporter related cached data
+    that might somehow later get used when attempting to load data from the new
+    distribution and thus cause such load operations to fail. Note that when
+    tracking down such remaining stale data, we can not catch every conceivable
+    usage from here, and we clear only those that we know of and have found to
+    cause problems if left alive. Any remaining caches should be updated by
+    whomever is in charge of maintaining them, i.e. they should be ready to
+    handle us replacing their zip archives with new distributions at runtime.
+
+    """
+    # There are several other known sources of stale zipimport.zipimporter
+    # instances that we do not clear here, but might if ever given a reason to
+    # do so:
+    # * Global setuptools pkg_resources.working_set (a.k.a. 'master working
+    # set') may contain distributions which may in turn contain their
+    #   zipimport.zipimporter loaders.
+    # * Several zipimport.zipimporter loaders held by local variables further
+    #   up the function call stack when running the setuptools installation.
+    # * Already loaded modules may have their __loader__ attribute set to the
+    #   exact loader instance used when importing them. Python 3.4 docs state
+    #   that this information is intended mostly for introspection and so is
+    #   not expected to cause us problems.
+    normalized_path = normalize_path(dist_path)
+    _uncache(normalized_path, sys.path_importer_cache)
+    if fix_zipimporter_caches:
+        _replace_zip_directory_cache_data(normalized_path)
+    else:
+        # Here, even though we do not want to fix existing and now stale
+        # zipimporter cache information, we still want to remove it. Related to
+        # Python's zip archive directory information cache, we clear each of
+        # its stale entries in two phases:
+        #   1. Clear the entry so attempting to access zip archive information
+        #      via any existing stale zipimport.zipimporter instances fails.
+        #   2. Remove the entry from the cache so any newly constructed
+        #      zipimport.zipimporter instances do not end up using old stale
+        #      zip archive directory information.
+        # This whole stale data removal step does not seem strictly necessary,
+        # but has been left in because it was done before we started replacing
+        # the zip archive directory information cache content if possible, and
+        # there are no relevant unit tests that we can depend on to tell us if
+        # this is really needed.
+        _remove_and_clear_zip_directory_cache_data(normalized_path)
+
+
+def _collect_zipimporter_cache_entries(normalized_path, cache):
+    """
+    Return zipimporter cache entry keys related to a given normalized path.
+
+    Alternative path spellings (e.g. those using different character case or
+    those using alternative path separators) related to the same path are
+    included. Any sub-path entries are included as well, i.e. those
+    corresponding to zip archives embedded in other zip archives.
+
+    """
+    result = []
+    prefix_len = len(normalized_path)
+    for p in cache:
+        np = normalize_path(p)
+        if (np.startswith(normalized_path) and
+                np[prefix_len:prefix_len + 1] in (os.sep, '')):
+            result.append(p)
+    return result
+
+
+def _update_zipimporter_cache(normalized_path, cache, updater=None):
+    """
+    Update zipimporter cache data for a given normalized path.
+
+    Any sub-path entries are processed as well, i.e. those corresponding to zip
+    archives embedded in other zip archives.
+
+    Given updater is a callable taking a cache entry key and the original entry
+    (after already removing the entry from the cache), and expected to update
+    the entry and possibly return a new one to be inserted in its place.
+    Returning None indicates that the entry should not be replaced with a new
+    one. If no updater is given, the cache entries are simply removed without
+    any additional processing, the same as if the updater simply returned None.
+
+    """
+    for p in _collect_zipimporter_cache_entries(normalized_path, cache):
+        # N.B. pypy's custom zipimport._zip_directory_cache implementation does
+        # not support the complete dict interface:
+        # * Does not support item assignment, thus not allowing this function
+        #    to be used only for removing existing cache entries.
+        #  * Does not support the dict.pop() method, forcing us to use the
+        #    get/del patterns instead. For more detailed information see the
+        #    following links:
+        #      https://github.com/pypa/setuptools/issues/202#issuecomment-202913420
+        #      http://bit.ly/2h9itJX
+        old_entry = cache[p]
+        del cache[p]
+        new_entry = updater and updater(p, old_entry)
+        if new_entry is not None:
+            cache[p] = new_entry
+
+
+def _uncache(normalized_path, cache):
+    _update_zipimporter_cache(normalized_path, cache)
+
+
+def _remove_and_clear_zip_directory_cache_data(normalized_path):
+    def clear_and_remove_cached_zip_archive_directory_data(path, old_entry):
+        old_entry.clear()
+
+    _update_zipimporter_cache(
+        normalized_path, zipimport._zip_directory_cache,
+        updater=clear_and_remove_cached_zip_archive_directory_data)
+
+
+# PyPy Python implementation does not allow directly writing to the
+# zipimport._zip_directory_cache and so prevents us from attempting to correct
+# its content. The best we can do there is clear the problematic cache content
+# and have PyPy repopulate it as needed. The downside is that if there are any
+# stale zipimport.zipimporter instances laying around, attempting to use them
+# will fail due to not having its zip archive directory information available
+# instead of being automatically corrected to use the new correct zip archive
+# directory information.
+if '__pypy__' in sys.builtin_module_names:
+    _replace_zip_directory_cache_data = \
+        _remove_and_clear_zip_directory_cache_data
+else:
+
+    def _replace_zip_directory_cache_data(normalized_path):
+        def replace_cached_zip_archive_directory_data(path, old_entry):
+            # N.B. In theory, we could load the zip directory information just
+            # once for all updated path spellings, and then copy it locally and
+            # update its contained path strings to contain the correct
+            # spelling, but that seems like a way too invasive move (this cache
+            # structure is not officially documented anywhere and could in
+            # theory change with new Python releases) for no significant
+            # benefit.
+            old_entry.clear()
+            zipimport.zipimporter(path)
+            old_entry.update(zipimport._zip_directory_cache[path])
+            return old_entry
+
+        _update_zipimporter_cache(
+            normalized_path, zipimport._zip_directory_cache,
+            updater=replace_cached_zip_archive_directory_data)
+
+
+def is_python(text, filename=''):
+    "Is this string a valid Python script?"
+    try:
+        compile(text, filename, 'exec')
+    except (SyntaxError, TypeError):
+        return False
+    else:
+        return True
+
+
+def is_sh(executable):
+    """Determine if the specified executable is a .sh (contains a #! line)"""
+    try:
+        with io.open(executable, encoding='latin-1') as fp:
+            magic = fp.read(2)
+    except (OSError, IOError):
+        return executable
+    return magic == '#!'
+
+
+def nt_quote_arg(arg):
+    """Quote a command line argument according to Windows parsing rules"""
+    return subprocess.list2cmdline([arg])
+
+
+def is_python_script(script_text, filename):
+    """Is this text, as a whole, a Python script? (as opposed to shell/bat/etc.
+    """
+    if filename.endswith('.py') or filename.endswith('.pyw'):
+        return True  # extension says it's Python
+    if is_python(script_text, filename):
+        return True  # it's syntactically valid Python
+    if script_text.startswith('#!'):
+        # It begins with a '#!' line, so check if 'python' is in it somewhere
+        return 'python' in script_text.splitlines()[0].lower()
+
+    return False  # Not any Python I can recognize
+
+
+try:
+    from os import chmod as _chmod
+except ImportError:
+    # Jython compatibility
+    def _chmod(*args):
+        pass
+
+
+def chmod(path, mode):
+    log.debug("changing mode of %s to %o", path, mode)
+    try:
+        _chmod(path, mode)
+    except os.error as e:
+        log.debug("chmod failed: %s", e)
+
+
+class CommandSpec(list):
+    """
+    A command spec for a #! header, specified as a list of arguments akin to
+    those passed to Popen.
+    """
+
+    options = []
+    split_args = dict()
+
+    @classmethod
+    def best(cls):
+        """
+        Choose the best CommandSpec class based on environmental conditions.
+        """
+        return cls
+
+    @classmethod
+    def _sys_executable(cls):
+        _default = os.path.normpath(sys.executable)
+        return os.environ.get('__PYVENV_LAUNCHER__', _default)
+
+    @classmethod
+    def from_param(cls, param):
+        """
+        Construct a CommandSpec from a parameter to build_scripts, which may
+        be None.
+        """
+        if isinstance(param, cls):
+            return param
+        if isinstance(param, list):
+            return cls(param)
+        if param is None:
+            return cls.from_environment()
+        # otherwise, assume it's a string.
+        return cls.from_string(param)
+
+    @classmethod
+    def from_environment(cls):
+        return cls([cls._sys_executable()])
+
+    @classmethod
+    def from_string(cls, string):
+        """
+        Construct a command spec from a simple string representing a command
+        line parseable by shlex.split.
+        """
+        items = shlex.split(string, **cls.split_args)
+        return cls(items)
+
+    def install_options(self, script_text):
+        self.options = shlex.split(self._extract_options(script_text))
+        cmdline = subprocess.list2cmdline(self)
+        if not isascii(cmdline):
+            self.options[:0] = ['-x']
+
+    @staticmethod
+    def _extract_options(orig_script):
+        """
+        Extract any options from the first line of the script.
+        """
+        first = (orig_script + '\n').splitlines()[0]
+        match = _first_line_re().match(first)
+        options = match.group(1) or '' if match else ''
+        return options.strip()
+
+    def as_header(self):
+        return self._render(self + list(self.options))
+
+    @staticmethod
+    def _strip_quotes(item):
+        _QUOTES = '"\''
+        for q in _QUOTES:
+            if item.startswith(q) and item.endswith(q):
+                return item[1:-1]
+        return item
+
+    @staticmethod
+    def _render(items):
+        cmdline = subprocess.list2cmdline(
+            CommandSpec._strip_quotes(item.strip()) for item in items)
+        return '#!' + cmdline + '\n'
+
+
+# For pbr compat; will be removed in a future version.
+sys_executable = CommandSpec._sys_executable()
+
+
+class WindowsCommandSpec(CommandSpec):
+    split_args = dict(posix=False)
+
+
+class ScriptWriter:
+    """
+    Encapsulates behavior around writing entry point scripts for console and
+    gui apps.
+    """
+
+    template = textwrap.dedent(r"""
+        # EASY-INSTALL-ENTRY-SCRIPT: %(spec)r,%(group)r,%(name)r
+        __requires__ = %(spec)r
+        import re
+        import sys
+        from pkg_resources import load_entry_point
+
+        if __name__ == '__main__':
+            sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0])
+            sys.exit(
+                load_entry_point(%(spec)r, %(group)r, %(name)r)()
+            )
+    """).lstrip()
+
+    command_spec_class = CommandSpec
+
+    @classmethod
+    def get_script_args(cls, dist, executable=None, wininst=False):
+        # for backward compatibility
+        warnings.warn("Use get_args", EasyInstallDeprecationWarning)
+        writer = (WindowsScriptWriter if wininst else ScriptWriter).best()
+        header = cls.get_script_header("", executable, wininst)
+        return writer.get_args(dist, header)
+
+    @classmethod
+    def get_script_header(cls, script_text, executable=None, wininst=False):
+        # for backward compatibility
+        warnings.warn("Use get_header", EasyInstallDeprecationWarning, stacklevel=2)
+        if wininst:
+            executable = "python.exe"
+        return cls.get_header(script_text, executable)
+
+    @classmethod
+    def get_args(cls, dist, header=None):
+        """
+        Yield write_script() argument tuples for a distribution's
+        console_scripts and gui_scripts entry points.
+        """
+        if header is None:
+            header = cls.get_header()
+        spec = str(dist.as_requirement())
+        for type_ in 'console', 'gui':
+            group = type_ + '_scripts'
+            for name, ep in dist.get_entry_map(group).items():
+                cls._ensure_safe_name(name)
+                script_text = cls.template % locals()
+                args = cls._get_script_args(type_, name, header, script_text)
+                for res in args:
+                    yield res
+
+    @staticmethod
+    def _ensure_safe_name(name):
+        """
+        Prevent paths in *_scripts entry point names.
+        """
+        has_path_sep = re.search(r'[\\/]', name)
+        if has_path_sep:
+            raise ValueError("Path separators not allowed in script names")
+
+    @classmethod
+    def get_writer(cls, force_windows):
+        # for backward compatibility
+        warnings.warn("Use best", EasyInstallDeprecationWarning)
+        return WindowsScriptWriter.best() if force_windows else cls.best()
+
+    @classmethod
+    def best(cls):
+        """
+        Select the best ScriptWriter for this environment.
+        """
+        if sys.platform == 'win32' or (os.name == 'java' and os._name == 'nt'):
+            return WindowsScriptWriter.best()
+        else:
+            return cls
+
+    @classmethod
+    def _get_script_args(cls, type_, name, header, script_text):
+        # Simply write the stub with no extension.
+        yield (name, header + script_text)
+
+    @classmethod
+    def get_header(cls, script_text="", executable=None):
+        """Create a #! line, getting options (if any) from script_text"""
+        cmd = cls.command_spec_class.best().from_param(executable)
+        cmd.install_options(script_text)
+        return cmd.as_header()
+
+
+class WindowsScriptWriter(ScriptWriter):
+    command_spec_class = WindowsCommandSpec
+
+    @classmethod
+    def get_writer(cls):
+        # for backward compatibility
+        warnings.warn("Use best", EasyInstallDeprecationWarning)
+        return cls.best()
+
+    @classmethod
+    def best(cls):
+        """
+        Select the best ScriptWriter suitable for Windows
+        """
+        writer_lookup = dict(
+            executable=WindowsExecutableLauncherWriter,
+            natural=cls,
+        )
+        # for compatibility, use the executable launcher by default
+        launcher = os.environ.get('SETUPTOOLS_LAUNCHER', 'executable')
+        return writer_lookup[launcher]
+
+    @classmethod
+    def _get_script_args(cls, type_, name, header, script_text):
+        "For Windows, add a .py extension"
+        ext = dict(console='.pya', gui='.pyw')[type_]
+        if ext not in os.environ['PATHEXT'].lower().split(';'):
+            msg = (
+                "{ext} not listed in PATHEXT; scripts will not be "
+                "recognized as executables."
+            ).format(**locals())
+            warnings.warn(msg, UserWarning)
+        old = ['.pya', '.py', '-script.py', '.pyc', '.pyo', '.pyw', '.exe']
+        old.remove(ext)
+        header = cls._adjust_header(type_, header)
+        blockers = [name + x for x in old]
+        yield name + ext, header + script_text, 't', blockers
+
+    @classmethod
+    def _adjust_header(cls, type_, orig_header):
+        """
+        Make sure 'pythonw' is used for gui and and 'python' is used for
+        console (regardless of what sys.executable is).
+        """
+        pattern = 'pythonw.exe'
+        repl = 'python.exe'
+        if type_ == 'gui':
+            pattern, repl = repl, pattern
+        pattern_ob = re.compile(re.escape(pattern), re.IGNORECASE)
+        new_header = pattern_ob.sub(string=orig_header, repl=repl)
+        return new_header if cls._use_header(new_header) else orig_header
+
+    @staticmethod
+    def _use_header(new_header):
+        """
+        Should _adjust_header use the replaced header?
+
+        On non-windows systems, always use. On
+        Windows systems, only use the replaced header if it resolves
+        to an executable on the system.
+        """
+        clean_header = new_header[2:-1].strip('"')
+        return sys.platform != 'win32' or find_executable(clean_header)
+
+
+class WindowsExecutableLauncherWriter(WindowsScriptWriter):
+    @classmethod
+    def _get_script_args(cls, type_, name, header, script_text):
+        """
+        For Windows, add a .py extension and an .exe launcher
+        """
+        if type_ == 'gui':
+            launcher_type = 'gui'
+            ext = '-script.pyw'
+            old = ['.pyw']
+        else:
+            launcher_type = 'cli'
+            ext = '-script.py'
+            old = ['.py', '.pyc', '.pyo']
+        hdr = cls._adjust_header(type_, header)
+        blockers = [name + x for x in old]
+        yield (name + ext, hdr + script_text, 't', blockers)
+        yield (
+            name + '.exe', get_win_launcher(launcher_type),
+            'b'  # write in binary mode
+        )
+        if not is_64bit():
+            # install a manifest for the launcher to prevent Windows
+            # from detecting it as an installer (which it will for
+            #  launchers like easy_install.exe). Consider only
+            #  adding a manifest for launchers detected as installers.
+            #  See Distribute #143 for details.
+            m_name = name + '.exe.manifest'
+            yield (m_name, load_launcher_manifest(name), 't')
+
+
+# for backward-compatibility
+get_script_args = ScriptWriter.get_script_args
+get_script_header = ScriptWriter.get_script_header
+
+
+def get_win_launcher(type):
+    """
+    Load the Windows launcher (executable) suitable for launching a script.
+
+    `type` should be either 'cli' or 'gui'
+
+    Returns the executable as a byte string.
+    """
+    launcher_fn = '%s.exe' % type
+    if is_64bit():
+        launcher_fn = launcher_fn.replace(".", "-64.")
+    else:
+        launcher_fn = launcher_fn.replace(".", "-32.")
+    return resource_string('setuptools', launcher_fn)
+
+
+def load_launcher_manifest(name):
+    manifest = pkg_resources.resource_string(__name__, 'launcher manifest.xml')
+    if six.PY2:
+        return manifest % vars()
+    else:
+        return manifest.decode('utf-8') % vars()
+
+
+def rmtree(path, ignore_errors=False, onerror=auto_chmod):
+    return shutil.rmtree(path, ignore_errors, onerror)
+
+
+def current_umask():
+    tmp = os.umask(0o022)
+    os.umask(tmp)
+    return tmp
+
+
+def bootstrap():
+    # This function is called when setuptools*.egg is run using /bin/sh
+    import setuptools
+
+    argv0 = os.path.dirname(setuptools.__path__[0])
+    sys.argv[0] = argv0
+    sys.argv.append(argv0)
+    main()
+
+
+def main(argv=None, **kw):
+    from setuptools import setup
+    from setuptools.dist import Distribution
+
+    class DistributionWithoutHelpCommands(Distribution):
+        common_usage = ""
+
+        def _show_help(self, *args, **kw):
+            with _patch_usage():
+                Distribution._show_help(self, *args, **kw)
+
+    if argv is None:
+        argv = sys.argv[1:]
+
+    with _patch_usage():
+        setup(
+            script_args=['-q', 'easy_install', '-v'] + argv,
+            script_name=sys.argv[0] or 'easy_install',
+            distclass=DistributionWithoutHelpCommands,
+            **kw
+        )
+
+
+@contextlib.contextmanager
+def _patch_usage():
+    import distutils.core
+    USAGE = textwrap.dedent("""
+        usage: %(script)s [options] requirement_or_url ...
+           or: %(script)s --help
+        """).lstrip()
+
+    def gen_usage(script_name):
+        return USAGE % dict(
+            script=os.path.basename(script_name),
+        )
+
+    saved = distutils.core.gen_usage
+    distutils.core.gen_usage = gen_usage
+    try:
+        yield
+    finally:
+        distutils.core.gen_usage = saved
+
+class EasyInstallDeprecationWarning(SetuptoolsDeprecationWarning):
+    """Class for warning about deprecations in EasyInstall in SetupTools. Not ignored by default, unlike DeprecationWarning."""
+    
diff --git a/ubuntu/venv/setuptools/command/egg_info.py b/ubuntu/venv/setuptools/command/egg_info.py
new file mode 100644
index 0000000..b767ef3
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/egg_info.py
@@ -0,0 +1,717 @@
+"""setuptools.command.egg_info
+
+Create a distribution's .egg-info directory and contents"""
+
+from distutils.filelist import FileList as _FileList
+from distutils.errors import DistutilsInternalError
+from distutils.util import convert_path
+from distutils import log
+import distutils.errors
+import distutils.filelist
+import os
+import re
+import sys
+import io
+import warnings
+import time
+import collections
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import map
+
+from setuptools import Command
+from setuptools.command.sdist import sdist
+from setuptools.command.sdist import walk_revctrl
+from setuptools.command.setopt import edit_config
+from setuptools.command import bdist_egg
+from pkg_resources import (
+    parse_requirements, safe_name, parse_version,
+    safe_version, yield_lines, EntryPoint, iter_entry_points, to_filename)
+import setuptools.unicode_utils as unicode_utils
+from setuptools.glob import glob
+
+from setuptools.extern import packaging
+from setuptools import SetuptoolsDeprecationWarning
+
+def translate_pattern(glob):
+    """
+    Translate a file path glob like '*.txt' in to a regular expression.
+    This differs from fnmatch.translate which allows wildcards to match
+    directory separators. It also knows about '**/' which matches any number of
+    directories.
+    """
+    pat = ''
+
+    # This will split on '/' within [character classes]. This is deliberate.
+    chunks = glob.split(os.path.sep)
+
+    sep = re.escape(os.sep)
+    valid_char = '[^%s]' % (sep,)
+
+    for c, chunk in enumerate(chunks):
+        last_chunk = c == len(chunks) - 1
+
+        # Chunks that are a literal ** are globstars. They match anything.
+        if chunk == '**':
+            if last_chunk:
+                # Match anything if this is the last component
+                pat += '.*'
+            else:
+                # Match '(name/)*'
+                pat += '(?:%s+%s)*' % (valid_char, sep)
+            continue  # Break here as the whole path component has been handled
+
+        # Find any special characters in the remainder
+        i = 0
+        chunk_len = len(chunk)
+        while i < chunk_len:
+            char = chunk[i]
+            if char == '*':
+                # Match any number of name characters
+                pat += valid_char + '*'
+            elif char == '?':
+                # Match a name character
+                pat += valid_char
+            elif char == '[':
+                # Character class
+                inner_i = i + 1
+                # Skip initial !/] chars
+                if inner_i < chunk_len and chunk[inner_i] == '!':
+                    inner_i = inner_i + 1
+                if inner_i < chunk_len and chunk[inner_i] == ']':
+                    inner_i = inner_i + 1
+
+                # Loop till the closing ] is found
+                while inner_i < chunk_len and chunk[inner_i] != ']':
+                    inner_i = inner_i + 1
+
+                if inner_i >= chunk_len:
+                    # Got to the end of the string without finding a closing ]
+                    # Do not treat this as a matching group, but as a literal [
+                    pat += re.escape(char)
+                else:
+                    # Grab the insides of the [brackets]
+                    inner = chunk[i + 1:inner_i]
+                    char_class = ''
+
+                    # Class negation
+                    if inner[0] == '!':
+                        char_class = '^'
+                        inner = inner[1:]
+
+                    char_class += re.escape(inner)
+                    pat += '[%s]' % (char_class,)
+
+                    # Skip to the end ]
+                    i = inner_i
+            else:
+                pat += re.escape(char)
+            i += 1
+
+        # Join each chunk with the dir separator
+        if not last_chunk:
+            pat += sep
+
+    pat += r'\Z'
+    return re.compile(pat, flags=re.MULTILINE|re.DOTALL)
+
+
+class InfoCommon:
+    tag_build = None
+    tag_date = None
+
+    @property
+    def name(self):
+        return safe_name(self.distribution.get_name())
+
+    def tagged_version(self):
+        version = self.distribution.get_version()
+        # egg_info may be called more than once for a distribution,
+        # in which case the version string already contains all tags.
+        if self.vtags and version.endswith(self.vtags):
+            return safe_version(version)
+        return safe_version(version + self.vtags)
+
+    def tags(self):
+        version = ''
+        if self.tag_build:
+            version += self.tag_build
+        if self.tag_date:
+            version += time.strftime("-%Y%m%d")
+        return version
+    vtags = property(tags)
+
+
+class egg_info(InfoCommon, Command):
+    description = "create a distribution's .egg-info directory"
+
+    user_options = [
+        ('egg-base=', 'e', "directory containing .egg-info directories"
+                           " (default: top of the source tree)"),
+        ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"),
+        ('tag-build=', 'b', "Specify explicit tag to add to version number"),
+        ('no-date', 'D', "Don't include date stamp [default]"),
+    ]
+
+    boolean_options = ['tag-date']
+    negative_opt = {
+        'no-date': 'tag-date',
+    }
+
+    def initialize_options(self):
+        self.egg_base = None
+        self.egg_name = None
+        self.egg_info = None
+        self.egg_version = None
+        self.broken_egg_info = False
+
+    ####################################
+    # allow the 'tag_svn_revision' to be detected and
+    # set, supporting sdists built on older Setuptools.
+    @property
+    def tag_svn_revision(self):
+        pass
+
+    @tag_svn_revision.setter
+    def tag_svn_revision(self, value):
+        pass
+    ####################################
+
+    def save_version_info(self, filename):
+        """
+        Materialize the value of date into the
+        build tag. Install build keys in a deterministic order
+        to avoid arbitrary reordering on subsequent builds.
+        """
+        egg_info = collections.OrderedDict()
+        # follow the order these keys would have been added
+        # when PYTHONHASHSEED=0
+        egg_info['tag_build'] = self.tags()
+        egg_info['tag_date'] = 0
+        edit_config(filename, dict(egg_info=egg_info))
+
+    def finalize_options(self):
+        # Note: we need to capture the current value returned
+        # by `self.tagged_version()`, so we can later update
+        # `self.distribution.metadata.version` without
+        # repercussions.
+        self.egg_name = self.name
+        self.egg_version = self.tagged_version()
+        parsed_version = parse_version(self.egg_version)
+
+        try:
+            is_version = isinstance(parsed_version, packaging.version.Version)
+            spec = (
+                "%s==%s" if is_version else "%s===%s"
+            )
+            list(
+                parse_requirements(spec % (self.egg_name, self.egg_version))
+            )
+        except ValueError:
+            raise distutils.errors.DistutilsOptionError(
+                "Invalid distribution name or version syntax: %s-%s" %
+                (self.egg_name, self.egg_version)
+            )
+
+        if self.egg_base is None:
+            dirs = self.distribution.package_dir
+            self.egg_base = (dirs or {}).get('', os.curdir)
+
+        self.ensure_dirname('egg_base')
+        self.egg_info = to_filename(self.egg_name) + '.egg-info'
+        if self.egg_base != os.curdir:
+            self.egg_info = os.path.join(self.egg_base, self.egg_info)
+        if '-' in self.egg_name:
+            self.check_broken_egg_info()
+
+        # Set package version for the benefit of dumber commands
+        # (e.g. sdist, bdist_wininst, etc.)
+        #
+        self.distribution.metadata.version = self.egg_version
+
+        # If we bootstrapped around the lack of a PKG-INFO, as might be the
+        # case in a fresh checkout, make sure that any special tags get added
+        # to the version info
+        #
+        pd = self.distribution._patched_dist
+        if pd is not None and pd.key == self.egg_name.lower():
+            pd._version = self.egg_version
+            pd._parsed_version = parse_version(self.egg_version)
+            self.distribution._patched_dist = None
+
+    def write_or_delete_file(self, what, filename, data, force=False):
+        """Write `data` to `filename` or delete if empty
+
+        If `data` is non-empty, this routine is the same as ``write_file()``.
+        If `data` is empty but not ``None``, this is the same as calling
+        ``delete_file(filename)`.  If `data` is ``None``, then this is a no-op
+        unless `filename` exists, in which case a warning is issued about the
+        orphaned file (if `force` is false), or deleted (if `force` is true).
+        """
+        if data:
+            self.write_file(what, filename, data)
+        elif os.path.exists(filename):
+            if data is None and not force:
+                log.warn(
+                    "%s not set in setup(), but %s exists", what, filename
+                )
+                return
+            else:
+                self.delete_file(filename)
+
+    def write_file(self, what, filename, data):
+        """Write `data` to `filename` (if not a dry run) after announcing it
+
+        `what` is used in a log message to identify what is being written
+        to the file.
+        """
+        log.info("writing %s to %s", what, filename)
+        if six.PY3:
+            data = data.encode("utf-8")
+        if not self.dry_run:
+            f = open(filename, 'wb')
+            f.write(data)
+            f.close()
+
+    def delete_file(self, filename):
+        """Delete `filename` (if not a dry run) after announcing it"""
+        log.info("deleting %s", filename)
+        if not self.dry_run:
+            os.unlink(filename)
+
+    def run(self):
+        self.mkpath(self.egg_info)
+        os.utime(self.egg_info, None)
+        installer = self.distribution.fetch_build_egg
+        for ep in iter_entry_points('egg_info.writers'):
+            ep.require(installer=installer)
+            writer = ep.resolve()
+            writer(self, ep.name, os.path.join(self.egg_info, ep.name))
+
+        # Get rid of native_libs.txt if it was put there by older bdist_egg
+        nl = os.path.join(self.egg_info, "native_libs.txt")
+        if os.path.exists(nl):
+            self.delete_file(nl)
+
+        self.find_sources()
+
+    def find_sources(self):
+        """Generate SOURCES.txt manifest file"""
+        manifest_filename = os.path.join(self.egg_info, "SOURCES.txt")
+        mm = manifest_maker(self.distribution)
+        mm.manifest = manifest_filename
+        mm.run()
+        self.filelist = mm.filelist
+
+    def check_broken_egg_info(self):
+        bei = self.egg_name + '.egg-info'
+        if self.egg_base != os.curdir:
+            bei = os.path.join(self.egg_base, bei)
+        if os.path.exists(bei):
+            log.warn(
+                "-" * 78 + '\n'
+                "Note: Your current .egg-info directory has a '-' in its name;"
+                '\nthis will not work correctly with "setup.py develop".\n\n'
+                'Please rename %s to %s to correct this problem.\n' + '-' * 78,
+                bei, self.egg_info
+            )
+            self.broken_egg_info = self.egg_info
+            self.egg_info = bei  # make it work for now
+
+
+class FileList(_FileList):
+    # Implementations of the various MANIFEST.in commands
+
+    def process_template_line(self, line):
+        # Parse the line: split it up, make sure the right number of words
+        # is there, and return the relevant words.  'action' is always
+        # defined: it's the first word of the line.  Which of the other
+        # three are defined depends on the action; it'll be either
+        # patterns, (dir and patterns), or (dir_pattern).
+        (action, patterns, dir, dir_pattern) = self._parse_template_line(line)
+
+        # OK, now we know that the action is valid and we have the
+        # right number of words on the line for that action -- so we
+        # can proceed with minimal error-checking.
+        if action == 'include':
+            self.debug_print("include " + ' '.join(patterns))
+            for pattern in patterns:
+                if not self.include(pattern):
+                    log.warn("warning: no files found matching '%s'", pattern)
+
+        elif action == 'exclude':
+            self.debug_print("exclude " + ' '.join(patterns))
+            for pattern in patterns:
+                if not self.exclude(pattern):
+                    log.warn(("warning: no previously-included files "
+                              "found matching '%s'"), pattern)
+
+        elif action == 'global-include':
+            self.debug_print("global-include " + ' '.join(patterns))
+            for pattern in patterns:
+                if not self.global_include(pattern):
+                    log.warn(("warning: no files found matching '%s' "
+                              "anywhere in distribution"), pattern)
+
+        elif action == 'global-exclude':
+            self.debug_print("global-exclude " + ' '.join(patterns))
+            for pattern in patterns:
+                if not self.global_exclude(pattern):
+                    log.warn(("warning: no previously-included files matching "
+                              "'%s' found anywhere in distribution"),
+                             pattern)
+
+        elif action == 'recursive-include':
+            self.debug_print("recursive-include %s %s" %
+                             (dir, ' '.join(patterns)))
+            for pattern in patterns:
+                if not self.recursive_include(dir, pattern):
+                    log.warn(("warning: no files found matching '%s' "
+                              "under directory '%s'"),
+                             pattern, dir)
+
+        elif action == 'recursive-exclude':
+            self.debug_print("recursive-exclude %s %s" %
+                             (dir, ' '.join(patterns)))
+            for pattern in patterns:
+                if not self.recursive_exclude(dir, pattern):
+                    log.warn(("warning: no previously-included files matching "
+                              "'%s' found under directory '%s'"),
+                             pattern, dir)
+
+        elif action == 'graft':
+            self.debug_print("graft " + dir_pattern)
+            if not self.graft(dir_pattern):
+                log.warn("warning: no directories found matching '%s'",
+                         dir_pattern)
+
+        elif action == 'prune':
+            self.debug_print("prune " + dir_pattern)
+            if not self.prune(dir_pattern):
+                log.warn(("no previously-included directories found "
+                          "matching '%s'"), dir_pattern)
+
+        else:
+            raise DistutilsInternalError(
+                "this cannot happen: invalid action '%s'" % action)
+
+    def _remove_files(self, predicate):
+        """
+        Remove all files from the file list that match the predicate.
+        Return True if any matching files were removed
+        """
+        found = False
+        for i in range(len(self.files) - 1, -1, -1):
+            if predicate(self.files[i]):
+                self.debug_print(" removing " + self.files[i])
+                del self.files[i]
+                found = True
+        return found
+
+    def include(self, pattern):
+        """Include files that match 'pattern'."""
+        found = [f for f in glob(pattern) if not os.path.isdir(f)]
+        self.extend(found)
+        return bool(found)
+
+    def exclude(self, pattern):
+        """Exclude files that match 'pattern'."""
+        match = translate_pattern(pattern)
+        return self._remove_files(match.match)
+
+    def recursive_include(self, dir, pattern):
+        """
+        Include all files anywhere in 'dir/' that match the pattern.
+        """
+        full_pattern = os.path.join(dir, '**', pattern)
+        found = [f for f in glob(full_pattern, recursive=True)
+                 if not os.path.isdir(f)]
+        self.extend(found)
+        return bool(found)
+
+    def recursive_exclude(self, dir, pattern):
+        """
+        Exclude any file anywhere in 'dir/' that match the pattern.
+        """
+        match = translate_pattern(os.path.join(dir, '**', pattern))
+        return self._remove_files(match.match)
+
+    def graft(self, dir):
+        """Include all files from 'dir/'."""
+        found = [
+            item
+            for match_dir in glob(dir)
+            for item in distutils.filelist.findall(match_dir)
+        ]
+        self.extend(found)
+        return bool(found)
+
+    def prune(self, dir):
+        """Filter out files from 'dir/'."""
+        match = translate_pattern(os.path.join(dir, '**'))
+        return self._remove_files(match.match)
+
+    def global_include(self, pattern):
+        """
+        Include all files anywhere in the current directory that match the
+        pattern. This is very inefficient on large file trees.
+        """
+        if self.allfiles is None:
+            self.findall()
+        match = translate_pattern(os.path.join('**', pattern))
+        found = [f for f in self.allfiles if match.match(f)]
+        self.extend(found)
+        return bool(found)
+
+    def global_exclude(self, pattern):
+        """
+        Exclude all files anywhere that match the pattern.
+        """
+        match = translate_pattern(os.path.join('**', pattern))
+        return self._remove_files(match.match)
+
+    def append(self, item):
+        if item.endswith('\r'):  # Fix older sdists built on Windows
+            item = item[:-1]
+        path = convert_path(item)
+
+        if self._safe_path(path):
+            self.files.append(path)
+
+    def extend(self, paths):
+        self.files.extend(filter(self._safe_path, paths))
+
+    def _repair(self):
+        """
+        Replace self.files with only safe paths
+
+        Because some owners of FileList manipulate the underlying
+        ``files`` attribute directly, this method must be called to
+        repair those paths.
+        """
+        self.files = list(filter(self._safe_path, self.files))
+
+    def _safe_path(self, path):
+        enc_warn = "'%s' not %s encodable -- skipping"
+
+        # To avoid accidental trans-codings errors, first to unicode
+        u_path = unicode_utils.filesys_decode(path)
+        if u_path is None:
+            log.warn("'%s' in unexpected encoding -- skipping" % path)
+            return False
+
+        # Must ensure utf-8 encodability
+        utf8_path = unicode_utils.try_encode(u_path, "utf-8")
+        if utf8_path is None:
+            log.warn(enc_warn, path, 'utf-8')
+            return False
+
+        try:
+            # accept is either way checks out
+            if os.path.exists(u_path) or os.path.exists(utf8_path):
+                return True
+        # this will catch any encode errors decoding u_path
+        except UnicodeEncodeError:
+            log.warn(enc_warn, path, sys.getfilesystemencoding())
+
+
+class manifest_maker(sdist):
+    template = "MANIFEST.in"
+
+    def initialize_options(self):
+        self.use_defaults = 1
+        self.prune = 1
+        self.manifest_only = 1
+        self.force_manifest = 1
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        self.filelist = FileList()
+        if not os.path.exists(self.manifest):
+            self.write_manifest()  # it must exist so it'll get in the list
+        self.add_defaults()
+        if os.path.exists(self.template):
+            self.read_template()
+        self.prune_file_list()
+        self.filelist.sort()
+        self.filelist.remove_duplicates()
+        self.write_manifest()
+
+    def _manifest_normalize(self, path):
+        path = unicode_utils.filesys_decode(path)
+        return path.replace(os.sep, '/')
+
+    def write_manifest(self):
+        """
+        Write the file list in 'self.filelist' to the manifest file
+        named by 'self.manifest'.
+        """
+        self.filelist._repair()
+
+        # Now _repairs should encodability, but not unicode
+        files = [self._manifest_normalize(f) for f in self.filelist.files]
+        msg = "writing manifest file '%s'" % self.manifest
+        self.execute(write_file, (self.manifest, files), msg)
+
+    def warn(self, msg):
+        if not self._should_suppress_warning(msg):
+            sdist.warn(self, msg)
+
+    @staticmethod
+    def _should_suppress_warning(msg):
+        """
+        suppress missing-file warnings from sdist
+        """
+        return re.match(r"standard file .*not found", msg)
+
+    def add_defaults(self):
+        sdist.add_defaults(self)
+        self.check_license()
+        self.filelist.append(self.template)
+        self.filelist.append(self.manifest)
+        rcfiles = list(walk_revctrl())
+        if rcfiles:
+            self.filelist.extend(rcfiles)
+        elif os.path.exists(self.manifest):
+            self.read_manifest()
+
+        if os.path.exists("setup.py"):
+            # setup.py should be included by default, even if it's not
+            # the script called to create the sdist
+            self.filelist.append("setup.py")
+
+        ei_cmd = self.get_finalized_command('egg_info')
+        self.filelist.graft(ei_cmd.egg_info)
+
+    def prune_file_list(self):
+        build = self.get_finalized_command('build')
+        base_dir = self.distribution.get_fullname()
+        self.filelist.prune(build.build_base)
+        self.filelist.prune(base_dir)
+        sep = re.escape(os.sep)
+        self.filelist.exclude_pattern(r'(^|' + sep + r')(RCS|CVS|\.svn)' + sep,
+                                      is_regex=1)
+
+
+def write_file(filename, contents):
+    """Create a file with the specified name and write 'contents' (a
+    sequence of strings without line terminators) to it.
+    """
+    contents = "\n".join(contents)
+
+    # assuming the contents has been vetted for utf-8 encoding
+    contents = contents.encode("utf-8")
+
+    with open(filename, "wb") as f:  # always write POSIX-style manifest
+        f.write(contents)
+
+
+def write_pkg_info(cmd, basename, filename):
+    log.info("writing %s", filename)
+    if not cmd.dry_run:
+        metadata = cmd.distribution.metadata
+        metadata.version, oldver = cmd.egg_version, metadata.version
+        metadata.name, oldname = cmd.egg_name, metadata.name
+
+        try:
+            # write unescaped data to PKG-INFO, so older pkg_resources
+            # can still parse it
+            metadata.write_pkg_info(cmd.egg_info)
+        finally:
+            metadata.name, metadata.version = oldname, oldver
+
+        safe = getattr(cmd.distribution, 'zip_safe', None)
+
+        bdist_egg.write_safety_flag(cmd.egg_info, safe)
+
+
+def warn_depends_obsolete(cmd, basename, filename):
+    if os.path.exists(filename):
+        log.warn(
+            "WARNING: 'depends.txt' is not used by setuptools 0.6!\n"
+            "Use the install_requires/extras_require setup() args instead."
+        )
+
+
+def _write_requirements(stream, reqs):
+    lines = yield_lines(reqs or ())
+    append_cr = lambda line: line + '\n'
+    lines = map(append_cr, sorted(lines))
+    stream.writelines(lines)
+
+
+def write_requirements(cmd, basename, filename):
+    dist = cmd.distribution
+    data = six.StringIO()
+    _write_requirements(data, dist.install_requires)
+    extras_require = dist.extras_require or {}
+    for extra in sorted(extras_require):
+        data.write('\n[{extra}]\n'.format(**vars()))
+        _write_requirements(data, extras_require[extra])
+    cmd.write_or_delete_file("requirements", filename, data.getvalue())
+
+
+def write_setup_requirements(cmd, basename, filename):
+    data = io.StringIO()
+    _write_requirements(data, cmd.distribution.setup_requires)
+    cmd.write_or_delete_file("setup-requirements", filename, data.getvalue())
+
+
+def write_toplevel_names(cmd, basename, filename):
+    pkgs = dict.fromkeys(
+        [
+            k.split('.', 1)[0]
+            for k in cmd.distribution.iter_distribution_names()
+        ]
+    )
+    cmd.write_file("top-level names", filename, '\n'.join(sorted(pkgs)) + '\n')
+
+
+def overwrite_arg(cmd, basename, filename):
+    write_arg(cmd, basename, filename, True)
+
+
+def write_arg(cmd, basename, filename, force=False):
+    argname = os.path.splitext(basename)[0]
+    value = getattr(cmd.distribution, argname, None)
+    if value is not None:
+        value = '\n'.join(value) + '\n'
+    cmd.write_or_delete_file(argname, filename, value, force)
+
+
+def write_entries(cmd, basename, filename):
+    ep = cmd.distribution.entry_points
+
+    if isinstance(ep, six.string_types) or ep is None:
+        data = ep
+    elif ep is not None:
+        data = []
+        for section, contents in sorted(ep.items()):
+            if not isinstance(contents, six.string_types):
+                contents = EntryPoint.parse_group(section, contents)
+                contents = '\n'.join(sorted(map(str, contents.values())))
+            data.append('[%s]\n%s\n\n' % (section, contents))
+        data = ''.join(data)
+
+    cmd.write_or_delete_file('entry points', filename, data, True)
+
+
+def get_pkg_info_revision():
+    """
+    Get a -r### off of PKG-INFO Version in case this is an sdist of
+    a subversion revision.
+    """
+    warnings.warn("get_pkg_info_revision is deprecated.", EggInfoDeprecationWarning)
+    if os.path.exists('PKG-INFO'):
+        with io.open('PKG-INFO') as f:
+            for line in f:
+                match = re.match(r"Version:.*-r(\d+)\s*$", line)
+                if match:
+                    return int(match.group(1))
+    return 0
+
+
+class EggInfoDeprecationWarning(SetuptoolsDeprecationWarning):
+    """Class for warning about deprecations in eggInfo in setupTools. Not ignored by default, unlike DeprecationWarning."""
diff --git a/ubuntu/venv/setuptools/command/install.py b/ubuntu/venv/setuptools/command/install.py
new file mode 100644
index 0000000..72b9a3e
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/install.py
@@ -0,0 +1,125 @@
+from distutils.errors import DistutilsArgError
+import inspect
+import glob
+import warnings
+import platform
+import distutils.command.install as orig
+
+import setuptools
+
+# Prior to numpy 1.9, NumPy relies on the '_install' name, so provide it for
+# now. See https://github.com/pypa/setuptools/issues/199/
+_install = orig.install
+
+
+class install(orig.install):
+    """Use easy_install to install the package, w/dependencies"""
+
+    user_options = orig.install.user_options + [
+        ('old-and-unmanageable', None, "Try not to use this!"),
+        ('single-version-externally-managed', None,
+         "used by system package builders to create 'flat' eggs"),
+    ]
+    boolean_options = orig.install.boolean_options + [
+        'old-and-unmanageable', 'single-version-externally-managed',
+    ]
+    new_commands = [
+        ('install_egg_info', lambda self: True),
+        ('install_scripts', lambda self: True),
+    ]
+    _nc = dict(new_commands)
+
+    def initialize_options(self):
+        orig.install.initialize_options(self)
+        self.old_and_unmanageable = None
+        self.single_version_externally_managed = None
+
+    def finalize_options(self):
+        orig.install.finalize_options(self)
+        if self.root:
+            self.single_version_externally_managed = True
+        elif self.single_version_externally_managed:
+            if not self.root and not self.record:
+                raise DistutilsArgError(
+                    "You must specify --record or --root when building system"
+                    " packages"
+                )
+
+    def handle_extra_path(self):
+        if self.root or self.single_version_externally_managed:
+            # explicit backward-compatibility mode, allow extra_path to work
+            return orig.install.handle_extra_path(self)
+
+        # Ignore extra_path when installing an egg (or being run by another
+        # command without --root or --single-version-externally-managed
+        self.path_file = None
+        self.extra_dirs = ''
+
+    def run(self):
+        # Explicit request for old-style install?  Just do it
+        if self.old_and_unmanageable or self.single_version_externally_managed:
+            return orig.install.run(self)
+
+        if not self._called_from_setup(inspect.currentframe()):
+            # Run in backward-compatibility mode to support bdist_* commands.
+            orig.install.run(self)
+        else:
+            self.do_egg_install()
+
+    @staticmethod
+    def _called_from_setup(run_frame):
+        """
+        Attempt to detect whether run() was called from setup() or by another
+        command.  If called by setup(), the parent caller will be the
+        'run_command' method in 'distutils.dist', and *its* caller will be
+        the 'run_commands' method.  If called any other way, the
+        immediate caller *might* be 'run_command', but it won't have been
+        called by 'run_commands'. Return True in that case or if a call stack
+        is unavailable. Return False otherwise.
+        """
+        if run_frame is None:
+            msg = "Call stack not available. bdist_* commands may fail."
+            warnings.warn(msg)
+            if platform.python_implementation() == 'IronPython':
+                msg = "For best results, pass -X:Frames to enable call stack."
+                warnings.warn(msg)
+            return True
+        res = inspect.getouterframes(run_frame)[2]
+        caller, = res[:1]
+        info = inspect.getframeinfo(caller)
+        caller_module = caller.f_globals.get('__name__', '')
+        return (
+            caller_module == 'distutils.dist'
+            and info.function == 'run_commands'
+        )
+
+    def do_egg_install(self):
+
+        easy_install = self.distribution.get_command_class('easy_install')
+
+        cmd = easy_install(
+            self.distribution, args="x", root=self.root, record=self.record,
+        )
+        cmd.ensure_finalized()  # finalize before bdist_egg munges install cmd
+        cmd.always_copy_from = '.'  # make sure local-dir eggs get installed
+
+        # pick up setup-dir .egg files only: no .egg-info
+        cmd.package_index.scan(glob.glob('*.egg'))
+
+        self.run_command('bdist_egg')
+        args = [self.distribution.get_command_obj('bdist_egg').egg_output]
+
+        if setuptools.bootstrap_install_from:
+            # Bootstrap self-installation of setuptools
+            args.insert(0, setuptools.bootstrap_install_from)
+
+        cmd.args = args
+        cmd.run(show_deprecation=False)
+        setuptools.bootstrap_install_from = None
+
+
+# XXX Python 3.1 doesn't see _nc if this is inside the class
+install.sub_commands = (
+    [cmd for cmd in orig.install.sub_commands if cmd[0] not in install._nc] +
+    install.new_commands
+)
diff --git a/ubuntu/venv/setuptools/command/install_egg_info.py b/ubuntu/venv/setuptools/command/install_egg_info.py
new file mode 100644
index 0000000..5f405bc
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/install_egg_info.py
@@ -0,0 +1,82 @@
+from distutils import log, dir_util
+import os, sys
+
+from setuptools import Command
+from setuptools import namespaces
+from setuptools.archive_util import unpack_archive
+import pkg_resources
+
+
+class install_egg_info(namespaces.Installer, Command):
+    """Install an .egg-info directory for the package"""
+
+    description = "Install an .egg-info directory for the package"
+
+    user_options = [
+        ('install-dir=', 'd', "directory to install to"),
+    ]
+
+    def initialize_options(self):
+        self.install_dir = None
+        self.install_layout = None
+        self.prefix_option = None
+
+    def finalize_options(self):
+        self.set_undefined_options('install_lib',
+                                   ('install_dir', 'install_dir'))
+        self.set_undefined_options('install',('install_layout','install_layout'))
+        if sys.hexversion > 0x2060000:
+            self.set_undefined_options('install',('prefix_option','prefix_option'))
+        ei_cmd = self.get_finalized_command("egg_info")
+        basename = pkg_resources.Distribution(
+            None, None, ei_cmd.egg_name, ei_cmd.egg_version
+        ).egg_name() + '.egg-info'
+
+        if self.install_layout:
+            if not self.install_layout.lower() in ['deb']:
+                raise DistutilsOptionError("unknown value for --install-layout")
+            self.install_layout = self.install_layout.lower()
+            basename = basename.replace('-py%s' % pkg_resources.PY_MAJOR, '')
+        elif self.prefix_option or 'real_prefix' in sys.__dict__:
+            # don't modify for virtualenv
+            pass
+        else:
+            basename = basename.replace('-py%s' % pkg_resources.PY_MAJOR, '')
+
+        self.source = ei_cmd.egg_info
+        self.target = os.path.join(self.install_dir, basename)
+        self.outputs = []
+
+    def run(self):
+        self.run_command('egg_info')
+        if os.path.isdir(self.target) and not os.path.islink(self.target):
+            dir_util.remove_tree(self.target, dry_run=self.dry_run)
+        elif os.path.exists(self.target):
+            self.execute(os.unlink, (self.target,), "Removing " + self.target)
+        if not self.dry_run:
+            pkg_resources.ensure_directory(self.target)
+        self.execute(
+            self.copytree, (), "Copying %s to %s" % (self.source, self.target)
+        )
+        self.install_namespaces()
+
+    def get_outputs(self):
+        return self.outputs
+
+    def copytree(self):
+        # Copy the .egg-info tree to site-packages
+        def skimmer(src, dst):
+            # filter out source-control directories; note that 'src' is always
+            # a '/'-separated path, regardless of platform.  'dst' is a
+            # platform-specific path.
+            for skip in '.svn/', 'CVS/':
+                if src.startswith(skip) or '/' + skip in src:
+                    return None
+            if self.install_layout and self.install_layout in ['deb'] and src.startswith('SOURCES.txt'):
+                log.info("Skipping SOURCES.txt")
+                return None
+            self.outputs.append(dst)
+            log.debug("Copying %s to %s", src, dst)
+            return dst
+
+        unpack_archive(self.source, self.target, skimmer)
diff --git a/ubuntu/venv/setuptools/command/install_lib.py b/ubuntu/venv/setuptools/command/install_lib.py
new file mode 100644
index 0000000..bf81519
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/install_lib.py
@@ -0,0 +1,147 @@
+import os
+import sys
+from itertools import product, starmap
+import distutils.command.install_lib as orig
+
+
+class install_lib(orig.install_lib):
+    """Don't add compiled flags to filenames of non-Python files"""
+
+    def initialize_options(self):
+        orig.install_lib.initialize_options(self)
+        self.multiarch = None
+        self.install_layout = None
+
+    def finalize_options(self):
+        orig.install_lib.finalize_options(self)
+        self.set_undefined_options('install',('install_layout','install_layout'))
+        if self.install_layout == 'deb' and sys.version_info[:2] >= (3, 3):
+            import sysconfig
+            self.multiarch = sysconfig.get_config_var('MULTIARCH')
+
+    def run(self):
+        self.build()
+        outfiles = self.install()
+        if outfiles is not None:
+            # always compile, in case we have any extension stubs to deal with
+            self.byte_compile(outfiles)
+
+    def get_exclusions(self):
+        """
+        Return a collections.Sized collections.Container of paths to be
+        excluded for single_version_externally_managed installations.
+        """
+        all_packages = (
+            pkg
+            for ns_pkg in self._get_SVEM_NSPs()
+            for pkg in self._all_packages(ns_pkg)
+        )
+
+        excl_specs = product(all_packages, self._gen_exclusion_paths())
+        return set(starmap(self._exclude_pkg_path, excl_specs))
+
+    def _exclude_pkg_path(self, pkg, exclusion_path):
+        """
+        Given a package name and exclusion path within that package,
+        compute the full exclusion path.
+        """
+        parts = pkg.split('.') + [exclusion_path]
+        return os.path.join(self.install_dir, *parts)
+
+    @staticmethod
+    def _all_packages(pkg_name):
+        """
+        >>> list(install_lib._all_packages('foo.bar.baz'))
+        ['foo.bar.baz', 'foo.bar', 'foo']
+        """
+        while pkg_name:
+            yield pkg_name
+            pkg_name, sep, child = pkg_name.rpartition('.')
+
+    def _get_SVEM_NSPs(self):
+        """
+        Get namespace packages (list) but only for
+        single_version_externally_managed installations and empty otherwise.
+        """
+        # TODO: is it necessary to short-circuit here? i.e. what's the cost
+        # if get_finalized_command is called even when namespace_packages is
+        # False?
+        if not self.distribution.namespace_packages:
+            return []
+
+        install_cmd = self.get_finalized_command('install')
+        svem = install_cmd.single_version_externally_managed
+
+        return self.distribution.namespace_packages if svem else []
+
+    @staticmethod
+    def _gen_exclusion_paths():
+        """
+        Generate file paths to be excluded for namespace packages (bytecode
+        cache files).
+        """
+        # always exclude the package module itself
+        yield '__init__.py'
+
+        yield '__init__.pyc'
+        yield '__init__.pyo'
+
+        if not hasattr(sys, 'implementation'):
+            return
+
+        base = os.path.join('__pycache__', '__init__.' + sys.implementation.cache_tag)
+        yield base + '.pyc'
+        yield base + '.pyo'
+        yield base + '.opt-1.pyc'
+        yield base + '.opt-2.pyc'
+
+    def copy_tree(
+            self, infile, outfile,
+            preserve_mode=1, preserve_times=1, preserve_symlinks=0, level=1
+    ):
+        assert preserve_mode and preserve_times and not preserve_symlinks
+        exclude = self.get_exclusions()
+
+        if not exclude:
+            import distutils.dir_util
+            distutils.dir_util._multiarch = self.multiarch
+            return orig.install_lib.copy_tree(self, infile, outfile)
+
+        # Exclude namespace package __init__.py* files from the output
+
+        from setuptools.archive_util import unpack_directory
+        from distutils import log
+
+        outfiles = []
+
+        if self.multiarch:
+            import sysconfig
+            ext_suffix = sysconfig.get_config_var ('EXT_SUFFIX')
+            if ext_suffix.endswith(self.multiarch + ext_suffix[-3:]):
+                new_suffix = None
+            else:
+                new_suffix = "%s-%s%s" % (ext_suffix[:-3], self.multiarch, ext_suffix[-3:])
+
+        def pf(src, dst):
+            if dst in exclude:
+                log.warn("Skipping installation of %s (namespace package)",
+                         dst)
+                return False
+
+            if self.multiarch and new_suffix and dst.endswith(ext_suffix) and not dst.endswith(new_suffix):
+                dst = dst.replace(ext_suffix, new_suffix)
+                log.info("renaming extension to %s", os.path.basename(dst))
+
+            log.info("copying %s -> %s", src, os.path.dirname(dst))
+            outfiles.append(dst)
+            return dst
+
+        unpack_directory(infile, outfile, pf)
+        return outfiles
+
+    def get_outputs(self):
+        outputs = orig.install_lib.get_outputs(self)
+        exclude = self.get_exclusions()
+        if exclude:
+            return [f for f in outputs if f not in exclude]
+        return outputs
diff --git a/ubuntu/venv/setuptools/command/install_scripts.py b/ubuntu/venv/setuptools/command/install_scripts.py
new file mode 100644
index 0000000..1623427
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/install_scripts.py
@@ -0,0 +1,65 @@
+from distutils import log
+import distutils.command.install_scripts as orig
+import os
+import sys
+
+from pkg_resources import Distribution, PathMetadata, ensure_directory
+
+
+class install_scripts(orig.install_scripts):
+    """Do normal script install, plus any egg_info wrapper scripts"""
+
+    def initialize_options(self):
+        orig.install_scripts.initialize_options(self)
+        self.no_ep = False
+
+    def run(self):
+        import setuptools.command.easy_install as ei
+
+        self.run_command("egg_info")
+        if self.distribution.scripts:
+            orig.install_scripts.run(self)  # run first to set up self.outfiles
+        else:
+            self.outfiles = []
+        if self.no_ep:
+            # don't install entry point scripts into .egg file!
+            return
+
+        ei_cmd = self.get_finalized_command("egg_info")
+        dist = Distribution(
+            ei_cmd.egg_base, PathMetadata(ei_cmd.egg_base, ei_cmd.egg_info),
+            ei_cmd.egg_name, ei_cmd.egg_version,
+        )
+        bs_cmd = self.get_finalized_command('build_scripts')
+        exec_param = getattr(bs_cmd, 'executable', None)
+        bw_cmd = self.get_finalized_command("bdist_wininst")
+        is_wininst = getattr(bw_cmd, '_is_running', False)
+        writer = ei.ScriptWriter
+        if is_wininst:
+            exec_param = "python.exe"
+            writer = ei.WindowsScriptWriter
+        if exec_param == sys.executable:
+            # In case the path to the Python executable contains a space, wrap
+            # it so it's not split up.
+            exec_param = [exec_param]
+        # resolve the writer to the environment
+        writer = writer.best()
+        cmd = writer.command_spec_class.best().from_param(exec_param)
+        for args in writer.get_args(dist, cmd.as_header()):
+            self.write_script(*args)
+
+    def write_script(self, script_name, contents, mode="t", *ignored):
+        """Write an executable file to the scripts directory"""
+        from setuptools.command.easy_install import chmod, current_umask
+
+        log.info("Installing %s script to %s", script_name, self.install_dir)
+        target = os.path.join(self.install_dir, script_name)
+        self.outfiles.append(target)
+
+        mask = current_umask()
+        if not self.dry_run:
+            ensure_directory(target)
+            f = open(target, "w" + mode)
+            f.write(contents)
+            f.close()
+            chmod(target, 0o777 - mask)
diff --git a/ubuntu/venv/setuptools/command/launcher manifest.xml b/ubuntu/venv/setuptools/command/launcher manifest.xml
new file mode 100644
index 0000000..5972a96
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/launcher manifest.xml	
@@ -0,0 +1,15 @@
+
+
+    
+    
+    
+        
+            
+                
+            
+        
+    
+
diff --git a/ubuntu/venv/setuptools/command/py36compat.py b/ubuntu/venv/setuptools/command/py36compat.py
new file mode 100644
index 0000000..61063e7
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/py36compat.py
@@ -0,0 +1,136 @@
+import os
+from glob import glob
+from distutils.util import convert_path
+from distutils.command import sdist
+
+from setuptools.extern.six.moves import filter
+
+
+class sdist_add_defaults:
+    """
+    Mix-in providing forward-compatibility for functionality as found in
+    distutils on Python 3.7.
+
+    Do not edit the code in this class except to update functionality
+    as implemented in distutils. Instead, override in the subclass.
+    """
+
+    def add_defaults(self):
+        """Add all the default files to self.filelist:
+          - README or README.txt
+          - setup.py
+          - test/test*.py
+          - all pure Python modules mentioned in setup script
+          - all files pointed by package_data (build_py)
+          - all files defined in data_files.
+          - all files defined as scripts.
+          - all C sources listed as part of extensions or C libraries
+            in the setup script (doesn't catch C headers!)
+        Warns if (README or README.txt) or setup.py are missing; everything
+        else is optional.
+        """
+        self._add_defaults_standards()
+        self._add_defaults_optional()
+        self._add_defaults_python()
+        self._add_defaults_data_files()
+        self._add_defaults_ext()
+        self._add_defaults_c_libs()
+        self._add_defaults_scripts()
+
+    @staticmethod
+    def _cs_path_exists(fspath):
+        """
+        Case-sensitive path existence check
+
+        >>> sdist_add_defaults._cs_path_exists(__file__)
+        True
+        >>> sdist_add_defaults._cs_path_exists(__file__.upper())
+        False
+        """
+        if not os.path.exists(fspath):
+            return False
+        # make absolute so we always have a directory
+        abspath = os.path.abspath(fspath)
+        directory, filename = os.path.split(abspath)
+        return filename in os.listdir(directory)
+
+    def _add_defaults_standards(self):
+        standards = [self.READMES, self.distribution.script_name]
+        for fn in standards:
+            if isinstance(fn, tuple):
+                alts = fn
+                got_it = False
+                for fn in alts:
+                    if self._cs_path_exists(fn):
+                        got_it = True
+                        self.filelist.append(fn)
+                        break
+
+                if not got_it:
+                    self.warn("standard file not found: should have one of " +
+                              ', '.join(alts))
+            else:
+                if self._cs_path_exists(fn):
+                    self.filelist.append(fn)
+                else:
+                    self.warn("standard file '%s' not found" % fn)
+
+    def _add_defaults_optional(self):
+        optional = ['test/test*.py', 'setup.cfg']
+        for pattern in optional:
+            files = filter(os.path.isfile, glob(pattern))
+            self.filelist.extend(files)
+
+    def _add_defaults_python(self):
+        # build_py is used to get:
+        #  - python modules
+        #  - files defined in package_data
+        build_py = self.get_finalized_command('build_py')
+
+        # getting python files
+        if self.distribution.has_pure_modules():
+            self.filelist.extend(build_py.get_source_files())
+
+        # getting package_data files
+        # (computed in build_py.data_files by build_py.finalize_options)
+        for pkg, src_dir, build_dir, filenames in build_py.data_files:
+            for filename in filenames:
+                self.filelist.append(os.path.join(src_dir, filename))
+
+    def _add_defaults_data_files(self):
+        # getting distribution.data_files
+        if self.distribution.has_data_files():
+            for item in self.distribution.data_files:
+                if isinstance(item, str):
+                    # plain file
+                    item = convert_path(item)
+                    if os.path.isfile(item):
+                        self.filelist.append(item)
+                else:
+                    # a (dirname, filenames) tuple
+                    dirname, filenames = item
+                    for f in filenames:
+                        f = convert_path(f)
+                        if os.path.isfile(f):
+                            self.filelist.append(f)
+
+    def _add_defaults_ext(self):
+        if self.distribution.has_ext_modules():
+            build_ext = self.get_finalized_command('build_ext')
+            self.filelist.extend(build_ext.get_source_files())
+
+    def _add_defaults_c_libs(self):
+        if self.distribution.has_c_libraries():
+            build_clib = self.get_finalized_command('build_clib')
+            self.filelist.extend(build_clib.get_source_files())
+
+    def _add_defaults_scripts(self):
+        if self.distribution.has_scripts():
+            build_scripts = self.get_finalized_command('build_scripts')
+            self.filelist.extend(build_scripts.get_source_files())
+
+
+if hasattr(sdist.sdist, '_add_defaults_standards'):
+    # disable the functionality already available upstream
+    class sdist_add_defaults:
+        pass
diff --git a/ubuntu/venv/setuptools/command/register.py b/ubuntu/venv/setuptools/command/register.py
new file mode 100644
index 0000000..b8266b9
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/register.py
@@ -0,0 +1,18 @@
+from distutils import log
+import distutils.command.register as orig
+
+from setuptools.errors import RemovedCommandError
+
+
+class register(orig.register):
+    """Formerly used to register packages on PyPI."""
+
+    def run(self):
+        msg = (
+            "The register command has been removed, use twine to upload "
+            + "instead (https://pypi.org/p/twine)"
+        )
+
+        self.announce("ERROR: " + msg, log.ERROR)
+
+        raise RemovedCommandError(msg)
diff --git a/ubuntu/venv/setuptools/command/rotate.py b/ubuntu/venv/setuptools/command/rotate.py
new file mode 100644
index 0000000..b89353f
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/rotate.py
@@ -0,0 +1,66 @@
+from distutils.util import convert_path
+from distutils import log
+from distutils.errors import DistutilsOptionError
+import os
+import shutil
+
+from setuptools.extern import six
+
+from setuptools import Command
+
+
+class rotate(Command):
+    """Delete older distributions"""
+
+    description = "delete older distributions, keeping N newest files"
+    user_options = [
+        ('match=', 'm', "patterns to match (required)"),
+        ('dist-dir=', 'd', "directory where the distributions are"),
+        ('keep=', 'k', "number of matching distributions to keep"),
+    ]
+
+    boolean_options = []
+
+    def initialize_options(self):
+        self.match = None
+        self.dist_dir = None
+        self.keep = None
+
+    def finalize_options(self):
+        if self.match is None:
+            raise DistutilsOptionError(
+                "Must specify one or more (comma-separated) match patterns "
+                "(e.g. '.zip' or '.egg')"
+            )
+        if self.keep is None:
+            raise DistutilsOptionError("Must specify number of files to keep")
+        try:
+            self.keep = int(self.keep)
+        except ValueError:
+            raise DistutilsOptionError("--keep must be an integer")
+        if isinstance(self.match, six.string_types):
+            self.match = [
+                convert_path(p.strip()) for p in self.match.split(',')
+            ]
+        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))
+
+    def run(self):
+        self.run_command("egg_info")
+        from glob import glob
+
+        for pattern in self.match:
+            pattern = self.distribution.get_name() + '*' + pattern
+            files = glob(os.path.join(self.dist_dir, pattern))
+            files = [(os.path.getmtime(f), f) for f in files]
+            files.sort()
+            files.reverse()
+
+            log.info("%d file(s) matching %s", len(files), pattern)
+            files = files[self.keep:]
+            for (t, f) in files:
+                log.info("Deleting %s", f)
+                if not self.dry_run:
+                    if os.path.isdir(f):
+                        shutil.rmtree(f)
+                    else:
+                        os.unlink(f)
diff --git a/ubuntu/venv/setuptools/command/saveopts.py b/ubuntu/venv/setuptools/command/saveopts.py
new file mode 100644
index 0000000..611cec5
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/saveopts.py
@@ -0,0 +1,22 @@
+from setuptools.command.setopt import edit_config, option_base
+
+
+class saveopts(option_base):
+    """Save command-line options to a file"""
+
+    description = "save supplied options to setup.cfg or other config file"
+
+    def run(self):
+        dist = self.distribution
+        settings = {}
+
+        for cmd in dist.command_options:
+
+            if cmd == 'saveopts':
+                continue  # don't save our own options!
+
+            for opt, (src, val) in dist.get_option_dict(cmd).items():
+                if src == "command line":
+                    settings.setdefault(cmd, {})[opt] = val
+
+        edit_config(self.filename, settings, self.dry_run)
diff --git a/ubuntu/venv/setuptools/command/sdist.py b/ubuntu/venv/setuptools/command/sdist.py
new file mode 100644
index 0000000..a851453
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/sdist.py
@@ -0,0 +1,252 @@
+from distutils import log
+import distutils.command.sdist as orig
+import os
+import sys
+import io
+import contextlib
+
+from setuptools.extern import six, ordered_set
+
+from .py36compat import sdist_add_defaults
+
+import pkg_resources
+
+_default_revctrl = list
+
+
+def walk_revctrl(dirname=''):
+    """Find all files under revision control"""
+    for ep in pkg_resources.iter_entry_points('setuptools.file_finders'):
+        for item in ep.load()(dirname):
+            yield item
+
+
+class sdist(sdist_add_defaults, orig.sdist):
+    """Smart sdist that finds anything supported by revision control"""
+
+    user_options = [
+        ('formats=', None,
+         "formats for source distribution (comma-separated list)"),
+        ('keep-temp', 'k',
+         "keep the distribution tree around after creating " +
+         "archive file(s)"),
+        ('dist-dir=', 'd',
+         "directory to put the source distribution archive(s) in "
+         "[default: dist]"),
+    ]
+
+    negative_opt = {}
+
+    README_EXTENSIONS = ['', '.rst', '.txt', '.md']
+    READMES = tuple('README{0}'.format(ext) for ext in README_EXTENSIONS)
+
+    def run(self):
+        self.run_command('egg_info')
+        ei_cmd = self.get_finalized_command('egg_info')
+        self.filelist = ei_cmd.filelist
+        self.filelist.append(os.path.join(ei_cmd.egg_info, 'SOURCES.txt'))
+        self.check_readme()
+
+        # Run sub commands
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        self.make_distribution()
+
+        dist_files = getattr(self.distribution, 'dist_files', [])
+        for file in self.archive_files:
+            data = ('sdist', '', file)
+            if data not in dist_files:
+                dist_files.append(data)
+
+    def initialize_options(self):
+        orig.sdist.initialize_options(self)
+
+        self._default_to_gztar()
+
+    def _default_to_gztar(self):
+        # only needed on Python prior to 3.6.
+        if sys.version_info >= (3, 6, 0, 'beta', 1):
+            return
+        self.formats = ['gztar']
+
+    def make_distribution(self):
+        """
+        Workaround for #516
+        """
+        with self._remove_os_link():
+            orig.sdist.make_distribution(self)
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _remove_os_link():
+        """
+        In a context, remove and restore os.link if it exists
+        """
+
+        class NoValue:
+            pass
+
+        orig_val = getattr(os, 'link', NoValue)
+        try:
+            del os.link
+        except Exception:
+            pass
+        try:
+            yield
+        finally:
+            if orig_val is not NoValue:
+                setattr(os, 'link', orig_val)
+
+    def __read_template_hack(self):
+        # This grody hack closes the template file (MANIFEST.in) if an
+        #  exception occurs during read_template.
+        # Doing so prevents an error when easy_install attempts to delete the
+        #  file.
+        try:
+            orig.sdist.read_template(self)
+        except Exception:
+            _, _, tb = sys.exc_info()
+            tb.tb_next.tb_frame.f_locals['template'].close()
+            raise
+
+    # Beginning with Python 2.7.2, 3.1.4, and 3.2.1, this leaky file handle
+    #  has been fixed, so only override the method if we're using an earlier
+    #  Python.
+    has_leaky_handle = (
+        sys.version_info < (2, 7, 2)
+        or (3, 0) <= sys.version_info < (3, 1, 4)
+        or (3, 2) <= sys.version_info < (3, 2, 1)
+    )
+    if has_leaky_handle:
+        read_template = __read_template_hack
+
+    def _add_defaults_optional(self):
+        if six.PY2:
+            sdist_add_defaults._add_defaults_optional(self)
+        else:
+            super()._add_defaults_optional()
+        if os.path.isfile('pyproject.toml'):
+            self.filelist.append('pyproject.toml')
+
+    def _add_defaults_python(self):
+        """getting python files"""
+        if self.distribution.has_pure_modules():
+            build_py = self.get_finalized_command('build_py')
+            self.filelist.extend(build_py.get_source_files())
+            self._add_data_files(self._safe_data_files(build_py))
+
+    def _safe_data_files(self, build_py):
+        """
+        Extracting data_files from build_py is known to cause
+        infinite recursion errors when `include_package_data`
+        is enabled, so suppress it in that case.
+        """
+        if self.distribution.include_package_data:
+            return ()
+        return build_py.data_files
+
+    def _add_data_files(self, data_files):
+        """
+        Add data files as found in build_py.data_files.
+        """
+        self.filelist.extend(
+            os.path.join(src_dir, name)
+            for _, src_dir, _, filenames in data_files
+            for name in filenames
+        )
+
+    def _add_defaults_data_files(self):
+        try:
+            if six.PY2:
+                sdist_add_defaults._add_defaults_data_files(self)
+            else:
+                super()._add_defaults_data_files()
+        except TypeError:
+            log.warn("data_files contains unexpected objects")
+
+    def check_readme(self):
+        for f in self.READMES:
+            if os.path.exists(f):
+                return
+        else:
+            self.warn(
+                "standard file not found: should have one of " +
+                ', '.join(self.READMES)
+            )
+
+    def make_release_tree(self, base_dir, files):
+        orig.sdist.make_release_tree(self, base_dir, files)
+
+        # Save any egg_info command line options used to create this sdist
+        dest = os.path.join(base_dir, 'setup.cfg')
+        if hasattr(os, 'link') and os.path.exists(dest):
+            # unlink and re-copy, since it might be hard-linked, and
+            # we don't want to change the source version
+            os.unlink(dest)
+            self.copy_file('setup.cfg', dest)
+
+        self.get_finalized_command('egg_info').save_version_info(dest)
+
+    def _manifest_is_not_generated(self):
+        # check for special comment used in 2.7.1 and higher
+        if not os.path.isfile(self.manifest):
+            return False
+
+        with io.open(self.manifest, 'rb') as fp:
+            first_line = fp.readline()
+        return (first_line !=
+                '# file GENERATED by distutils, do NOT edit\n'.encode())
+
+    def read_manifest(self):
+        """Read the manifest file (named by 'self.manifest') and use it to
+        fill in 'self.filelist', the list of files to include in the source
+        distribution.
+        """
+        log.info("reading manifest file '%s'", self.manifest)
+        manifest = open(self.manifest, 'rb')
+        for line in manifest:
+            # The manifest must contain UTF-8. See #303.
+            if six.PY3:
+                try:
+                    line = line.decode('UTF-8')
+                except UnicodeDecodeError:
+                    log.warn("%r not UTF-8 decodable -- skipping" % line)
+                    continue
+            # ignore comments and blank lines
+            line = line.strip()
+            if line.startswith('#') or not line:
+                continue
+            self.filelist.append(line)
+        manifest.close()
+
+    def check_license(self):
+        """Checks if license_file' or 'license_files' is configured and adds any
+        valid paths to 'self.filelist'.
+        """
+
+        files = ordered_set.OrderedSet()
+
+        opts = self.distribution.get_option_dict('metadata')
+
+        # ignore the source of the value
+        _, license_file = opts.get('license_file', (None, None))
+
+        if license_file is None:
+            log.debug("'license_file' option was not specified")
+        else:
+            files.add(license_file)
+
+        try:
+            files.update(self.distribution.metadata.license_files)
+        except TypeError:
+            log.warn("warning: 'license_files' option is malformed")
+
+        for f in files:
+            if not os.path.exists(f):
+                log.warn(
+                    "warning: Failed to find the configured license file '%s'",
+                    f)
+                files.remove(f)
+
+        self.filelist.extend(files)
diff --git a/ubuntu/venv/setuptools/command/setopt.py b/ubuntu/venv/setuptools/command/setopt.py
new file mode 100644
index 0000000..7e57cc0
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/setopt.py
@@ -0,0 +1,149 @@
+from distutils.util import convert_path
+from distutils import log
+from distutils.errors import DistutilsOptionError
+import distutils
+import os
+
+from setuptools.extern.six.moves import configparser
+
+from setuptools import Command
+
+__all__ = ['config_file', 'edit_config', 'option_base', 'setopt']
+
+
+def config_file(kind="local"):
+    """Get the filename of the distutils, local, global, or per-user config
+
+    `kind` must be one of "local", "global", or "user"
+    """
+    if kind == 'local':
+        return 'setup.cfg'
+    if kind == 'global':
+        return os.path.join(
+            os.path.dirname(distutils.__file__), 'distutils.cfg'
+        )
+    if kind == 'user':
+        dot = os.name == 'posix' and '.' or ''
+        return os.path.expanduser(convert_path("~/%spydistutils.cfg" % dot))
+    raise ValueError(
+        "config_file() type must be 'local', 'global', or 'user'", kind
+    )
+
+
+def edit_config(filename, settings, dry_run=False):
+    """Edit a configuration file to include `settings`
+
+    `settings` is a dictionary of dictionaries or ``None`` values, keyed by
+    command/section name.  A ``None`` value means to delete the entire section,
+    while a dictionary lists settings to be changed or deleted in that section.
+    A setting of ``None`` means to delete that setting.
+    """
+    log.debug("Reading configuration from %s", filename)
+    opts = configparser.RawConfigParser()
+    opts.read([filename])
+    for section, options in settings.items():
+        if options is None:
+            log.info("Deleting section [%s] from %s", section, filename)
+            opts.remove_section(section)
+        else:
+            if not opts.has_section(section):
+                log.debug("Adding new section [%s] to %s", section, filename)
+                opts.add_section(section)
+            for option, value in options.items():
+                if value is None:
+                    log.debug(
+                        "Deleting %s.%s from %s",
+                        section, option, filename
+                    )
+                    opts.remove_option(section, option)
+                    if not opts.options(section):
+                        log.info("Deleting empty [%s] section from %s",
+                                 section, filename)
+                        opts.remove_section(section)
+                else:
+                    log.debug(
+                        "Setting %s.%s to %r in %s",
+                        section, option, value, filename
+                    )
+                    opts.set(section, option, value)
+
+    log.info("Writing %s", filename)
+    if not dry_run:
+        with open(filename, 'w') as f:
+            opts.write(f)
+
+
+class option_base(Command):
+    """Abstract base class for commands that mess with config files"""
+
+    user_options = [
+        ('global-config', 'g',
+         "save options to the site-wide distutils.cfg file"),
+        ('user-config', 'u',
+         "save options to the current user's pydistutils.cfg file"),
+        ('filename=', 'f',
+         "configuration file to use (default=setup.cfg)"),
+    ]
+
+    boolean_options = [
+        'global-config', 'user-config',
+    ]
+
+    def initialize_options(self):
+        self.global_config = None
+        self.user_config = None
+        self.filename = None
+
+    def finalize_options(self):
+        filenames = []
+        if self.global_config:
+            filenames.append(config_file('global'))
+        if self.user_config:
+            filenames.append(config_file('user'))
+        if self.filename is not None:
+            filenames.append(self.filename)
+        if not filenames:
+            filenames.append(config_file('local'))
+        if len(filenames) > 1:
+            raise DistutilsOptionError(
+                "Must specify only one configuration file option",
+                filenames
+            )
+        self.filename, = filenames
+
+
+class setopt(option_base):
+    """Save command-line options to a file"""
+
+    description = "set an option in setup.cfg or another config file"
+
+    user_options = [
+        ('command=', 'c', 'command to set an option for'),
+        ('option=', 'o', 'option to set'),
+        ('set-value=', 's', 'value of the option'),
+        ('remove', 'r', 'remove (unset) the value'),
+    ] + option_base.user_options
+
+    boolean_options = option_base.boolean_options + ['remove']
+
+    def initialize_options(self):
+        option_base.initialize_options(self)
+        self.command = None
+        self.option = None
+        self.set_value = None
+        self.remove = None
+
+    def finalize_options(self):
+        option_base.finalize_options(self)
+        if self.command is None or self.option is None:
+            raise DistutilsOptionError("Must specify --command *and* --option")
+        if self.set_value is None and not self.remove:
+            raise DistutilsOptionError("Must specify --set-value or --remove")
+
+    def run(self):
+        edit_config(
+            self.filename, {
+                self.command: {self.option.replace('-', '_'): self.set_value}
+            },
+            self.dry_run
+        )
diff --git a/ubuntu/venv/setuptools/command/test.py b/ubuntu/venv/setuptools/command/test.py
new file mode 100644
index 0000000..c148b38
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/test.py
@@ -0,0 +1,279 @@
+import os
+import operator
+import sys
+import contextlib
+import itertools
+import unittest
+from distutils.errors import DistutilsError, DistutilsOptionError
+from distutils import log
+from unittest import TestLoader
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import map, filter
+
+from pkg_resources import (resource_listdir, resource_exists, normalize_path,
+                           working_set, _namespace_packages, evaluate_marker,
+                           add_activation_listener, require, EntryPoint)
+from setuptools import Command
+from .build_py import _unique_everseen
+
+__metaclass__ = type
+
+
+class ScanningLoader(TestLoader):
+
+    def __init__(self):
+        TestLoader.__init__(self)
+        self._visited = set()
+
+    def loadTestsFromModule(self, module, pattern=None):
+        """Return a suite of all tests cases contained in the given module
+
+        If the module is a package, load tests from all the modules in it.
+        If the module has an ``additional_tests`` function, call it and add
+        the return value to the tests.
+        """
+        if module in self._visited:
+            return None
+        self._visited.add(module)
+
+        tests = []
+        tests.append(TestLoader.loadTestsFromModule(self, module))
+
+        if hasattr(module, "additional_tests"):
+            tests.append(module.additional_tests())
+
+        if hasattr(module, '__path__'):
+            for file in resource_listdir(module.__name__, ''):
+                if file.endswith('.py') and file != '__init__.py':
+                    submodule = module.__name__ + '.' + file[:-3]
+                else:
+                    if resource_exists(module.__name__, file + '/__init__.py'):
+                        submodule = module.__name__ + '.' + file
+                    else:
+                        continue
+                tests.append(self.loadTestsFromName(submodule))
+
+        if len(tests) != 1:
+            return self.suiteClass(tests)
+        else:
+            return tests[0]  # don't create a nested suite for only one return
+
+
+# adapted from jaraco.classes.properties:NonDataProperty
+class NonDataProperty:
+    def __init__(self, fget):
+        self.fget = fget
+
+    def __get__(self, obj, objtype=None):
+        if obj is None:
+            return self
+        return self.fget(obj)
+
+
+class test(Command):
+    """Command to run unit tests after in-place build"""
+
+    description = "run unit tests after in-place build (deprecated)"
+
+    user_options = [
+        ('test-module=', 'm', "Run 'test_suite' in specified module"),
+        ('test-suite=', 's',
+         "Run single test, case or suite (e.g. 'module.test_suite')"),
+        ('test-runner=', 'r', "Test runner to use"),
+    ]
+
+    def initialize_options(self):
+        self.test_suite = None
+        self.test_module = None
+        self.test_loader = None
+        self.test_runner = None
+
+    def finalize_options(self):
+
+        if self.test_suite and self.test_module:
+            msg = "You may specify a module or a suite, but not both"
+            raise DistutilsOptionError(msg)
+
+        if self.test_suite is None:
+            if self.test_module is None:
+                self.test_suite = self.distribution.test_suite
+            else:
+                self.test_suite = self.test_module + ".test_suite"
+
+        if self.test_loader is None:
+            self.test_loader = getattr(self.distribution, 'test_loader', None)
+        if self.test_loader is None:
+            self.test_loader = "setuptools.command.test:ScanningLoader"
+        if self.test_runner is None:
+            self.test_runner = getattr(self.distribution, 'test_runner', None)
+
+    @NonDataProperty
+    def test_args(self):
+        return list(self._test_args())
+
+    def _test_args(self):
+        if not self.test_suite and sys.version_info >= (2, 7):
+            yield 'discover'
+        if self.verbose:
+            yield '--verbose'
+        if self.test_suite:
+            yield self.test_suite
+
+    def with_project_on_sys_path(self, func):
+        """
+        Backward compatibility for project_on_sys_path context.
+        """
+        with self.project_on_sys_path():
+            func()
+
+    @contextlib.contextmanager
+    def project_on_sys_path(self, include_dists=[]):
+        with_2to3 = six.PY3 and getattr(self.distribution, 'use_2to3', False)
+
+        if with_2to3:
+            # If we run 2to3 we can not do this inplace:
+
+            # Ensure metadata is up-to-date
+            self.reinitialize_command('build_py', inplace=0)
+            self.run_command('build_py')
+            bpy_cmd = self.get_finalized_command("build_py")
+            build_path = normalize_path(bpy_cmd.build_lib)
+
+            # Build extensions
+            self.reinitialize_command('egg_info', egg_base=build_path)
+            self.run_command('egg_info')
+
+            self.reinitialize_command('build_ext', inplace=0)
+            self.run_command('build_ext')
+        else:
+            # Without 2to3 inplace works fine:
+            self.run_command('egg_info')
+
+            # Build extensions in-place
+            self.reinitialize_command('build_ext', inplace=1)
+            self.run_command('build_ext')
+
+        ei_cmd = self.get_finalized_command("egg_info")
+
+        old_path = sys.path[:]
+        old_modules = sys.modules.copy()
+
+        try:
+            project_path = normalize_path(ei_cmd.egg_base)
+            sys.path.insert(0, project_path)
+            working_set.__init__()
+            add_activation_listener(lambda dist: dist.activate())
+            require('%s==%s' % (ei_cmd.egg_name, ei_cmd.egg_version))
+            with self.paths_on_pythonpath([project_path]):
+                yield
+        finally:
+            sys.path[:] = old_path
+            sys.modules.clear()
+            sys.modules.update(old_modules)
+            working_set.__init__()
+
+    @staticmethod
+    @contextlib.contextmanager
+    def paths_on_pythonpath(paths):
+        """
+        Add the indicated paths to the head of the PYTHONPATH environment
+        variable so that subprocesses will also see the packages at
+        these paths.
+
+        Do this in a context that restores the value on exit.
+        """
+        nothing = object()
+        orig_pythonpath = os.environ.get('PYTHONPATH', nothing)
+        current_pythonpath = os.environ.get('PYTHONPATH', '')
+        try:
+            prefix = os.pathsep.join(_unique_everseen(paths))
+            to_join = filter(None, [prefix, current_pythonpath])
+            new_path = os.pathsep.join(to_join)
+            if new_path:
+                os.environ['PYTHONPATH'] = new_path
+            yield
+        finally:
+            if orig_pythonpath is nothing:
+                os.environ.pop('PYTHONPATH', None)
+            else:
+                os.environ['PYTHONPATH'] = orig_pythonpath
+
+    @staticmethod
+    def install_dists(dist):
+        """
+        Install the requirements indicated by self.distribution and
+        return an iterable of the dists that were built.
+        """
+        ir_d = dist.fetch_build_eggs(dist.install_requires)
+        tr_d = dist.fetch_build_eggs(dist.tests_require or [])
+        er_d = dist.fetch_build_eggs(
+            v for k, v in dist.extras_require.items()
+            if k.startswith(':') and evaluate_marker(k[1:])
+        )
+        return itertools.chain(ir_d, tr_d, er_d)
+
+    def run(self):
+        self.announce(
+            "WARNING: Testing via this command is deprecated and will be "
+            "removed in a future version. Users looking for a generic test "
+            "entry point independent of test runner are encouraged to use "
+            "tox.",
+            log.WARN,
+        )
+
+        installed_dists = self.install_dists(self.distribution)
+
+        cmd = ' '.join(self._argv)
+        if self.dry_run:
+            self.announce('skipping "%s" (dry run)' % cmd)
+            return
+
+        self.announce('running "%s"' % cmd)
+
+        paths = map(operator.attrgetter('location'), installed_dists)
+        with self.paths_on_pythonpath(paths):
+            with self.project_on_sys_path():
+                self.run_tests()
+
+    def run_tests(self):
+        # Purge modules under test from sys.modules. The test loader will
+        # re-import them from the build location. Required when 2to3 is used
+        # with namespace packages.
+        if six.PY3 and getattr(self.distribution, 'use_2to3', False):
+            module = self.test_suite.split('.')[0]
+            if module in _namespace_packages:
+                del_modules = []
+                if module in sys.modules:
+                    del_modules.append(module)
+                module += '.'
+                for name in sys.modules:
+                    if name.startswith(module):
+                        del_modules.append(name)
+                list(map(sys.modules.__delitem__, del_modules))
+
+        test = unittest.main(
+            None, None, self._argv,
+            testLoader=self._resolve_as_ep(self.test_loader),
+            testRunner=self._resolve_as_ep(self.test_runner),
+            exit=False,
+        )
+        if not test.result.wasSuccessful():
+            msg = 'Test failed: %s' % test.result
+            self.announce(msg, log.ERROR)
+            raise DistutilsError(msg)
+
+    @property
+    def _argv(self):
+        return ['unittest'] + self.test_args
+
+    @staticmethod
+    def _resolve_as_ep(val):
+        """
+        Load the indicated attribute value, called, as a as if it were
+        specified as an entry point.
+        """
+        if val is None:
+            return
+        parsed = EntryPoint.parse("x=" + val)
+        return parsed.resolve()()
diff --git a/ubuntu/venv/setuptools/command/upload.py b/ubuntu/venv/setuptools/command/upload.py
new file mode 100644
index 0000000..ec7f81e
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/upload.py
@@ -0,0 +1,17 @@
+from distutils import log
+from distutils.command import upload as orig
+
+from setuptools.errors import RemovedCommandError
+
+
+class upload(orig.upload):
+    """Formerly used to upload packages to PyPI."""
+
+    def run(self):
+        msg = (
+            "The upload command has been removed, use twine to upload "
+            + "instead (https://pypi.org/p/twine)"
+        )
+
+        self.announce("ERROR: " + msg, log.ERROR)
+        raise RemovedCommandError(msg)
diff --git a/ubuntu/venv/setuptools/command/upload_docs.py b/ubuntu/venv/setuptools/command/upload_docs.py
new file mode 100644
index 0000000..07aa564
--- /dev/null
+++ b/ubuntu/venv/setuptools/command/upload_docs.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+"""upload_docs
+
+Implements a Distutils 'upload_docs' subcommand (upload documentation to
+PyPI's pythonhosted.org).
+"""
+
+from base64 import standard_b64encode
+from distutils import log
+from distutils.errors import DistutilsOptionError
+import os
+import socket
+import zipfile
+import tempfile
+import shutil
+import itertools
+import functools
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import http_client, urllib
+
+from pkg_resources import iter_entry_points
+from .upload import upload
+
+
+def _encode(s):
+    errors = 'surrogateescape' if six.PY3 else 'strict'
+    return s.encode('utf-8', errors)
+
+
+class upload_docs(upload):
+    # override the default repository as upload_docs isn't
+    # supported by Warehouse (and won't be).
+    DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi/'
+
+    description = 'Upload documentation to PyPI'
+
+    user_options = [
+        ('repository=', 'r',
+         "url of repository [default: %s]" % upload.DEFAULT_REPOSITORY),
+        ('show-response', None,
+         'display full response text from server'),
+        ('upload-dir=', None, 'directory to upload'),
+    ]
+    boolean_options = upload.boolean_options
+
+    def has_sphinx(self):
+        if self.upload_dir is None:
+            for ep in iter_entry_points('distutils.commands', 'build_sphinx'):
+                return True
+
+    sub_commands = [('build_sphinx', has_sphinx)]
+
+    def initialize_options(self):
+        upload.initialize_options(self)
+        self.upload_dir = None
+        self.target_dir = None
+
+    def finalize_options(self):
+        upload.finalize_options(self)
+        if self.upload_dir is None:
+            if self.has_sphinx():
+                build_sphinx = self.get_finalized_command('build_sphinx')
+                self.target_dir = build_sphinx.builder_target_dir
+            else:
+                build = self.get_finalized_command('build')
+                self.target_dir = os.path.join(build.build_base, 'docs')
+        else:
+            self.ensure_dirname('upload_dir')
+            self.target_dir = self.upload_dir
+        if 'pypi.python.org' in self.repository:
+            log.warn("Upload_docs command is deprecated. Use RTD instead.")
+        self.announce('Using upload directory %s' % self.target_dir)
+
+    def create_zipfile(self, filename):
+        zip_file = zipfile.ZipFile(filename, "w")
+        try:
+            self.mkpath(self.target_dir)  # just in case
+            for root, dirs, files in os.walk(self.target_dir):
+                if root == self.target_dir and not files:
+                    tmpl = "no files found in upload directory '%s'"
+                    raise DistutilsOptionError(tmpl % self.target_dir)
+                for name in files:
+                    full = os.path.join(root, name)
+                    relative = root[len(self.target_dir):].lstrip(os.path.sep)
+                    dest = os.path.join(relative, name)
+                    zip_file.write(full, dest)
+        finally:
+            zip_file.close()
+
+    def run(self):
+        # Run sub commands
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
+
+        tmp_dir = tempfile.mkdtemp()
+        name = self.distribution.metadata.get_name()
+        zip_file = os.path.join(tmp_dir, "%s.zip" % name)
+        try:
+            self.create_zipfile(zip_file)
+            self.upload_file(zip_file)
+        finally:
+            shutil.rmtree(tmp_dir)
+
+    @staticmethod
+    def _build_part(item, sep_boundary):
+        key, values = item
+        title = '\nContent-Disposition: form-data; name="%s"' % key
+        # handle multiple entries for the same name
+        if not isinstance(values, list):
+            values = [values]
+        for value in values:
+            if isinstance(value, tuple):
+                title += '; filename="%s"' % value[0]
+                value = value[1]
+            else:
+                value = _encode(value)
+            yield sep_boundary
+            yield _encode(title)
+            yield b"\n\n"
+            yield value
+            if value and value[-1:] == b'\r':
+                yield b'\n'  # write an extra newline (lurve Macs)
+
+    @classmethod
+    def _build_multipart(cls, data):
+        """
+        Build up the MIME payload for the POST data
+        """
+        boundary = b'--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
+        sep_boundary = b'\n--' + boundary
+        end_boundary = sep_boundary + b'--'
+        end_items = end_boundary, b"\n",
+        builder = functools.partial(
+            cls._build_part,
+            sep_boundary=sep_boundary,
+        )
+        part_groups = map(builder, data.items())
+        parts = itertools.chain.from_iterable(part_groups)
+        body_items = itertools.chain(parts, end_items)
+        content_type = 'multipart/form-data; boundary=%s' % boundary.decode('ascii')
+        return b''.join(body_items), content_type
+
+    def upload_file(self, filename):
+        with open(filename, 'rb') as f:
+            content = f.read()
+        meta = self.distribution.metadata
+        data = {
+            ':action': 'doc_upload',
+            'name': meta.get_name(),
+            'content': (os.path.basename(filename), content),
+        }
+        # set up the authentication
+        credentials = _encode(self.username + ':' + self.password)
+        credentials = standard_b64encode(credentials)
+        if six.PY3:
+            credentials = credentials.decode('ascii')
+        auth = "Basic " + credentials
+
+        body, ct = self._build_multipart(data)
+
+        msg = "Submitting documentation to %s" % (self.repository)
+        self.announce(msg, log.INFO)
+
+        # build the Request
+        # We can't use urllib2 since we need to send the Basic
+        # auth right with the first request
+        schema, netloc, url, params, query, fragments = \
+            urllib.parse.urlparse(self.repository)
+        assert not params and not query and not fragments
+        if schema == 'http':
+            conn = http_client.HTTPConnection(netloc)
+        elif schema == 'https':
+            conn = http_client.HTTPSConnection(netloc)
+        else:
+            raise AssertionError("unsupported schema " + schema)
+
+        data = ''
+        try:
+            conn.connect()
+            conn.putrequest("POST", url)
+            content_type = ct
+            conn.putheader('Content-type', content_type)
+            conn.putheader('Content-length', str(len(body)))
+            conn.putheader('Authorization', auth)
+            conn.endheaders()
+            conn.send(body)
+        except socket.error as e:
+            self.announce(str(e), log.ERROR)
+            return
+
+        r = conn.getresponse()
+        if r.status == 200:
+            msg = 'Server response (%s): %s' % (r.status, r.reason)
+            self.announce(msg, log.INFO)
+        elif r.status == 301:
+            location = r.getheader('Location')
+            if location is None:
+                location = 'https://pythonhosted.org/%s/' % meta.get_name()
+            msg = 'Upload successful. Visit %s' % location
+            self.announce(msg, log.INFO)
+        else:
+            msg = 'Upload failed (%s): %s' % (r.status, r.reason)
+            self.announce(msg, log.ERROR)
+        if self.show_response:
+            print('-' * 75, r.read(), '-' * 75)
diff --git a/ubuntu/venv/setuptools/config.py b/ubuntu/venv/setuptools/config.py
new file mode 100644
index 0000000..9b9a0c4
--- /dev/null
+++ b/ubuntu/venv/setuptools/config.py
@@ -0,0 +1,659 @@
+from __future__ import absolute_import, unicode_literals
+import io
+import os
+import sys
+
+import warnings
+import functools
+from collections import defaultdict
+from functools import partial
+from functools import wraps
+from importlib import import_module
+
+from distutils.errors import DistutilsOptionError, DistutilsFileError
+from setuptools.extern.packaging.version import LegacyVersion, parse
+from setuptools.extern.packaging.specifiers import SpecifierSet
+from setuptools.extern.six import string_types, PY3
+
+
+__metaclass__ = type
+
+
+def read_configuration(
+        filepath, find_others=False, ignore_option_errors=False):
+    """Read given configuration file and returns options from it as a dict.
+
+    :param str|unicode filepath: Path to configuration file
+        to get options from.
+
+    :param bool find_others: Whether to search for other configuration files
+        which could be on in various places.
+
+    :param bool ignore_option_errors: Whether to silently ignore
+        options, values of which could not be resolved (e.g. due to exceptions
+        in directives such as file:, attr:, etc.).
+        If False exceptions are propagated as expected.
+
+    :rtype: dict
+    """
+    from setuptools.dist import Distribution, _Distribution
+
+    filepath = os.path.abspath(filepath)
+
+    if not os.path.isfile(filepath):
+        raise DistutilsFileError(
+            'Configuration file %s does not exist.' % filepath)
+
+    current_directory = os.getcwd()
+    os.chdir(os.path.dirname(filepath))
+
+    try:
+        dist = Distribution()
+
+        filenames = dist.find_config_files() if find_others else []
+        if filepath not in filenames:
+            filenames.append(filepath)
+
+        _Distribution.parse_config_files(dist, filenames=filenames)
+
+        handlers = parse_configuration(
+            dist, dist.command_options,
+            ignore_option_errors=ignore_option_errors)
+
+    finally:
+        os.chdir(current_directory)
+
+    return configuration_to_dict(handlers)
+
+
+def _get_option(target_obj, key):
+    """
+    Given a target object and option key, get that option from
+    the target object, either through a get_{key} method or
+    from an attribute directly.
+    """
+    getter_name = 'get_{key}'.format(**locals())
+    by_attribute = functools.partial(getattr, target_obj, key)
+    getter = getattr(target_obj, getter_name, by_attribute)
+    return getter()
+
+
+def configuration_to_dict(handlers):
+    """Returns configuration data gathered by given handlers as a dict.
+
+    :param list[ConfigHandler] handlers: Handlers list,
+        usually from parse_configuration()
+
+    :rtype: dict
+    """
+    config_dict = defaultdict(dict)
+
+    for handler in handlers:
+        for option in handler.set_options:
+            value = _get_option(handler.target_obj, option)
+            config_dict[handler.section_prefix][option] = value
+
+    return config_dict
+
+
+def parse_configuration(
+        distribution, command_options, ignore_option_errors=False):
+    """Performs additional parsing of configuration options
+    for a distribution.
+
+    Returns a list of used option handlers.
+
+    :param Distribution distribution:
+    :param dict command_options:
+    :param bool ignore_option_errors: Whether to silently ignore
+        options, values of which could not be resolved (e.g. due to exceptions
+        in directives such as file:, attr:, etc.).
+        If False exceptions are propagated as expected.
+    :rtype: list
+    """
+    options = ConfigOptionsHandler(
+        distribution, command_options, ignore_option_errors)
+    options.parse()
+
+    meta = ConfigMetadataHandler(
+        distribution.metadata, command_options, ignore_option_errors,
+        distribution.package_dir)
+    meta.parse()
+
+    return meta, options
+
+
+class ConfigHandler:
+    """Handles metadata supplied in configuration files."""
+
+    section_prefix = None
+    """Prefix for config sections handled by this handler.
+    Must be provided by class heirs.
+
+    """
+
+    aliases = {}
+    """Options aliases.
+    For compatibility with various packages. E.g.: d2to1 and pbr.
+    Note: `-` in keys is replaced with `_` by config parser.
+
+    """
+
+    def __init__(self, target_obj, options, ignore_option_errors=False):
+        sections = {}
+
+        section_prefix = self.section_prefix
+        for section_name, section_options in options.items():
+            if not section_name.startswith(section_prefix):
+                continue
+
+            section_name = section_name.replace(section_prefix, '').strip('.')
+            sections[section_name] = section_options
+
+        self.ignore_option_errors = ignore_option_errors
+        self.target_obj = target_obj
+        self.sections = sections
+        self.set_options = []
+
+    @property
+    def parsers(self):
+        """Metadata item name to parser function mapping."""
+        raise NotImplementedError(
+            '%s must provide .parsers property' % self.__class__.__name__)
+
+    def __setitem__(self, option_name, value):
+        unknown = tuple()
+        target_obj = self.target_obj
+
+        # Translate alias into real name.
+        option_name = self.aliases.get(option_name, option_name)
+
+        current_value = getattr(target_obj, option_name, unknown)
+
+        if current_value is unknown:
+            raise KeyError(option_name)
+
+        if current_value:
+            # Already inhabited. Skipping.
+            return
+
+        skip_option = False
+        parser = self.parsers.get(option_name)
+        if parser:
+            try:
+                value = parser(value)
+
+            except Exception:
+                skip_option = True
+                if not self.ignore_option_errors:
+                    raise
+
+        if skip_option:
+            return
+
+        setter = getattr(target_obj, 'set_%s' % option_name, None)
+        if setter is None:
+            setattr(target_obj, option_name, value)
+        else:
+            setter(value)
+
+        self.set_options.append(option_name)
+
+    @classmethod
+    def _parse_list(cls, value, separator=','):
+        """Represents value as a list.
+
+        Value is split either by separator (defaults to comma) or by lines.
+
+        :param value:
+        :param separator: List items separator character.
+        :rtype: list
+        """
+        if isinstance(value, list):  # _get_parser_compound case
+            return value
+
+        if '\n' in value:
+            value = value.splitlines()
+        else:
+            value = value.split(separator)
+
+        return [chunk.strip() for chunk in value if chunk.strip()]
+
+    @classmethod
+    def _parse_dict(cls, value):
+        """Represents value as a dict.
+
+        :param value:
+        :rtype: dict
+        """
+        separator = '='
+        result = {}
+        for line in cls._parse_list(value):
+            key, sep, val = line.partition(separator)
+            if sep != separator:
+                raise DistutilsOptionError(
+                    'Unable to parse option value to dict: %s' % value)
+            result[key.strip()] = val.strip()
+
+        return result
+
+    @classmethod
+    def _parse_bool(cls, value):
+        """Represents value as boolean.
+
+        :param value:
+        :rtype: bool
+        """
+        value = value.lower()
+        return value in ('1', 'true', 'yes')
+
+    @classmethod
+    def _exclude_files_parser(cls, key):
+        """Returns a parser function to make sure field inputs
+        are not files.
+
+        Parses a value after getting the key so error messages are
+        more informative.
+
+        :param key:
+        :rtype: callable
+        """
+        def parser(value):
+            exclude_directive = 'file:'
+            if value.startswith(exclude_directive):
+                raise ValueError(
+                    'Only strings are accepted for the {0} field, '
+                    'files are not accepted'.format(key))
+            return value
+        return parser
+
+    @classmethod
+    def _parse_file(cls, value):
+        """Represents value as a string, allowing including text
+        from nearest files using `file:` directive.
+
+        Directive is sandboxed and won't reach anything outside
+        directory with setup.py.
+
+        Examples:
+            file: README.rst, CHANGELOG.md, src/file.txt
+
+        :param str value:
+        :rtype: str
+        """
+        include_directive = 'file:'
+
+        if not isinstance(value, string_types):
+            return value
+
+        if not value.startswith(include_directive):
+            return value
+
+        spec = value[len(include_directive):]
+        filepaths = (os.path.abspath(path.strip()) for path in spec.split(','))
+        return '\n'.join(
+            cls._read_file(path)
+            for path in filepaths
+            if (cls._assert_local(path) or True)
+            and os.path.isfile(path)
+        )
+
+    @staticmethod
+    def _assert_local(filepath):
+        if not filepath.startswith(os.getcwd()):
+            raise DistutilsOptionError(
+                '`file:` directive can not access %s' % filepath)
+
+    @staticmethod
+    def _read_file(filepath):
+        with io.open(filepath, encoding='utf-8') as f:
+            return f.read()
+
+    @classmethod
+    def _parse_attr(cls, value, package_dir=None):
+        """Represents value as a module attribute.
+
+        Examples:
+            attr: package.attr
+            attr: package.module.attr
+
+        :param str value:
+        :rtype: str
+        """
+        attr_directive = 'attr:'
+        if not value.startswith(attr_directive):
+            return value
+
+        attrs_path = value.replace(attr_directive, '').strip().split('.')
+        attr_name = attrs_path.pop()
+
+        module_name = '.'.join(attrs_path)
+        module_name = module_name or '__init__'
+
+        parent_path = os.getcwd()
+        if package_dir:
+            if attrs_path[0] in package_dir:
+                # A custom path was specified for the module we want to import
+                custom_path = package_dir[attrs_path[0]]
+                parts = custom_path.rsplit('/', 1)
+                if len(parts) > 1:
+                    parent_path = os.path.join(os.getcwd(), parts[0])
+                    module_name = parts[1]
+                else:
+                    module_name = custom_path
+            elif '' in package_dir:
+                # A custom parent directory was specified for all root modules
+                parent_path = os.path.join(os.getcwd(), package_dir[''])
+        sys.path.insert(0, parent_path)
+        try:
+            module = import_module(module_name)
+            value = getattr(module, attr_name)
+
+        finally:
+            sys.path = sys.path[1:]
+
+        return value
+
+    @classmethod
+    def _get_parser_compound(cls, *parse_methods):
+        """Returns parser function to represents value as a list.
+
+        Parses a value applying given methods one after another.
+
+        :param parse_methods:
+        :rtype: callable
+        """
+        def parse(value):
+            parsed = value
+
+            for method in parse_methods:
+                parsed = method(parsed)
+
+            return parsed
+
+        return parse
+
+    @classmethod
+    def _parse_section_to_dict(cls, section_options, values_parser=None):
+        """Parses section options into a dictionary.
+
+        Optionally applies a given parser to values.
+
+        :param dict section_options:
+        :param callable values_parser:
+        :rtype: dict
+        """
+        value = {}
+        values_parser = values_parser or (lambda val: val)
+        for key, (_, val) in section_options.items():
+            value[key] = values_parser(val)
+        return value
+
+    def parse_section(self, section_options):
+        """Parses configuration file section.
+
+        :param dict section_options:
+        """
+        for (name, (_, value)) in section_options.items():
+            try:
+                self[name] = value
+
+            except KeyError:
+                pass  # Keep silent for a new option may appear anytime.
+
+    def parse(self):
+        """Parses configuration file items from one
+        or more related sections.
+
+        """
+        for section_name, section_options in self.sections.items():
+
+            method_postfix = ''
+            if section_name:  # [section.option] variant
+                method_postfix = '_%s' % section_name
+
+            section_parser_method = getattr(
+                self,
+                # Dots in section names are translated into dunderscores.
+                ('parse_section%s' % method_postfix).replace('.', '__'),
+                None)
+
+            if section_parser_method is None:
+                raise DistutilsOptionError(
+                    'Unsupported distribution option section: [%s.%s]' % (
+                        self.section_prefix, section_name))
+
+            section_parser_method(section_options)
+
+    def _deprecated_config_handler(self, func, msg, warning_class):
+        """ this function will wrap around parameters that are deprecated
+
+        :param msg: deprecation message
+        :param warning_class: class of warning exception to be raised
+        :param func: function to be wrapped around
+        """
+        @wraps(func)
+        def config_handler(*args, **kwargs):
+            warnings.warn(msg, warning_class)
+            return func(*args, **kwargs)
+
+        return config_handler
+
+
+class ConfigMetadataHandler(ConfigHandler):
+
+    section_prefix = 'metadata'
+
+    aliases = {
+        'home_page': 'url',
+        'summary': 'description',
+        'classifier': 'classifiers',
+        'platform': 'platforms',
+    }
+
+    strict_mode = False
+    """We need to keep it loose, to be partially compatible with
+    `pbr` and `d2to1` packages which also uses `metadata` section.
+
+    """
+
+    def __init__(self, target_obj, options, ignore_option_errors=False,
+                 package_dir=None):
+        super(ConfigMetadataHandler, self).__init__(target_obj, options,
+                                                    ignore_option_errors)
+        self.package_dir = package_dir
+
+    @property
+    def parsers(self):
+        """Metadata item name to parser function mapping."""
+        parse_list = self._parse_list
+        parse_file = self._parse_file
+        parse_dict = self._parse_dict
+        exclude_files_parser = self._exclude_files_parser
+
+        return {
+            'platforms': parse_list,
+            'keywords': parse_list,
+            'provides': parse_list,
+            'requires': self._deprecated_config_handler(
+                parse_list,
+                "The requires parameter is deprecated, please use "
+                "install_requires for runtime dependencies.",
+                DeprecationWarning),
+            'obsoletes': parse_list,
+            'classifiers': self._get_parser_compound(parse_file, parse_list),
+            'license': exclude_files_parser('license'),
+            'license_files': parse_list,
+            'description': parse_file,
+            'long_description': parse_file,
+            'version': self._parse_version,
+            'project_urls': parse_dict,
+        }
+
+    def _parse_version(self, value):
+        """Parses `version` option value.
+
+        :param value:
+        :rtype: str
+
+        """
+        version = self._parse_file(value)
+
+        if version != value:
+            version = version.strip()
+            # Be strict about versions loaded from file because it's easy to
+            # accidentally include newlines and other unintended content
+            if isinstance(parse(version), LegacyVersion):
+                tmpl = (
+                    'Version loaded from {value} does not '
+                    'comply with PEP 440: {version}'
+                )
+                raise DistutilsOptionError(tmpl.format(**locals()))
+
+            return version
+
+        version = self._parse_attr(value, self.package_dir)
+
+        if callable(version):
+            version = version()
+
+        if not isinstance(version, string_types):
+            if hasattr(version, '__iter__'):
+                version = '.'.join(map(str, version))
+            else:
+                version = '%s' % version
+
+        return version
+
+
+class ConfigOptionsHandler(ConfigHandler):
+
+    section_prefix = 'options'
+
+    @property
+    def parsers(self):
+        """Metadata item name to parser function mapping."""
+        parse_list = self._parse_list
+        parse_list_semicolon = partial(self._parse_list, separator=';')
+        parse_bool = self._parse_bool
+        parse_dict = self._parse_dict
+
+        return {
+            'zip_safe': parse_bool,
+            'use_2to3': parse_bool,
+            'include_package_data': parse_bool,
+            'package_dir': parse_dict,
+            'use_2to3_fixers': parse_list,
+            'use_2to3_exclude_fixers': parse_list,
+            'convert_2to3_doctests': parse_list,
+            'scripts': parse_list,
+            'eager_resources': parse_list,
+            'dependency_links': parse_list,
+            'namespace_packages': parse_list,
+            'install_requires': parse_list_semicolon,
+            'setup_requires': parse_list_semicolon,
+            'tests_require': parse_list_semicolon,
+            'packages': self._parse_packages,
+            'entry_points': self._parse_file,
+            'py_modules': parse_list,
+            'python_requires': SpecifierSet,
+        }
+
+    def _parse_packages(self, value):
+        """Parses `packages` option value.
+
+        :param value:
+        :rtype: list
+        """
+        find_directives = ['find:', 'find_namespace:']
+        trimmed_value = value.strip()
+
+        if trimmed_value not in find_directives:
+            return self._parse_list(value)
+
+        findns = trimmed_value == find_directives[1]
+        if findns and not PY3:
+            raise DistutilsOptionError(
+                'find_namespace: directive is unsupported on Python < 3.3')
+
+        # Read function arguments from a dedicated section.
+        find_kwargs = self.parse_section_packages__find(
+            self.sections.get('packages.find', {}))
+
+        if findns:
+            from setuptools import find_namespace_packages as find_packages
+        else:
+            from setuptools import find_packages
+
+        return find_packages(**find_kwargs)
+
+    def parse_section_packages__find(self, section_options):
+        """Parses `packages.find` configuration file section.
+
+        To be used in conjunction with _parse_packages().
+
+        :param dict section_options:
+        """
+        section_data = self._parse_section_to_dict(
+            section_options, self._parse_list)
+
+        valid_keys = ['where', 'include', 'exclude']
+
+        find_kwargs = dict(
+            [(k, v) for k, v in section_data.items() if k in valid_keys and v])
+
+        where = find_kwargs.get('where')
+        if where is not None:
+            find_kwargs['where'] = where[0]  # cast list to single val
+
+        return find_kwargs
+
+    def parse_section_entry_points(self, section_options):
+        """Parses `entry_points` configuration file section.
+
+        :param dict section_options:
+        """
+        parsed = self._parse_section_to_dict(section_options, self._parse_list)
+        self['entry_points'] = parsed
+
+    def _parse_package_data(self, section_options):
+        parsed = self._parse_section_to_dict(section_options, self._parse_list)
+
+        root = parsed.get('*')
+        if root:
+            parsed[''] = root
+            del parsed['*']
+
+        return parsed
+
+    def parse_section_package_data(self, section_options):
+        """Parses `package_data` configuration file section.
+
+        :param dict section_options:
+        """
+        self['package_data'] = self._parse_package_data(section_options)
+
+    def parse_section_exclude_package_data(self, section_options):
+        """Parses `exclude_package_data` configuration file section.
+
+        :param dict section_options:
+        """
+        self['exclude_package_data'] = self._parse_package_data(
+            section_options)
+
+    def parse_section_extras_require(self, section_options):
+        """Parses `extras_require` configuration file section.
+
+        :param dict section_options:
+        """
+        parse_list = partial(self._parse_list, separator=';')
+        self['extras_require'] = self._parse_section_to_dict(
+            section_options, parse_list)
+
+    def parse_section_data_files(self, section_options):
+        """Parses `data_files` configuration file section.
+
+        :param dict section_options:
+        """
+        parsed = self._parse_section_to_dict(section_options, self._parse_list)
+        self['data_files'] = [(k, v) for k, v in parsed.items()]
diff --git a/ubuntu/venv/setuptools/dep_util.py b/ubuntu/venv/setuptools/dep_util.py
new file mode 100644
index 0000000..2931c13
--- /dev/null
+++ b/ubuntu/venv/setuptools/dep_util.py
@@ -0,0 +1,23 @@
+from distutils.dep_util import newer_group
+
+# yes, this is was almost entirely copy-pasted from
+# 'newer_pairwise()', this is just another convenience
+# function.
+def newer_pairwise_group(sources_groups, targets):
+    """Walk both arguments in parallel, testing if each source group is newer
+    than its corresponding target. Returns a pair of lists (sources_groups,
+    targets) where sources is newer than target, according to the semantics
+    of 'newer_group()'.
+    """
+    if len(sources_groups) != len(targets):
+        raise ValueError("'sources_group' and 'targets' must be the same length")
+
+    # build a pair of lists (sources_groups, targets) where source is newer
+    n_sources = []
+    n_targets = []
+    for i in range(len(sources_groups)):
+        if newer_group(sources_groups[i], targets[i]):
+            n_sources.append(sources_groups[i])
+            n_targets.append(targets[i])
+
+    return n_sources, n_targets
diff --git a/ubuntu/venv/setuptools/depends.py b/ubuntu/venv/setuptools/depends.py
new file mode 100644
index 0000000..a37675c
--- /dev/null
+++ b/ubuntu/venv/setuptools/depends.py
@@ -0,0 +1,176 @@
+import sys
+import marshal
+import contextlib
+from distutils.version import StrictVersion
+
+from .py33compat import Bytecode
+
+from .py27compat import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE
+from . import py27compat
+
+
+__all__ = [
+    'Require', 'find_module', 'get_module_constant', 'extract_constant'
+]
+
+
+class Require:
+    """A prerequisite to building or installing a distribution"""
+
+    def __init__(
+            self, name, requested_version, module, homepage='',
+            attribute=None, format=None):
+
+        if format is None and requested_version is not None:
+            format = StrictVersion
+
+        if format is not None:
+            requested_version = format(requested_version)
+            if attribute is None:
+                attribute = '__version__'
+
+        self.__dict__.update(locals())
+        del self.self
+
+    def full_name(self):
+        """Return full package/distribution name, w/version"""
+        if self.requested_version is not None:
+            return '%s-%s' % (self.name, self.requested_version)
+        return self.name
+
+    def version_ok(self, version):
+        """Is 'version' sufficiently up-to-date?"""
+        return self.attribute is None or self.format is None or \
+            str(version) != "unknown" and version >= self.requested_version
+
+    def get_version(self, paths=None, default="unknown"):
+        """Get version number of installed module, 'None', or 'default'
+
+        Search 'paths' for module.  If not found, return 'None'.  If found,
+        return the extracted version attribute, or 'default' if no version
+        attribute was specified, or the value cannot be determined without
+        importing the module.  The version is formatted according to the
+        requirement's version format (if any), unless it is 'None' or the
+        supplied 'default'.
+        """
+
+        if self.attribute is None:
+            try:
+                f, p, i = find_module(self.module, paths)
+                if f:
+                    f.close()
+                return default
+            except ImportError:
+                return None
+
+        v = get_module_constant(self.module, self.attribute, default, paths)
+
+        if v is not None and v is not default and self.format is not None:
+            return self.format(v)
+
+        return v
+
+    def is_present(self, paths=None):
+        """Return true if dependency is present on 'paths'"""
+        return self.get_version(paths) is not None
+
+    def is_current(self, paths=None):
+        """Return true if dependency is present and up-to-date on 'paths'"""
+        version = self.get_version(paths)
+        if version is None:
+            return False
+        return self.version_ok(version)
+
+
+def maybe_close(f):
+    @contextlib.contextmanager
+    def empty():
+        yield
+        return
+    if not f:
+        return empty()
+
+    return contextlib.closing(f)
+
+
+def get_module_constant(module, symbol, default=-1, paths=None):
+    """Find 'module' by searching 'paths', and extract 'symbol'
+
+    Return 'None' if 'module' does not exist on 'paths', or it does not define
+    'symbol'.  If the module defines 'symbol' as a constant, return the
+    constant.  Otherwise, return 'default'."""
+
+    try:
+        f, path, (suffix, mode, kind) = info = find_module(module, paths)
+    except ImportError:
+        # Module doesn't exist
+        return None
+
+    with maybe_close(f):
+        if kind == PY_COMPILED:
+            f.read(8)  # skip magic & date
+            code = marshal.load(f)
+        elif kind == PY_FROZEN:
+            code = py27compat.get_frozen_object(module, paths)
+        elif kind == PY_SOURCE:
+            code = compile(f.read(), path, 'exec')
+        else:
+            # Not something we can parse; we'll have to import it.  :(
+            imported = py27compat.get_module(module, paths, info)
+            return getattr(imported, symbol, None)
+
+    return extract_constant(code, symbol, default)
+
+
+def extract_constant(code, symbol, default=-1):
+    """Extract the constant value of 'symbol' from 'code'
+
+    If the name 'symbol' is bound to a constant value by the Python code
+    object 'code', return that value.  If 'symbol' is bound to an expression,
+    return 'default'.  Otherwise, return 'None'.
+
+    Return value is based on the first assignment to 'symbol'.  'symbol' must
+    be a global, or at least a non-"fast" local in the code block.  That is,
+    only 'STORE_NAME' and 'STORE_GLOBAL' opcodes are checked, and 'symbol'
+    must be present in 'code.co_names'.
+    """
+    if symbol not in code.co_names:
+        # name's not there, can't possibly be an assignment
+        return None
+
+    name_idx = list(code.co_names).index(symbol)
+
+    STORE_NAME = 90
+    STORE_GLOBAL = 97
+    LOAD_CONST = 100
+
+    const = default
+
+    for byte_code in Bytecode(code):
+        op = byte_code.opcode
+        arg = byte_code.arg
+
+        if op == LOAD_CONST:
+            const = code.co_consts[arg]
+        elif arg == name_idx and (op == STORE_NAME or op == STORE_GLOBAL):
+            return const
+        else:
+            const = default
+
+
+def _update_globals():
+    """
+    Patch the globals to remove the objects not available on some platforms.
+
+    XXX it'd be better to test assertions about bytecode instead.
+    """
+
+    if not sys.platform.startswith('java') and sys.platform != 'cli':
+        return
+    incompatible = 'extract_constant', 'get_module_constant'
+    for name in incompatible:
+        del globals()[name]
+        __all__.remove(name)
+
+
+_update_globals()
diff --git a/ubuntu/venv/setuptools/dist.py b/ubuntu/venv/setuptools/dist.py
new file mode 100644
index 0000000..f22429e
--- /dev/null
+++ b/ubuntu/venv/setuptools/dist.py
@@ -0,0 +1,1274 @@
+# -*- coding: utf-8 -*-
+__all__ = ['Distribution']
+
+import io
+import sys
+import re
+import os
+import warnings
+import numbers
+import distutils.log
+import distutils.core
+import distutils.cmd
+import distutils.dist
+from distutils.util import strtobool
+from distutils.debug import DEBUG
+from distutils.fancy_getopt import translate_longopt
+import itertools
+
+from collections import defaultdict
+from email import message_from_file
+
+from distutils.errors import (
+    DistutilsOptionError, DistutilsPlatformError, DistutilsSetupError,
+)
+from distutils.util import rfc822_escape
+from distutils.version import StrictVersion
+
+from setuptools.extern import six
+from setuptools.extern import packaging
+from setuptools.extern import ordered_set
+from setuptools.extern.six.moves import map, filter, filterfalse
+
+from . import SetuptoolsDeprecationWarning
+
+from setuptools.depends import Require
+from setuptools import windows_support
+from setuptools.monkey import get_unpatched
+from setuptools.config import parse_configuration
+import pkg_resources
+
+__import__('setuptools.extern.packaging.specifiers')
+__import__('setuptools.extern.packaging.version')
+
+
+def _get_unpatched(cls):
+    warnings.warn("Do not call this function", DistDeprecationWarning)
+    return get_unpatched(cls)
+
+
+def get_metadata_version(self):
+    mv = getattr(self, 'metadata_version', None)
+
+    if mv is None:
+        if self.long_description_content_type or self.provides_extras:
+            mv = StrictVersion('2.1')
+        elif (self.maintainer is not None or
+              self.maintainer_email is not None or
+              getattr(self, 'python_requires', None) is not None or
+              self.project_urls):
+            mv = StrictVersion('1.2')
+        elif (self.provides or self.requires or self.obsoletes or
+                self.classifiers or self.download_url):
+            mv = StrictVersion('1.1')
+        else:
+            mv = StrictVersion('1.0')
+
+        self.metadata_version = mv
+
+    return mv
+
+
+def read_pkg_file(self, file):
+    """Reads the metadata values from a file object."""
+    msg = message_from_file(file)
+
+    def _read_field(name):
+        value = msg[name]
+        if value == 'UNKNOWN':
+            return None
+        return value
+
+    def _read_list(name):
+        values = msg.get_all(name, None)
+        if values == []:
+            return None
+        return values
+
+    self.metadata_version = StrictVersion(msg['metadata-version'])
+    self.name = _read_field('name')
+    self.version = _read_field('version')
+    self.description = _read_field('summary')
+    # we are filling author only.
+    self.author = _read_field('author')
+    self.maintainer = None
+    self.author_email = _read_field('author-email')
+    self.maintainer_email = None
+    self.url = _read_field('home-page')
+    self.license = _read_field('license')
+
+    if 'download-url' in msg:
+        self.download_url = _read_field('download-url')
+    else:
+        self.download_url = None
+
+    self.long_description = _read_field('description')
+    self.description = _read_field('summary')
+
+    if 'keywords' in msg:
+        self.keywords = _read_field('keywords').split(',')
+
+    self.platforms = _read_list('platform')
+    self.classifiers = _read_list('classifier')
+
+    # PEP 314 - these fields only exist in 1.1
+    if self.metadata_version == StrictVersion('1.1'):
+        self.requires = _read_list('requires')
+        self.provides = _read_list('provides')
+        self.obsoletes = _read_list('obsoletes')
+    else:
+        self.requires = None
+        self.provides = None
+        self.obsoletes = None
+
+
+# Based on Python 3.5 version
+def write_pkg_file(self, file):
+    """Write the PKG-INFO format data to a file object.
+    """
+    version = self.get_metadata_version()
+
+    if six.PY2:
+        def write_field(key, value):
+            file.write("%s: %s\n" % (key, self._encode_field(value)))
+    else:
+        def write_field(key, value):
+            file.write("%s: %s\n" % (key, value))
+
+    write_field('Metadata-Version', str(version))
+    write_field('Name', self.get_name())
+    write_field('Version', self.get_version())
+    write_field('Summary', self.get_description())
+    write_field('Home-page', self.get_url())
+
+    if version < StrictVersion('1.2'):
+        write_field('Author', self.get_contact())
+        write_field('Author-email', self.get_contact_email())
+    else:
+        optional_fields = (
+            ('Author', 'author'),
+            ('Author-email', 'author_email'),
+            ('Maintainer', 'maintainer'),
+            ('Maintainer-email', 'maintainer_email'),
+        )
+
+        for field, attr in optional_fields:
+            attr_val = getattr(self, attr)
+
+            if attr_val is not None:
+                write_field(field, attr_val)
+
+    write_field('License', self.get_license())
+    if self.download_url:
+        write_field('Download-URL', self.download_url)
+    for project_url in self.project_urls.items():
+        write_field('Project-URL',  '%s, %s' % project_url)
+
+    long_desc = rfc822_escape(self.get_long_description())
+    write_field('Description', long_desc)
+
+    keywords = ','.join(self.get_keywords())
+    if keywords:
+        write_field('Keywords', keywords)
+
+    if version >= StrictVersion('1.2'):
+        for platform in self.get_platforms():
+            write_field('Platform', platform)
+    else:
+        self._write_list(file, 'Platform', self.get_platforms())
+
+    self._write_list(file, 'Classifier', self.get_classifiers())
+
+    # PEP 314
+    self._write_list(file, 'Requires', self.get_requires())
+    self._write_list(file, 'Provides', self.get_provides())
+    self._write_list(file, 'Obsoletes', self.get_obsoletes())
+
+    # Setuptools specific for PEP 345
+    if hasattr(self, 'python_requires'):
+        write_field('Requires-Python', self.python_requires)
+
+    # PEP 566
+    if self.long_description_content_type:
+        write_field(
+            'Description-Content-Type',
+            self.long_description_content_type
+        )
+    if self.provides_extras:
+        for extra in sorted(self.provides_extras):
+            write_field('Provides-Extra', extra)
+
+
+sequence = tuple, list
+
+
+def check_importable(dist, attr, value):
+    try:
+        ep = pkg_resources.EntryPoint.parse('x=' + value)
+        assert not ep.extras
+    except (TypeError, ValueError, AttributeError, AssertionError):
+        raise DistutilsSetupError(
+            "%r must be importable 'module:attrs' string (got %r)"
+            % (attr, value)
+        )
+
+
+def assert_string_list(dist, attr, value):
+    """Verify that value is a string list"""
+    try:
+        # verify that value is a list or tuple to exclude unordered
+        # or single-use iterables
+        assert isinstance(value, (list, tuple))
+        # verify that elements of value are strings
+        assert ''.join(value) != value
+    except (TypeError, ValueError, AttributeError, AssertionError):
+        raise DistutilsSetupError(
+            "%r must be a list of strings (got %r)" % (attr, value)
+        )
+
+
+def check_nsp(dist, attr, value):
+    """Verify that namespace packages are valid"""
+    ns_packages = value
+    assert_string_list(dist, attr, ns_packages)
+    for nsp in ns_packages:
+        if not dist.has_contents_for(nsp):
+            raise DistutilsSetupError(
+                "Distribution contains no modules or packages for " +
+                "namespace package %r" % nsp
+            )
+        parent, sep, child = nsp.rpartition('.')
+        if parent and parent not in ns_packages:
+            distutils.log.warn(
+                "WARNING: %r is declared as a package namespace, but %r"
+                " is not: please correct this in setup.py", nsp, parent
+            )
+
+
+def check_extras(dist, attr, value):
+    """Verify that extras_require mapping is valid"""
+    try:
+        list(itertools.starmap(_check_extra, value.items()))
+    except (TypeError, ValueError, AttributeError):
+        raise DistutilsSetupError(
+            "'extras_require' must be a dictionary whose values are "
+            "strings or lists of strings containing valid project/version "
+            "requirement specifiers."
+        )
+
+
+def _check_extra(extra, reqs):
+    name, sep, marker = extra.partition(':')
+    if marker and pkg_resources.invalid_marker(marker):
+        raise DistutilsSetupError("Invalid environment marker: " + marker)
+    list(pkg_resources.parse_requirements(reqs))
+
+
+def assert_bool(dist, attr, value):
+    """Verify that value is True, False, 0, or 1"""
+    if bool(value) != value:
+        tmpl = "{attr!r} must be a boolean value (got {value!r})"
+        raise DistutilsSetupError(tmpl.format(attr=attr, value=value))
+
+
+def check_requirements(dist, attr, value):
+    """Verify that install_requires is a valid requirements list"""
+    try:
+        list(pkg_resources.parse_requirements(value))
+        if isinstance(value, (dict, set)):
+            raise TypeError("Unordered types are not allowed")
+    except (TypeError, ValueError) as error:
+        tmpl = (
+            "{attr!r} must be a string or list of strings "
+            "containing valid project/version requirement specifiers; {error}"
+        )
+        raise DistutilsSetupError(tmpl.format(attr=attr, error=error))
+
+
+def check_specifier(dist, attr, value):
+    """Verify that value is a valid version specifier"""
+    try:
+        packaging.specifiers.SpecifierSet(value)
+    except packaging.specifiers.InvalidSpecifier as error:
+        tmpl = (
+            "{attr!r} must be a string "
+            "containing valid version specifiers; {error}"
+        )
+        raise DistutilsSetupError(tmpl.format(attr=attr, error=error))
+
+
+def check_entry_points(dist, attr, value):
+    """Verify that entry_points map is parseable"""
+    try:
+        pkg_resources.EntryPoint.parse_map(value)
+    except ValueError as e:
+        raise DistutilsSetupError(e)
+
+
+def check_test_suite(dist, attr, value):
+    if not isinstance(value, six.string_types):
+        raise DistutilsSetupError("test_suite must be a string")
+
+
+def check_package_data(dist, attr, value):
+    """Verify that value is a dictionary of package names to glob lists"""
+    if not isinstance(value, dict):
+        raise DistutilsSetupError(
+            "{!r} must be a dictionary mapping package names to lists of "
+            "string wildcard patterns".format(attr))
+    for k, v in value.items():
+        if not isinstance(k, six.string_types):
+            raise DistutilsSetupError(
+                "keys of {!r} dict must be strings (got {!r})"
+                .format(attr, k)
+            )
+        assert_string_list(dist, 'values of {!r} dict'.format(attr), v)
+
+
+def check_packages(dist, attr, value):
+    for pkgname in value:
+        if not re.match(r'\w+(\.\w+)*', pkgname):
+            distutils.log.warn(
+                "WARNING: %r not a valid package name; please use only "
+                ".-separated package names in setup.py", pkgname
+            )
+
+
+_Distribution = get_unpatched(distutils.core.Distribution)
+
+
+class Distribution(_Distribution):
+    """Distribution with support for features, tests, and package data
+
+    This is an enhanced version of 'distutils.dist.Distribution' that
+    effectively adds the following new optional keyword arguments to 'setup()':
+
+     'install_requires' -- a string or sequence of strings specifying project
+        versions that the distribution requires when installed, in the format
+        used by 'pkg_resources.require()'.  They will be installed
+        automatically when the package is installed.  If you wish to use
+        packages that are not available in PyPI, or want to give your users an
+        alternate download location, you can add a 'find_links' option to the
+        '[easy_install]' section of your project's 'setup.cfg' file, and then
+        setuptools will scan the listed web pages for links that satisfy the
+        requirements.
+
+     'extras_require' -- a dictionary mapping names of optional "extras" to the
+        additional requirement(s) that using those extras incurs. For example,
+        this::
+
+            extras_require = dict(reST = ["docutils>=0.3", "reSTedit"])
+
+        indicates that the distribution can optionally provide an extra
+        capability called "reST", but it can only be used if docutils and
+        reSTedit are installed.  If the user installs your package using
+        EasyInstall and requests one of your extras, the corresponding
+        additional requirements will be installed if needed.
+
+     'features' **deprecated** -- a dictionary mapping option names to
+        'setuptools.Feature'
+        objects.  Features are a portion of the distribution that can be
+        included or excluded based on user options, inter-feature dependencies,
+        and availability on the current system.  Excluded features are omitted
+        from all setup commands, including source and binary distributions, so
+        you can create multiple distributions from the same source tree.
+        Feature names should be valid Python identifiers, except that they may
+        contain the '-' (minus) sign.  Features can be included or excluded
+        via the command line options '--with-X' and '--without-X', where 'X' is
+        the name of the feature.  Whether a feature is included by default, and
+        whether you are allowed to control this from the command line, is
+        determined by the Feature object.  See the 'Feature' class for more
+        information.
+
+     'test_suite' -- the name of a test suite to run for the 'test' command.
+        If the user runs 'python setup.py test', the package will be installed,
+        and the named test suite will be run.  The format is the same as
+        would be used on a 'unittest.py' command line.  That is, it is the
+        dotted name of an object to import and call to generate a test suite.
+
+     'package_data' -- a dictionary mapping package names to lists of filenames
+        or globs to use to find data files contained in the named packages.
+        If the dictionary has filenames or globs listed under '""' (the empty
+        string), those names will be searched for in every package, in addition
+        to any names for the specific package.  Data files found using these
+        names/globs will be installed along with the package, in the same
+        location as the package.  Note that globs are allowed to reference
+        the contents of non-package subdirectories, as long as you use '/' as
+        a path separator.  (Globs are automatically converted to
+        platform-specific paths at runtime.)
+
+    In addition to these new keywords, this class also has several new methods
+    for manipulating the distribution's contents.  For example, the 'include()'
+    and 'exclude()' methods can be thought of as in-place add and subtract
+    commands that add or remove packages, modules, extensions, and so on from
+    the distribution.  They are used by the feature subsystem to configure the
+    distribution for the included and excluded features.
+    """
+
+    _DISTUTILS_UNSUPPORTED_METADATA = {
+        'long_description_content_type': None,
+        'project_urls': dict,
+        'provides_extras': ordered_set.OrderedSet,
+        'license_files': ordered_set.OrderedSet,
+    }
+
+    _patched_dist = None
+
+    def patch_missing_pkg_info(self, attrs):
+        # Fake up a replacement for the data that would normally come from
+        # PKG-INFO, but which might not yet be built if this is a fresh
+        # checkout.
+        #
+        if not attrs or 'name' not in attrs or 'version' not in attrs:
+            return
+        key = pkg_resources.safe_name(str(attrs['name'])).lower()
+        dist = pkg_resources.working_set.by_key.get(key)
+        if dist is not None and not dist.has_metadata('PKG-INFO'):
+            dist._version = pkg_resources.safe_version(str(attrs['version']))
+            self._patched_dist = dist
+
+    def __init__(self, attrs=None):
+        have_package_data = hasattr(self, "package_data")
+        if not have_package_data:
+            self.package_data = {}
+        attrs = attrs or {}
+        if 'features' in attrs or 'require_features' in attrs:
+            Feature.warn_deprecated()
+        self.require_features = []
+        self.features = {}
+        self.dist_files = []
+        # Filter-out setuptools' specific options.
+        self.src_root = attrs.pop("src_root", None)
+        self.patch_missing_pkg_info(attrs)
+        self.dependency_links = attrs.pop('dependency_links', [])
+        self.setup_requires = attrs.pop('setup_requires', [])
+        for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'):
+            vars(self).setdefault(ep.name, None)
+        _Distribution.__init__(self, {
+            k: v for k, v in attrs.items()
+            if k not in self._DISTUTILS_UNSUPPORTED_METADATA
+        })
+
+        # Fill-in missing metadata fields not supported by distutils.
+        # Note some fields may have been set by other tools (e.g. pbr)
+        # above; they are taken preferrentially to setup() arguments
+        for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items():
+            for source in self.metadata.__dict__, attrs:
+                if option in source:
+                    value = source[option]
+                    break
+            else:
+                value = default() if default else None
+            setattr(self.metadata, option, value)
+
+        if isinstance(self.metadata.version, numbers.Number):
+            # Some people apparently take "version number" too literally :)
+            self.metadata.version = str(self.metadata.version)
+
+        if self.metadata.version is not None:
+            try:
+                ver = packaging.version.Version(self.metadata.version)
+                normalized_version = str(ver)
+                if self.metadata.version != normalized_version:
+                    warnings.warn(
+                        "Normalizing '%s' to '%s'" % (
+                            self.metadata.version,
+                            normalized_version,
+                        )
+                    )
+                    self.metadata.version = normalized_version
+            except (packaging.version.InvalidVersion, TypeError):
+                warnings.warn(
+                    "The version specified (%r) is an invalid version, this "
+                    "may not work as expected with newer versions of "
+                    "setuptools, pip, and PyPI. Please see PEP 440 for more "
+                    "details." % self.metadata.version
+                )
+        self._finalize_requires()
+
+    def _finalize_requires(self):
+        """
+        Set `metadata.python_requires` and fix environment markers
+        in `install_requires` and `extras_require`.
+        """
+        if getattr(self, 'python_requires', None):
+            self.metadata.python_requires = self.python_requires
+
+        if getattr(self, 'extras_require', None):
+            for extra in self.extras_require.keys():
+                # Since this gets called multiple times at points where the
+                # keys have become 'converted' extras, ensure that we are only
+                # truly adding extras we haven't seen before here.
+                extra = extra.split(':')[0]
+                if extra:
+                    self.metadata.provides_extras.add(extra)
+
+        self._convert_extras_requirements()
+        self._move_install_requirements_markers()
+
+    def _convert_extras_requirements(self):
+        """
+        Convert requirements in `extras_require` of the form
+        `"extra": ["barbazquux; {marker}"]` to
+        `"extra:{marker}": ["barbazquux"]`.
+        """
+        spec_ext_reqs = getattr(self, 'extras_require', None) or {}
+        self._tmp_extras_require = defaultdict(list)
+        for section, v in spec_ext_reqs.items():
+            # Do not strip empty sections.
+            self._tmp_extras_require[section]
+            for r in pkg_resources.parse_requirements(v):
+                suffix = self._suffix_for(r)
+                self._tmp_extras_require[section + suffix].append(r)
+
+    @staticmethod
+    def _suffix_for(req):
+        """
+        For a requirement, return the 'extras_require' suffix for
+        that requirement.
+        """
+        return ':' + str(req.marker) if req.marker else ''
+
+    def _move_install_requirements_markers(self):
+        """
+        Move requirements in `install_requires` that are using environment
+        markers `extras_require`.
+        """
+
+        # divide the install_requires into two sets, simple ones still
+        # handled by install_requires and more complex ones handled
+        # by extras_require.
+
+        def is_simple_req(req):
+            return not req.marker
+
+        spec_inst_reqs = getattr(self, 'install_requires', None) or ()
+        inst_reqs = list(pkg_resources.parse_requirements(spec_inst_reqs))
+        simple_reqs = filter(is_simple_req, inst_reqs)
+        complex_reqs = filterfalse(is_simple_req, inst_reqs)
+        self.install_requires = list(map(str, simple_reqs))
+
+        for r in complex_reqs:
+            self._tmp_extras_require[':' + str(r.marker)].append(r)
+        self.extras_require = dict(
+            (k, [str(r) for r in map(self._clean_req, v)])
+            for k, v in self._tmp_extras_require.items()
+        )
+
+    def _clean_req(self, req):
+        """
+        Given a Requirement, remove environment markers and return it.
+        """
+        req.marker = None
+        return req
+
+    def _parse_config_files(self, filenames=None):
+        """
+        Adapted from distutils.dist.Distribution.parse_config_files,
+        this method provides the same functionality in subtly-improved
+        ways.
+        """
+        from setuptools.extern.six.moves.configparser import ConfigParser
+
+        # Ignore install directory options if we have a venv
+        if six.PY3 and sys.prefix != sys.base_prefix:
+            ignore_options = [
+                'install-base', 'install-platbase', 'install-lib',
+                'install-platlib', 'install-purelib', 'install-headers',
+                'install-scripts', 'install-data', 'prefix', 'exec-prefix',
+                'home', 'user', 'root']
+        else:
+            ignore_options = []
+
+        ignore_options = frozenset(ignore_options)
+
+        if filenames is None:
+            filenames = self.find_config_files()
+
+        if DEBUG:
+            self.announce("Distribution.parse_config_files():")
+
+        parser = ConfigParser()
+        for filename in filenames:
+            with io.open(filename, encoding='utf-8') as reader:
+                if DEBUG:
+                    self.announce("  reading {filename}".format(**locals()))
+                (parser.read_file if six.PY3 else parser.readfp)(reader)
+            for section in parser.sections():
+                options = parser.options(section)
+                opt_dict = self.get_option_dict(section)
+
+                for opt in options:
+                    if opt != '__name__' and opt not in ignore_options:
+                        val = self._try_str(parser.get(section, opt))
+                        opt = opt.replace('-', '_')
+                        opt_dict[opt] = (filename, val)
+
+            # Make the ConfigParser forget everything (so we retain
+            # the original filenames that options come from)
+            parser.__init__()
+
+        # If there was a "global" section in the config file, use it
+        # to set Distribution options.
+
+        if 'global' in self.command_options:
+            for (opt, (src, val)) in self.command_options['global'].items():
+                alias = self.negative_opt.get(opt)
+                try:
+                    if alias:
+                        setattr(self, alias, not strtobool(val))
+                    elif opt in ('verbose', 'dry_run'):  # ugh!
+                        setattr(self, opt, strtobool(val))
+                    else:
+                        setattr(self, opt, val)
+                except ValueError as msg:
+                    raise DistutilsOptionError(msg)
+
+    @staticmethod
+    def _try_str(val):
+        """
+        On Python 2, much of distutils relies on string values being of
+        type 'str' (bytes) and not unicode text. If the value can be safely
+        encoded to bytes using the default encoding, prefer that.
+
+        Why the default encoding? Because that value can be implicitly
+        decoded back to text if needed.
+
+        Ref #1653
+        """
+        if six.PY3:
+            return val
+        try:
+            return val.encode()
+        except UnicodeEncodeError:
+            pass
+        return val
+
+    def _set_command_options(self, command_obj, option_dict=None):
+        """
+        Set the options for 'command_obj' from 'option_dict'.  Basically
+        this means copying elements of a dictionary ('option_dict') to
+        attributes of an instance ('command').
+
+        'command_obj' must be a Command instance.  If 'option_dict' is not
+        supplied, uses the standard option dictionary for this command
+        (from 'self.command_options').
+
+        (Adopted from distutils.dist.Distribution._set_command_options)
+        """
+        command_name = command_obj.get_command_name()
+        if option_dict is None:
+            option_dict = self.get_option_dict(command_name)
+
+        if DEBUG:
+            self.announce("  setting options for '%s' command:" % command_name)
+        for (option, (source, value)) in option_dict.items():
+            if DEBUG:
+                self.announce("    %s = %s (from %s)" % (option, value,
+                                                         source))
+            try:
+                bool_opts = [translate_longopt(o)
+                             for o in command_obj.boolean_options]
+            except AttributeError:
+                bool_opts = []
+            try:
+                neg_opt = command_obj.negative_opt
+            except AttributeError:
+                neg_opt = {}
+
+            try:
+                is_string = isinstance(value, six.string_types)
+                if option in neg_opt and is_string:
+                    setattr(command_obj, neg_opt[option], not strtobool(value))
+                elif option in bool_opts and is_string:
+                    setattr(command_obj, option, strtobool(value))
+                elif hasattr(command_obj, option):
+                    setattr(command_obj, option, value)
+                else:
+                    raise DistutilsOptionError(
+                        "error in %s: command '%s' has no such option '%s'"
+                        % (source, command_name, option))
+            except ValueError as msg:
+                raise DistutilsOptionError(msg)
+
+    def parse_config_files(self, filenames=None, ignore_option_errors=False):
+        """Parses configuration files from various levels
+        and loads configuration.
+
+        """
+        self._parse_config_files(filenames=filenames)
+
+        parse_configuration(self, self.command_options,
+                            ignore_option_errors=ignore_option_errors)
+        self._finalize_requires()
+
+    def parse_command_line(self):
+        """Process features after parsing command line options"""
+        result = _Distribution.parse_command_line(self)
+        if self.features:
+            self._finalize_features()
+        return result
+
+    def _feature_attrname(self, name):
+        """Convert feature name to corresponding option attribute name"""
+        return 'with_' + name.replace('-', '_')
+
+    def fetch_build_eggs(self, requires):
+        """Resolve pre-setup requirements"""
+        resolved_dists = pkg_resources.working_set.resolve(
+            pkg_resources.parse_requirements(requires),
+            installer=self.fetch_build_egg,
+            replace_conflicting=True,
+        )
+        for dist in resolved_dists:
+            pkg_resources.working_set.add(dist, replace=True)
+        return resolved_dists
+
+    def finalize_options(self):
+        """
+        Allow plugins to apply arbitrary operations to the
+        distribution. Each hook may optionally define a 'order'
+        to influence the order of execution. Smaller numbers
+        go first and the default is 0.
+        """
+        hook_key = 'setuptools.finalize_distribution_options'
+
+        def by_order(hook):
+            return getattr(hook, 'order', 0)
+        eps = pkg_resources.iter_entry_points(hook_key)
+        for ep in sorted(eps, key=by_order):
+            ep.load()(self)
+
+    def _finalize_setup_keywords(self):
+        for ep in pkg_resources.iter_entry_points('distutils.setup_keywords'):
+            value = getattr(self, ep.name, None)
+            if value is not None:
+                ep.require(installer=self.fetch_build_egg)
+                ep.load()(self, ep.name, value)
+
+    def _finalize_2to3_doctests(self):
+        if getattr(self, 'convert_2to3_doctests', None):
+            # XXX may convert to set here when we can rely on set being builtin
+            self.convert_2to3_doctests = [
+                os.path.abspath(p)
+                for p in self.convert_2to3_doctests
+            ]
+        else:
+            self.convert_2to3_doctests = []
+
+    def get_egg_cache_dir(self):
+        egg_cache_dir = os.path.join(os.curdir, '.eggs')
+        if not os.path.exists(egg_cache_dir):
+            os.mkdir(egg_cache_dir)
+            windows_support.hide_file(egg_cache_dir)
+            readme_txt_filename = os.path.join(egg_cache_dir, 'README.txt')
+            with open(readme_txt_filename, 'w') as f:
+                f.write('This directory contains eggs that were downloaded '
+                        'by setuptools to build, test, and run plug-ins.\n\n')
+                f.write('This directory caches those eggs to prevent '
+                        'repeated downloads.\n\n')
+                f.write('However, it is safe to delete this directory.\n\n')
+
+        return egg_cache_dir
+
+    def fetch_build_egg(self, req):
+        """Fetch an egg needed for building"""
+        from setuptools.installer import fetch_build_egg
+        return fetch_build_egg(self, req)
+
+    def _finalize_feature_opts(self):
+        """Add --with-X/--without-X options based on optional features"""
+
+        if not self.features:
+            return
+
+        go = []
+        no = self.negative_opt.copy()
+
+        for name, feature in self.features.items():
+            self._set_feature(name, None)
+            feature.validate(self)
+
+            if feature.optional:
+                descr = feature.description
+                incdef = ' (default)'
+                excdef = ''
+                if not feature.include_by_default():
+                    excdef, incdef = incdef, excdef
+
+                new = (
+                    ('with-' + name, None, 'include ' + descr + incdef),
+                    ('without-' + name, None, 'exclude ' + descr + excdef),
+                )
+                go.extend(new)
+                no['without-' + name] = 'with-' + name
+
+        self.global_options = self.feature_options = go + self.global_options
+        self.negative_opt = self.feature_negopt = no
+
+    def _finalize_features(self):
+        """Add/remove features and resolve dependencies between them"""
+
+        # First, flag all the enabled items (and thus their dependencies)
+        for name, feature in self.features.items():
+            enabled = self.feature_is_included(name)
+            if enabled or (enabled is None and feature.include_by_default()):
+                feature.include_in(self)
+                self._set_feature(name, 1)
+
+        # Then disable the rest, so that off-by-default features don't
+        # get flagged as errors when they're required by an enabled feature
+        for name, feature in self.features.items():
+            if not self.feature_is_included(name):
+                feature.exclude_from(self)
+                self._set_feature(name, 0)
+
+    def get_command_class(self, command):
+        """Pluggable version of get_command_class()"""
+        if command in self.cmdclass:
+            return self.cmdclass[command]
+
+        eps = pkg_resources.iter_entry_points('distutils.commands', command)
+        for ep in eps:
+            ep.require(installer=self.fetch_build_egg)
+            self.cmdclass[command] = cmdclass = ep.load()
+            return cmdclass
+        else:
+            return _Distribution.get_command_class(self, command)
+
+    def print_commands(self):
+        for ep in pkg_resources.iter_entry_points('distutils.commands'):
+            if ep.name not in self.cmdclass:
+                # don't require extras as the commands won't be invoked
+                cmdclass = ep.resolve()
+                self.cmdclass[ep.name] = cmdclass
+        return _Distribution.print_commands(self)
+
+    def get_command_list(self):
+        for ep in pkg_resources.iter_entry_points('distutils.commands'):
+            if ep.name not in self.cmdclass:
+                # don't require extras as the commands won't be invoked
+                cmdclass = ep.resolve()
+                self.cmdclass[ep.name] = cmdclass
+        return _Distribution.get_command_list(self)
+
+    def _set_feature(self, name, status):
+        """Set feature's inclusion status"""
+        setattr(self, self._feature_attrname(name), status)
+
+    def feature_is_included(self, name):
+        """Return 1 if feature is included, 0 if excluded, 'None' if unknown"""
+        return getattr(self, self._feature_attrname(name))
+
+    def include_feature(self, name):
+        """Request inclusion of feature named 'name'"""
+
+        if self.feature_is_included(name) == 0:
+            descr = self.features[name].description
+            raise DistutilsOptionError(
+                descr + " is required, but was excluded or is not available"
+            )
+        self.features[name].include_in(self)
+        self._set_feature(name, 1)
+
+    def include(self, **attrs):
+        """Add items to distribution that are named in keyword arguments
+
+        For example, 'dist.include(py_modules=["x"])' would add 'x' to
+        the distribution's 'py_modules' attribute, if it was not already
+        there.
+
+        Currently, this method only supports inclusion for attributes that are
+        lists or tuples.  If you need to add support for adding to other
+        attributes in this or a subclass, you can add an '_include_X' method,
+        where 'X' is the name of the attribute.  The method will be called with
+        the value passed to 'include()'.  So, 'dist.include(foo={"bar":"baz"})'
+        will try to call 'dist._include_foo({"bar":"baz"})', which can then
+        handle whatever special inclusion logic is needed.
+        """
+        for k, v in attrs.items():
+            include = getattr(self, '_include_' + k, None)
+            if include:
+                include(v)
+            else:
+                self._include_misc(k, v)
+
+    def exclude_package(self, package):
+        """Remove packages, modules, and extensions in named package"""
+
+        pfx = package + '.'
+        if self.packages:
+            self.packages = [
+                p for p in self.packages
+                if p != package and not p.startswith(pfx)
+            ]
+
+        if self.py_modules:
+            self.py_modules = [
+                p for p in self.py_modules
+                if p != package and not p.startswith(pfx)
+            ]
+
+        if self.ext_modules:
+            self.ext_modules = [
+                p for p in self.ext_modules
+                if p.name != package and not p.name.startswith(pfx)
+            ]
+
+    def has_contents_for(self, package):
+        """Return true if 'exclude_package(package)' would do something"""
+
+        pfx = package + '.'
+
+        for p in self.iter_distribution_names():
+            if p == package or p.startswith(pfx):
+                return True
+
+    def _exclude_misc(self, name, value):
+        """Handle 'exclude()' for list/tuple attrs without a special handler"""
+        if not isinstance(value, sequence):
+            raise DistutilsSetupError(
+                "%s: setting must be a list or tuple (%r)" % (name, value)
+            )
+        try:
+            old = getattr(self, name)
+        except AttributeError:
+            raise DistutilsSetupError(
+                "%s: No such distribution setting" % name
+            )
+        if old is not None and not isinstance(old, sequence):
+            raise DistutilsSetupError(
+                name + ": this setting cannot be changed via include/exclude"
+            )
+        elif old:
+            setattr(self, name, [item for item in old if item not in value])
+
+    def _include_misc(self, name, value):
+        """Handle 'include()' for list/tuple attrs without a special handler"""
+
+        if not isinstance(value, sequence):
+            raise DistutilsSetupError(
+                "%s: setting must be a list (%r)" % (name, value)
+            )
+        try:
+            old = getattr(self, name)
+        except AttributeError:
+            raise DistutilsSetupError(
+                "%s: No such distribution setting" % name
+            )
+        if old is None:
+            setattr(self, name, value)
+        elif not isinstance(old, sequence):
+            raise DistutilsSetupError(
+                name + ": this setting cannot be changed via include/exclude"
+            )
+        else:
+            new = [item for item in value if item not in old]
+            setattr(self, name, old + new)
+
+    def exclude(self, **attrs):
+        """Remove items from distribution that are named in keyword arguments
+
+        For example, 'dist.exclude(py_modules=["x"])' would remove 'x' from
+        the distribution's 'py_modules' attribute.  Excluding packages uses
+        the 'exclude_package()' method, so all of the package's contained
+        packages, modules, and extensions are also excluded.
+
+        Currently, this method only supports exclusion from attributes that are
+        lists or tuples.  If you need to add support for excluding from other
+        attributes in this or a subclass, you can add an '_exclude_X' method,
+        where 'X' is the name of the attribute.  The method will be called with
+        the value passed to 'exclude()'.  So, 'dist.exclude(foo={"bar":"baz"})'
+        will try to call 'dist._exclude_foo({"bar":"baz"})', which can then
+        handle whatever special exclusion logic is needed.
+        """
+        for k, v in attrs.items():
+            exclude = getattr(self, '_exclude_' + k, None)
+            if exclude:
+                exclude(v)
+            else:
+                self._exclude_misc(k, v)
+
+    def _exclude_packages(self, packages):
+        if not isinstance(packages, sequence):
+            raise DistutilsSetupError(
+                "packages: setting must be a list or tuple (%r)" % (packages,)
+            )
+        list(map(self.exclude_package, packages))
+
+    def _parse_command_opts(self, parser, args):
+        # Remove --with-X/--without-X options when processing command args
+        self.global_options = self.__class__.global_options
+        self.negative_opt = self.__class__.negative_opt
+
+        # First, expand any aliases
+        command = args[0]
+        aliases = self.get_option_dict('aliases')
+        while command in aliases:
+            src, alias = aliases[command]
+            del aliases[command]  # ensure each alias can expand only once!
+            import shlex
+            args[:1] = shlex.split(alias, True)
+            command = args[0]
+
+        nargs = _Distribution._parse_command_opts(self, parser, args)
+
+        # Handle commands that want to consume all remaining arguments
+        cmd_class = self.get_command_class(command)
+        if getattr(cmd_class, 'command_consumes_arguments', None):
+            self.get_option_dict(command)['args'] = ("command line", nargs)
+            if nargs is not None:
+                return []
+
+        return nargs
+
+    def get_cmdline_options(self):
+        """Return a '{cmd: {opt:val}}' map of all command-line options
+
+        Option names are all long, but do not include the leading '--', and
+        contain dashes rather than underscores.  If the option doesn't take
+        an argument (e.g. '--quiet'), the 'val' is 'None'.
+
+        Note that options provided by config files are intentionally excluded.
+        """
+
+        d = {}
+
+        for cmd, opts in self.command_options.items():
+
+            for opt, (src, val) in opts.items():
+
+                if src != "command line":
+                    continue
+
+                opt = opt.replace('_', '-')
+
+                if val == 0:
+                    cmdobj = self.get_command_obj(cmd)
+                    neg_opt = self.negative_opt.copy()
+                    neg_opt.update(getattr(cmdobj, 'negative_opt', {}))
+                    for neg, pos in neg_opt.items():
+                        if pos == opt:
+                            opt = neg
+                            val = None
+                            break
+                    else:
+                        raise AssertionError("Shouldn't be able to get here")
+
+                elif val == 1:
+                    val = None
+
+                d.setdefault(cmd, {})[opt] = val
+
+        return d
+
+    def iter_distribution_names(self):
+        """Yield all packages, modules, and extension names in distribution"""
+
+        for pkg in self.packages or ():
+            yield pkg
+
+        for module in self.py_modules or ():
+            yield module
+
+        for ext in self.ext_modules or ():
+            if isinstance(ext, tuple):
+                name, buildinfo = ext
+            else:
+                name = ext.name
+            if name.endswith('module'):
+                name = name[:-6]
+            yield name
+
+    def handle_display_options(self, option_order):
+        """If there were any non-global "display-only" options
+        (--help-commands or the metadata display options) on the command
+        line, display the requested info and return true; else return
+        false.
+        """
+        import sys
+
+        if six.PY2 or self.help_commands:
+            return _Distribution.handle_display_options(self, option_order)
+
+        # Stdout may be StringIO (e.g. in tests)
+        if not isinstance(sys.stdout, io.TextIOWrapper):
+            return _Distribution.handle_display_options(self, option_order)
+
+        # Don't wrap stdout if utf-8 is already the encoding. Provides
+        #  workaround for #334.
+        if sys.stdout.encoding.lower() in ('utf-8', 'utf8'):
+            return _Distribution.handle_display_options(self, option_order)
+
+        # Print metadata in UTF-8 no matter the platform
+        encoding = sys.stdout.encoding
+        errors = sys.stdout.errors
+        newline = sys.platform != 'win32' and '\n' or None
+        line_buffering = sys.stdout.line_buffering
+
+        sys.stdout = io.TextIOWrapper(
+            sys.stdout.detach(), 'utf-8', errors, newline, line_buffering)
+        try:
+            return _Distribution.handle_display_options(self, option_order)
+        finally:
+            sys.stdout = io.TextIOWrapper(
+                sys.stdout.detach(), encoding, errors, newline, line_buffering)
+
+
+class Feature:
+    """
+    **deprecated** -- The `Feature` facility was never completely implemented
+    or supported, `has reported issues
+    `_ and will be removed in
+    a future version.
+
+    A subset of the distribution that can be excluded if unneeded/wanted
+
+    Features are created using these keyword arguments:
+
+      'description' -- a short, human readable description of the feature, to
+         be used in error messages, and option help messages.
+
+      'standard' -- if true, the feature is included by default if it is
+         available on the current system.  Otherwise, the feature is only
+         included if requested via a command line '--with-X' option, or if
+         another included feature requires it.  The default setting is 'False'.
+
+      'available' -- if true, the feature is available for installation on the
+         current system.  The default setting is 'True'.
+
+      'optional' -- if true, the feature's inclusion can be controlled from the
+         command line, using the '--with-X' or '--without-X' options.  If
+         false, the feature's inclusion status is determined automatically,
+         based on 'availabile', 'standard', and whether any other feature
+         requires it.  The default setting is 'True'.
+
+      'require_features' -- a string or sequence of strings naming features
+         that should also be included if this feature is included.  Defaults to
+         empty list.  May also contain 'Require' objects that should be
+         added/removed from the distribution.
+
+      'remove' -- a string or list of strings naming packages to be removed
+         from the distribution if this feature is *not* included.  If the
+         feature *is* included, this argument is ignored.  This argument exists
+         to support removing features that "crosscut" a distribution, such as
+         defining a 'tests' feature that removes all the 'tests' subpackages
+         provided by other features.  The default for this argument is an empty
+         list.  (Note: the named package(s) or modules must exist in the base
+         distribution when the 'setup()' function is initially called.)
+
+      other keywords -- any other keyword arguments are saved, and passed to
+         the distribution's 'include()' and 'exclude()' methods when the
+         feature is included or excluded, respectively.  So, for example, you
+         could pass 'packages=["a","b"]' to cause packages 'a' and 'b' to be
+         added or removed from the distribution as appropriate.
+
+    A feature must include at least one 'requires', 'remove', or other
+    keyword argument.  Otherwise, it can't affect the distribution in any way.
+    Note also that you can subclass 'Feature' to create your own specialized
+    feature types that modify the distribution in other ways when included or
+    excluded.  See the docstrings for the various methods here for more detail.
+    Aside from the methods, the only feature attributes that distributions look
+    at are 'description' and 'optional'.
+    """
+
+    @staticmethod
+    def warn_deprecated():
+        msg = (
+            "Features are deprecated and will be removed in a future "
+            "version. See https://github.com/pypa/setuptools/issues/65."
+        )
+        warnings.warn(msg, DistDeprecationWarning, stacklevel=3)
+
+    def __init__(
+            self, description, standard=False, available=True,
+            optional=True, require_features=(), remove=(), **extras):
+        self.warn_deprecated()
+
+        self.description = description
+        self.standard = standard
+        self.available = available
+        self.optional = optional
+        if isinstance(require_features, (str, Require)):
+            require_features = require_features,
+
+        self.require_features = [
+            r for r in require_features if isinstance(r, str)
+        ]
+        er = [r for r in require_features if not isinstance(r, str)]
+        if er:
+            extras['require_features'] = er
+
+        if isinstance(remove, str):
+            remove = remove,
+        self.remove = remove
+        self.extras = extras
+
+        if not remove and not require_features and not extras:
+            raise DistutilsSetupError(
+                "Feature %s: must define 'require_features', 'remove', or "
+                "at least one of 'packages', 'py_modules', etc."
+            )
+
+    def include_by_default(self):
+        """Should this feature be included by default?"""
+        return self.available and self.standard
+
+    def include_in(self, dist):
+        """Ensure feature and its requirements are included in distribution
+
+        You may override this in a subclass to perform additional operations on
+        the distribution.  Note that this method may be called more than once
+        per feature, and so should be idempotent.
+
+        """
+
+        if not self.available:
+            raise DistutilsPlatformError(
+                self.description + " is required, "
+                "but is not available on this platform"
+            )
+
+        dist.include(**self.extras)
+
+        for f in self.require_features:
+            dist.include_feature(f)
+
+    def exclude_from(self, dist):
+        """Ensure feature is excluded from distribution
+
+        You may override this in a subclass to perform additional operations on
+        the distribution.  This method will be called at most once per
+        feature, and only after all included features have been asked to
+        include themselves.
+        """
+
+        dist.exclude(**self.extras)
+
+        if self.remove:
+            for item in self.remove:
+                dist.exclude_package(item)
+
+    def validate(self, dist):
+        """Verify that feature makes sense in context of distribution
+
+        This method is called by the distribution just before it parses its
+        command line.  It checks to ensure that the 'remove' attribute, if any,
+        contains only valid package/module names that are present in the base
+        distribution when 'setup()' is called.  You may override it in a
+        subclass to perform any other required validation of the feature
+        against a target distribution.
+        """
+
+        for item in self.remove:
+            if not dist.has_contents_for(item):
+                raise DistutilsSetupError(
+                    "%s wants to be able to remove %s, but the distribution"
+                    " doesn't contain any packages or modules under %s"
+                    % (self.description, item, item)
+                )
+
+
+class DistDeprecationWarning(SetuptoolsDeprecationWarning):
+    """Class for warning about deprecations in dist in
+    setuptools. Not ignored by default, unlike DeprecationWarning."""
diff --git a/ubuntu/venv/setuptools/errors.py b/ubuntu/venv/setuptools/errors.py
new file mode 100644
index 0000000..2701747
--- /dev/null
+++ b/ubuntu/venv/setuptools/errors.py
@@ -0,0 +1,16 @@
+"""setuptools.errors
+
+Provides exceptions used by setuptools modules.
+"""
+
+from distutils.errors import DistutilsError
+
+
+class RemovedCommandError(DistutilsError, RuntimeError):
+    """Error used for commands that have been removed in setuptools.
+
+    Since ``setuptools`` is built on ``distutils``, simply removing a command
+    from ``setuptools`` will make the behavior fall back to ``distutils``; this
+    error is raised if a command exists in ``distutils`` but has been actively
+    removed in ``setuptools``.
+    """
diff --git a/ubuntu/venv/setuptools/extension.py b/ubuntu/venv/setuptools/extension.py
new file mode 100644
index 0000000..2946889
--- /dev/null
+++ b/ubuntu/venv/setuptools/extension.py
@@ -0,0 +1,57 @@
+import re
+import functools
+import distutils.core
+import distutils.errors
+import distutils.extension
+
+from setuptools.extern.six.moves import map
+
+from .monkey import get_unpatched
+
+
+def _have_cython():
+    """
+    Return True if Cython can be imported.
+    """
+    cython_impl = 'Cython.Distutils.build_ext'
+    try:
+        # from (cython_impl) import build_ext
+        __import__(cython_impl, fromlist=['build_ext']).build_ext
+        return True
+    except Exception:
+        pass
+    return False
+
+
+# for compatibility
+have_pyrex = _have_cython
+
+_Extension = get_unpatched(distutils.core.Extension)
+
+
+class Extension(_Extension):
+    """Extension that uses '.c' files in place of '.pyx' files"""
+
+    def __init__(self, name, sources, *args, **kw):
+        # The *args is needed for compatibility as calls may use positional
+        # arguments. py_limited_api may be set only via keyword.
+        self.py_limited_api = kw.pop("py_limited_api", False)
+        _Extension.__init__(self, name, sources, *args, **kw)
+
+    def _convert_pyx_sources_to_lang(self):
+        """
+        Replace sources with .pyx extensions to sources with the target
+        language extension. This mechanism allows language authors to supply
+        pre-converted sources but to prefer the .pyx sources.
+        """
+        if _have_cython():
+            # the build has Cython, so allow it to compile the .pyx files
+            return
+        lang = self.language or ''
+        target_ext = '.cpp' if lang.lower() == 'c++' else '.c'
+        sub = functools.partial(re.sub, '.pyx$', target_ext)
+        self.sources = list(map(sub, self.sources))
+
+
+class Library(Extension):
+    """Just like a regular Extension, but built as a library instead"""
diff --git a/ubuntu/venv/setuptools/extern/__init__.py b/ubuntu/venv/setuptools/extern/__init__.py
new file mode 100644
index 0000000..e8c616f
--- /dev/null
+++ b/ubuntu/venv/setuptools/extern/__init__.py
@@ -0,0 +1,73 @@
+import sys
+
+
+class VendorImporter:
+    """
+    A PEP 302 meta path importer for finding optionally-vendored
+    or otherwise naturally-installed packages from root_name.
+    """
+
+    def __init__(self, root_name, vendored_names=(), vendor_pkg=None):
+        self.root_name = root_name
+        self.vendored_names = set(vendored_names)
+        self.vendor_pkg = vendor_pkg or root_name.replace('extern', '_vendor')
+
+    @property
+    def search_path(self):
+        """
+        Search first the vendor package then as a natural package.
+        """
+        yield self.vendor_pkg + '.'
+        yield ''
+
+    def find_module(self, fullname, path=None):
+        """
+        Return self when fullname starts with root_name and the
+        target module is one vendored through this importer.
+        """
+        root, base, target = fullname.partition(self.root_name + '.')
+        if root:
+            return
+        if not any(map(target.startswith, self.vendored_names)):
+            return
+        return self
+
+    def load_module(self, fullname):
+        """
+        Iterate over the search path to locate and load fullname.
+        """
+        root, base, target = fullname.partition(self.root_name + '.')
+        for prefix in self.search_path:
+            try:
+                extant = prefix + target
+                __import__(extant)
+                mod = sys.modules[extant]
+                sys.modules[fullname] = mod
+                # mysterious hack:
+                # Remove the reference to the extant package/module
+                # on later Python versions to cause relative imports
+                # in the vendor package to resolve the same modules
+                # as those going through this importer.
+                if sys.version_info >= (3, ):
+                    del sys.modules[extant]
+                return mod
+            except ImportError:
+                pass
+        else:
+            raise ImportError(
+                "The '{target}' package is required; "
+                "normally this is bundled with this package so if you get "
+                "this warning, consult the packager of your "
+                "distribution.".format(**locals())
+            )
+
+    def install(self):
+        """
+        Install this importer into sys.meta_path if not already present.
+        """
+        if self not in sys.meta_path:
+            sys.meta_path.append(self)
+
+
+names = 'six', 'packaging', 'pyparsing', 'ordered_set',
+VendorImporter(__name__, names, 'setuptools._vendor').install()
diff --git a/ubuntu/venv/setuptools/glob.py b/ubuntu/venv/setuptools/glob.py
new file mode 100644
index 0000000..9d7cbc5
--- /dev/null
+++ b/ubuntu/venv/setuptools/glob.py
@@ -0,0 +1,174 @@
+"""
+Filename globbing utility. Mostly a copy of `glob` from Python 3.5.
+
+Changes include:
+ * `yield from` and PEP3102 `*` removed.
+ * Hidden files are not ignored.
+"""
+
+import os
+import re
+import fnmatch
+
+__all__ = ["glob", "iglob", "escape"]
+
+
+def glob(pathname, recursive=False):
+    """Return a list of paths matching a pathname pattern.
+
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    return list(iglob(pathname, recursive=recursive))
+
+
+def iglob(pathname, recursive=False):
+    """Return an iterator which yields the paths matching a pathname pattern.
+
+    The pattern may contain simple shell-style wildcards a la
+    fnmatch. However, unlike fnmatch, filenames starting with a
+    dot are special cases that are not matched by '*' and '?'
+    patterns.
+
+    If recursive is true, the pattern '**' will match any files and
+    zero or more directories and subdirectories.
+    """
+    it = _iglob(pathname, recursive)
+    if recursive and _isrecursive(pathname):
+        s = next(it)  # skip empty string
+        assert not s
+    return it
+
+
+def _iglob(pathname, recursive):
+    dirname, basename = os.path.split(pathname)
+    if not has_magic(pathname):
+        if basename:
+            if os.path.lexists(pathname):
+                yield pathname
+        else:
+            # Patterns ending with a slash should match only directories
+            if os.path.isdir(dirname):
+                yield pathname
+        return
+    if not dirname:
+        if recursive and _isrecursive(basename):
+            for x in glob2(dirname, basename):
+                yield x
+        else:
+            for x in glob1(dirname, basename):
+                yield x
+        return
+    # `os.path.split()` returns the argument itself as a dirname if it is a
+    # drive or UNC path.  Prevent an infinite recursion if a drive or UNC path
+    # contains magic characters (i.e. r'\\?\C:').
+    if dirname != pathname and has_magic(dirname):
+        dirs = _iglob(dirname, recursive)
+    else:
+        dirs = [dirname]
+    if has_magic(basename):
+        if recursive and _isrecursive(basename):
+            glob_in_dir = glob2
+        else:
+            glob_in_dir = glob1
+    else:
+        glob_in_dir = glob0
+    for dirname in dirs:
+        for name in glob_in_dir(dirname, basename):
+            yield os.path.join(dirname, name)
+
+
+# These 2 helper functions non-recursively glob inside a literal directory.
+# They return a list of basenames. `glob1` accepts a pattern while `glob0`
+# takes a literal basename (so it only has to check for its existence).
+
+
+def glob1(dirname, pattern):
+    if not dirname:
+        if isinstance(pattern, bytes):
+            dirname = os.curdir.encode('ASCII')
+        else:
+            dirname = os.curdir
+    try:
+        names = os.listdir(dirname)
+    except OSError:
+        return []
+    return fnmatch.filter(names, pattern)
+
+
+def glob0(dirname, basename):
+    if not basename:
+        # `os.path.split()` returns an empty basename for paths ending with a
+        # directory separator.  'q*x/' should match only directories.
+        if os.path.isdir(dirname):
+            return [basename]
+    else:
+        if os.path.lexists(os.path.join(dirname, basename)):
+            return [basename]
+    return []
+
+
+# This helper function recursively yields relative pathnames inside a literal
+# directory.
+
+
+def glob2(dirname, pattern):
+    assert _isrecursive(pattern)
+    yield pattern[:0]
+    for x in _rlistdir(dirname):
+        yield x
+
+
+# Recursively yields relative pathnames inside a literal directory.
+def _rlistdir(dirname):
+    if not dirname:
+        if isinstance(dirname, bytes):
+            dirname = os.curdir.encode('ASCII')
+        else:
+            dirname = os.curdir
+    try:
+        names = os.listdir(dirname)
+    except os.error:
+        return
+    for x in names:
+        yield x
+        path = os.path.join(dirname, x) if dirname else x
+        for y in _rlistdir(path):
+            yield os.path.join(x, y)
+
+
+magic_check = re.compile('([*?[])')
+magic_check_bytes = re.compile(b'([*?[])')
+
+
+def has_magic(s):
+    if isinstance(s, bytes):
+        match = magic_check_bytes.search(s)
+    else:
+        match = magic_check.search(s)
+    return match is not None
+
+
+def _isrecursive(pattern):
+    if isinstance(pattern, bytes):
+        return pattern == b'**'
+    else:
+        return pattern == '**'
+
+
+def escape(pathname):
+    """Escape all special characters.
+    """
+    # Escaping is done by wrapping any of "*?[" between square brackets.
+    # Metacharacters do not work in the drive part and shouldn't be escaped.
+    drive, pathname = os.path.splitdrive(pathname)
+    if isinstance(pathname, bytes):
+        pathname = magic_check_bytes.sub(br'[\1]', pathname)
+    else:
+        pathname = magic_check.sub(r'[\1]', pathname)
+    return drive + pathname
diff --git a/ubuntu/venv/setuptools/installer.py b/ubuntu/venv/setuptools/installer.py
new file mode 100644
index 0000000..9f8be2e
--- /dev/null
+++ b/ubuntu/venv/setuptools/installer.py
@@ -0,0 +1,150 @@
+import glob
+import os
+import subprocess
+import sys
+from distutils import log
+from distutils.errors import DistutilsError
+
+import pkg_resources
+from setuptools.command.easy_install import easy_install
+from setuptools.extern import six
+from setuptools.wheel import Wheel
+
+from .py31compat import TemporaryDirectory
+
+
+def _fixup_find_links(find_links):
+    """Ensure find-links option end-up being a list of strings."""
+    if isinstance(find_links, six.string_types):
+        return find_links.split()
+    assert isinstance(find_links, (tuple, list))
+    return find_links
+
+
+def _legacy_fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Legacy path using EasyInstall.
+    """
+    tmp_dist = dist.__class__({'script_args': ['easy_install']})
+    opts = tmp_dist.get_option_dict('easy_install')
+    opts.clear()
+    opts.update(
+        (k, v)
+        for k, v in dist.get_option_dict('easy_install').items()
+        if k in (
+            # don't use any other settings
+            'find_links', 'site_dirs', 'index_url',
+            'optimize', 'site_dirs', 'allow_hosts',
+        ))
+    if dist.dependency_links:
+        links = dist.dependency_links[:]
+        if 'find_links' in opts:
+            links = _fixup_find_links(opts['find_links'][1]) + links
+        opts['find_links'] = ('setup', links)
+    install_dir = dist.get_egg_cache_dir()
+    cmd = easy_install(
+        tmp_dist, args=["x"], install_dir=install_dir,
+        exclude_scripts=True,
+        always_copy=False, build_directory=None, editable=False,
+        upgrade=False, multi_version=True, no_report=True, user=False
+    )
+    cmd.ensure_finalized()
+    return cmd.easy_install(req)
+
+
+def fetch_build_egg(dist, req):
+    """Fetch an egg needed for building.
+
+    Use pip/wheel to fetch/build a wheel."""
+    # Check pip is available.
+    try:
+        pkg_resources.get_distribution('pip')
+    except pkg_resources.DistributionNotFound:
+        dist.announce(
+            'WARNING: The pip package is not available, falling back '
+            'to EasyInstall for handling setup_requires/test_requires; '
+            'this is deprecated and will be removed in a future version.'
+            , log.WARN
+        )
+        return _legacy_fetch_build_egg(dist, req)
+    # Warn if wheel is not.
+    try:
+        pkg_resources.get_distribution('wheel')
+    except pkg_resources.DistributionNotFound:
+        dist.announce('WARNING: The wheel package is not available.', log.WARN)
+    # Ignore environment markers; if supplied, it is required.
+    req = strip_marker(req)
+    # Take easy_install options into account, but do not override relevant
+    # pip environment variables (like PIP_INDEX_URL or PIP_QUIET); they'll
+    # take precedence.
+    opts = dist.get_option_dict('easy_install')
+    if 'allow_hosts' in opts:
+        raise DistutilsError('the `allow-hosts` option is not supported '
+                             'when using pip to install requirements.')
+    if 'PIP_QUIET' in os.environ or 'PIP_VERBOSE' in os.environ:
+        quiet = False
+    else:
+        quiet = True
+    if 'PIP_INDEX_URL' in os.environ:
+        index_url = None
+    elif 'index_url' in opts:
+        index_url = opts['index_url'][1]
+    else:
+        index_url = None
+    if 'find_links' in opts:
+        find_links = _fixup_find_links(opts['find_links'][1])[:]
+    else:
+        find_links = []
+    if dist.dependency_links:
+        find_links.extend(dist.dependency_links)
+    eggs_dir = os.path.realpath(dist.get_egg_cache_dir())
+    environment = pkg_resources.Environment()
+    for egg_dist in pkg_resources.find_distributions(eggs_dir):
+        if egg_dist in req and environment.can_add(egg_dist):
+            return egg_dist
+    with TemporaryDirectory() as tmpdir:
+        cmd = [
+            sys.executable, '-m', 'pip',
+            '--disable-pip-version-check',
+            'wheel', '--no-deps',
+            '-w', tmpdir,
+        ]
+        if quiet:
+            cmd.append('--quiet')
+        if index_url is not None:
+            cmd.extend(('--index-url', index_url))
+        if find_links is not None:
+            for link in find_links:
+                cmd.extend(('--find-links', link))
+        # If requirement is a PEP 508 direct URL, directly pass
+        # the URL to pip, as `req @ url` does not work on the
+        # command line.
+        if req.url:
+            cmd.append(req.url)
+        else:
+            cmd.append(str(req))
+        try:
+            subprocess.check_call(cmd)
+        except subprocess.CalledProcessError as e:
+            raise DistutilsError(str(e))
+        wheel = Wheel(glob.glob(os.path.join(tmpdir, '*.whl'))[0])
+        dist_location = os.path.join(eggs_dir, wheel.egg_name())
+        wheel.install_as_egg(dist_location)
+        dist_metadata = pkg_resources.PathMetadata(
+            dist_location, os.path.join(dist_location, 'EGG-INFO'))
+        dist = pkg_resources.Distribution.from_filename(
+            dist_location, metadata=dist_metadata)
+        return dist
+
+
+def strip_marker(req):
+    """
+    Return a new requirement without the environment marker to avoid
+    calling pip with something like `babel; extra == "i18n"`, which
+    would always be ignored.
+    """
+    # create a copy to avoid mutating the input
+    req = pkg_resources.Requirement.parse(str(req))
+    req.marker = None
+    return req
diff --git a/ubuntu/venv/setuptools/launch.py b/ubuntu/venv/setuptools/launch.py
new file mode 100644
index 0000000..308283e
--- /dev/null
+++ b/ubuntu/venv/setuptools/launch.py
@@ -0,0 +1,35 @@
+"""
+Launch the Python script on the command line after
+setuptools is bootstrapped via import.
+"""
+
+# Note that setuptools gets imported implicitly by the
+# invocation of this script using python -m setuptools.launch
+
+import tokenize
+import sys
+
+
+def run():
+    """
+    Run the script in sys.argv[1] as if it had
+    been invoked naturally.
+    """
+    __builtins__
+    script_name = sys.argv[1]
+    namespace = dict(
+        __file__=script_name,
+        __name__='__main__',
+        __doc__=None,
+    )
+    sys.argv[:] = sys.argv[1:]
+
+    open_ = getattr(tokenize, 'open', open)
+    script = open_(script_name).read()
+    norm_script = script.replace('\\r\\n', '\\n')
+    code = compile(norm_script, script_name, 'exec')
+    exec(code, namespace)
+
+
+if __name__ == '__main__':
+    run()
diff --git a/ubuntu/venv/setuptools/lib2to3_ex.py b/ubuntu/venv/setuptools/lib2to3_ex.py
new file mode 100644
index 0000000..4b1a73f
--- /dev/null
+++ b/ubuntu/venv/setuptools/lib2to3_ex.py
@@ -0,0 +1,62 @@
+"""
+Customized Mixin2to3 support:
+
+ - adds support for converting doctests
+
+
+This module raises an ImportError on Python 2.
+"""
+
+from distutils.util import Mixin2to3 as _Mixin2to3
+from distutils import log
+from lib2to3.refactor import RefactoringTool, get_fixers_from_package
+
+import setuptools
+
+
+class DistutilsRefactoringTool(RefactoringTool):
+    def log_error(self, msg, *args, **kw):
+        log.error(msg, *args)
+
+    def log_message(self, msg, *args):
+        log.info(msg, *args)
+
+    def log_debug(self, msg, *args):
+        log.debug(msg, *args)
+
+
+class Mixin2to3(_Mixin2to3):
+    def run_2to3(self, files, doctests=False):
+        # See of the distribution option has been set, otherwise check the
+        # setuptools default.
+        if self.distribution.use_2to3 is not True:
+            return
+        if not files:
+            return
+        log.info("Fixing " + " ".join(files))
+        self.__build_fixer_names()
+        self.__exclude_fixers()
+        if doctests:
+            if setuptools.run_2to3_on_doctests:
+                r = DistutilsRefactoringTool(self.fixer_names)
+                r.refactor(files, write=True, doctests_only=True)
+        else:
+            _Mixin2to3.run_2to3(self, files)
+
+    def __build_fixer_names(self):
+        if self.fixer_names:
+            return
+        self.fixer_names = []
+        for p in setuptools.lib2to3_fixer_packages:
+            self.fixer_names.extend(get_fixers_from_package(p))
+        if self.distribution.use_2to3_fixers is not None:
+            for p in self.distribution.use_2to3_fixers:
+                self.fixer_names.extend(get_fixers_from_package(p))
+
+    def __exclude_fixers(self):
+        excluded_fixers = getattr(self, 'exclude_fixers', [])
+        if self.distribution.use_2to3_exclude_fixers is not None:
+            excluded_fixers.extend(self.distribution.use_2to3_exclude_fixers)
+        for fixer_name in excluded_fixers:
+            if fixer_name in self.fixer_names:
+                self.fixer_names.remove(fixer_name)
diff --git a/ubuntu/venv/setuptools/monkey.py b/ubuntu/venv/setuptools/monkey.py
new file mode 100644
index 0000000..3c77f8c
--- /dev/null
+++ b/ubuntu/venv/setuptools/monkey.py
@@ -0,0 +1,179 @@
+"""
+Monkey patching of distutils.
+"""
+
+import sys
+import distutils.filelist
+import platform
+import types
+import functools
+from importlib import import_module
+import inspect
+
+from setuptools.extern import six
+
+import setuptools
+
+__all__ = []
+"""
+Everything is private. Contact the project team
+if you think you need this functionality.
+"""
+
+
+def _get_mro(cls):
+    """
+    Returns the bases classes for cls sorted by the MRO.
+
+    Works around an issue on Jython where inspect.getmro will not return all
+    base classes if multiple classes share the same name. Instead, this
+    function will return a tuple containing the class itself, and the contents
+    of cls.__bases__. See https://github.com/pypa/setuptools/issues/1024.
+    """
+    if platform.python_implementation() == "Jython":
+        return (cls,) + cls.__bases__
+    return inspect.getmro(cls)
+
+
+def get_unpatched(item):
+    lookup = (
+        get_unpatched_class if isinstance(item, six.class_types) else
+        get_unpatched_function if isinstance(item, types.FunctionType) else
+        lambda item: None
+    )
+    return lookup(item)
+
+
+def get_unpatched_class(cls):
+    """Protect against re-patching the distutils if reloaded
+
+    Also ensures that no other distutils extension monkeypatched the distutils
+    first.
+    """
+    external_bases = (
+        cls
+        for cls in _get_mro(cls)
+        if not cls.__module__.startswith('setuptools')
+    )
+    base = next(external_bases)
+    if not base.__module__.startswith('distutils'):
+        msg = "distutils has already been patched by %r" % cls
+        raise AssertionError(msg)
+    return base
+
+
+def patch_all():
+    # we can't patch distutils.cmd, alas
+    distutils.core.Command = setuptools.Command
+
+    has_issue_12885 = sys.version_info <= (3, 5, 3)
+
+    if has_issue_12885:
+        # fix findall bug in distutils (http://bugs.python.org/issue12885)
+        distutils.filelist.findall = setuptools.findall
+
+    needs_warehouse = (
+        sys.version_info < (2, 7, 13)
+        or
+        (3, 4) < sys.version_info < (3, 4, 6)
+        or
+        (3, 5) < sys.version_info <= (3, 5, 3)
+    )
+
+    if needs_warehouse:
+        warehouse = 'https://upload.pypi.org/legacy/'
+        distutils.config.PyPIRCCommand.DEFAULT_REPOSITORY = warehouse
+
+    _patch_distribution_metadata()
+
+    # Install Distribution throughout the distutils
+    for module in distutils.dist, distutils.core, distutils.cmd:
+        module.Distribution = setuptools.dist.Distribution
+
+    # Install the patched Extension
+    distutils.core.Extension = setuptools.extension.Extension
+    distutils.extension.Extension = setuptools.extension.Extension
+    if 'distutils.command.build_ext' in sys.modules:
+        sys.modules['distutils.command.build_ext'].Extension = (
+            setuptools.extension.Extension
+        )
+
+    patch_for_msvc_specialized_compiler()
+
+
+def _patch_distribution_metadata():
+    """Patch write_pkg_file and read_pkg_file for higher metadata standards"""
+    for attr in ('write_pkg_file', 'read_pkg_file', 'get_metadata_version'):
+        new_val = getattr(setuptools.dist, attr)
+        setattr(distutils.dist.DistributionMetadata, attr, new_val)
+
+
+def patch_func(replacement, target_mod, func_name):
+    """
+    Patch func_name in target_mod with replacement
+
+    Important - original must be resolved by name to avoid
+    patching an already patched function.
+    """
+    original = getattr(target_mod, func_name)
+
+    # set the 'unpatched' attribute on the replacement to
+    # point to the original.
+    vars(replacement).setdefault('unpatched', original)
+
+    # replace the function in the original module
+    setattr(target_mod, func_name, replacement)
+
+
+def get_unpatched_function(candidate):
+    return getattr(candidate, 'unpatched')
+
+
+def patch_for_msvc_specialized_compiler():
+    """
+    Patch functions in distutils to use standalone Microsoft Visual C++
+    compilers.
+    """
+    # import late to avoid circular imports on Python < 3.5
+    msvc = import_module('setuptools.msvc')
+
+    if platform.system() != 'Windows':
+        # Compilers only availables on Microsoft Windows
+        return
+
+    def patch_params(mod_name, func_name):
+        """
+        Prepare the parameters for patch_func to patch indicated function.
+        """
+        repl_prefix = 'msvc9_' if 'msvc9' in mod_name else 'msvc14_'
+        repl_name = repl_prefix + func_name.lstrip('_')
+        repl = getattr(msvc, repl_name)
+        mod = import_module(mod_name)
+        if not hasattr(mod, func_name):
+            raise ImportError(func_name)
+        return repl, mod, func_name
+
+    # Python 2.7 to 3.4
+    msvc9 = functools.partial(patch_params, 'distutils.msvc9compiler')
+
+    # Python 3.5+
+    msvc14 = functools.partial(patch_params, 'distutils._msvccompiler')
+
+    try:
+        # Patch distutils.msvc9compiler
+        patch_func(*msvc9('find_vcvarsall'))
+        patch_func(*msvc9('query_vcvarsall'))
+    except ImportError:
+        pass
+
+    try:
+        # Patch distutils._msvccompiler._get_vc_env
+        patch_func(*msvc14('_get_vc_env'))
+    except ImportError:
+        pass
+
+    try:
+        # Patch distutils._msvccompiler.gen_lib_options for Numpy
+        patch_func(*msvc14('gen_lib_options'))
+    except ImportError:
+        pass
diff --git a/ubuntu/venv/setuptools/msvc.py b/ubuntu/venv/setuptools/msvc.py
new file mode 100644
index 0000000..2ffe1c8
--- /dev/null
+++ b/ubuntu/venv/setuptools/msvc.py
@@ -0,0 +1,1679 @@
+"""
+Improved support for Microsoft Visual C++ compilers.
+
+Known supported compilers:
+--------------------------
+Microsoft Visual C++ 9.0:
+    Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64)
+    Microsoft Windows SDK 6.1 (x86, x64, ia64)
+    Microsoft Windows SDK 7.0 (x86, x64, ia64)
+
+Microsoft Visual C++ 10.0:
+    Microsoft Windows SDK 7.1 (x86, x64, ia64)
+
+Microsoft Visual C++ 14.X:
+    Microsoft Visual C++ Build Tools 2015 (x86, x64, arm)
+    Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64)
+    Microsoft Visual Studio Build Tools 2019 (x86, x64, arm, arm64)
+
+This may also support compilers shipped with compatible Visual Studio versions.
+"""
+
+import json
+from io import open
+from os import listdir, pathsep
+from os.path import join, isfile, isdir, dirname
+import sys
+import platform
+import itertools
+import distutils.errors
+from setuptools.extern.packaging.version import LegacyVersion
+
+from setuptools.extern.six.moves import filterfalse
+
+from .monkey import get_unpatched
+
+if platform.system() == 'Windows':
+    from setuptools.extern.six.moves import winreg
+    from os import environ
+else:
+    # Mock winreg and environ so the module can be imported on this platform.
+
+    class winreg:
+        HKEY_USERS = None
+        HKEY_CURRENT_USER = None
+        HKEY_LOCAL_MACHINE = None
+        HKEY_CLASSES_ROOT = None
+
+    environ = dict()
+
+_msvc9_suppress_errors = (
+    # msvc9compiler isn't available on some platforms
+    ImportError,
+
+    # msvc9compiler raises DistutilsPlatformError in some
+    # environments. See #1118.
+    distutils.errors.DistutilsPlatformError,
+)
+
+try:
+    from distutils.msvc9compiler import Reg
+except _msvc9_suppress_errors:
+    pass
+
+
+def msvc9_find_vcvarsall(version):
+    """
+    Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone
+    compiler build for Python
+    (VCForPython / Microsoft Visual C++ Compiler for Python 2.7).
+
+    Fall back to original behavior when the standalone compiler is not
+    available.
+
+    Redirect the path of "vcvarsall.bat".
+
+    Parameters
+    ----------
+    version: float
+        Required Microsoft Visual C++ version.
+
+    Return
+    ------
+    str
+        vcvarsall.bat path
+    """
+    vc_base = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f'
+    key = vc_base % ('', version)
+    try:
+        # Per-user installs register the compiler path here
+        productdir = Reg.get_value(key, "installdir")
+    except KeyError:
+        try:
+            # All-user installs on a 64-bit system register here
+            key = vc_base % ('Wow6432Node\\', version)
+            productdir = Reg.get_value(key, "installdir")
+        except KeyError:
+            productdir = None
+
+    if productdir:
+        vcvarsall = join(productdir, "vcvarsall.bat")
+        if isfile(vcvarsall):
+            return vcvarsall
+
+    return get_unpatched(msvc9_find_vcvarsall)(version)
+
+
+def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs):
+    """
+    Patched "distutils.msvc9compiler.query_vcvarsall" for support extra
+    Microsoft Visual C++ 9.0 and 10.0 compilers.
+
+    Set environment without use of "vcvarsall.bat".
+
+    Parameters
+    ----------
+    ver: float
+        Required Microsoft Visual C++ version.
+    arch: str
+        Target architecture.
+
+    Return
+    ------
+    dict
+        environment
+    """
+    # Try to get environment from vcvarsall.bat (Classical way)
+    try:
+        orig = get_unpatched(msvc9_query_vcvarsall)
+        return orig(ver, arch, *args, **kwargs)
+    except distutils.errors.DistutilsPlatformError:
+        # Pass error if Vcvarsall.bat is missing
+        pass
+    except ValueError:
+        # Pass error if environment not set after executing vcvarsall.bat
+        pass
+
+    # If error, try to set environment directly
+    try:
+        return EnvironmentInfo(arch, ver).return_env()
+    except distutils.errors.DistutilsPlatformError as exc:
+        _augment_exception(exc, ver, arch)
+        raise
+
+
+def msvc14_get_vc_env(plat_spec):
+    """
+    Patched "distutils._msvccompiler._get_vc_env" for support extra
+    Microsoft Visual C++ 14.X compilers.
+
+    Set environment without use of "vcvarsall.bat".
+
+    Parameters
+    ----------
+    plat_spec: str
+        Target architecture.
+
+    Return
+    ------
+    dict
+        environment
+    """
+    # Try to get environment from vcvarsall.bat (Classical way)
+    try:
+        return get_unpatched(msvc14_get_vc_env)(plat_spec)
+    except distutils.errors.DistutilsPlatformError:
+        # Pass error Vcvarsall.bat is missing
+        pass
+
+    # If error, try to set environment directly
+    try:
+        return EnvironmentInfo(plat_spec, vc_min_ver=14.0).return_env()
+    except distutils.errors.DistutilsPlatformError as exc:
+        _augment_exception(exc, 14.0)
+        raise
+
+
+def msvc14_gen_lib_options(*args, **kwargs):
+    """
+    Patched "distutils._msvccompiler.gen_lib_options" for fix
+    compatibility between "numpy.distutils" and "distutils._msvccompiler"
+    (for Numpy < 1.11.2)
+    """
+    if "numpy.distutils" in sys.modules:
+        import numpy as np
+        if LegacyVersion(np.__version__) < LegacyVersion('1.11.2'):
+            return np.distutils.ccompiler.gen_lib_options(*args, **kwargs)
+    return get_unpatched(msvc14_gen_lib_options)(*args, **kwargs)
+
+
+def _augment_exception(exc, version, arch=''):
+    """
+    Add details to the exception message to help guide the user
+    as to what action will resolve it.
+    """
+    # Error if MSVC++ directory not found or environment not set
+    message = exc.args[0]
+
+    if "vcvarsall" in message.lower() or "visual c" in message.lower():
+        # Special error message if MSVC++ not installed
+        tmpl = 'Microsoft Visual C++ {version:0.1f} is required.'
+        message = tmpl.format(**locals())
+        msdownload = 'www.microsoft.com/download/details.aspx?id=%d'
+        if version == 9.0:
+            if arch.lower().find('ia64') > -1:
+                # For VC++ 9.0, if IA64 support is needed, redirect user
+                # to Windows SDK 7.0.
+                # Note: No download link available from Microsoft.
+                message += ' Get it with "Microsoft Windows SDK 7.0"'
+            else:
+                # For VC++ 9.0 redirect user to Vc++ for Python 2.7 :
+                # This redirection link is maintained by Microsoft.
+                # Contact vspython@microsoft.com if it needs updating.
+                message += ' Get it from http://aka.ms/vcpython27'
+        elif version == 10.0:
+            # For VC++ 10.0 Redirect user to Windows SDK 7.1
+            message += ' Get it with "Microsoft Windows SDK 7.1": '
+            message += msdownload % 8279
+        elif version >= 14.0:
+            # For VC++ 14.X Redirect user to latest Visual C++ Build Tools
+            message += (' Get it with "Build Tools for Visual Studio": '
+                        r'https://visualstudio.microsoft.com/downloads/')
+
+    exc.args = (message, )
+
+
+class PlatformInfo:
+    """
+    Current and Target Architectures information.
+
+    Parameters
+    ----------
+    arch: str
+        Target architecture.
+    """
+    current_cpu = environ.get('processor_architecture', '').lower()
+
+    def __init__(self, arch):
+        self.arch = arch.lower().replace('x64', 'amd64')
+
+    @property
+    def target_cpu(self):
+        """
+        Return Target CPU architecture.
+
+        Return
+        ------
+        str
+            Target CPU
+        """
+        return self.arch[self.arch.find('_') + 1:]
+
+    def target_is_x86(self):
+        """
+        Return True if target CPU is x86 32 bits..
+
+        Return
+        ------
+        bool
+            CPU is x86 32 bits
+        """
+        return self.target_cpu == 'x86'
+
+    def current_is_x86(self):
+        """
+        Return True if current CPU is x86 32 bits..
+
+        Return
+        ------
+        bool
+            CPU is x86 32 bits
+        """
+        return self.current_cpu == 'x86'
+
+    def current_dir(self, hidex86=False, x64=False):
+        """
+        Current platform specific subfolder.
+
+        Parameters
+        ----------
+        hidex86: bool
+            return '' and not '\x86' if architecture is x86.
+        x64: bool
+            return '\x64' and not '\amd64' if architecture is amd64.
+
+        Return
+        ------
+        str
+            subfolder: '\target', or '' (see hidex86 parameter)
+        """
+        return (
+            '' if (self.current_cpu == 'x86' and hidex86) else
+            r'\x64' if (self.current_cpu == 'amd64' and x64) else
+            r'\%s' % self.current_cpu
+        )
+
+    def target_dir(self, hidex86=False, x64=False):
+        r"""
+        Target platform specific subfolder.
+
+        Parameters
+        ----------
+        hidex86: bool
+            return '' and not '\x86' if architecture is x86.
+        x64: bool
+            return '\x64' and not '\amd64' if architecture is amd64.
+
+        Return
+        ------
+        str
+            subfolder: '\current', or '' (see hidex86 parameter)
+        """
+        return (
+            '' if (self.target_cpu == 'x86' and hidex86) else
+            r'\x64' if (self.target_cpu == 'amd64' and x64) else
+            r'\%s' % self.target_cpu
+        )
+
+    def cross_dir(self, forcex86=False):
+        r"""
+        Cross platform specific subfolder.
+
+        Parameters
+        ----------
+        forcex86: bool
+            Use 'x86' as current architecture even if current architecture is
+            not x86.
+
+        Return
+        ------
+        str
+            subfolder: '' if target architecture is current architecture,
+            '\current_target' if not.
+        """
+        current = 'x86' if forcex86 else self.current_cpu
+        return (
+            '' if self.target_cpu == current else
+            self.target_dir().replace('\\', '\\%s_' % current)
+        )
+
+
+class RegistryInfo:
+    """
+    Microsoft Visual Studio related registry information.
+
+    Parameters
+    ----------
+    platform_info: PlatformInfo
+        "PlatformInfo" instance.
+    """
+    HKEYS = (winreg.HKEY_USERS,
+             winreg.HKEY_CURRENT_USER,
+             winreg.HKEY_LOCAL_MACHINE,
+             winreg.HKEY_CLASSES_ROOT)
+
+    def __init__(self, platform_info):
+        self.pi = platform_info
+
+    @property
+    def visualstudio(self):
+        """
+        Microsoft Visual Studio root registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return 'VisualStudio'
+
+    @property
+    def sxs(self):
+        """
+        Microsoft Visual Studio SxS registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return join(self.visualstudio, 'SxS')
+
+    @property
+    def vc(self):
+        """
+        Microsoft Visual C++ VC7 registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return join(self.sxs, 'VC7')
+
+    @property
+    def vs(self):
+        """
+        Microsoft Visual Studio VS7 registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return join(self.sxs, 'VS7')
+
+    @property
+    def vc_for_python(self):
+        """
+        Microsoft Visual C++ for Python registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return r'DevDiv\VCForPython'
+
+    @property
+    def microsoft_sdk(self):
+        """
+        Microsoft SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return 'Microsoft SDKs'
+
+    @property
+    def windows_sdk(self):
+        """
+        Microsoft Windows/Platform SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return join(self.microsoft_sdk, 'Windows')
+
+    @property
+    def netfx_sdk(self):
+        """
+        Microsoft .NET Framework SDK registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return join(self.microsoft_sdk, 'NETFXSDK')
+
+    @property
+    def windows_kits_roots(self):
+        """
+        Microsoft Windows Kits Roots registry key.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        return r'Windows Kits\Installed Roots'
+
+    def microsoft(self, key, x86=False):
+        """
+        Return key in Microsoft software registry.
+
+        Parameters
+        ----------
+        key: str
+            Registry key path where look.
+        x86: str
+            Force x86 software registry.
+
+        Return
+        ------
+        str
+            Registry key
+        """
+        node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node'
+        return join('Software', node64, 'Microsoft', key)
+
+    def lookup(self, key, name):
+        """
+        Look for values in registry in Microsoft software registry.
+
+        Parameters
+        ----------
+        key: str
+            Registry key path where look.
+        name: str
+            Value name to find.
+
+        Return
+        ------
+        str
+            value
+        """
+        key_read = winreg.KEY_READ
+        openkey = winreg.OpenKey
+        ms = self.microsoft
+        for hkey in self.HKEYS:
+            try:
+                bkey = openkey(hkey, ms(key), 0, key_read)
+            except (OSError, IOError):
+                if not self.pi.current_is_x86():
+                    try:
+                        bkey = openkey(hkey, ms(key, True), 0, key_read)
+                    except (OSError, IOError):
+                        continue
+                else:
+                    continue
+            try:
+                return winreg.QueryValueEx(bkey, name)[0]
+            except (OSError, IOError):
+                pass
+
+
+class SystemInfo:
+    """
+    Microsoft Windows and Visual Studio related system information.
+
+    Parameters
+    ----------
+    registry_info: RegistryInfo
+        "RegistryInfo" instance.
+    vc_ver: float
+        Required Microsoft Visual C++ version.
+    """
+
+    # Variables and properties in this class use originals CamelCase variables
+    # names from Microsoft source files for more easy comparison.
+    WinDir = environ.get('WinDir', '')
+    ProgramFiles = environ.get('ProgramFiles', '')
+    ProgramFilesx86 = environ.get('ProgramFiles(x86)', ProgramFiles)
+
+    def __init__(self, registry_info, vc_ver=None):
+        self.ri = registry_info
+        self.pi = self.ri.pi
+
+        self.known_vs_paths = self.find_programdata_vs_vers()
+
+        # Except for VS15+, VC version is aligned with VS version
+        self.vs_ver = self.vc_ver = (
+                vc_ver or self._find_latest_available_vs_ver())
+
+    def _find_latest_available_vs_ver(self):
+        """
+        Find the latest VC version
+
+        Return
+        ------
+        float
+            version
+        """
+        reg_vc_vers = self.find_reg_vs_vers()
+
+        if not (reg_vc_vers or self.known_vs_paths):
+            raise distutils.errors.DistutilsPlatformError(
+                'No Microsoft Visual C++ version found')
+
+        vc_vers = set(reg_vc_vers)
+        vc_vers.update(self.known_vs_paths)
+        return sorted(vc_vers)[-1]
+
+    def find_reg_vs_vers(self):
+        """
+        Find Microsoft Visual Studio versions available in registry.
+
+        Return
+        ------
+        list of float
+            Versions
+        """
+        ms = self.ri.microsoft
+        vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs)
+        vs_vers = []
+        for hkey in self.ri.HKEYS:
+            for key in vckeys:
+                try:
+                    bkey = winreg.OpenKey(hkey, ms(key), 0, winreg.KEY_READ)
+                except (OSError, IOError):
+                    continue
+                subkeys, values, _ = winreg.QueryInfoKey(bkey)
+                for i in range(values):
+                    try:
+                        ver = float(winreg.EnumValue(bkey, i)[0])
+                        if ver not in vs_vers:
+                            vs_vers.append(ver)
+                    except ValueError:
+                        pass
+                for i in range(subkeys):
+                    try:
+                        ver = float(winreg.EnumKey(bkey, i))
+                        if ver not in vs_vers:
+                            vs_vers.append(ver)
+                    except ValueError:
+                        pass
+        return sorted(vs_vers)
+
+    def find_programdata_vs_vers(self):
+        r"""
+        Find Visual studio 2017+ versions from information in
+        "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances".
+
+        Return
+        ------
+        dict
+            float version as key, path as value.
+        """
+        vs_versions = {}
+        instances_dir = \
+            r'C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances'
+
+        try:
+            hashed_names = listdir(instances_dir)
+
+        except (OSError, IOError):
+            # Directory not exists with all Visual Studio versions
+            return vs_versions
+
+        for name in hashed_names:
+            try:
+                # Get VS installation path from "state.json" file
+                state_path = join(instances_dir, name, 'state.json')
+                with open(state_path, 'rt', encoding='utf-8') as state_file:
+                    state = json.load(state_file)
+                vs_path = state['installationPath']
+
+                # Raises OSError if this VS installation does not contain VC
+                listdir(join(vs_path, r'VC\Tools\MSVC'))
+
+                # Store version and path
+                vs_versions[self._as_float_version(
+                    state['installationVersion'])] = vs_path
+
+            except (OSError, IOError, KeyError):
+                # Skip if "state.json" file is missing or bad format
+                continue
+
+        return vs_versions
+
+    @staticmethod
+    def _as_float_version(version):
+        """
+        Return a string version as a simplified float version (major.minor)
+
+        Parameters
+        ----------
+        version: str
+            Version.
+
+        Return
+        ------
+        float
+            version
+        """
+        return float('.'.join(version.split('.')[:2]))
+
+    @property
+    def VSInstallDir(self):
+        """
+        Microsoft Visual Studio directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        # Default path
+        default = join(self.ProgramFilesx86,
+                       'Microsoft Visual Studio %0.1f' % self.vs_ver)
+
+        # Try to get path from registry, if fail use default path
+        return self.ri.lookup(self.ri.vs, '%0.1f' % self.vs_ver) or default
+
+    @property
+    def VCInstallDir(self):
+        """
+        Microsoft Visual C++ directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        path = self._guess_vc() or self._guess_vc_legacy()
+
+        if not isdir(path):
+            msg = 'Microsoft Visual C++ directory not found'
+            raise distutils.errors.DistutilsPlatformError(msg)
+
+        return path
+
+    def _guess_vc(self):
+        """
+        Locate Visual C++ for VS2017+.
+
+        Return
+        ------
+        str
+            path
+        """
+        if self.vs_ver <= 14.0:
+            return ''
+
+        try:
+            # First search in known VS paths
+            vs_dir = self.known_vs_paths[self.vs_ver]
+        except KeyError:
+            # Else, search with path from registry
+            vs_dir = self.VSInstallDir
+
+        guess_vc = join(vs_dir, r'VC\Tools\MSVC')
+
+        # Subdir with VC exact version as name
+        try:
+            # Update the VC version with real one instead of VS version
+            vc_ver = listdir(guess_vc)[-1]
+            self.vc_ver = self._as_float_version(vc_ver)
+            return join(guess_vc, vc_ver)
+        except (OSError, IOError, IndexError):
+            return ''
+
+    def _guess_vc_legacy(self):
+        """
+        Locate Visual C++ for versions prior to 2017.
+
+        Return
+        ------
+        str
+            path
+        """
+        default = join(self.ProgramFilesx86,
+                       r'Microsoft Visual Studio %0.1f\VC' % self.vs_ver)
+
+        # Try to get "VC++ for Python" path from registry as default path
+        reg_path = join(self.ri.vc_for_python, '%0.1f' % self.vs_ver)
+        python_vc = self.ri.lookup(reg_path, 'installdir')
+        default_vc = join(python_vc, 'VC') if python_vc else default
+
+        # Try to get path from registry, if fail use default path
+        return self.ri.lookup(self.ri.vc, '%0.1f' % self.vs_ver) or default_vc
+
+    @property
+    def WindowsSdkVersion(self):
+        """
+        Microsoft Windows SDK versions for specified MSVC++ version.
+
+        Return
+        ------
+        tuple of str
+            versions
+        """
+        if self.vs_ver <= 9.0:
+            return '7.0', '6.1', '6.0a'
+        elif self.vs_ver == 10.0:
+            return '7.1', '7.0a'
+        elif self.vs_ver == 11.0:
+            return '8.0', '8.0a'
+        elif self.vs_ver == 12.0:
+            return '8.1', '8.1a'
+        elif self.vs_ver >= 14.0:
+            return '10.0', '8.1'
+
+    @property
+    def WindowsSdkLastVersion(self):
+        """
+        Microsoft Windows SDK last version.
+
+        Return
+        ------
+        str
+            version
+        """
+        return self._use_last_dir_name(join(self.WindowsSdkDir, 'lib'))
+
+    @property
+    def WindowsSdkDir(self):
+        """
+        Microsoft Windows SDK directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        sdkdir = ''
+        for ver in self.WindowsSdkVersion:
+            # Try to get it from registry
+            loc = join(self.ri.windows_sdk, 'v%s' % ver)
+            sdkdir = self.ri.lookup(loc, 'installationfolder')
+            if sdkdir:
+                break
+        if not sdkdir or not isdir(sdkdir):
+            # Try to get "VC++ for Python" version from registry
+            path = join(self.ri.vc_for_python, '%0.1f' % self.vc_ver)
+            install_base = self.ri.lookup(path, 'installdir')
+            if install_base:
+                sdkdir = join(install_base, 'WinSDK')
+        if not sdkdir or not isdir(sdkdir):
+            # If fail, use default new path
+            for ver in self.WindowsSdkVersion:
+                intver = ver[:ver.rfind('.')]
+                path = r'Microsoft SDKs\Windows Kits\%s' % intver
+                d = join(self.ProgramFiles, path)
+                if isdir(d):
+                    sdkdir = d
+        if not sdkdir or not isdir(sdkdir):
+            # If fail, use default old path
+            for ver in self.WindowsSdkVersion:
+                path = r'Microsoft SDKs\Windows\v%s' % ver
+                d = join(self.ProgramFiles, path)
+                if isdir(d):
+                    sdkdir = d
+        if not sdkdir:
+            # If fail, use Platform SDK
+            sdkdir = join(self.VCInstallDir, 'PlatformSDK')
+        return sdkdir
+
+    @property
+    def WindowsSDKExecutablePath(self):
+        """
+        Microsoft Windows SDK executable directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        # Find WinSDK NetFx Tools registry dir name
+        if self.vs_ver <= 11.0:
+            netfxver = 35
+            arch = ''
+        else:
+            netfxver = 40
+            hidex86 = True if self.vs_ver <= 12.0 else False
+            arch = self.pi.current_dir(x64=True, hidex86=hidex86)
+        fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-'))
+
+        # list all possibles registry paths
+        regpaths = []
+        if self.vs_ver >= 14.0:
+            for ver in self.NetFxSdkVersion:
+                regpaths += [join(self.ri.netfx_sdk, ver, fx)]
+
+        for ver in self.WindowsSdkVersion:
+            regpaths += [join(self.ri.windows_sdk, 'v%sA' % ver, fx)]
+
+        # Return installation folder from the more recent path
+        for path in regpaths:
+            execpath = self.ri.lookup(path, 'installationfolder')
+            if execpath:
+                return execpath
+
+    @property
+    def FSharpInstallDir(self):
+        """
+        Microsoft Visual F# directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        path = join(self.ri.visualstudio, r'%0.1f\Setup\F#' % self.vs_ver)
+        return self.ri.lookup(path, 'productdir') or ''
+
+    @property
+    def UniversalCRTSdkDir(self):
+        """
+        Microsoft Universal CRT SDK directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        # Set Kit Roots versions for specified MSVC++ version
+        vers = ('10', '81') if self.vs_ver >= 14.0 else ()
+
+        # Find path of the more recent Kit
+        for ver in vers:
+            sdkdir = self.ri.lookup(self.ri.windows_kits_roots,
+                                    'kitsroot%s' % ver)
+            if sdkdir:
+                return sdkdir or ''
+
+    @property
+    def UniversalCRTSdkLastVersion(self):
+        """
+        Microsoft Universal C Runtime SDK last version.
+
+        Return
+        ------
+        str
+            version
+        """
+        return self._use_last_dir_name(join(self.UniversalCRTSdkDir, 'lib'))
+
+    @property
+    def NetFxSdkVersion(self):
+        """
+        Microsoft .NET Framework SDK versions.
+
+        Return
+        ------
+        tuple of str
+            versions
+        """
+        # Set FxSdk versions for specified VS version
+        return (('4.7.2', '4.7.1', '4.7',
+                 '4.6.2', '4.6.1', '4.6',
+                 '4.5.2', '4.5.1', '4.5')
+                if self.vs_ver >= 14.0 else ())
+
+    @property
+    def NetFxSdkDir(self):
+        """
+        Microsoft .NET Framework SDK directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        sdkdir = ''
+        for ver in self.NetFxSdkVersion:
+            loc = join(self.ri.netfx_sdk, ver)
+            sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder')
+            if sdkdir:
+                break
+        return sdkdir
+
+    @property
+    def FrameworkDir32(self):
+        """
+        Microsoft .NET Framework 32bit directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        # Default path
+        guess_fw = join(self.WinDir, r'Microsoft.NET\Framework')
+
+        # Try to get path from registry, if fail use default path
+        return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw
+
+    @property
+    def FrameworkDir64(self):
+        """
+        Microsoft .NET Framework 64bit directory.
+
+        Return
+        ------
+        str
+            path
+        """
+        # Default path
+        guess_fw = join(self.WinDir, r'Microsoft.NET\Framework64')
+
+        # Try to get path from registry, if fail use default path
+        return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw
+
+    @property
+    def FrameworkVersion32(self):
+        """
+        Microsoft .NET Framework 32bit versions.
+
+        Return
+        ------
+        tuple of str
+            versions
+        """
+        return self._find_dot_net_versions(32)
+
+    @property
+    def FrameworkVersion64(self):
+        """
+        Microsoft .NET Framework 64bit versions.
+
+        Return
+        ------
+        tuple of str
+            versions
+        """
+        return self._find_dot_net_versions(64)
+
+    def _find_dot_net_versions(self, bits):
+        """
+        Find Microsoft .NET Framework versions.
+
+        Parameters
+        ----------
+        bits: int
+            Platform number of bits: 32 or 64.
+
+        Return
+        ------
+        tuple of str
+            versions
+        """
+        # Find actual .NET version in registry
+        reg_ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits)
+        dot_net_dir = getattr(self, 'FrameworkDir%d' % bits)
+        ver = reg_ver or self._use_last_dir_name(dot_net_dir, 'v') or ''
+
+        # Set .NET versions for specified MSVC++ version
+        if self.vs_ver >= 12.0:
+            return ver, 'v4.0'
+        elif self.vs_ver >= 10.0:
+            return 'v4.0.30319' if ver.lower()[:2] != 'v4' else ver, 'v3.5'
+        elif self.vs_ver == 9.0:
+            return 'v3.5', 'v2.0.50727'
+        elif self.vs_ver == 8.0:
+            return 'v3.0', 'v2.0.50727'
+
+    @staticmethod
+    def _use_last_dir_name(path, prefix=''):
+        """
+        Return name of the last dir in path or '' if no dir found.
+
+        Parameters
+        ----------
+        path: str
+            Use dirs in this path
+        prefix: str
+            Use only dirs starting by this prefix
+
+        Return
+        ------
+        str
+            name
+        """
+        matching_dirs = (
+            dir_name
+            for dir_name in reversed(listdir(path))
+            if isdir(join(path, dir_name)) and
+            dir_name.startswith(prefix)
+        )
+        return next(matching_dirs, None) or ''
+
+
+class EnvironmentInfo:
+    """
+    Return environment variables for specified Microsoft Visual C++ version
+    and platform : Lib, Include, Path and libpath.
+
+    This function is compatible with Microsoft Visual C++ 9.0 to 14.X.
+
+    Script created by analysing Microsoft environment configuration files like
+    "vcvars[...].bat", "SetEnv.Cmd", "vcbuildtools.bat", ...
+
+    Parameters
+    ----------
+    arch: str
+        Target architecture.
+    vc_ver: float
+        Required Microsoft Visual C++ version. If not set, autodetect the last
+        version.
+    vc_min_ver: float
+        Minimum Microsoft Visual C++ version.
+    """
+
+    # Variables and properties in this class use originals CamelCase variables
+    # names from Microsoft source files for more easy comparison.
+
+    def __init__(self, arch, vc_ver=None, vc_min_ver=0):
+        self.pi = PlatformInfo(arch)
+        self.ri = RegistryInfo(self.pi)
+        self.si = SystemInfo(self.ri, vc_ver)
+
+        if self.vc_ver < vc_min_ver:
+            err = 'No suitable Microsoft Visual C++ version found'
+            raise distutils.errors.DistutilsPlatformError(err)
+
+    @property
+    def vs_ver(self):
+        """
+        Microsoft Visual Studio.
+
+        Return
+        ------
+        float
+            version
+        """
+        return self.si.vs_ver
+
+    @property
+    def vc_ver(self):
+        """
+        Microsoft Visual C++ version.
+
+        Return
+        ------
+        float
+            version
+        """
+        return self.si.vc_ver
+
+    @property
+    def VSTools(self):
+        """
+        Microsoft Visual Studio Tools.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        paths = [r'Common7\IDE', r'Common7\Tools']
+
+        if self.vs_ver >= 14.0:
+            arch_subdir = self.pi.current_dir(hidex86=True, x64=True)
+            paths += [r'Common7\IDE\CommonExtensions\Microsoft\TestWindow']
+            paths += [r'Team Tools\Performance Tools']
+            paths += [r'Team Tools\Performance Tools%s' % arch_subdir]
+
+        return [join(self.si.VSInstallDir, path) for path in paths]
+
+    @property
+    def VCIncludes(self):
+        """
+        Microsoft Visual C++ & Microsoft Foundation Class Includes.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        return [join(self.si.VCInstallDir, 'Include'),
+                join(self.si.VCInstallDir, r'ATLMFC\Include')]
+
+    @property
+    def VCLibraries(self):
+        """
+        Microsoft Visual C++ & Microsoft Foundation Class Libraries.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver >= 15.0:
+            arch_subdir = self.pi.target_dir(x64=True)
+        else:
+            arch_subdir = self.pi.target_dir(hidex86=True)
+        paths = ['Lib%s' % arch_subdir, r'ATLMFC\Lib%s' % arch_subdir]
+
+        if self.vs_ver >= 14.0:
+            paths += [r'Lib\store%s' % arch_subdir]
+
+        return [join(self.si.VCInstallDir, path) for path in paths]
+
+    @property
+    def VCStoreRefs(self):
+        """
+        Microsoft Visual C++ store references Libraries.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 14.0:
+            return []
+        return [join(self.si.VCInstallDir, r'Lib\store\references')]
+
+    @property
+    def VCTools(self):
+        """
+        Microsoft Visual C++ Tools.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        si = self.si
+        tools = [join(si.VCInstallDir, 'VCPackages')]
+
+        forcex86 = True if self.vs_ver <= 10.0 else False
+        arch_subdir = self.pi.cross_dir(forcex86)
+        if arch_subdir:
+            tools += [join(si.VCInstallDir, 'Bin%s' % arch_subdir)]
+
+        if self.vs_ver == 14.0:
+            path = 'Bin%s' % self.pi.current_dir(hidex86=True)
+            tools += [join(si.VCInstallDir, path)]
+
+        elif self.vs_ver >= 15.0:
+            host_dir = (r'bin\HostX86%s' if self.pi.current_is_x86() else
+                        r'bin\HostX64%s')
+            tools += [join(
+                si.VCInstallDir, host_dir % self.pi.target_dir(x64=True))]
+
+            if self.pi.current_cpu != self.pi.target_cpu:
+                tools += [join(
+                    si.VCInstallDir, host_dir % self.pi.current_dir(x64=True))]
+
+        else:
+            tools += [join(si.VCInstallDir, 'Bin')]
+
+        return tools
+
+    @property
+    def OSLibraries(self):
+        """
+        Microsoft Windows SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver <= 10.0:
+            arch_subdir = self.pi.target_dir(hidex86=True, x64=True)
+            return [join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)]
+
+        else:
+            arch_subdir = self.pi.target_dir(x64=True)
+            lib = join(self.si.WindowsSdkDir, 'lib')
+            libver = self._sdk_subdir
+            return [join(lib, '%sum%s' % (libver , arch_subdir))]
+
+    @property
+    def OSIncludes(self):
+        """
+        Microsoft Windows SDK Include.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        include = join(self.si.WindowsSdkDir, 'include')
+
+        if self.vs_ver <= 10.0:
+            return [include, join(include, 'gl')]
+
+        else:
+            if self.vs_ver >= 14.0:
+                sdkver = self._sdk_subdir
+            else:
+                sdkver = ''
+            return [join(include, '%sshared' % sdkver),
+                    join(include, '%sum' % sdkver),
+                    join(include, '%swinrt' % sdkver)]
+
+    @property
+    def OSLibpath(self):
+        """
+        Microsoft Windows SDK Libraries Paths.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        ref = join(self.si.WindowsSdkDir, 'References')
+        libpath = []
+
+        if self.vs_ver <= 9.0:
+            libpath += self.OSLibraries
+
+        if self.vs_ver >= 11.0:
+            libpath += [join(ref, r'CommonConfiguration\Neutral')]
+
+        if self.vs_ver >= 14.0:
+            libpath += [
+                ref,
+                join(self.si.WindowsSdkDir, 'UnionMetadata'),
+                join(ref, 'Windows.Foundation.UniversalApiContract', '1.0.0.0'),
+                join(ref, 'Windows.Foundation.FoundationContract', '1.0.0.0'),
+                join(ref,'Windows.Networking.Connectivity.WwanContract',
+                     '1.0.0.0'),
+                join(self.si.WindowsSdkDir, 'ExtensionSDKs', 'Microsoft.VCLibs',
+                     '%0.1f' % self.vs_ver, 'References', 'CommonConfiguration',
+                     'neutral'),
+            ]
+        return libpath
+
+    @property
+    def SdkTools(self):
+        """
+        Microsoft Windows SDK Tools.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        return list(self._sdk_tools())
+
+    def _sdk_tools(self):
+        """
+        Microsoft Windows SDK Tools paths generator.
+
+        Return
+        ------
+        generator of str
+            paths
+        """
+        if self.vs_ver < 15.0:
+            bin_dir = 'Bin' if self.vs_ver <= 11.0 else r'Bin\x86'
+            yield join(self.si.WindowsSdkDir, bin_dir)
+
+        if not self.pi.current_is_x86():
+            arch_subdir = self.pi.current_dir(x64=True)
+            path = 'Bin%s' % arch_subdir
+            yield join(self.si.WindowsSdkDir, path)
+
+        if self.vs_ver in (10.0, 11.0):
+            if self.pi.target_is_x86():
+                arch_subdir = ''
+            else:
+                arch_subdir = self.pi.current_dir(hidex86=True, x64=True)
+            path = r'Bin\NETFX 4.0 Tools%s' % arch_subdir
+            yield join(self.si.WindowsSdkDir, path)
+
+        elif self.vs_ver >= 15.0:
+            path = join(self.si.WindowsSdkDir, 'Bin')
+            arch_subdir = self.pi.current_dir(x64=True)
+            sdkver = self.si.WindowsSdkLastVersion
+            yield join(path, '%s%s' % (sdkver, arch_subdir))
+
+        if self.si.WindowsSDKExecutablePath:
+            yield self.si.WindowsSDKExecutablePath
+
+    @property
+    def _sdk_subdir(self):
+        """
+        Microsoft Windows SDK version subdir.
+
+        Return
+        ------
+        str
+            subdir
+        """
+        ucrtver = self.si.WindowsSdkLastVersion
+        return ('%s\\' % ucrtver) if ucrtver else ''
+
+    @property
+    def SdkSetup(self):
+        """
+        Microsoft Windows SDK Setup.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver > 9.0:
+            return []
+
+        return [join(self.si.WindowsSdkDir, 'Setup')]
+
+    @property
+    def FxTools(self):
+        """
+        Microsoft .NET Framework Tools.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        pi = self.pi
+        si = self.si
+
+        if self.vs_ver <= 10.0:
+            include32 = True
+            include64 = not pi.target_is_x86() and not pi.current_is_x86()
+        else:
+            include32 = pi.target_is_x86() or pi.current_is_x86()
+            include64 = pi.current_cpu == 'amd64' or pi.target_cpu == 'amd64'
+
+        tools = []
+        if include32:
+            tools += [join(si.FrameworkDir32, ver)
+                      for ver in si.FrameworkVersion32]
+        if include64:
+            tools += [join(si.FrameworkDir64, ver)
+                      for ver in si.FrameworkVersion64]
+        return tools
+
+    @property
+    def NetFxSDKLibraries(self):
+        """
+        Microsoft .Net Framework SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 14.0 or not self.si.NetFxSdkDir:
+            return []
+
+        arch_subdir = self.pi.target_dir(x64=True)
+        return [join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)]
+
+    @property
+    def NetFxSDKIncludes(self):
+        """
+        Microsoft .Net Framework SDK Includes.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 14.0 or not self.si.NetFxSdkDir:
+            return []
+
+        return [join(self.si.NetFxSdkDir, r'include\um')]
+
+    @property
+    def VsTDb(self):
+        """
+        Microsoft Visual Studio Team System Database.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        return [join(self.si.VSInstallDir, r'VSTSDB\Deploy')]
+
+    @property
+    def MSBuild(self):
+        """
+        Microsoft Build Engine.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 12.0:
+            return []
+        elif self.vs_ver < 15.0:
+            base_path = self.si.ProgramFilesx86
+            arch_subdir = self.pi.current_dir(hidex86=True)
+        else:
+            base_path = self.si.VSInstallDir
+            arch_subdir = ''
+
+        path = r'MSBuild\%0.1f\bin%s' % (self.vs_ver, arch_subdir)
+        build = [join(base_path, path)]
+
+        if self.vs_ver >= 15.0:
+            # Add Roslyn C# & Visual Basic Compiler
+            build += [join(base_path, path, 'Roslyn')]
+
+        return build
+
+    @property
+    def HTMLHelpWorkshop(self):
+        """
+        Microsoft HTML Help Workshop.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 11.0:
+            return []
+
+        return [join(self.si.ProgramFilesx86, 'HTML Help Workshop')]
+
+    @property
+    def UCRTLibraries(self):
+        """
+        Microsoft Universal C Runtime SDK Libraries.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 14.0:
+            return []
+
+        arch_subdir = self.pi.target_dir(x64=True)
+        lib = join(self.si.UniversalCRTSdkDir, 'lib')
+        ucrtver = self._ucrt_subdir
+        return [join(lib, '%sucrt%s' % (ucrtver, arch_subdir))]
+
+    @property
+    def UCRTIncludes(self):
+        """
+        Microsoft Universal C Runtime SDK Include.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if self.vs_ver < 14.0:
+            return []
+
+        include = join(self.si.UniversalCRTSdkDir, 'include')
+        return [join(include, '%sucrt' % self._ucrt_subdir)]
+
+    @property
+    def _ucrt_subdir(self):
+        """
+        Microsoft Universal C Runtime SDK version subdir.
+
+        Return
+        ------
+        str
+            subdir
+        """
+        ucrtver = self.si.UniversalCRTSdkLastVersion
+        return ('%s\\' % ucrtver) if ucrtver else ''
+
+    @property
+    def FSharp(self):
+        """
+        Microsoft Visual F#.
+
+        Return
+        ------
+        list of str
+            paths
+        """
+        if 11.0 > self.vs_ver > 12.0:
+            return []
+
+        return [self.si.FSharpInstallDir]
+
+    @property
+    def VCRuntimeRedist(self):
+        """
+        Microsoft Visual C++ runtime redistributable dll.
+
+        Return
+        ------
+        str
+            path
+        """
+        vcruntime = 'vcruntime%d0.dll' % self.vc_ver
+        arch_subdir = self.pi.target_dir(x64=True).strip('\\')
+
+        # Installation prefixes candidates
+        prefixes = []
+        tools_path = self.si.VCInstallDir
+        redist_path = dirname(tools_path.replace(r'\Tools', r'\Redist'))
+        if isdir(redist_path):
+            # Redist version may not be exactly the same as tools
+            redist_path = join(redist_path, listdir(redist_path)[-1])
+            prefixes += [redist_path, join(redist_path, 'onecore')]
+
+        prefixes += [join(tools_path, 'redist')]  # VS14 legacy path
+
+        # CRT directory
+        crt_dirs = ('Microsoft.VC%d.CRT' % (self.vc_ver * 10),
+                    # Sometime store in directory with VS version instead of VC
+                    'Microsoft.VC%d.CRT' % (int(self.vs_ver) * 10))
+
+        # vcruntime path
+        for prefix, crt_dir in itertools.product(prefixes, crt_dirs):
+            path = join(prefix, arch_subdir, crt_dir, vcruntime)
+            if isfile(path):
+                return path
+
+    def return_env(self, exists=True):
+        """
+        Return environment dict.
+
+        Parameters
+        ----------
+        exists: bool
+            It True, only return existing paths.
+
+        Return
+        ------
+        dict
+            environment
+        """
+        env = dict(
+            include=self._build_paths('include',
+                                      [self.VCIncludes,
+                                       self.OSIncludes,
+                                       self.UCRTIncludes,
+                                       self.NetFxSDKIncludes],
+                                      exists),
+            lib=self._build_paths('lib',
+                                  [self.VCLibraries,
+                                   self.OSLibraries,
+                                   self.FxTools,
+                                   self.UCRTLibraries,
+                                   self.NetFxSDKLibraries],
+                                  exists),
+            libpath=self._build_paths('libpath',
+                                      [self.VCLibraries,
+                                       self.FxTools,
+                                       self.VCStoreRefs,
+                                       self.OSLibpath],
+                                      exists),
+            path=self._build_paths('path',
+                                   [self.VCTools,
+                                    self.VSTools,
+                                    self.VsTDb,
+                                    self.SdkTools,
+                                    self.SdkSetup,
+                                    self.FxTools,
+                                    self.MSBuild,
+                                    self.HTMLHelpWorkshop,
+                                    self.FSharp],
+                                   exists),
+        )
+        if self.vs_ver >= 14 and isfile(self.VCRuntimeRedist):
+            env['py_vcruntime_redist'] = self.VCRuntimeRedist
+        return env
+
+    def _build_paths(self, name, spec_path_lists, exists):
+        """
+        Given an environment variable name and specified paths,
+        return a pathsep-separated string of paths containing
+        unique, extant, directories from those paths and from
+        the environment variable. Raise an error if no paths
+        are resolved.
+
+        Parameters
+        ----------
+        name: str
+            Environment variable name
+        spec_path_lists: list of str
+            Paths
+        exists: bool
+            It True, only return existing paths.
+
+        Return
+        ------
+        str
+            Pathsep-separated paths
+        """
+        # flatten spec_path_lists
+        spec_paths = itertools.chain.from_iterable(spec_path_lists)
+        env_paths = environ.get(name, '').split(pathsep)
+        paths = itertools.chain(spec_paths, env_paths)
+        extant_paths = list(filter(isdir, paths)) if exists else paths
+        if not extant_paths:
+            msg = "%s environment variable is empty" % name.upper()
+            raise distutils.errors.DistutilsPlatformError(msg)
+        unique_paths = self._unique_everseen(extant_paths)
+        return pathsep.join(unique_paths)
+
+    # from Python docs
+    @staticmethod
+    def _unique_everseen(iterable, key=None):
+        """
+        List unique elements, preserving order.
+        Remember all elements ever seen.
+
+        _unique_everseen('AAAABBBCCDAABBB') --> A B C D
+
+        _unique_everseen('ABBCcAD', str.lower) --> A B C D
+        """
+        seen = set()
+        seen_add = seen.add
+        if key is None:
+            for element in filterfalse(seen.__contains__, iterable):
+                seen_add(element)
+                yield element
+        else:
+            for element in iterable:
+                k = key(element)
+                if k not in seen:
+                    seen_add(k)
+                    yield element
diff --git a/ubuntu/venv/setuptools/namespaces.py b/ubuntu/venv/setuptools/namespaces.py
new file mode 100644
index 0000000..dc16106
--- /dev/null
+++ b/ubuntu/venv/setuptools/namespaces.py
@@ -0,0 +1,107 @@
+import os
+from distutils import log
+import itertools
+
+from setuptools.extern.six.moves import map
+
+
+flatten = itertools.chain.from_iterable
+
+
+class Installer:
+
+    nspkg_ext = '-nspkg.pth'
+
+    def install_namespaces(self):
+        nsp = self._get_all_ns_packages()
+        if not nsp:
+            return
+        filename, ext = os.path.splitext(self._get_target())
+        filename += self.nspkg_ext
+        self.outputs.append(filename)
+        log.info("Installing %s", filename)
+        lines = map(self._gen_nspkg_line, nsp)
+
+        if self.dry_run:
+            # always generate the lines, even in dry run
+            list(lines)
+            return
+
+        with open(filename, 'wt') as f:
+            f.writelines(lines)
+
+    def uninstall_namespaces(self):
+        filename, ext = os.path.splitext(self._get_target())
+        filename += self.nspkg_ext
+        if not os.path.exists(filename):
+            return
+        log.info("Removing %s", filename)
+        os.remove(filename)
+
+    def _get_target(self):
+        return self.target
+
+    _nspkg_tmpl = (
+        "import sys, types, os",
+        "has_mfs = sys.version_info > (3, 5)",
+        "p = os.path.join(%(root)s, *%(pth)r)",
+        "importlib = has_mfs and __import__('importlib.util')",
+        "has_mfs and __import__('importlib.machinery')",
+        "m = has_mfs and "
+            "sys.modules.setdefault(%(pkg)r, "
+                "importlib.util.module_from_spec("
+                    "importlib.machinery.PathFinder.find_spec(%(pkg)r, "
+                        "[os.path.dirname(p)])))",
+        "m = m or "
+            "sys.modules.setdefault(%(pkg)r, types.ModuleType(%(pkg)r))",
+        "mp = (m or []) and m.__dict__.setdefault('__path__',[])",
+        "(p not in mp) and mp.append(p)",
+    )
+    "lines for the namespace installer"
+
+    _nspkg_tmpl_multi = (
+        'm and setattr(sys.modules[%(parent)r], %(child)r, m)',
+    )
+    "additional line(s) when a parent package is indicated"
+
+    def _get_root(self):
+        return "sys._getframe(1).f_locals['sitedir']"
+
+    def _gen_nspkg_line(self, pkg):
+        # ensure pkg is not a unicode string under Python 2.7
+        pkg = str(pkg)
+        pth = tuple(pkg.split('.'))
+        root = self._get_root()
+        tmpl_lines = self._nspkg_tmpl
+        parent, sep, child = pkg.rpartition('.')
+        if parent:
+            tmpl_lines += self._nspkg_tmpl_multi
+        return ';'.join(tmpl_lines) % locals() + '\n'
+
+    def _get_all_ns_packages(self):
+        """Return sorted list of all package namespaces"""
+        pkgs = self.distribution.namespace_packages or []
+        return sorted(flatten(map(self._pkg_names, pkgs)))
+
+    @staticmethod
+    def _pkg_names(pkg):
+        """
+        Given a namespace package, yield the components of that
+        package.
+
+        >>> names = Installer._pkg_names('a.b.c')
+        >>> set(names) == set(['a', 'a.b', 'a.b.c'])
+        True
+        """
+        parts = pkg.split('.')
+        while parts:
+            yield '.'.join(parts)
+            parts.pop()
+
+
+class DevelopInstaller(Installer):
+    def _get_root(self):
+        return repr(str(self.egg_path))
+
+    def _get_target(self):
+        return self.egg_link
diff --git a/ubuntu/venv/setuptools/package_index.py b/ubuntu/venv/setuptools/package_index.py
new file mode 100644
index 0000000..f419d47
--- /dev/null
+++ b/ubuntu/venv/setuptools/package_index.py
@@ -0,0 +1,1136 @@
+"""PyPI and direct package downloading"""
+import sys
+import os
+import re
+import shutil
+import socket
+import base64
+import hashlib
+import itertools
+import warnings
+from functools import wraps
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import urllib, http_client, configparser, map
+
+import setuptools
+from pkg_resources import (
+    CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
+    Environment, find_distributions, safe_name, safe_version,
+    to_filename, Requirement, DEVELOP_DIST, EGG_DIST,
+)
+from setuptools import ssl_support
+from distutils import log
+from distutils.errors import DistutilsError
+from fnmatch import translate
+from setuptools.py27compat import get_all_headers
+from setuptools.py33compat import unescape
+from setuptools.wheel import Wheel
+
+__metaclass__ = type
+
+EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$')
+HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I)
+PYPI_MD5 = re.compile(
+    r'([^<]+)\n\s+\(md5\)'
+)
+URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match
+EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
+
+__all__ = [
+    'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
+    'interpret_distro_name',
+]
+
+_SOCKET_TIMEOUT = 15
+
+_tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}"
+user_agent = _tmpl.format(py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools)
+
+
+def parse_requirement_arg(spec):
+    try:
+        return Requirement.parse(spec)
+    except ValueError:
+        raise DistutilsError(
+            "Not a URL, existing file, or requirement spec: %r" % (spec,)
+        )
+
+
+def parse_bdist_wininst(name):
+    """Return (base,pyversion) or (None,None) for possible .exe name"""
+
+    lower = name.lower()
+    base, py_ver, plat = None, None, None
+
+    if lower.endswith('.exe'):
+        if lower.endswith('.win32.exe'):
+            base = name[:-10]
+            plat = 'win32'
+        elif lower.startswith('.win32-py', -16):
+            py_ver = name[-7:-4]
+            base = name[:-16]
+            plat = 'win32'
+        elif lower.endswith('.win-amd64.exe'):
+            base = name[:-14]
+            plat = 'win-amd64'
+        elif lower.startswith('.win-amd64-py', -20):
+            py_ver = name[-7:-4]
+            base = name[:-20]
+            plat = 'win-amd64'
+    return base, py_ver, plat
+
+
+def egg_info_for_url(url):
+    parts = urllib.parse.urlparse(url)
+    scheme, server, path, parameters, query, fragment = parts
+    base = urllib.parse.unquote(path.split('/')[-1])
+    if server == 'sourceforge.net' and base == 'download':  # XXX Yuck
+        base = urllib.parse.unquote(path.split('/')[-2])
+    if '#' in base:
+        base, fragment = base.split('#', 1)
+    return base, fragment
+
+
+def distros_for_url(url, metadata=None):
+    """Yield egg or source distribution objects that might be found at a URL"""
+    base, fragment = egg_info_for_url(url)
+    for dist in distros_for_location(url, base, metadata):
+        yield dist
+    if fragment:
+        match = EGG_FRAGMENT.match(fragment)
+        if match:
+            for dist in interpret_distro_name(
+                url, match.group(1), metadata, precedence=CHECKOUT_DIST
+            ):
+                yield dist
+
+
+def distros_for_location(location, basename, metadata=None):
+    """Yield egg or source distribution objects based on basename"""
+    if basename.endswith('.egg.zip'):
+        basename = basename[:-4]  # strip the .zip
+    if basename.endswith('.egg') and '-' in basename:
+        # only one, unambiguous interpretation
+        return [Distribution.from_location(location, basename, metadata)]
+    if basename.endswith('.whl') and '-' in basename:
+        wheel = Wheel(basename)
+        if not wheel.is_compatible():
+            return []
+        return [Distribution(
+            location=location,
+            project_name=wheel.project_name,
+            version=wheel.version,
+            # Increase priority over eggs.
+            precedence=EGG_DIST + 1,
+        )]
+    if basename.endswith('.exe'):
+        win_base, py_ver, platform = parse_bdist_wininst(basename)
+        if win_base is not None:
+            return interpret_distro_name(
+                location, win_base, metadata, py_ver, BINARY_DIST, platform
+            )
+    # Try source distro extensions (.zip, .tgz, etc.)
+    #
+    for ext in EXTENSIONS:
+        if basename.endswith(ext):
+            basename = basename[:-len(ext)]
+            return interpret_distro_name(location, basename, metadata)
+    return []  # no extension matched
+
+
+def distros_for_filename(filename, metadata=None):
+    """Yield possible egg or source distribution objects based on a filename"""
+    return distros_for_location(
+        normalize_path(filename), os.path.basename(filename), metadata
+    )
+
+
+def interpret_distro_name(
+        location, basename, metadata, py_version=None, precedence=SOURCE_DIST,
+        platform=None
+):
+    """Generate alternative interpretations of a source distro name
+
+    Note: if `location` is a filesystem filename, you should call
+    ``pkg_resources.normalize_path()`` on it before passing it to this
+    routine!
+    """
+    # Generate alternative interpretations of a source distro name
+    # Because some packages are ambiguous as to name/versions split
+    # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
+    # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
+    # "adns-python, 1.1.0", and "adns-python-1.1.0, no version").  In practice,
+    # the spurious interpretations should be ignored, because in the event
+    # there's also an "adns" package, the spurious "python-1.1.0" version will
+    # compare lower than any numeric version number, and is therefore unlikely
+    # to match a request for it.  It's still a potential problem, though, and
+    # in the long run PyPI and the distutils should go for "safe" names and
+    # versions in distribution archive names (sdist and bdist).
+
+    parts = basename.split('-')
+    if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]):
+        # it is a bdist_dumb, not an sdist -- bail out
+        return
+
+    for p in range(1, len(parts) + 1):
+        yield Distribution(
+            location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
+            py_version=py_version, precedence=precedence,
+            platform=platform
+        )
+
+
+# From Python 2.7 docs
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in six.moves.filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
+
+
+def unique_values(func):
+    """
+    Wrap a function returning an iterable such that the resulting iterable
+    only ever yields unique items.
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return unique_everseen(func(*args, **kwargs))
+
+    return wrapper
+
+
+REL = re.compile(r"""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
+# this line is here to fix emacs' cruddy broken syntax highlighting
+
+
+@unique_values
+def find_external_links(url, page):
+    """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
+
+    for match in REL.finditer(page):
+        tag, rel = match.groups()
+        rels = set(map(str.strip, rel.lower().split(',')))
+        if 'homepage' in rels or 'download' in rels:
+            for match in HREF.finditer(tag):
+                yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
+
+    for tag in ("Home Page", "Download URL"):
+        pos = page.find(tag)
+        if pos != -1:
+            match = HREF.search(page, pos)
+            if match:
+                yield urllib.parse.urljoin(url, htmldecode(match.group(1)))
+
+
+class ContentChecker:
+    """
+    A null content checker that defines the interface for checking content
+    """
+
+    def feed(self, block):
+        """
+        Feed a block of data to the hash.
+        """
+        return
+
+    def is_valid(self):
+        """
+        Check the hash. Return False if validation fails.
+        """
+        return True
+
+    def report(self, reporter, template):
+        """
+        Call reporter with information about the checker (hash name)
+        substituted into the template.
+        """
+        return
+
+
+class HashChecker(ContentChecker):
+    pattern = re.compile(
+        r'(?Psha1|sha224|sha384|sha256|sha512|md5)='
+        r'(?P[a-f0-9]+)'
+    )
+
+    def __init__(self, hash_name, expected):
+        self.hash_name = hash_name
+        self.hash = hashlib.new(hash_name)
+        self.expected = expected
+
+    @classmethod
+    def from_url(cls, url):
+        "Construct a (possibly null) ContentChecker from a URL"
+        fragment = urllib.parse.urlparse(url)[-1]
+        if not fragment:
+            return ContentChecker()
+        match = cls.pattern.search(fragment)
+        if not match:
+            return ContentChecker()
+        return cls(**match.groupdict())
+
+    def feed(self, block):
+        self.hash.update(block)
+
+    def is_valid(self):
+        return self.hash.hexdigest() == self.expected
+
+    def report(self, reporter, template):
+        msg = template % self.hash_name
+        return reporter(msg)
+
+
+class PackageIndex(Environment):
+    """A distribution index that scans web pages for download URLs"""
+
+    def __init__(
+            self, index_url="https://pypi.org/simple/", hosts=('*',),
+            ca_bundle=None, verify_ssl=True, *args, **kw
+    ):
+        Environment.__init__(self, *args, **kw)
+        self.index_url = index_url + "/" [:not index_url.endswith('/')]
+        self.scanned_urls = {}
+        self.fetched_urls = {}
+        self.package_pages = {}
+        self.allows = re.compile('|'.join(map(translate, hosts))).match
+        self.to_scan = []
+        use_ssl = (
+            verify_ssl
+            and ssl_support.is_available
+            and (ca_bundle or ssl_support.find_ca_bundle())
+        )
+        if use_ssl:
+            self.opener = ssl_support.opener_for(ca_bundle)
+        else:
+            self.opener = urllib.request.urlopen
+
+    def process_url(self, url, retrieve=False):
+        """Evaluate a URL as a possible download, and maybe retrieve it"""
+        if url in self.scanned_urls and not retrieve:
+            return
+        self.scanned_urls[url] = True
+        if not URL_SCHEME(url):
+            self.process_filename(url)
+            return
+        else:
+            dists = list(distros_for_url(url))
+            if dists:
+                if not self.url_ok(url):
+                    return
+                self.debug("Found link: %s", url)
+
+        if dists or not retrieve or url in self.fetched_urls:
+            list(map(self.add, dists))
+            return  # don't need the actual page
+
+        if not self.url_ok(url):
+            self.fetched_urls[url] = True
+            return
+
+        self.info("Reading %s", url)
+        self.fetched_urls[url] = True  # prevent multiple fetch attempts
+        tmpl = "Download error on %s: %%s -- Some packages may not be found!"
+        f = self.open_url(url, tmpl % url)
+        if f is None:
+            return
+        self.fetched_urls[f.url] = True
+        if 'html' not in f.headers.get('content-type', '').lower():
+            f.close()  # not html, we can't process it
+            return
+
+        base = f.url  # handle redirects
+        page = f.read()
+        if not isinstance(page, str):
+            # In Python 3 and got bytes but want str.
+            if isinstance(f, urllib.error.HTTPError):
+                # Errors have no charset, assume latin1:
+                charset = 'latin-1'
+            else:
+                charset = f.headers.get_param('charset') or 'latin-1'
+            page = page.decode(charset, "ignore")
+        f.close()
+        for match in HREF.finditer(page):
+            link = urllib.parse.urljoin(base, htmldecode(match.group(1)))
+            self.process_url(link)
+        if url.startswith(self.index_url) and getattr(f, 'code', None) != 404:
+            page = self.process_index(url, page)
+
+    def process_filename(self, fn, nested=False):
+        # process filenames or directories
+        if not os.path.exists(fn):
+            self.warn("Not found: %s", fn)
+            return
+
+        if os.path.isdir(fn) and not nested:
+            path = os.path.realpath(fn)
+            for item in os.listdir(path):
+                self.process_filename(os.path.join(path, item), True)
+
+        dists = distros_for_filename(fn)
+        if dists:
+            self.debug("Found: %s", fn)
+            list(map(self.add, dists))
+
+    def url_ok(self, url, fatal=False):
+        s = URL_SCHEME(url)
+        is_file = s and s.group(1).lower() == 'file'
+        if is_file or self.allows(urllib.parse.urlparse(url)[1]):
+            return True
+        msg = (
+            "\nNote: Bypassing %s (disallowed host; see "
+            "http://bit.ly/2hrImnY for details).\n")
+        if fatal:
+            raise DistutilsError(msg % url)
+        else:
+            self.warn(msg, url)
+
+    def scan_egg_links(self, search_path):
+        dirs = filter(os.path.isdir, search_path)
+        egg_links = (
+            (path, entry)
+            for path in dirs
+            for entry in os.listdir(path)
+            if entry.endswith('.egg-link')
+        )
+        list(itertools.starmap(self.scan_egg_link, egg_links))
+
+    def scan_egg_link(self, path, entry):
+        with open(os.path.join(path, entry)) as raw_lines:
+            # filter non-empty lines
+            lines = list(filter(None, map(str.strip, raw_lines)))
+
+        if len(lines) != 2:
+            # format is not recognized; punt
+            return
+
+        egg_path, setup_path = lines
+
+        for dist in find_distributions(os.path.join(path, egg_path)):
+            dist.location = os.path.join(path, *lines)
+            dist.precedence = SOURCE_DIST
+            self.add(dist)
+
+    def process_index(self, url, page):
+        """Process the contents of a PyPI page"""
+
+        def scan(link):
+            # Process a URL to see if it's for a package page
+            if link.startswith(self.index_url):
+                parts = list(map(
+                    urllib.parse.unquote, link[len(self.index_url):].split('/')
+                ))
+                if len(parts) == 2 and '#' not in parts[1]:
+                    # it's a package page, sanitize and index it
+                    pkg = safe_name(parts[0])
+                    ver = safe_version(parts[1])
+                    self.package_pages.setdefault(pkg.lower(), {})[link] = True
+                    return to_filename(pkg), to_filename(ver)
+            return None, None
+
+        # process an index page into the package-page index
+        for match in HREF.finditer(page):
+            try:
+                scan(urllib.parse.urljoin(url, htmldecode(match.group(1))))
+            except ValueError:
+                pass
+
+        pkg, ver = scan(url)  # ensure this page is in the page index
+        if pkg:
+            # process individual package page
+            for new_url in find_external_links(url, page):
+                # Process the found URL
+                base, frag = egg_info_for_url(new_url)
+                if base.endswith('.py') and not frag:
+                    if ver:
+                        new_url += '#egg=%s-%s' % (pkg, ver)
+                    else:
+                        self.need_version_info(url)
+                self.scan_url(new_url)
+
+            return PYPI_MD5.sub(
+                lambda m: '%s' % m.group(1, 3, 2), page
+            )
+        else:
+            return ""  # no sense double-scanning non-package pages
+
+    def need_version_info(self, url):
+        self.scan_all(
+            "Page at %s links to .py file(s) without version info; an index "
+            "scan is required.", url
+        )
+
+    def scan_all(self, msg=None, *args):
+        if self.index_url not in self.fetched_urls:
+            if msg:
+                self.warn(msg, *args)
+            self.info(
+                "Scanning index of all packages (this may take a while)"
+            )
+        self.scan_url(self.index_url)
+
+    def find_packages(self, requirement):
+        self.scan_url(self.index_url + requirement.unsafe_name + '/')
+
+        if not self.package_pages.get(requirement.key):
+            # Fall back to safe version of the name
+            self.scan_url(self.index_url + requirement.project_name + '/')
+
+        if not self.package_pages.get(requirement.key):
+            # We couldn't find the target package, so search the index page too
+            self.not_found_in_index(requirement)
+
+        for url in list(self.package_pages.get(requirement.key, ())):
+            # scan each page that might be related to the desired package
+            self.scan_url(url)
+
+    def obtain(self, requirement, installer=None):
+        self.prescan()
+        self.find_packages(requirement)
+        for dist in self[requirement.key]:
+            if dist in requirement:
+                return dist
+            self.debug("%s does not match %s", requirement, dist)
+        return super(PackageIndex, self).obtain(requirement, installer)
+
+    def check_hash(self, checker, filename, tfp):
+        """
+        checker is a ContentChecker
+        """
+        checker.report(
+            self.debug,
+            "Validating %%s checksum for %s" % filename)
+        if not checker.is_valid():
+            tfp.close()
+            os.unlink(filename)
+            raise DistutilsError(
+                "%s validation failed for %s; "
+                "possible download problem?"
+                % (checker.hash.name, os.path.basename(filename))
+            )
+
+    def add_find_links(self, urls):
+        """Add `urls` to the list that will be prescanned for searches"""
+        for url in urls:
+            if (
+                self.to_scan is None  # if we have already "gone online"
+                or not URL_SCHEME(url)  # or it's a local file/directory
+                or url.startswith('file:')
+                or list(distros_for_url(url))  # or a direct package link
+            ):
+                # then go ahead and process it now
+                self.scan_url(url)
+            else:
+                # otherwise, defer retrieval till later
+                self.to_scan.append(url)
+
+    def prescan(self):
+        """Scan urls scheduled for prescanning (e.g. --find-links)"""
+        if self.to_scan:
+            list(map(self.scan_url, self.to_scan))
+        self.to_scan = None  # from now on, go ahead and process immediately
+
+    def not_found_in_index(self, requirement):
+        if self[requirement.key]:  # we've seen at least one distro
+            meth, msg = self.info, "Couldn't retrieve index page for %r"
+        else:  # no distros seen for this name, might be misspelled
+            meth, msg = (
+                self.warn,
+                "Couldn't find index page for %r (maybe misspelled?)")
+        meth(msg, requirement.unsafe_name)
+        self.scan_all()
+
+    def download(self, spec, tmpdir):
+        """Locate and/or download `spec` to `tmpdir`, returning a local path
+
+        `spec` may be a ``Requirement`` object, or a string containing a URL,
+        an existing local filename, or a project/version requirement spec
+        (i.e. the string form of a ``Requirement`` object).  If it is the URL
+        of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
+        that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
+        automatically created alongside the downloaded file.
+
+        If `spec` is a ``Requirement`` object or a string containing a
+        project/version requirement spec, this method returns the location of
+        a matching distribution (possibly after downloading it to `tmpdir`).
+        If `spec` is a locally existing file or directory name, it is simply
+        returned unchanged.  If `spec` is a URL, it is downloaded to a subpath
+        of `tmpdir`, and the local filename is returned.  Various errors may be
+        raised if a problem occurs during downloading.
+        """
+        if not isinstance(spec, Requirement):
+            scheme = URL_SCHEME(spec)
+            if scheme:
+                # It's a url, download it to tmpdir
+                found = self._download_url(scheme.group(1), spec, tmpdir)
+                base, fragment = egg_info_for_url(spec)
+                if base.endswith('.py'):
+                    found = self.gen_setup(found, fragment, tmpdir)
+                return found
+            elif os.path.exists(spec):
+                # Existing file or directory, just return it
+                return spec
+            else:
+                spec = parse_requirement_arg(spec)
+        return getattr(self.fetch_distribution(spec, tmpdir), 'location', None)
+
+    def fetch_distribution(
+            self, requirement, tmpdir, force_scan=False, source=False,
+            develop_ok=False, local_index=None):
+        """Obtain a distribution suitable for fulfilling `requirement`
+
+        `requirement` must be a ``pkg_resources.Requirement`` instance.
+        If necessary, or if the `force_scan` flag is set, the requirement is
+        searched for in the (online) package index as well as the locally
+        installed packages.  If a distribution matching `requirement` is found,
+        the returned distribution's ``location`` is the value you would have
+        gotten from calling the ``download()`` method with the matching
+        distribution's URL or filename.  If no matching distribution is found,
+        ``None`` is returned.
+
+        If the `source` flag is set, only source distributions and source
+        checkout links will be considered.  Unless the `develop_ok` flag is
+        set, development and system eggs (i.e., those using the ``.egg-info``
+        format) will be ignored.
+        """
+        # process a Requirement
+        self.info("Searching for %s", requirement)
+        skipped = {}
+        dist = None
+
+        def find(req, env=None):
+            if env is None:
+                env = self
+            # Find a matching distribution; may be called more than once
+
+            for dist in env[req.key]:
+
+                if dist.precedence == DEVELOP_DIST and not develop_ok:
+                    if dist not in skipped:
+                        self.warn(
+                            "Skipping development or system egg: %s", dist,
+                        )
+                        skipped[dist] = 1
+                    continue
+
+                test = (
+                    dist in req
+                    and (dist.precedence <= SOURCE_DIST or not source)
+                )
+                if test:
+                    loc = self.download(dist.location, tmpdir)
+                    dist.download_location = loc
+                    if os.path.exists(dist.download_location):
+                        return dist
+
+        if force_scan:
+            self.prescan()
+            self.find_packages(requirement)
+            dist = find(requirement)
+
+        if not dist and local_index is not None:
+            dist = find(requirement, local_index)
+
+        if dist is None:
+            if self.to_scan is not None:
+                self.prescan()
+            dist = find(requirement)
+
+        if dist is None and not force_scan:
+            self.find_packages(requirement)
+            dist = find(requirement)
+
+        if dist is None:
+            self.warn(
+                "No local packages or working download links found for %s%s",
+                (source and "a source distribution of " or ""),
+                requirement,
+            )
+        else:
+            self.info("Best match: %s", dist)
+            return dist.clone(location=dist.download_location)
+
+    def fetch(self, requirement, tmpdir, force_scan=False, source=False):
+        """Obtain a file suitable for fulfilling `requirement`
+
+        DEPRECATED; use the ``fetch_distribution()`` method now instead.  For
+        backward compatibility, this routine is identical but returns the
+        ``location`` of the downloaded distribution instead of a distribution
+        object.
+        """
+        dist = self.fetch_distribution(requirement, tmpdir, force_scan, source)
+        if dist is not None:
+            return dist.location
+        return None
+
+    def gen_setup(self, filename, fragment, tmpdir):
+        match = EGG_FRAGMENT.match(fragment)
+        dists = match and [
+            d for d in
+            interpret_distro_name(filename, match.group(1), None) if d.version
+        ] or []
+
+        if len(dists) == 1:  # unambiguous ``#egg`` fragment
+            basename = os.path.basename(filename)
+
+            # Make sure the file has been downloaded to the temp dir.
+            if os.path.dirname(filename) != tmpdir:
+                dst = os.path.join(tmpdir, basename)
+                from setuptools.command.easy_install import samefile
+                if not samefile(filename, dst):
+                    shutil.copy2(filename, dst)
+                    filename = dst
+
+            with open(os.path.join(tmpdir, 'setup.py'), 'w') as file:
+                file.write(
+                    "from setuptools import setup\n"
+                    "setup(name=%r, version=%r, py_modules=[%r])\n"
+                    % (
+                        dists[0].project_name, dists[0].version,
+                        os.path.splitext(basename)[0]
+                    )
+                )
+            return filename
+
+        elif match:
+            raise DistutilsError(
+                "Can't unambiguously interpret project/version identifier %r; "
+                "any dashes in the name or version should be escaped using "
+                "underscores. %r" % (fragment, dists)
+            )
+        else:
+            raise DistutilsError(
+                "Can't process plain .py files without an '#egg=name-version'"
+                " suffix to enable automatic setup script generation."
+            )
+
+    dl_blocksize = 8192
+
+    def _download_to(self, url, filename):
+        self.info("Downloading %s", url)
+        # Download the file
+        fp = None
+        try:
+            checker = HashChecker.from_url(url)
+            fp = self.open_url(url)
+            if isinstance(fp, urllib.error.HTTPError):
+                raise DistutilsError(
+                    "Can't download %s: %s %s" % (url, fp.code, fp.msg)
+                )
+            headers = fp.info()
+            blocknum = 0
+            bs = self.dl_blocksize
+            size = -1
+            if "content-length" in headers:
+                # Some servers return multiple Content-Length headers :(
+                sizes = get_all_headers(headers, 'Content-Length')
+                size = max(map(int, sizes))
+                self.reporthook(url, filename, blocknum, bs, size)
+            with open(filename, 'wb') as tfp:
+                while True:
+                    block = fp.read(bs)
+                    if block:
+                        checker.feed(block)
+                        tfp.write(block)
+                        blocknum += 1
+                        self.reporthook(url, filename, blocknum, bs, size)
+                    else:
+                        break
+                self.check_hash(checker, filename, tfp)
+            return headers
+        finally:
+            if fp:
+                fp.close()
+
+    def reporthook(self, url, filename, blocknum, blksize, size):
+        pass  # no-op
+
+    def open_url(self, url, warning=None):
+        if url.startswith('file:'):
+            return local_open(url)
+        try:
+            return open_with_auth(url, self.opener)
+        except (ValueError, http_client.InvalidURL) as v:
+            msg = ' '.join([str(arg) for arg in v.args])
+            if warning:
+                self.warn(warning, msg)
+            else:
+                raise DistutilsError('%s %s' % (url, msg))
+        except urllib.error.HTTPError as v:
+            return v
+        except urllib.error.URLError as v:
+            if warning:
+                self.warn(warning, v.reason)
+            else:
+                raise DistutilsError("Download error for %s: %s"
+                                     % (url, v.reason))
+        except http_client.BadStatusLine as v:
+            if warning:
+                self.warn(warning, v.line)
+            else:
+                raise DistutilsError(
+                    '%s returned a bad status line. The server might be '
+                    'down, %s' %
+                    (url, v.line)
+                )
+        except (http_client.HTTPException, socket.error) as v:
+            if warning:
+                self.warn(warning, v)
+            else:
+                raise DistutilsError("Download error for %s: %s"
+                                     % (url, v))
+
+    def _download_url(self, scheme, url, tmpdir):
+        # Determine download filename
+        #
+        name, fragment = egg_info_for_url(url)
+        if name:
+            while '..' in name:
+                name = name.replace('..', '.').replace('\\', '_')
+        else:
+            name = "__downloaded__"  # default if URL has no path contents
+
+        if name.endswith('.egg.zip'):
+            name = name[:-4]  # strip the extra .zip before download
+
+        filename = os.path.join(tmpdir, name)
+
+        # Download the file
+        #
+        if scheme == 'svn' or scheme.startswith('svn+'):
+            return self._download_svn(url, filename)
+        elif scheme == 'git' or scheme.startswith('git+'):
+            return self._download_git(url, filename)
+        elif scheme.startswith('hg+'):
+            return self._download_hg(url, filename)
+        elif scheme == 'file':
+            return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
+        else:
+            self.url_ok(url, True)  # raises error if not allowed
+            return self._attempt_download(url, filename)
+
+    def scan_url(self, url):
+        self.process_url(url, True)
+
+    def _attempt_download(self, url, filename):
+        headers = self._download_to(url, filename)
+        if 'html' in headers.get('content-type', '').lower():
+            return self._download_html(url, headers, filename)
+        else:
+            return filename
+
+    def _download_html(self, url, headers, filename):
+        file = open(filename)
+        for line in file:
+            if line.strip():
+                # Check for a subversion index page
+                if re.search(r'([^- ]+ - )?Revision \d+:', line):
+                    # it's a subversion index page:
+                    file.close()
+                    os.unlink(filename)
+                    return self._download_svn(url, filename)
+                break  # not an index page
+        file.close()
+        os.unlink(filename)
+        raise DistutilsError("Unexpected HTML page found at " + url)
+
+    def _download_svn(self, url, filename):
+        warnings.warn("SVN download support is deprecated", UserWarning)
+        url = url.split('#', 1)[0]  # remove any fragment for svn's sake
+        creds = ''
+        if url.lower().startswith('svn:') and '@' in url:
+            scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
+            if not netloc and path.startswith('//') and '/' in path[2:]:
+                netloc, path = path[2:].split('/', 1)
+                auth, host = _splituser(netloc)
+                if auth:
+                    if ':' in auth:
+                        user, pw = auth.split(':', 1)
+                        creds = " --username=%s --password=%s" % (user, pw)
+                    else:
+                        creds = " --username=" + auth
+                    netloc = host
+                    parts = scheme, netloc, url, p, q, f
+                    url = urllib.parse.urlunparse(parts)
+        self.info("Doing subversion checkout from %s to %s", url, filename)
+        os.system("svn checkout%s -q %s %s" % (creds, url, filename))
+        return filename
+
+    @staticmethod
+    def _vcs_split_rev_from_url(url, pop_prefix=False):
+        scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)
+
+        scheme = scheme.split('+', 1)[-1]
+
+        # Some fragment identification fails
+        path = path.split('#', 1)[0]
+
+        rev = None
+        if '@' in path:
+            path, rev = path.rsplit('@', 1)
+
+        # Also, discard fragment
+        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))
+
+        return url, rev
+
+    def _download_git(self, url, filename):
+        filename = filename.split('#', 1)[0]
+        url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
+
+        self.info("Doing git clone from %s to %s", url, filename)
+        os.system("git clone --quiet %s %s" % (url, filename))
+
+        if rev is not None:
+            self.info("Checking out %s", rev)
+            os.system("git -C %s checkout --quiet %s" % (
+                filename,
+                rev,
+            ))
+
+        return filename
+
+    def _download_hg(self, url, filename):
+        filename = filename.split('#', 1)[0]
+        url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
+
+        self.info("Doing hg clone from %s to %s", url, filename)
+        os.system("hg clone --quiet %s %s" % (url, filename))
+
+        if rev is not None:
+            self.info("Updating to %s", rev)
+            os.system("hg --cwd %s up -C -r %s -q" % (
+                filename,
+                rev,
+            ))
+
+        return filename
+
+    def debug(self, msg, *args):
+        log.debug(msg, *args)
+
+    def info(self, msg, *args):
+        log.info(msg, *args)
+
+    def warn(self, msg, *args):
+        log.warn(msg, *args)
+
+
+# This pattern matches a character entity reference (a decimal numeric
+# references, a hexadecimal numeric reference, or a named reference).
+entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
+
+
+def decode_entity(match):
+    what = match.group(0)
+    return unescape(what)
+
+
+def htmldecode(text):
+    """
+    Decode HTML entities in the given text.
+
+    >>> htmldecode(
+    ...     'https://../package_name-0.1.2.tar.gz'
+    ...     '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz')
+    'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz'
+    """
+    return entity_sub(decode_entity, text)
+
+
+def socket_timeout(timeout=15):
+    def _socket_timeout(func):
+        def _socket_timeout(*args, **kwargs):
+            old_timeout = socket.getdefaulttimeout()
+            socket.setdefaulttimeout(timeout)
+            try:
+                return func(*args, **kwargs)
+            finally:
+                socket.setdefaulttimeout(old_timeout)
+
+        return _socket_timeout
+
+    return _socket_timeout
+
+
+def _encode_auth(auth):
+    """
+    A function compatible with Python 2.3-3.3 that will encode
+    auth from a URL suitable for an HTTP header.
+    >>> str(_encode_auth('username%3Apassword'))
+    'dXNlcm5hbWU6cGFzc3dvcmQ='
+
+    Long auth strings should not cause a newline to be inserted.
+    >>> long_auth = 'username:' + 'password'*10
+    >>> chr(10) in str(_encode_auth(long_auth))
+    False
+    """
+    auth_s = urllib.parse.unquote(auth)
+    # convert to bytes
+    auth_bytes = auth_s.encode()
+    encoded_bytes = base64.b64encode(auth_bytes)
+    # convert back to a string
+    encoded = encoded_bytes.decode()
+    # strip the trailing carriage return
+    return encoded.replace('\n', '')
+
+
+class Credential:
+    """
+    A username/password pair. Use like a namedtuple.
+    """
+
+    def __init__(self, username, password):
+        self.username = username
+        self.password = password
+
+    def __iter__(self):
+        yield self.username
+        yield self.password
+
+    def __str__(self):
+        return '%(username)s:%(password)s' % vars(self)
+
+
+class PyPIConfig(configparser.RawConfigParser):
+    def __init__(self):
+        """
+        Load from ~/.pypirc
+        """
+        defaults = dict.fromkeys(['username', 'password', 'repository'], '')
+        configparser.RawConfigParser.__init__(self, defaults)
+
+        rc = os.path.join(os.path.expanduser('~'), '.pypirc')
+        if os.path.exists(rc):
+            self.read(rc)
+
+    @property
+    def creds_by_repository(self):
+        sections_with_repositories = [
+            section for section in self.sections()
+            if self.get(section, 'repository').strip()
+        ]
+
+        return dict(map(self._get_repo_cred, sections_with_repositories))
+
+    def _get_repo_cred(self, section):
+        repo = self.get(section, 'repository').strip()
+        return repo, Credential(
+            self.get(section, 'username').strip(),
+            self.get(section, 'password').strip(),
+        )
+
+    def find_credential(self, url):
+        """
+        If the URL indicated appears to be a repository defined in this
+        config, return the credential for that repository.
+        """
+        for repository, cred in self.creds_by_repository.items():
+            if url.startswith(repository):
+                return cred
+
+
+def open_with_auth(url, opener=urllib.request.urlopen):
+    """Open a urllib2 request, handling HTTP authentication"""
+
+    parsed = urllib.parse.urlparse(url)
+    scheme, netloc, path, params, query, frag = parsed
+
+    # Double scheme does not raise on Mac OS X as revealed by a
+    # failing test. We would expect "nonnumeric port". Refs #20.
+    if netloc.endswith(':'):
+        raise http_client.InvalidURL("nonnumeric port: ''")
+
+    if scheme in ('http', 'https'):
+        auth, address = _splituser(netloc)
+    else:
+        auth = None
+
+    if not auth:
+        cred = PyPIConfig().find_credential(url)
+        if cred:
+            auth = str(cred)
+            info = cred.username, url
+            log.info('Authenticating as %s for %s (from .pypirc)', *info)
+
+    if auth:
+        auth = "Basic " + _encode_auth(auth)
+        parts = scheme, address, path, params, query, frag
+        new_url = urllib.parse.urlunparse(parts)
+        request = urllib.request.Request(new_url)
+        request.add_header("Authorization", auth)
+    else:
+        request = urllib.request.Request(url)
+
+    request.add_header('User-Agent', user_agent)
+    fp = opener(request)
+
+    if auth:
+        # Put authentication info back into request URL if same host,
+        # so that links found on the page will work
+        s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url)
+        if s2 == scheme and h2 == address:
+            parts = s2, netloc, path2, param2, query2, frag2
+            fp.url = urllib.parse.urlunparse(parts)
+
+    return fp
+
+
+# copy of urllib.parse._splituser from Python 3.8
+def _splituser(host):
+    """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
+    user, delim, host = host.rpartition('@')
+    return (user if delim else None), host
+
+
+# adding a timeout to avoid freezing package_index
+open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
+
+
+def fix_sf_url(url):
+    return url  # backward compatibility
+
+
+def local_open(url):
+    """Read a local path, with special support for directories"""
+    scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
+    filename = urllib.request.url2pathname(path)
+    if os.path.isfile(filename):
+        return urllib.request.urlopen(url)
+    elif path.endswith('/') and os.path.isdir(filename):
+        files = []
+        for f in os.listdir(filename):
+            filepath = os.path.join(filename, f)
+            if f == 'index.html':
+                with open(filepath, 'r') as fp:
+                    body = fp.read()
+                break
+            elif os.path.isdir(filepath):
+                f += '/'
+            files.append('<a href="{name}">{name}</a>'.format(name=f))
+        else:
+            tmpl = (
+                "<html><head><title>{url}"
+                "{files}")
+            body = tmpl.format(url=url, files='\n'.join(files))
+        status, message = 200, "OK"
+    else:
+        status, message, body = 404, "Path not found", "Not found"
+
+    headers = {'content-type': 'text/html'}
+    body_stream = six.StringIO(body)
+    return urllib.error.HTTPError(url, status, message, headers, body_stream)
diff --git a/ubuntu/venv/setuptools/py27compat.py b/ubuntu/venv/setuptools/py27compat.py
new file mode 100644
index 0000000..1d57360
--- /dev/null
+++ b/ubuntu/venv/setuptools/py27compat.py
@@ -0,0 +1,60 @@
+"""
+Compatibility Support for Python 2.7 and earlier
+"""
+
+import sys
+import platform
+
+from setuptools.extern import six
+
+
+def get_all_headers(message, key):
+    """
+    Given an HTTPMessage, return all headers matching a given key.
+    """
+    return message.get_all(key)
+
+
+if six.PY2:
+    def get_all_headers(message, key):
+        return message.getheaders(key)
+
+
+linux_py2_ascii = (
+    platform.system() == 'Linux' and
+    six.PY2
+)
+
+rmtree_safe = str if linux_py2_ascii else lambda x: x
+"""Workaround for http://bugs.python.org/issue24672"""
+
+
+try:
+    from ._imp import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE
+    from ._imp import get_frozen_object, get_module
+except ImportError:
+    import imp
+    from imp import PY_COMPILED, PY_FROZEN, PY_SOURCE  # noqa
+
+    def find_module(module, paths=None):
+        """Just like 'imp.find_module()', but with package support"""
+        parts = module.split('.')
+        while parts:
+            part = parts.pop(0)
+            f, path, (suffix, mode, kind) = info = imp.find_module(part, paths)
+
+            if kind == imp.PKG_DIRECTORY:
+                parts = parts or ['__init__']
+                paths = [path]
+
+            elif parts:
+                raise ImportError("Can't find %r in %s" % (parts, module))
+
+        return info
+
+    def get_frozen_object(module, paths):
+        return imp.get_frozen_object(module)
+
+    def get_module(module, paths, info):
+        imp.load_module(module, *info)
+        return sys.modules[module]
diff --git a/ubuntu/venv/setuptools/py31compat.py b/ubuntu/venv/setuptools/py31compat.py
new file mode 100644
index 0000000..e1da7ee
--- /dev/null
+++ b/ubuntu/venv/setuptools/py31compat.py
@@ -0,0 +1,32 @@
+__all__ = []
+
+__metaclass__ = type
+
+
+try:
+    # Python >=3.2
+    from tempfile import TemporaryDirectory
+except ImportError:
+    import shutil
+    import tempfile
+
+    class TemporaryDirectory:
+        """
+        Very simple temporary directory context manager.
+        Will try to delete afterward, but will also ignore OS and similar
+        errors on deletion.
+        """
+
+        def __init__(self, **kwargs):
+            self.name = None  # Handle mkdtemp raising an exception
+            self.name = tempfile.mkdtemp(**kwargs)
+
+        def __enter__(self):
+            return self.name
+
+        def __exit__(self, exctype, excvalue, exctrace):
+            try:
+                shutil.rmtree(self.name, True)
+            except OSError:  # removal errors are not the only possible
+                pass
+            self.name = None
diff --git a/ubuntu/venv/setuptools/py33compat.py b/ubuntu/venv/setuptools/py33compat.py
new file mode 100644
index 0000000..cb69443
--- /dev/null
+++ b/ubuntu/venv/setuptools/py33compat.py
@@ -0,0 +1,59 @@
+import dis
+import array
+import collections
+
+try:
+    import html
+except ImportError:
+    html = None
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import html_parser
+
+__metaclass__ = type
+
+OpArg = collections.namedtuple('OpArg', 'opcode arg')
+
+
+class Bytecode_compat:
+    def __init__(self, code):
+        self.code = code
+
+    def __iter__(self):
+        """Yield '(op,arg)' pair for each operation in code object 'code'"""
+
+        bytes = array.array('b', self.code.co_code)
+        eof = len(self.code.co_code)
+
+        ptr = 0
+        extended_arg = 0
+
+        while ptr < eof:
+
+            op = bytes[ptr]
+
+            if op >= dis.HAVE_ARGUMENT:
+
+                arg = bytes[ptr + 1] + bytes[ptr + 2] * 256 + extended_arg
+                ptr += 3
+
+                if op == dis.EXTENDED_ARG:
+                    long_type = six.integer_types[-1]
+                    extended_arg = arg * long_type(65536)
+                    continue
+
+            else:
+                arg = None
+                ptr += 1
+
+            yield OpArg(op, arg)
+
+
+Bytecode = getattr(dis, 'Bytecode', Bytecode_compat)
+
+
+unescape = getattr(html, 'unescape', None)
+if unescape is None:
+    # HTMLParser.unescape is deprecated since Python 3.4, and will be removed
+    # from 3.9.
+    unescape = html_parser.HTMLParser().unescape
diff --git a/ubuntu/venv/setuptools/py34compat.py b/ubuntu/venv/setuptools/py34compat.py
new file mode 100644
index 0000000..3ad9172
--- /dev/null
+++ b/ubuntu/venv/setuptools/py34compat.py
@@ -0,0 +1,13 @@
+import importlib
+
+try:
+    import importlib.util
+except ImportError:
+    pass
+
+
+try:
+    module_from_spec = importlib.util.module_from_spec
+except AttributeError:
+    def module_from_spec(spec):
+        return spec.loader.load_module(spec.name)
diff --git a/ubuntu/venv/setuptools/sandbox.py b/ubuntu/venv/setuptools/sandbox.py
new file mode 100644
index 0000000..685f3f7
--- /dev/null
+++ b/ubuntu/venv/setuptools/sandbox.py
@@ -0,0 +1,491 @@
+import os
+import sys
+import tempfile
+import operator
+import functools
+import itertools
+import re
+import contextlib
+import pickle
+import textwrap
+
+from setuptools.extern import six
+from setuptools.extern.six.moves import builtins, map
+
+import pkg_resources.py31compat
+
+if sys.platform.startswith('java'):
+    import org.python.modules.posix.PosixModule as _os
+else:
+    _os = sys.modules[os.name]
+try:
+    _file = file
+except NameError:
+    _file = None
+_open = open
+from distutils.errors import DistutilsError
+from pkg_resources import working_set
+
+
+__all__ = [
+    "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
+]
+
+
+def _execfile(filename, globals, locals=None):
+    """
+    Python 3 implementation of execfile.
+    """
+    mode = 'rb'
+    with open(filename, mode) as stream:
+        script = stream.read()
+    if locals is None:
+        locals = globals
+    code = compile(script, filename, 'exec')
+    exec(code, globals, locals)
+
+
+@contextlib.contextmanager
+def save_argv(repl=None):
+    saved = sys.argv[:]
+    if repl is not None:
+        sys.argv[:] = repl
+    try:
+        yield saved
+    finally:
+        sys.argv[:] = saved
+
+
+@contextlib.contextmanager
+def save_path():
+    saved = sys.path[:]
+    try:
+        yield saved
+    finally:
+        sys.path[:] = saved
+
+
+@contextlib.contextmanager
+def override_temp(replacement):
+    """
+    Monkey-patch tempfile.tempdir with replacement, ensuring it exists
+    """
+    pkg_resources.py31compat.makedirs(replacement, exist_ok=True)
+
+    saved = tempfile.tempdir
+
+    tempfile.tempdir = replacement
+
+    try:
+        yield
+    finally:
+        tempfile.tempdir = saved
+
+
+@contextlib.contextmanager
+def pushd(target):
+    saved = os.getcwd()
+    os.chdir(target)
+    try:
+        yield saved
+    finally:
+        os.chdir(saved)
+
+
+class UnpickleableException(Exception):
+    """
+    An exception representing another Exception that could not be pickled.
+    """
+
+    @staticmethod
+    def dump(type, exc):
+        """
+        Always return a dumped (pickled) type and exc. If exc can't be pickled,
+        wrap it in UnpickleableException first.
+        """
+        try:
+            return pickle.dumps(type), pickle.dumps(exc)
+        except Exception:
+            # get UnpickleableException inside the sandbox
+            from setuptools.sandbox import UnpickleableException as cls
+            return cls.dump(cls, cls(repr(exc)))
+
+
+class ExceptionSaver:
+    """
+    A Context Manager that will save an exception, serialized, and restore it
+    later.
+    """
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, exc, tb):
+        if not exc:
+            return
+
+        # dump the exception
+        self._saved = UnpickleableException.dump(type, exc)
+        self._tb = tb
+
+        # suppress the exception
+        return True
+
+    def resume(self):
+        "restore and re-raise any exception"
+
+        if '_saved' not in vars(self):
+            return
+
+        type, exc = map(pickle.loads, self._saved)
+        six.reraise(type, exc, self._tb)
+
+
+@contextlib.contextmanager
+def save_modules():
+    """
+    Context in which imported modules are saved.
+
+    Translates exceptions internal to the context into the equivalent exception
+    outside the context.
+    """
+    saved = sys.modules.copy()
+    with ExceptionSaver() as saved_exc:
+        yield saved
+
+    sys.modules.update(saved)
+    # remove any modules imported since
+    del_modules = (
+        mod_name for mod_name in sys.modules
+        if mod_name not in saved
+        # exclude any encodings modules. See #285
+        and not mod_name.startswith('encodings.')
+    )
+    _clear_modules(del_modules)
+
+    saved_exc.resume()
+
+
+def _clear_modules(module_names):
+    for mod_name in list(module_names):
+        del sys.modules[mod_name]
+
+
+@contextlib.contextmanager
+def save_pkg_resources_state():
+    saved = pkg_resources.__getstate__()
+    try:
+        yield saved
+    finally:
+        pkg_resources.__setstate__(saved)
+
+
+@contextlib.contextmanager
+def setup_context(setup_dir):
+    temp_dir = os.path.join(setup_dir, 'temp')
+    with save_pkg_resources_state():
+        with save_modules():
+            hide_setuptools()
+            with save_path():
+                with save_argv():
+                    with override_temp(temp_dir):
+                        with pushd(setup_dir):
+                            # ensure setuptools commands are available
+                            __import__('setuptools')
+                            yield
+
+
+def _needs_hiding(mod_name):
+    """
+    >>> _needs_hiding('setuptools')
+    True
+    >>> _needs_hiding('pkg_resources')
+    True
+    >>> _needs_hiding('setuptools_plugin')
+    False
+    >>> _needs_hiding('setuptools.__init__')
+    True
+    >>> _needs_hiding('distutils')
+    True
+    >>> _needs_hiding('os')
+    False
+    >>> _needs_hiding('Cython')
+    True
+    """
+    pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)')
+    return bool(pattern.match(mod_name))
+
+
+def hide_setuptools():
+    """
+    Remove references to setuptools' modules from sys.modules to allow the
+    invocation to import the most appropriate setuptools. This technique is
+    necessary to avoid issues such as #315 where setuptools upgrading itself
+    would fail to find a function declared in the metadata.
+    """
+    modules = filter(_needs_hiding, sys.modules)
+    _clear_modules(modules)
+
+
+def run_setup(setup_script, args):
+    """Run a distutils setup script, sandboxed in its directory"""
+    setup_dir = os.path.abspath(os.path.dirname(setup_script))
+    with setup_context(setup_dir):
+        try:
+            sys.argv[:] = [setup_script] + list(args)
+            sys.path.insert(0, setup_dir)
+            # reset to include setup dir, w/clean callback list
+            working_set.__init__()
+            working_set.callbacks.append(lambda dist: dist.activate())
+
+            # __file__ should be a byte string on Python 2 (#712)
+            dunder_file = (
+                setup_script
+                if isinstance(setup_script, str) else
+                setup_script.encode(sys.getfilesystemencoding())
+            )
+
+            with DirectorySandbox(setup_dir):
+                ns = dict(__file__=dunder_file, __name__='__main__')
+                _execfile(setup_script, ns)
+        except SystemExit as v:
+            if v.args and v.args[0]:
+                raise
+            # Normal exit, just return
+
+
+class AbstractSandbox:
+    """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
+
+    _active = False
+
+    def __init__(self):
+        self._attrs = [
+            name for name in dir(_os)
+            if not name.startswith('_') and hasattr(self, name)
+        ]
+
+    def _copy(self, source):
+        for name in self._attrs:
+            setattr(os, name, getattr(source, name))
+
+    def __enter__(self):
+        self._copy(self)
+        if _file:
+            builtins.file = self._file
+        builtins.open = self._open
+        self._active = True
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._active = False
+        if _file:
+            builtins.file = _file
+        builtins.open = _open
+        self._copy(_os)
+
+    def run(self, func):
+        """Run 'func' under os sandboxing"""
+        with self:
+            return func()
+
+    def _mk_dual_path_wrapper(name):
+        original = getattr(_os, name)
+
+        def wrap(self, src, dst, *args, **kw):
+            if self._active:
+                src, dst = self._remap_pair(name, src, dst, *args, **kw)
+            return original(src, dst, *args, **kw)
+
+        return wrap
+
+    for name in ["rename", "link", "symlink"]:
+        if hasattr(_os, name):
+            locals()[name] = _mk_dual_path_wrapper(name)
+
+    def _mk_single_path_wrapper(name, original=None):
+        original = original or getattr(_os, name)
+
+        def wrap(self, path, *args, **kw):
+            if self._active:
+                path = self._remap_input(name, path, *args, **kw)
+            return original(path, *args, **kw)
+
+        return wrap
+
+    if _file:
+        _file = _mk_single_path_wrapper('file', _file)
+    _open = _mk_single_path_wrapper('open', _open)
+    for name in [
+        "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
+        "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
+        "startfile", "mkfifo", "mknod", "pathconf", "access"
+    ]:
+        if hasattr(_os, name):
+            locals()[name] = _mk_single_path_wrapper(name)
+
+    def _mk_single_with_return(name):
+        original = getattr(_os, name)
+
+        def wrap(self, path, *args, **kw):
+            if self._active:
+                path = self._remap_input(name, path, *args, **kw)
+                return self._remap_output(name, original(path, *args, **kw))
+            return original(path, *args, **kw)
+
+        return wrap
+
+    for name in ['readlink', 'tempnam']:
+        if hasattr(_os, name):
+            locals()[name] = _mk_single_with_return(name)
+
+    def _mk_query(name):
+        original = getattr(_os, name)
+
+        def wrap(self, *args, **kw):
+            retval = original(*args, **kw)
+            if self._active:
+                return self._remap_output(name, retval)
+            return retval
+
+        return wrap
+
+    for name in ['getcwd', 'tmpnam']:
+        if hasattr(_os, name):
+            locals()[name] = _mk_query(name)
+
+    def _validate_path(self, path):
+        """Called to remap or validate any path, whether input or output"""
+        return path
+
+    def _remap_input(self, operation, path, *args, **kw):
+        """Called for path inputs"""
+        return self._validate_path(path)
+
+    def _remap_output(self, operation, path):
+        """Called for path outputs"""
+        return self._validate_path(path)
+
+    def _remap_pair(self, operation, src, dst, *args, **kw):
+        """Called for path pairs like rename, link, and symlink operations"""
+        return (
+            self._remap_input(operation + '-from', src, *args, **kw),
+            self._remap_input(operation + '-to', dst, *args, **kw)
+        )
+
+
+if hasattr(os, 'devnull'):
+    _EXCEPTIONS = [os.devnull,]
+else:
+    _EXCEPTIONS = []
+
+
+class DirectorySandbox(AbstractSandbox):
+    """Restrict operations to a single subdirectory - pseudo-chroot"""
+
+    write_ops = dict.fromkeys([
+        "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
+        "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
+    ])
+
+    _exception_patterns = [
+        # Allow lib2to3 to attempt to save a pickled grammar object (#121)
+        r'.*lib2to3.*\.pickle$',
+    ]
+    "exempt writing to paths that match the pattern"
+
+    def __init__(self, sandbox, exceptions=_EXCEPTIONS):
+        self._sandbox = os.path.normcase(os.path.realpath(sandbox))
+        self._prefix = os.path.join(self._sandbox, '')
+        self._exceptions = [
+            os.path.normcase(os.path.realpath(path))
+            for path in exceptions
+        ]
+        AbstractSandbox.__init__(self)
+
+    def _violation(self, operation, *args, **kw):
+        from setuptools.sandbox import SandboxViolation
+        raise SandboxViolation(operation, args, kw)
+
+    if _file:
+
+        def _file(self, path, mode='r', *args, **kw):
+            if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
+                self._violation("file", path, mode, *args, **kw)
+            return _file(path, mode, *args, **kw)
+
+    def _open(self, path, mode='r', *args, **kw):
+        if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
+            self._violation("open", path, mode, *args, **kw)
+        return _open(path, mode, *args, **kw)
+
+    def tmpnam(self):
+        self._violation("tmpnam")
+
+    def _ok(self, path):
+        active = self._active
+        try:
+            self._active = False
+            realpath = os.path.normcase(os.path.realpath(path))
+            return (
+                self._exempted(realpath)
+                or realpath == self._sandbox
+                or realpath.startswith(self._prefix)
+            )
+        finally:
+            self._active = active
+
+    def _exempted(self, filepath):
+        start_matches = (
+            filepath.startswith(exception)
+            for exception in self._exceptions
+        )
+        pattern_matches = (
+            re.match(pattern, filepath)
+            for pattern in self._exception_patterns
+        )
+        candidates = itertools.chain(start_matches, pattern_matches)
+        return any(candidates)
+
+    def _remap_input(self, operation, path, *args, **kw):
+        """Called for path inputs"""
+        if operation in self.write_ops and not self._ok(path):
+            self._violation(operation, os.path.realpath(path), *args, **kw)
+        return path
+
+    def _remap_pair(self, operation, src, dst, *args, **kw):
+        """Called for path pairs like rename, link, and symlink operations"""
+        if not self._ok(src) or not self._ok(dst):
+            self._violation(operation, src, dst, *args, **kw)
+        return (src, dst)
+
+    def open(self, file, flags, mode=0o777, *args, **kw):
+        """Called for low-level os.open()"""
+        if flags & WRITE_FLAGS and not self._ok(file):
+            self._violation("os.open", file, flags, mode, *args, **kw)
+        return _os.open(file, flags, mode, *args, **kw)
+
+
+WRITE_FLAGS = functools.reduce(
+    operator.or_, [getattr(_os, a, 0) for a in
+        "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
+)
+
+
+class SandboxViolation(DistutilsError):
+    """A setup script attempted to modify the filesystem outside the sandbox"""
+
+    tmpl = textwrap.dedent("""
+        SandboxViolation: {cmd}{args!r} {kwargs}
+
+        The package setup script has attempted to modify files on your system
+        that are not within the EasyInstall build area, and has been aborted.
+
+        This package cannot be safely installed by EasyInstall, and may not
+        support alternate installation locations even if you run its setup
+        script by hand.  Please inform the package's author and the EasyInstall
+        maintainers to find out if a fix or workaround is available.
+        """).lstrip()
+
+    def __str__(self):
+        cmd, args, kwargs = self.args
+        return self.tmpl.format(**locals())
diff --git a/ubuntu/venv/setuptools/script (dev).tmpl b/ubuntu/venv/setuptools/script (dev).tmpl
new file mode 100644
index 0000000..39a24b0
--- /dev/null
+++ b/ubuntu/venv/setuptools/script (dev).tmpl	
@@ -0,0 +1,6 @@
+# EASY-INSTALL-DEV-SCRIPT: %(spec)r,%(script_name)r
+__requires__ = %(spec)r
+__import__('pkg_resources').require(%(spec)r)
+__file__ = %(dev_path)r
+with open(__file__) as f:
+    exec(compile(f.read(), __file__, 'exec'))
diff --git a/ubuntu/venv/setuptools/script.tmpl b/ubuntu/venv/setuptools/script.tmpl
new file mode 100644
index 0000000..ff5efbc
--- /dev/null
+++ b/ubuntu/venv/setuptools/script.tmpl
@@ -0,0 +1,3 @@
+# EASY-INSTALL-SCRIPT: %(spec)r,%(script_name)r
+__requires__ = %(spec)r
+__import__('pkg_resources').run_script(%(spec)r, %(script_name)r)
diff --git a/ubuntu/venv/setuptools/site-patch.py b/ubuntu/venv/setuptools/site-patch.py
new file mode 100644
index 0000000..40b00de
--- /dev/null
+++ b/ubuntu/venv/setuptools/site-patch.py
@@ -0,0 +1,74 @@
+def __boot():
+    import sys
+    import os
+    PYTHONPATH = os.environ.get('PYTHONPATH')
+    if PYTHONPATH is None or (sys.platform == 'win32' and not PYTHONPATH):
+        PYTHONPATH = []
+    else:
+        PYTHONPATH = PYTHONPATH.split(os.pathsep)
+
+    pic = getattr(sys, 'path_importer_cache', {})
+    stdpath = sys.path[len(PYTHONPATH):]
+    mydir = os.path.dirname(__file__)
+
+    for item in stdpath:
+        if item == mydir or not item:
+            continue  # skip if current dir. on Windows, or my own directory
+        importer = pic.get(item)
+        if importer is not None:
+            loader = importer.find_module('site')
+            if loader is not None:
+                # This should actually reload the current module
+                loader.load_module('site')
+                break
+        else:
+            try:
+                import imp  # Avoid import loop in Python 3
+                stream, path, descr = imp.find_module('site', [item])
+            except ImportError:
+                continue
+            if stream is None:
+                continue
+            try:
+                # This should actually reload the current module
+                imp.load_module('site', stream, path, descr)
+            finally:
+                stream.close()
+            break
+    else:
+        raise ImportError("Couldn't find the real 'site' module")
+
+    known_paths = dict([(makepath(item)[1], 1) for item in sys.path])  # 2.2 comp
+
+    oldpos = getattr(sys, '__egginsert', 0)  # save old insertion position
+    sys.__egginsert = 0  # and reset the current one
+
+    for item in PYTHONPATH:
+        addsitedir(item)
+
+    sys.__egginsert += oldpos  # restore effective old position
+
+    d, nd = makepath(stdpath[0])
+    insert_at = None
+    new_path = []
+
+    for item in sys.path:
+        p, np = makepath(item)
+
+        if np == nd and insert_at is None:
+            # We've hit the first 'system' path entry, so added entries go here
+            insert_at = len(new_path)
+
+        if np in known_paths or insert_at is None:
+            new_path.append(item)
+        else:
+            # new path after the insert point, back-insert it
+            new_path.insert(insert_at, item)
+            insert_at += 1
+
+    sys.path[:] = new_path
+
+
+if __name__ == 'site':
+    __boot()
+    del __boot
diff --git a/ubuntu/venv/setuptools/ssl_support.py b/ubuntu/venv/setuptools/ssl_support.py
new file mode 100644
index 0000000..226db69
--- /dev/null
+++ b/ubuntu/venv/setuptools/ssl_support.py
@@ -0,0 +1,260 @@
+import os
+import socket
+import atexit
+import re
+import functools
+
+from setuptools.extern.six.moves import urllib, http_client, map, filter
+
+from pkg_resources import ResolutionError, ExtractionError
+
+try:
+    import ssl
+except ImportError:
+    ssl = None
+
+__all__ = [
+    'VerifyingHTTPSHandler', 'find_ca_bundle', 'is_available', 'cert_paths',
+    'opener_for'
+]
+
+cert_paths = """
+/etc/pki/tls/certs/ca-bundle.crt
+/etc/ssl/certs/ca-certificates.crt
+/usr/share/ssl/certs/ca-bundle.crt
+/usr/local/share/certs/ca-root.crt
+/etc/ssl/cert.pem
+/System/Library/OpenSSL/certs/cert.pem
+/usr/local/share/certs/ca-root-nss.crt
+/etc/ssl/ca-bundle.pem
+""".strip().split()
+
+try:
+    HTTPSHandler = urllib.request.HTTPSHandler
+    HTTPSConnection = http_client.HTTPSConnection
+except AttributeError:
+    HTTPSHandler = HTTPSConnection = object
+
+is_available = ssl is not None and object not in (HTTPSHandler, HTTPSConnection)
+
+
+try:
+    from ssl import CertificateError, match_hostname
+except ImportError:
+    try:
+        from backports.ssl_match_hostname import CertificateError
+        from backports.ssl_match_hostname import match_hostname
+    except ImportError:
+        CertificateError = None
+        match_hostname = None
+
+if not CertificateError:
+
+    class CertificateError(ValueError):
+        pass
+
+
+if not match_hostname:
+
+    def _dnsname_match(dn, hostname, max_wildcards=1):
+        """Matching according to RFC 6125, section 6.4.3
+
+        https://tools.ietf.org/html/rfc6125#section-6.4.3
+        """
+        pats = []
+        if not dn:
+            return False
+
+        # Ported from python3-syntax:
+        # leftmost, *remainder = dn.split(r'.')
+        parts = dn.split(r'.')
+        leftmost = parts[0]
+        remainder = parts[1:]
+
+        wildcards = leftmost.count('*')
+        if wildcards > max_wildcards:
+            # Issue #17980: avoid denials of service by refusing more
+            # than one wildcard per fragment.  A survey of established
+            # policy among SSL implementations showed it to be a
+            # reasonable choice.
+            raise CertificateError(
+                "too many wildcards in certificate DNS name: " + repr(dn))
+
+        # speed up common case w/o wildcards
+        if not wildcards:
+            return dn.lower() == hostname.lower()
+
+        # RFC 6125, section 6.4.3, subitem 1.
+        # The client SHOULD NOT attempt to match a presented identifier in which
+        # the wildcard character comprises a label other than the left-most label.
+        if leftmost == '*':
+            # When '*' is a fragment by itself, it matches a non-empty dotless
+            # fragment.
+            pats.append('[^.]+')
+        elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
+            # RFC 6125, section 6.4.3, subitem 3.
+            # The client SHOULD NOT attempt to match a presented identifier
+            # where the wildcard character is embedded within an A-label or
+            # U-label of an internationalized domain name.
+            pats.append(re.escape(leftmost))
+        else:
+            # Otherwise, '*' matches any dotless string, e.g. www*
+            pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
+
+        # add the remaining fragments, ignore any wildcards
+        for frag in remainder:
+            pats.append(re.escape(frag))
+
+        pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+        return pat.match(hostname)
+
+    def match_hostname(cert, hostname):
+        """Verify that *cert* (in decoded format as returned by
+        SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
+        rules are followed, but IP addresses are not accepted for *hostname*.
+
+        CertificateError is raised on failure. On success, the function
+        returns nothing.
+        """
+        if not cert:
+            raise ValueError("empty or no certificate")
+        dnsnames = []
+        san = cert.get('subjectAltName', ())
+        for key, value in san:
+            if key == 'DNS':
+                if _dnsname_match(value, hostname):
+                    return
+                dnsnames.append(value)
+        if not dnsnames:
+            # The subject is only checked when there is no dNSName entry
+            # in subjectAltName
+            for sub in cert.get('subject', ()):
+                for key, value in sub:
+                    # XXX according to RFC 2818, the most specific Common Name
+                    # must be used.
+                    if key == 'commonName':
+                        if _dnsname_match(value, hostname):
+                            return
+                        dnsnames.append(value)
+        if len(dnsnames) > 1:
+            raise CertificateError("hostname %r "
+                "doesn't match either of %s"
+                % (hostname, ', '.join(map(repr, dnsnames))))
+        elif len(dnsnames) == 1:
+            raise CertificateError("hostname %r "
+                "doesn't match %r"
+                % (hostname, dnsnames[0]))
+        else:
+            raise CertificateError("no appropriate commonName or "
+                "subjectAltName fields were found")
+
+
+class VerifyingHTTPSHandler(HTTPSHandler):
+    """Simple verifying handler: no auth, subclasses, timeouts, etc."""
+
+    def __init__(self, ca_bundle):
+        self.ca_bundle = ca_bundle
+        HTTPSHandler.__init__(self)
+
+    def https_open(self, req):
+        return self.do_open(
+            lambda host, **kw: VerifyingHTTPSConn(host, self.ca_bundle, **kw), req
+        )
+
+
+class VerifyingHTTPSConn(HTTPSConnection):
+    """Simple verifying connection: no auth, subclasses, timeouts, etc."""
+
+    def __init__(self, host, ca_bundle, **kw):
+        HTTPSConnection.__init__(self, host, **kw)
+        self.ca_bundle = ca_bundle
+
+    def connect(self):
+        sock = socket.create_connection(
+            (self.host, self.port), getattr(self, 'source_address', None)
+        )
+
+        # Handle the socket if a (proxy) tunnel is present
+        if hasattr(self, '_tunnel') and getattr(self, '_tunnel_host', None):
+            self.sock = sock
+            self._tunnel()
+            # http://bugs.python.org/issue7776: Python>=3.4.1 and >=2.7.7
+            # change self.host to mean the proxy server host when tunneling is
+            # being used. Adapt, since we are interested in the destination
+            # host for the match_hostname() comparison.
+            actual_host = self._tunnel_host
+        else:
+            actual_host = self.host
+
+        if hasattr(ssl, 'create_default_context'):
+            ctx = ssl.create_default_context(cafile=self.ca_bundle)
+            self.sock = ctx.wrap_socket(sock, server_hostname=actual_host)
+        else:
+            # This is for python < 2.7.9 and < 3.4?
+            self.sock = ssl.wrap_socket(
+                sock, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_bundle
+            )
+        try:
+            match_hostname(self.sock.getpeercert(), actual_host)
+        except CertificateError:
+            self.sock.shutdown(socket.SHUT_RDWR)
+            self.sock.close()
+            raise
+
+
+def opener_for(ca_bundle=None):
+    """Get a urlopen() replacement that uses ca_bundle for verification"""
+    return urllib.request.build_opener(
+        VerifyingHTTPSHandler(ca_bundle or find_ca_bundle())
+    ).open
+
+
+# from jaraco.functools
+def once(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if not hasattr(func, 'always_returns'):
+            func.always_returns = func(*args, **kwargs)
+        return func.always_returns
+    return wrapper
+
+
+@once
+def get_win_certfile():
+    try:
+        import wincertstore
+    except ImportError:
+        return None
+
+    class CertFile(wincertstore.CertFile):
+        def __init__(self):
+            super(CertFile, self).__init__()
+            atexit.register(self.close)
+
+        def close(self):
+            try:
+                super(CertFile, self).close()
+            except OSError:
+                pass
+
+    _wincerts = CertFile()
+    _wincerts.addstore('CA')
+    _wincerts.addstore('ROOT')
+    return _wincerts.name
+
+
+def find_ca_bundle():
+    """Return an existing CA bundle path, or None"""
+    extant_cert_paths = filter(os.path.isfile, cert_paths)
+    return (
+        get_win_certfile()
+        or next(extant_cert_paths, None)
+        or _certifi_where()
+    )
+
+
+def _certifi_where():
+    try:
+        return __import__('certifi').where()
+    except (ImportError, ResolutionError, ExtractionError):
+        pass
diff --git a/ubuntu/venv/setuptools/unicode_utils.py b/ubuntu/venv/setuptools/unicode_utils.py
new file mode 100644
index 0000000..7c63efd
--- /dev/null
+++ b/ubuntu/venv/setuptools/unicode_utils.py
@@ -0,0 +1,44 @@
+import unicodedata
+import sys
+
+from setuptools.extern import six
+
+
+# HFS Plus uses decomposed UTF-8
+def decompose(path):
+    if isinstance(path, six.text_type):
+        return unicodedata.normalize('NFD', path)
+    try:
+        path = path.decode('utf-8')
+        path = unicodedata.normalize('NFD', path)
+        path = path.encode('utf-8')
+    except UnicodeError:
+        pass  # Not UTF-8
+    return path
+
+
+def filesys_decode(path):
+    """
+    Ensure that the given path is decoded,
+    NONE when no expected encoding works
+    """
+
+    if isinstance(path, six.text_type):
+        return path
+
+    fs_enc = sys.getfilesystemencoding() or 'utf-8'
+    candidates = fs_enc, 'utf-8'
+
+    for enc in candidates:
+        try:
+            return path.decode(enc)
+        except UnicodeDecodeError:
+            continue
+
+
+def try_encode(string, enc):
+    "turn unicode encoding into a functional routine"
+    try:
+        return string.encode(enc)
+    except UnicodeEncodeError:
+        return None
diff --git a/ubuntu/venv/setuptools/version.py b/ubuntu/venv/setuptools/version.py
new file mode 100644
index 0000000..95e1869
--- /dev/null
+++ b/ubuntu/venv/setuptools/version.py
@@ -0,0 +1,6 @@
+import pkg_resources
+
+try:
+    __version__ = pkg_resources.get_distribution('setuptools').version
+except Exception:
+    __version__ = 'unknown'
diff --git a/ubuntu/venv/setuptools/wheel.py b/ubuntu/venv/setuptools/wheel.py
new file mode 100644
index 0000000..025aaa8
--- /dev/null
+++ b/ubuntu/venv/setuptools/wheel.py
@@ -0,0 +1,220 @@
+"""Wheels support."""
+
+from distutils.util import get_platform
+from distutils import log
+import email
+import itertools
+import os
+import posixpath
+import re
+import zipfile
+
+import pkg_resources
+import setuptools
+from pkg_resources import parse_version
+from setuptools.extern.packaging.tags import sys_tags
+from setuptools.extern.packaging.utils import canonicalize_name
+from setuptools.extern.six import PY3
+from setuptools.command.egg_info import write_requirements
+
+
+__metaclass__ = type
+
+
+WHEEL_NAME = re.compile(
+    r"""^(?P.+?)-(?P\d.*?)
+    ((-(?P\d.*?))?-(?P.+?)-(?P.+?)-(?P.+?)
+    )\.whl$""",
+    re.VERBOSE).match
+
+NAMESPACE_PACKAGE_INIT = '''\
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+'''
+
+
+def unpack(src_dir, dst_dir):
+    '''Move everything under `src_dir` to `dst_dir`, and delete the former.'''
+    for dirpath, dirnames, filenames in os.walk(src_dir):
+        subdir = os.path.relpath(dirpath, src_dir)
+        for f in filenames:
+            src = os.path.join(dirpath, f)
+            dst = os.path.join(dst_dir, subdir, f)
+            os.renames(src, dst)
+        for n, d in reversed(list(enumerate(dirnames))):
+            src = os.path.join(dirpath, d)
+            dst = os.path.join(dst_dir, subdir, d)
+            if not os.path.exists(dst):
+                # Directory does not exist in destination,
+                # rename it and prune it from os.walk list.
+                os.renames(src, dst)
+                del dirnames[n]
+    # Cleanup.
+    for dirpath, dirnames, filenames in os.walk(src_dir, topdown=True):
+        assert not filenames
+        os.rmdir(dirpath)
+
+
+class Wheel:
+
+    def __init__(self, filename):
+        match = WHEEL_NAME(os.path.basename(filename))
+        if match is None:
+            raise ValueError('invalid wheel name: %r' % filename)
+        self.filename = filename
+        for k, v in match.groupdict().items():
+            setattr(self, k, v)
+
+    def tags(self):
+        '''List tags (py_version, abi, platform) supported by this wheel.'''
+        return itertools.product(
+            self.py_version.split('.'),
+            self.abi.split('.'),
+            self.platform.split('.'),
+        )
+
+    def is_compatible(self):
+        '''Is the wheel is compatible with the current platform?'''
+        supported_tags = set((t.interpreter, t.abi, t.platform) for t in sys_tags())
+        return next((True for t in self.tags() if t in supported_tags), False)
+
+    def egg_name(self):
+        return pkg_resources.Distribution(
+            project_name=self.project_name, version=self.version,
+            platform=(None if self.platform == 'any' else get_platform()),
+        ).egg_name() + '.egg'
+
+    def get_dist_info(self, zf):
+        # find the correct name of the .dist-info dir in the wheel file
+        for member in zf.namelist():
+            dirname = posixpath.dirname(member)
+            if (dirname.endswith('.dist-info') and
+                    canonicalize_name(dirname).startswith(
+                        canonicalize_name(self.project_name))):
+                return dirname
+        raise ValueError("unsupported wheel format. .dist-info not found")
+
+    def install_as_egg(self, destination_eggdir):
+        '''Install wheel as an egg directory.'''
+        with zipfile.ZipFile(self.filename) as zf:
+            self._install_as_egg(destination_eggdir, zf)
+
+    def _install_as_egg(self, destination_eggdir, zf):
+        dist_basename = '%s-%s' % (self.project_name, self.version)
+        dist_info = self.get_dist_info(zf)
+        dist_data = '%s.data' % dist_basename
+        egg_info = os.path.join(destination_eggdir, 'EGG-INFO')
+
+        self._convert_metadata(zf, destination_eggdir, dist_info, egg_info)
+        self._move_data_entries(destination_eggdir, dist_data)
+        self._fix_namespace_packages(egg_info, destination_eggdir)
+
+    @staticmethod
+    def _convert_metadata(zf, destination_eggdir, dist_info, egg_info):
+        def get_metadata(name):
+            with zf.open(posixpath.join(dist_info, name)) as fp:
+                value = fp.read().decode('utf-8') if PY3 else fp.read()
+                return email.parser.Parser().parsestr(value)
+
+        wheel_metadata = get_metadata('WHEEL')
+        # Check wheel format version is supported.
+        wheel_version = parse_version(wheel_metadata.get('Wheel-Version'))
+        wheel_v1 = (
+            parse_version('1.0') <= wheel_version < parse_version('2.0dev0')
+        )
+        if not wheel_v1:
+            raise ValueError(
+                'unsupported wheel format version: %s' % wheel_version)
+        # Extract to target directory.
+        os.mkdir(destination_eggdir)
+        zf.extractall(destination_eggdir)
+        # Convert metadata.
+        dist_info = os.path.join(destination_eggdir, dist_info)
+        dist = pkg_resources.Distribution.from_location(
+            destination_eggdir, dist_info,
+            metadata=pkg_resources.PathMetadata(destination_eggdir, dist_info),
+        )
+
+        # Note: Evaluate and strip markers now,
+        # as it's difficult to convert back from the syntax:
+        # foobar; "linux" in sys_platform and extra == 'test'
+        def raw_req(req):
+            req.marker = None
+            return str(req)
+        install_requires = list(sorted(map(raw_req, dist.requires())))
+        extras_require = {
+            extra: sorted(
+                req
+                for req in map(raw_req, dist.requires((extra,)))
+                if req not in install_requires
+            )
+            for extra in dist.extras
+        }
+        os.rename(dist_info, egg_info)
+        os.rename(
+            os.path.join(egg_info, 'METADATA'),
+            os.path.join(egg_info, 'PKG-INFO'),
+        )
+        setup_dist = setuptools.Distribution(
+            attrs=dict(
+                install_requires=install_requires,
+                extras_require=extras_require,
+            ),
+        )
+        # Temporarily disable info traces.
+        log_threshold = log._global_log.threshold
+        log.set_threshold(log.WARN)
+        try:
+            write_requirements(
+                setup_dist.get_command_obj('egg_info'),
+                None,
+                os.path.join(egg_info, 'requires.txt'),
+            )
+        finally:
+            log.set_threshold(log_threshold)
+
+    @staticmethod
+    def _move_data_entries(destination_eggdir, dist_data):
+        """Move data entries to their correct location."""
+        dist_data = os.path.join(destination_eggdir, dist_data)
+        dist_data_scripts = os.path.join(dist_data, 'scripts')
+        if os.path.exists(dist_data_scripts):
+            egg_info_scripts = os.path.join(
+                destination_eggdir, 'EGG-INFO', 'scripts')
+            os.mkdir(egg_info_scripts)
+            for entry in os.listdir(dist_data_scripts):
+                # Remove bytecode, as it's not properly handled
+                # during easy_install scripts install phase.
+                if entry.endswith('.pyc'):
+                    os.unlink(os.path.join(dist_data_scripts, entry))
+                else:
+                    os.rename(
+                        os.path.join(dist_data_scripts, entry),
+                        os.path.join(egg_info_scripts, entry),
+                    )
+            os.rmdir(dist_data_scripts)
+        for subdir in filter(os.path.exists, (
+            os.path.join(dist_data, d)
+            for d in ('data', 'headers', 'purelib', 'platlib')
+        )):
+            unpack(subdir, destination_eggdir)
+        if os.path.exists(dist_data):
+            os.rmdir(dist_data)
+
+    @staticmethod
+    def _fix_namespace_packages(egg_info, destination_eggdir):
+        namespace_packages = os.path.join(
+            egg_info, 'namespace_packages.txt')
+        if os.path.exists(namespace_packages):
+            with open(namespace_packages) as fp:
+                namespace_packages = fp.read().split()
+            for mod in namespace_packages:
+                mod_dir = os.path.join(destination_eggdir, *mod.split('.'))
+                mod_init = os.path.join(mod_dir, '__init__.py')
+                if not os.path.exists(mod_dir):
+                    os.mkdir(mod_dir)
+                if not os.path.exists(mod_init):
+                    with open(mod_init, 'w') as fp:
+                        fp.write(NAMESPACE_PACKAGE_INIT)
diff --git a/ubuntu/venv/setuptools/windows_support.py b/ubuntu/venv/setuptools/windows_support.py
new file mode 100644
index 0000000..cb977cf
--- /dev/null
+++ b/ubuntu/venv/setuptools/windows_support.py
@@ -0,0 +1,29 @@
+import platform
+import ctypes
+
+
+def windows_only(func):
+    if platform.system() != 'Windows':
+        return lambda *args, **kwargs: None
+    return func
+
+
+@windows_only
+def hide_file(path):
+    """
+    Set the hidden attribute on a file or directory.
+
+    From http://stackoverflow.com/questions/19622133/
+
+    `path` must be text.
+    """
+    __import__('ctypes.wintypes')
+    SetFileAttributes = ctypes.windll.kernel32.SetFileAttributesW
+    SetFileAttributes.argtypes = ctypes.wintypes.LPWSTR, ctypes.wintypes.DWORD
+    SetFileAttributes.restype = ctypes.wintypes.BOOL
+
+    FILE_ATTRIBUTE_HIDDEN = 0x02
+
+    ret = SetFileAttributes(path, FILE_ATTRIBUTE_HIDDEN)
+    if not ret:
+        raise ctypes.WinError()
diff --git a/ubuntu/venv/yaml/__init__.py b/ubuntu/venv/yaml/__init__.py
new file mode 100644
index 0000000..465041d
--- /dev/null
+++ b/ubuntu/venv/yaml/__init__.py
@@ -0,0 +1,390 @@
+
+from .error import *
+
+from .tokens import *
+from .events import *
+from .nodes import *
+
+from .loader import *
+from .dumper import *
+
+__version__ = '6.0'
+try:
+    from .cyaml import *
+    __with_libyaml__ = True
+except ImportError:
+    __with_libyaml__ = False
+
+import io
+
+#------------------------------------------------------------------------------
+# XXX "Warnings control" is now deprecated. Leaving in the API function to not
+# break code that uses it.
+#------------------------------------------------------------------------------
+def warnings(settings=None):
+    if settings is None:
+        return {}
+
+#------------------------------------------------------------------------------
+def scan(stream, Loader=Loader):
+    """
+    Scan a YAML stream and produce scanning tokens.
+    """
+    loader = Loader(stream)
+    try:
+        while loader.check_token():
+            yield loader.get_token()
+    finally:
+        loader.dispose()
+
+def parse(stream, Loader=Loader):
+    """
+    Parse a YAML stream and produce parsing events.
+    """
+    loader = Loader(stream)
+    try:
+        while loader.check_event():
+            yield loader.get_event()
+    finally:
+        loader.dispose()
+
+def compose(stream, Loader=Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding representation tree.
+    """
+    loader = Loader(stream)
+    try:
+        return loader.get_single_node()
+    finally:
+        loader.dispose()
+
+def compose_all(stream, Loader=Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding representation trees.
+    """
+    loader = Loader(stream)
+    try:
+        while loader.check_node():
+            yield loader.get_node()
+    finally:
+        loader.dispose()
+
+def load(stream, Loader):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+    """
+    loader = Loader(stream)
+    try:
+        return loader.get_single_data()
+    finally:
+        loader.dispose()
+
+def load_all(stream, Loader):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+    """
+    loader = Loader(stream)
+    try:
+        while loader.check_data():
+            yield loader.get_data()
+    finally:
+        loader.dispose()
+
+def full_load(stream):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+
+    Resolve all tags except those known to be
+    unsafe on untrusted input.
+    """
+    return load(stream, FullLoader)
+
+def full_load_all(stream):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+
+    Resolve all tags except those known to be
+    unsafe on untrusted input.
+    """
+    return load_all(stream, FullLoader)
+
+def safe_load(stream):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+
+    Resolve only basic YAML tags. This is known
+    to be safe for untrusted input.
+    """
+    return load(stream, SafeLoader)
+
+def safe_load_all(stream):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+
+    Resolve only basic YAML tags. This is known
+    to be safe for untrusted input.
+    """
+    return load_all(stream, SafeLoader)
+
+def unsafe_load(stream):
+    """
+    Parse the first YAML document in a stream
+    and produce the corresponding Python object.
+
+    Resolve all tags, even those known to be
+    unsafe on untrusted input.
+    """
+    return load(stream, UnsafeLoader)
+
+def unsafe_load_all(stream):
+    """
+    Parse all YAML documents in a stream
+    and produce corresponding Python objects.
+
+    Resolve all tags, even those known to be
+    unsafe on untrusted input.
+    """
+    return load_all(stream, UnsafeLoader)
+
+def emit(events, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None):
+    """
+    Emit YAML parsing events into a stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        stream = io.StringIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break)
+    try:
+        for event in events:
+            dumper.emit(event)
+    finally:
+        dumper.dispose()
+    if getvalue:
+        return getvalue()
+
+def serialize_all(nodes, stream=None, Dumper=Dumper,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding=None, explicit_start=None, explicit_end=None,
+        version=None, tags=None):
+    """
+    Serialize a sequence of representation trees into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        if encoding is None:
+            stream = io.StringIO()
+        else:
+            stream = io.BytesIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end)
+    try:
+        dumper.open()
+        for node in nodes:
+            dumper.serialize(node)
+        dumper.close()
+    finally:
+        dumper.dispose()
+    if getvalue:
+        return getvalue()
+
+def serialize(node, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a representation tree into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return serialize_all([node], stream, Dumper=Dumper, **kwds)
+
+def dump_all(documents, stream=None, Dumper=Dumper,
+        default_style=None, default_flow_style=False,
+        canonical=None, indent=None, width=None,
+        allow_unicode=None, line_break=None,
+        encoding=None, explicit_start=None, explicit_end=None,
+        version=None, tags=None, sort_keys=True):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    getvalue = None
+    if stream is None:
+        if encoding is None:
+            stream = io.StringIO()
+        else:
+            stream = io.BytesIO()
+        getvalue = stream.getvalue
+    dumper = Dumper(stream, default_style=default_style,
+            default_flow_style=default_flow_style,
+            canonical=canonical, indent=indent, width=width,
+            allow_unicode=allow_unicode, line_break=line_break,
+            encoding=encoding, version=version, tags=tags,
+            explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys)
+    try:
+        dumper.open()
+        for data in documents:
+            dumper.represent(data)
+        dumper.close()
+    finally:
+        dumper.dispose()
+    if getvalue:
+        return getvalue()
+
+def dump(data, stream=None, Dumper=Dumper, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=Dumper, **kwds)
+
+def safe_dump_all(documents, stream=None, **kwds):
+    """
+    Serialize a sequence of Python objects into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all(documents, stream, Dumper=SafeDumper, **kwds)
+
+def safe_dump(data, stream=None, **kwds):
+    """
+    Serialize a Python object into a YAML stream.
+    Produce only basic YAML tags.
+    If stream is None, return the produced string instead.
+    """
+    return dump_all([data], stream, Dumper=SafeDumper, **kwds)
+
+def add_implicit_resolver(tag, regexp, first=None,
+        Loader=None, Dumper=Dumper):
+    """
+    Add an implicit scalar detector.
+    If an implicit scalar value matches the given regexp,
+    the corresponding tag is assigned to the scalar.
+    first is a sequence of possible initial characters or None.
+    """
+    if Loader is None:
+        loader.Loader.add_implicit_resolver(tag, regexp, first)
+        loader.FullLoader.add_implicit_resolver(tag, regexp, first)
+        loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first)
+    else:
+        Loader.add_implicit_resolver(tag, regexp, first)
+    Dumper.add_implicit_resolver(tag, regexp, first)
+
+def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper):
+    """
+    Add a path based resolver for the given tag.
+    A path is a list of keys that forms a path
+    to a node in the representation tree.
+    Keys can be string values, integers, or None.
+    """
+    if Loader is None:
+        loader.Loader.add_path_resolver(tag, path, kind)
+        loader.FullLoader.add_path_resolver(tag, path, kind)
+        loader.UnsafeLoader.add_path_resolver(tag, path, kind)
+    else:
+        Loader.add_path_resolver(tag, path, kind)
+    Dumper.add_path_resolver(tag, path, kind)
+
+def add_constructor(tag, constructor, Loader=None):
+    """
+    Add a constructor for the given tag.
+    Constructor is a function that accepts a Loader instance
+    and a node object and produces the corresponding Python object.
+    """
+    if Loader is None:
+        loader.Loader.add_constructor(tag, constructor)
+        loader.FullLoader.add_constructor(tag, constructor)
+        loader.UnsafeLoader.add_constructor(tag, constructor)
+    else:
+        Loader.add_constructor(tag, constructor)
+
+def add_multi_constructor(tag_prefix, multi_constructor, Loader=None):
+    """
+    Add a multi-constructor for the given tag prefix.
+    Multi-constructor is called for a node if its tag starts with tag_prefix.
+    Multi-constructor accepts a Loader instance, a tag suffix,
+    and a node object and produces the corresponding Python object.
+    """
+    if Loader is None:
+        loader.Loader.add_multi_constructor(tag_prefix, multi_constructor)
+        loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor)
+        loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor)
+    else:
+        Loader.add_multi_constructor(tag_prefix, multi_constructor)
+
+def add_representer(data_type, representer, Dumper=Dumper):
+    """
+    Add a representer for the given type.
+    Representer is a function accepting a Dumper instance
+    and an instance of the given data type
+    and producing the corresponding representation node.
+    """
+    Dumper.add_representer(data_type, representer)
+
+def add_multi_representer(data_type, multi_representer, Dumper=Dumper):
+    """
+    Add a representer for the given type.
+    Multi-representer is a function accepting a Dumper instance
+    and an instance of the given data type or subtype
+    and producing the corresponding representation node.
+    """
+    Dumper.add_multi_representer(data_type, multi_representer)
+
+class YAMLObjectMetaclass(type):
+    """
+    The metaclass for YAMLObject.
+    """
+    def __init__(cls, name, bases, kwds):
+        super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds)
+        if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None:
+            if isinstance(cls.yaml_loader, list):
+                for loader in cls.yaml_loader:
+                    loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+            else:
+                cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml)
+
+            cls.yaml_dumper.add_representer(cls, cls.to_yaml)
+
+class YAMLObject(metaclass=YAMLObjectMetaclass):
+    """
+    An object that can dump itself to a YAML stream
+    and load itself from a YAML stream.
+    """
+
+    __slots__ = ()  # no direct instantiation, so allow immutable subclasses
+
+    yaml_loader = [Loader, FullLoader, UnsafeLoader]
+    yaml_dumper = Dumper
+
+    yaml_tag = None
+    yaml_flow_style = None
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        """
+        Convert a representation node to a Python object.
+        """
+        return loader.construct_yaml_object(node, cls)
+
+    @classmethod
+    def to_yaml(cls, dumper, data):
+        """
+        Convert a Python object to a representation node.
+        """
+        return dumper.represent_yaml_object(cls.yaml_tag, data, cls,
+                flow_style=cls.yaml_flow_style)
+
diff --git a/ubuntu/venv/yaml/composer.py b/ubuntu/venv/yaml/composer.py
new file mode 100644
index 0000000..6d15cb4
--- /dev/null
+++ b/ubuntu/venv/yaml/composer.py
@@ -0,0 +1,139 @@
+
+__all__ = ['Composer', 'ComposerError']
+
+from .error import MarkedYAMLError
+from .events import *
+from .nodes import *
+
+class ComposerError(MarkedYAMLError):
+    pass
+
+class Composer:
+
+    def __init__(self):
+        self.anchors = {}
+
+    def check_node(self):
+        # Drop the STREAM-START event.
+        if self.check_event(StreamStartEvent):
+            self.get_event()
+
+        # If there are more documents available?
+        return not self.check_event(StreamEndEvent)
+
+    def get_node(self):
+        # Get the root node of the next document.
+        if not self.check_event(StreamEndEvent):
+            return self.compose_document()
+
+    def get_single_node(self):
+        # Drop the STREAM-START event.
+        self.get_event()
+
+        # Compose a document if the stream is not empty.
+        document = None
+        if not self.check_event(StreamEndEvent):
+            document = self.compose_document()
+
+        # Ensure that the stream contains no more documents.
+        if not self.check_event(StreamEndEvent):
+            event = self.get_event()
+            raise ComposerError("expected a single document in the stream",
+                    document.start_mark, "but found another document",
+                    event.start_mark)
+
+        # Drop the STREAM-END event.
+        self.get_event()
+
+        return document
+
+    def compose_document(self):
+        # Drop the DOCUMENT-START event.
+        self.get_event()
+
+        # Compose the root node.
+        node = self.compose_node(None, None)
+
+        # Drop the DOCUMENT-END event.
+        self.get_event()
+
+        self.anchors = {}
+        return node
+
+    def compose_node(self, parent, index):
+        if self.check_event(AliasEvent):
+            event = self.get_event()
+            anchor = event.anchor
+            if anchor not in self.anchors:
+                raise ComposerError(None, None, "found undefined alias %r"
+                        % anchor, event.start_mark)
+            return self.anchors[anchor]
+        event = self.peek_event()
+        anchor = event.anchor
+        if anchor is not None:
+            if anchor in self.anchors:
+                raise ComposerError("found duplicate anchor %r; first occurrence"
+                        % anchor, self.anchors[anchor].start_mark,
+                        "second occurrence", event.start_mark)
+        self.descend_resolver(parent, index)
+        if self.check_event(ScalarEvent):
+            node = self.compose_scalar_node(anchor)
+        elif self.check_event(SequenceStartEvent):
+            node = self.compose_sequence_node(anchor)
+        elif self.check_event(MappingStartEvent):
+            node = self.compose_mapping_node(anchor)
+        self.ascend_resolver()
+        return node
+
+    def compose_scalar_node(self, anchor):
+        event = self.get_event()
+        tag = event.tag
+        if tag is None or tag == '!':
+            tag = self.resolve(ScalarNode, event.value, event.implicit)
+        node = ScalarNode(tag, event.value,
+                event.start_mark, event.end_mark, style=event.style)
+        if anchor is not None:
+            self.anchors[anchor] = node
+        return node
+
+    def compose_sequence_node(self, anchor):
+        start_event = self.get_event()
+        tag = start_event.tag
+        if tag is None or tag == '!':
+            tag = self.resolve(SequenceNode, None, start_event.implicit)
+        node = SequenceNode(tag, [],
+                start_event.start_mark, None,
+                flow_style=start_event.flow_style)
+        if anchor is not None:
+            self.anchors[anchor] = node
+        index = 0
+        while not self.check_event(SequenceEndEvent):
+            node.value.append(self.compose_node(node, index))
+            index += 1
+        end_event = self.get_event()
+        node.end_mark = end_event.end_mark
+        return node
+
+    def compose_mapping_node(self, anchor):
+        start_event = self.get_event()
+        tag = start_event.tag
+        if tag is None or tag == '!':
+            tag = self.resolve(MappingNode, None, start_event.implicit)
+        node = MappingNode(tag, [],
+                start_event.start_mark, None,
+                flow_style=start_event.flow_style)
+        if anchor is not None:
+            self.anchors[anchor] = node
+        while not self.check_event(MappingEndEvent):
+            #key_event = self.peek_event()
+            item_key = self.compose_node(node, None)
+            #if item_key in node.value:
+            #    raise ComposerError("while composing a mapping", start_event.start_mark,
+            #            "found duplicate key", key_event.start_mark)
+            item_value = self.compose_node(node, item_key)
+            #node.value[item_key] = item_value
+            node.value.append((item_key, item_value))
+        end_event = self.get_event()
+        node.end_mark = end_event.end_mark
+        return node
+
diff --git a/ubuntu/venv/yaml/constructor.py b/ubuntu/venv/yaml/constructor.py
new file mode 100644
index 0000000..619acd3
--- /dev/null
+++ b/ubuntu/venv/yaml/constructor.py
@@ -0,0 +1,748 @@
+
+__all__ = [
+    'BaseConstructor',
+    'SafeConstructor',
+    'FullConstructor',
+    'UnsafeConstructor',
+    'Constructor',
+    'ConstructorError'
+]
+
+from .error import *
+from .nodes import *
+
+import collections.abc, datetime, base64, binascii, re, sys, types
+
+class ConstructorError(MarkedYAMLError):
+    pass
+
+class BaseConstructor:
+
+    yaml_constructors = {}
+    yaml_multi_constructors = {}
+
+    def __init__(self):
+        self.constructed_objects = {}
+        self.recursive_objects = {}
+        self.state_generators = []
+        self.deep_construct = False
+
+    def check_data(self):
+        # If there are more documents available?
+        return self.check_node()
+
+    def check_state_key(self, key):
+        """Block special attributes/methods from being set in a newly created
+        object, to prevent user-controlled methods from being called during
+        deserialization"""
+        if self.get_state_keys_blacklist_regexp().match(key):
+            raise ConstructorError(None, None,
+                "blacklisted key '%s' in instance state found" % (key,), None)
+
+    def get_data(self):
+        # Construct and return the next document.
+        if self.check_node():
+            return self.construct_document(self.get_node())
+
+    def get_single_data(self):
+        # Ensure that the stream contains a single document and construct it.
+        node = self.get_single_node()
+        if node is not None:
+            return self.construct_document(node)
+        return None
+
+    def construct_document(self, node):
+        data = self.construct_object(node)
+        while self.state_generators:
+            state_generators = self.state_generators
+            self.state_generators = []
+            for generator in state_generators:
+                for dummy in generator:
+                    pass
+        self.constructed_objects = {}
+        self.recursive_objects = {}
+        self.deep_construct = False
+        return data
+
+    def construct_object(self, node, deep=False):
+        if node in self.constructed_objects:
+            return self.constructed_objects[node]
+        if deep:
+            old_deep = self.deep_construct
+            self.deep_construct = True
+        if node in self.recursive_objects:
+            raise ConstructorError(None, None,
+                    "found unconstructable recursive node", node.start_mark)
+        self.recursive_objects[node] = None
+        constructor = None
+        tag_suffix = None
+        if node.tag in self.yaml_constructors:
+            constructor = self.yaml_constructors[node.tag]
+        else:
+            for tag_prefix in self.yaml_multi_constructors:
+                if tag_prefix is not None and node.tag.startswith(tag_prefix):
+                    tag_suffix = node.tag[len(tag_prefix):]
+                    constructor = self.yaml_multi_constructors[tag_prefix]
+                    break
+            else:
+                if None in self.yaml_multi_constructors:
+                    tag_suffix = node.tag
+                    constructor = self.yaml_multi_constructors[None]
+                elif None in self.yaml_constructors:
+                    constructor = self.yaml_constructors[None]
+                elif isinstance(node, ScalarNode):
+                    constructor = self.__class__.construct_scalar
+                elif isinstance(node, SequenceNode):
+                    constructor = self.__class__.construct_sequence
+                elif isinstance(node, MappingNode):
+                    constructor = self.__class__.construct_mapping
+        if tag_suffix is None:
+            data = constructor(self, node)
+        else:
+            data = constructor(self, tag_suffix, node)
+        if isinstance(data, types.GeneratorType):
+            generator = data
+            data = next(generator)
+            if self.deep_construct:
+                for dummy in generator:
+                    pass
+            else:
+                self.state_generators.append(generator)
+        self.constructed_objects[node] = data
+        del self.recursive_objects[node]
+        if deep:
+            self.deep_construct = old_deep
+        return data
+
+    def construct_scalar(self, node):
+        if not isinstance(node, ScalarNode):
+            raise ConstructorError(None, None,
+                    "expected a scalar node, but found %s" % node.id,
+                    node.start_mark)
+        return node.value
+
+    def construct_sequence(self, node, deep=False):
+        if not isinstance(node, SequenceNode):
+            raise ConstructorError(None, None,
+                    "expected a sequence node, but found %s" % node.id,
+                    node.start_mark)
+        return [self.construct_object(child, deep=deep)
+                for child in node.value]
+
+    def construct_mapping(self, node, deep=False):
+        if not isinstance(node, MappingNode):
+            raise ConstructorError(None, None,
+                    "expected a mapping node, but found %s" % node.id,
+                    node.start_mark)
+        mapping = {}
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if not isinstance(key, collections.abc.Hashable):
+                raise ConstructorError("while constructing a mapping", node.start_mark,
+                        "found unhashable key", key_node.start_mark)
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
+
+    def construct_pairs(self, node, deep=False):
+        if not isinstance(node, MappingNode):
+            raise ConstructorError(None, None,
+                    "expected a mapping node, but found %s" % node.id,
+                    node.start_mark)
+        pairs = []
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            value = self.construct_object(value_node, deep=deep)
+            pairs.append((key, value))
+        return pairs
+
+    @classmethod
+    def add_constructor(cls, tag, constructor):
+        if not 'yaml_constructors' in cls.__dict__:
+            cls.yaml_constructors = cls.yaml_constructors.copy()
+        cls.yaml_constructors[tag] = constructor
+
+    @classmethod
+    def add_multi_constructor(cls, tag_prefix, multi_constructor):
+        if not 'yaml_multi_constructors' in cls.__dict__:
+            cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
+        cls.yaml_multi_constructors[tag_prefix] = multi_constructor
+
+class SafeConstructor(BaseConstructor):
+
+    def construct_scalar(self, node):
+        if isinstance(node, MappingNode):
+            for key_node, value_node in node.value:
+                if key_node.tag == 'tag:yaml.org,2002:value':
+                    return self.construct_scalar(value_node)
+        return super().construct_scalar(node)
+
+    def flatten_mapping(self, node):
+        merge = []
+        index = 0
+        while index < len(node.value):
+            key_node, value_node = node.value[index]
+            if key_node.tag == 'tag:yaml.org,2002:merge':
+                del node.value[index]
+                if isinstance(value_node, MappingNode):
+                    self.flatten_mapping(value_node)
+                    merge.extend(value_node.value)
+                elif isinstance(value_node, SequenceNode):
+                    submerge = []
+                    for subnode in value_node.value:
+                        if not isinstance(subnode, MappingNode):
+                            raise ConstructorError("while constructing a mapping",
+                                    node.start_mark,
+                                    "expected a mapping for merging, but found %s"
+                                    % subnode.id, subnode.start_mark)
+                        self.flatten_mapping(subnode)
+                        submerge.append(subnode.value)
+                    submerge.reverse()
+                    for value in submerge:
+                        merge.extend(value)
+                else:
+                    raise ConstructorError("while constructing a mapping", node.start_mark,
+                            "expected a mapping or list of mappings for merging, but found %s"
+                            % value_node.id, value_node.start_mark)
+            elif key_node.tag == 'tag:yaml.org,2002:value':
+                key_node.tag = 'tag:yaml.org,2002:str'
+                index += 1
+            else:
+                index += 1
+        if merge:
+            node.value = merge + node.value
+
+    def construct_mapping(self, node, deep=False):
+        if isinstance(node, MappingNode):
+            self.flatten_mapping(node)
+        return super().construct_mapping(node, deep=deep)
+
+    def construct_yaml_null(self, node):
+        self.construct_scalar(node)
+        return None
+
+    bool_values = {
+        'yes':      True,
+        'no':       False,
+        'true':     True,
+        'false':    False,
+        'on':       True,
+        'off':      False,
+    }
+
+    def construct_yaml_bool(self, node):
+        value = self.construct_scalar(node)
+        return self.bool_values[value.lower()]
+
+    def construct_yaml_int(self, node):
+        value = self.construct_scalar(node)
+        value = value.replace('_', '')
+        sign = +1
+        if value[0] == '-':
+            sign = -1
+        if value[0] in '+-':
+            value = value[1:]
+        if value == '0':
+            return 0
+        elif value.startswith('0b'):
+            return sign*int(value[2:], 2)
+        elif value.startswith('0x'):
+            return sign*int(value[2:], 16)
+        elif value[0] == '0':
+            return sign*int(value, 8)
+        elif ':' in value:
+            digits = [int(part) for part in value.split(':')]
+            digits.reverse()
+            base = 1
+            value = 0
+            for digit in digits:
+                value += digit*base
+                base *= 60
+            return sign*value
+        else:
+            return sign*int(value)
+
+    inf_value = 1e300
+    while inf_value != inf_value*inf_value:
+        inf_value *= inf_value
+    nan_value = -inf_value/inf_value   # Trying to make a quiet NaN (like C99).
+
+    def construct_yaml_float(self, node):
+        value = self.construct_scalar(node)
+        value = value.replace('_', '').lower()
+        sign = +1
+        if value[0] == '-':
+            sign = -1
+        if value[0] in '+-':
+            value = value[1:]
+        if value == '.inf':
+            return sign*self.inf_value
+        elif value == '.nan':
+            return self.nan_value
+        elif ':' in value:
+            digits = [float(part) for part in value.split(':')]
+            digits.reverse()
+            base = 1
+            value = 0.0
+            for digit in digits:
+                value += digit*base
+                base *= 60
+            return sign*value
+        else:
+            return sign*float(value)
+
+    def construct_yaml_binary(self, node):
+        try:
+            value = self.construct_scalar(node).encode('ascii')
+        except UnicodeEncodeError as exc:
+            raise ConstructorError(None, None,
+                    "failed to convert base64 data into ascii: %s" % exc,
+                    node.start_mark)
+        try:
+            if hasattr(base64, 'decodebytes'):
+                return base64.decodebytes(value)
+            else:
+                return base64.decodestring(value)
+        except binascii.Error as exc:
+            raise ConstructorError(None, None,
+                    "failed to decode base64 data: %s" % exc, node.start_mark)
+
+    timestamp_regexp = re.compile(
+            r'''^(?P[0-9][0-9][0-9][0-9])
+                -(?P[0-9][0-9]?)
+                -(?P[0-9][0-9]?)
+                (?:(?:[Tt]|[ \t]+)
+                (?P[0-9][0-9]?)
+                :(?P[0-9][0-9])
+                :(?P[0-9][0-9])
+                (?:\.(?P[0-9]*))?
+                (?:[ \t]*(?PZ|(?P[-+])(?P[0-9][0-9]?)
+                (?::(?P[0-9][0-9]))?))?)?$''', re.X)
+
+    def construct_yaml_timestamp(self, node):
+        value = self.construct_scalar(node)
+        match = self.timestamp_regexp.match(node.value)
+        values = match.groupdict()
+        year = int(values['year'])
+        month = int(values['month'])
+        day = int(values['day'])
+        if not values['hour']:
+            return datetime.date(year, month, day)
+        hour = int(values['hour'])
+        minute = int(values['minute'])
+        second = int(values['second'])
+        fraction = 0
+        tzinfo = None
+        if values['fraction']:
+            fraction = values['fraction'][:6]
+            while len(fraction) < 6:
+                fraction += '0'
+            fraction = int(fraction)
+        if values['tz_sign']:
+            tz_hour = int(values['tz_hour'])
+            tz_minute = int(values['tz_minute'] or 0)
+            delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
+            if values['tz_sign'] == '-':
+                delta = -delta
+            tzinfo = datetime.timezone(delta)
+        elif values['tz']:
+            tzinfo = datetime.timezone.utc
+        return datetime.datetime(year, month, day, hour, minute, second, fraction,
+                                 tzinfo=tzinfo)
+
+    def construct_yaml_omap(self, node):
+        # Note: we do not check for duplicate keys, because it's too
+        # CPU-expensive.
+        omap = []
+        yield omap
+        if not isinstance(node, SequenceNode):
+            raise ConstructorError("while constructing an ordered map", node.start_mark,
+                    "expected a sequence, but found %s" % node.id, node.start_mark)
+        for subnode in node.value:
+            if not isinstance(subnode, MappingNode):
+                raise ConstructorError("while constructing an ordered map", node.start_mark,
+                        "expected a mapping of length 1, but found %s" % subnode.id,
+                        subnode.start_mark)
+            if len(subnode.value) != 1:
+                raise ConstructorError("while constructing an ordered map", node.start_mark,
+                        "expected a single mapping item, but found %d items" % len(subnode.value),
+                        subnode.start_mark)
+            key_node, value_node = subnode.value[0]
+            key = self.construct_object(key_node)
+            value = self.construct_object(value_node)
+            omap.append((key, value))
+
+    def construct_yaml_pairs(self, node):
+        # Note: the same code as `construct_yaml_omap`.
+        pairs = []
+        yield pairs
+        if not isinstance(node, SequenceNode):
+            raise ConstructorError("while constructing pairs", node.start_mark,
+                    "expected a sequence, but found %s" % node.id, node.start_mark)
+        for subnode in node.value:
+            if not isinstance(subnode, MappingNode):
+                raise ConstructorError("while constructing pairs", node.start_mark,
+                        "expected a mapping of length 1, but found %s" % subnode.id,
+                        subnode.start_mark)
+            if len(subnode.value) != 1:
+                raise ConstructorError("while constructing pairs", node.start_mark,
+                        "expected a single mapping item, but found %d items" % len(subnode.value),
+                        subnode.start_mark)
+            key_node, value_node = subnode.value[0]
+            key = self.construct_object(key_node)
+            value = self.construct_object(value_node)
+            pairs.append((key, value))
+
+    def construct_yaml_set(self, node):
+        data = set()
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_yaml_str(self, node):
+        return self.construct_scalar(node)
+
+    def construct_yaml_seq(self, node):
+        data = []
+        yield data
+        data.extend(self.construct_sequence(node))
+
+    def construct_yaml_map(self, node):
+        data = {}
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_yaml_object(self, node, cls):
+        data = cls.__new__(cls)
+        yield data
+        if hasattr(data, '__setstate__'):
+            state = self.construct_mapping(node, deep=True)
+            data.__setstate__(state)
+        else:
+            state = self.construct_mapping(node)
+            data.__dict__.update(state)
+
+    def construct_undefined(self, node):
+        raise ConstructorError(None, None,
+                "could not determine a constructor for the tag %r" % node.tag,
+                node.start_mark)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:null',
+        SafeConstructor.construct_yaml_null)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:bool',
+        SafeConstructor.construct_yaml_bool)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:int',
+        SafeConstructor.construct_yaml_int)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:float',
+        SafeConstructor.construct_yaml_float)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:binary',
+        SafeConstructor.construct_yaml_binary)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:timestamp',
+        SafeConstructor.construct_yaml_timestamp)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:omap',
+        SafeConstructor.construct_yaml_omap)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:pairs',
+        SafeConstructor.construct_yaml_pairs)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:set',
+        SafeConstructor.construct_yaml_set)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:str',
+        SafeConstructor.construct_yaml_str)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:seq',
+        SafeConstructor.construct_yaml_seq)
+
+SafeConstructor.add_constructor(
+        'tag:yaml.org,2002:map',
+        SafeConstructor.construct_yaml_map)
+
+SafeConstructor.add_constructor(None,
+        SafeConstructor.construct_undefined)
+
+class FullConstructor(SafeConstructor):
+    # 'extend' is blacklisted because it is used by
+    # construct_python_object_apply to add `listitems` to a newly generate
+    # python instance
+    def get_state_keys_blacklist(self):
+        return ['^extend$', '^__.*__$']
+
+    def get_state_keys_blacklist_regexp(self):
+        if not hasattr(self, 'state_keys_blacklist_regexp'):
+            self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')')
+        return self.state_keys_blacklist_regexp
+
+    def construct_python_str(self, node):
+        return self.construct_scalar(node)
+
+    def construct_python_unicode(self, node):
+        return self.construct_scalar(node)
+
+    def construct_python_bytes(self, node):
+        try:
+            value = self.construct_scalar(node).encode('ascii')
+        except UnicodeEncodeError as exc:
+            raise ConstructorError(None, None,
+                    "failed to convert base64 data into ascii: %s" % exc,
+                    node.start_mark)
+        try:
+            if hasattr(base64, 'decodebytes'):
+                return base64.decodebytes(value)
+            else:
+                return base64.decodestring(value)
+        except binascii.Error as exc:
+            raise ConstructorError(None, None,
+                    "failed to decode base64 data: %s" % exc, node.start_mark)
+
+    def construct_python_long(self, node):
+        return self.construct_yaml_int(node)
+
+    def construct_python_complex(self, node):
+       return complex(self.construct_scalar(node))
+
+    def construct_python_tuple(self, node):
+        return tuple(self.construct_sequence(node))
+
+    def find_python_module(self, name, mark, unsafe=False):
+        if not name:
+            raise ConstructorError("while constructing a Python module", mark,
+                    "expected non-empty name appended to the tag", mark)
+        if unsafe:
+            try:
+                __import__(name)
+            except ImportError as exc:
+                raise ConstructorError("while constructing a Python module", mark,
+                        "cannot find module %r (%s)" % (name, exc), mark)
+        if name not in sys.modules:
+            raise ConstructorError("while constructing a Python module", mark,
+                    "module %r is not imported" % name, mark)
+        return sys.modules[name]
+
+    def find_python_name(self, name, mark, unsafe=False):
+        if not name:
+            raise ConstructorError("while constructing a Python object", mark,
+                    "expected non-empty name appended to the tag", mark)
+        if '.' in name:
+            module_name, object_name = name.rsplit('.', 1)
+        else:
+            module_name = 'builtins'
+            object_name = name
+        if unsafe:
+            try:
+                __import__(module_name)
+            except ImportError as exc:
+                raise ConstructorError("while constructing a Python object", mark,
+                        "cannot find module %r (%s)" % (module_name, exc), mark)
+        if module_name not in sys.modules:
+            raise ConstructorError("while constructing a Python object", mark,
+                    "module %r is not imported" % module_name, mark)
+        module = sys.modules[module_name]
+        if not hasattr(module, object_name):
+            raise ConstructorError("while constructing a Python object", mark,
+                    "cannot find %r in the module %r"
+                    % (object_name, module.__name__), mark)
+        return getattr(module, object_name)
+
+    def construct_python_name(self, suffix, node):
+        value = self.construct_scalar(node)
+        if value:
+            raise ConstructorError("while constructing a Python name", node.start_mark,
+                    "expected the empty value, but found %r" % value, node.start_mark)
+        return self.find_python_name(suffix, node.start_mark)
+
+    def construct_python_module(self, suffix, node):
+        value = self.construct_scalar(node)
+        if value:
+            raise ConstructorError("while constructing a Python module", node.start_mark,
+                    "expected the empty value, but found %r" % value, node.start_mark)
+        return self.find_python_module(suffix, node.start_mark)
+
+    def make_python_instance(self, suffix, node,
+            args=None, kwds=None, newobj=False, unsafe=False):
+        if not args:
+            args = []
+        if not kwds:
+            kwds = {}
+        cls = self.find_python_name(suffix, node.start_mark)
+        if not (unsafe or isinstance(cls, type)):
+            raise ConstructorError("while constructing a Python instance", node.start_mark,
+                    "expected a class, but found %r" % type(cls),
+                    node.start_mark)
+        if newobj and isinstance(cls, type):
+            return cls.__new__(cls, *args, **kwds)
+        else:
+            return cls(*args, **kwds)
+
+    def set_python_instance_state(self, instance, state, unsafe=False):
+        if hasattr(instance, '__setstate__'):
+            instance.__setstate__(state)
+        else:
+            slotstate = {}
+            if isinstance(state, tuple) and len(state) == 2:
+                state, slotstate = state
+            if hasattr(instance, '__dict__'):
+                if not unsafe and state:
+                    for key in state.keys():
+                        self.check_state_key(key)
+                instance.__dict__.update(state)
+            elif state:
+                slotstate.update(state)
+            for key, value in slotstate.items():
+                if not unsafe:
+                    self.check_state_key(key)
+                setattr(instance, key, value)
+
+    def construct_python_object(self, suffix, node):
+        # Format:
+        #   !!python/object:module.name { ... state ... }
+        instance = self.make_python_instance(suffix, node, newobj=True)
+        yield instance
+        deep = hasattr(instance, '__setstate__')
+        state = self.construct_mapping(node, deep=deep)
+        self.set_python_instance_state(instance, state)
+
+    def construct_python_object_apply(self, suffix, node, newobj=False):
+        # Format:
+        #   !!python/object/apply       # (or !!python/object/new)
+        #   args: [ ... arguments ... ]
+        #   kwds: { ... keywords ... }
+        #   state: ... state ...
+        #   listitems: [ ... listitems ... ]
+        #   dictitems: { ... dictitems ... }
+        # or short format:
+        #   !!python/object/apply [ ... arguments ... ]
+        # The difference between !!python/object/apply and !!python/object/new
+        # is how an object is created, check make_python_instance for details.
+        if isinstance(node, SequenceNode):
+            args = self.construct_sequence(node, deep=True)
+            kwds = {}
+            state = {}
+            listitems = []
+            dictitems = {}
+        else:
+            value = self.construct_mapping(node, deep=True)
+            args = value.get('args', [])
+            kwds = value.get('kwds', {})
+            state = value.get('state', {})
+            listitems = value.get('listitems', [])
+            dictitems = value.get('dictitems', {})
+        instance = self.make_python_instance(suffix, node, args, kwds, newobj)
+        if state:
+            self.set_python_instance_state(instance, state)
+        if listitems:
+            instance.extend(listitems)
+        if dictitems:
+            for key in dictitems:
+                instance[key] = dictitems[key]
+        return instance
+
+    def construct_python_object_new(self, suffix, node):
+        return self.construct_python_object_apply(suffix, node, newobj=True)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/none',
+    FullConstructor.construct_yaml_null)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/bool',
+    FullConstructor.construct_yaml_bool)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/str',
+    FullConstructor.construct_python_str)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/unicode',
+    FullConstructor.construct_python_unicode)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/bytes',
+    FullConstructor.construct_python_bytes)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/int',
+    FullConstructor.construct_yaml_int)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/long',
+    FullConstructor.construct_python_long)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/float',
+    FullConstructor.construct_yaml_float)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/complex',
+    FullConstructor.construct_python_complex)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/list',
+    FullConstructor.construct_yaml_seq)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/tuple',
+    FullConstructor.construct_python_tuple)
+
+FullConstructor.add_constructor(
+    'tag:yaml.org,2002:python/dict',
+    FullConstructor.construct_yaml_map)
+
+FullConstructor.add_multi_constructor(
+    'tag:yaml.org,2002:python/name:',
+    FullConstructor.construct_python_name)
+
+class UnsafeConstructor(FullConstructor):
+
+    def find_python_module(self, name, mark):
+        return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True)
+
+    def find_python_name(self, name, mark):
+        return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True)
+
+    def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False):
+        return super(UnsafeConstructor, self).make_python_instance(
+            suffix, node, args, kwds, newobj, unsafe=True)
+
+    def set_python_instance_state(self, instance, state):
+        return super(UnsafeConstructor, self).set_python_instance_state(
+            instance, state, unsafe=True)
+
+UnsafeConstructor.add_multi_constructor(
+    'tag:yaml.org,2002:python/module:',
+    UnsafeConstructor.construct_python_module)
+
+UnsafeConstructor.add_multi_constructor(
+    'tag:yaml.org,2002:python/object:',
+    UnsafeConstructor.construct_python_object)
+
+UnsafeConstructor.add_multi_constructor(
+    'tag:yaml.org,2002:python/object/new:',
+    UnsafeConstructor.construct_python_object_new)
+
+UnsafeConstructor.add_multi_constructor(
+    'tag:yaml.org,2002:python/object/apply:',
+    UnsafeConstructor.construct_python_object_apply)
+
+# Constructor is same as UnsafeConstructor. Need to leave this in place in case
+# people have extended it directly.
+class Constructor(UnsafeConstructor):
+    pass
diff --git a/ubuntu/venv/yaml/cyaml.py b/ubuntu/venv/yaml/cyaml.py
new file mode 100644
index 0000000..0c21345
--- /dev/null
+++ b/ubuntu/venv/yaml/cyaml.py
@@ -0,0 +1,101 @@
+
+__all__ = [
+    'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader',
+    'CBaseDumper', 'CSafeDumper', 'CDumper'
+]
+
+from yaml._yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        BaseConstructor.__init__(self)
+        BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        SafeConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class CFullLoader(CParser, FullConstructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        FullConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class CUnsafeLoader(CParser, UnsafeConstructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        UnsafeConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        Constructor.__init__(self)
+        Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        SafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
diff --git a/ubuntu/venv/yaml/dumper.py b/ubuntu/venv/yaml/dumper.py
new file mode 100644
index 0000000..6aadba5
--- /dev/null
+++ b/ubuntu/venv/yaml/dumper.py
@@ -0,0 +1,62 @@
+
+__all__ = ['BaseDumper', 'SafeDumper', 'Dumper']
+
+from .emitter import *
+from .serializer import *
+from .representer import *
+from .resolver import *
+
+class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
+class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        SafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
+class Dumper(Emitter, Serializer, Representer, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=False,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None, sort_keys=True):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style, sort_keys=sort_keys)
+        Resolver.__init__(self)
+
diff --git a/ubuntu/venv/yaml/emitter.py b/ubuntu/venv/yaml/emitter.py
new file mode 100644
index 0000000..a664d01
--- /dev/null
+++ b/ubuntu/venv/yaml/emitter.py
@@ -0,0 +1,1137 @@
+
+# Emitter expects events obeying the following grammar:
+# stream ::= STREAM-START document* STREAM-END
+# document ::= DOCUMENT-START node DOCUMENT-END
+# node ::= SCALAR | sequence | mapping
+# sequence ::= SEQUENCE-START node* SEQUENCE-END
+# mapping ::= MAPPING-START (node node)* MAPPING-END
+
+__all__ = ['Emitter', 'EmitterError']
+
+from .error import YAMLError
+from .events import *
+
+class EmitterError(YAMLError):
+    pass
+
+class ScalarAnalysis:
+    def __init__(self, scalar, empty, multiline,
+            allow_flow_plain, allow_block_plain,
+            allow_single_quoted, allow_double_quoted,
+            allow_block):
+        self.scalar = scalar
+        self.empty = empty
+        self.multiline = multiline
+        self.allow_flow_plain = allow_flow_plain
+        self.allow_block_plain = allow_block_plain
+        self.allow_single_quoted = allow_single_quoted
+        self.allow_double_quoted = allow_double_quoted
+        self.allow_block = allow_block
+
+class Emitter:
+
+    DEFAULT_TAG_PREFIXES = {
+        '!' : '!',
+        'tag:yaml.org,2002:' : '!!',
+    }
+
+    def __init__(self, stream, canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None):
+
+        # The stream should have the methods `write` and possibly `flush`.
+        self.stream = stream
+
+        # Encoding can be overridden by STREAM-START.
+        self.encoding = None
+
+        # Emitter is a state machine with a stack of states to handle nested
+        # structures.
+        self.states = []
+        self.state = self.expect_stream_start
+
+        # Current event and the event queue.
+        self.events = []
+        self.event = None
+
+        # The current indentation level and the stack of previous indents.
+        self.indents = []
+        self.indent = None
+
+        # Flow level.
+        self.flow_level = 0
+
+        # Contexts.
+        self.root_context = False
+        self.sequence_context = False
+        self.mapping_context = False
+        self.simple_key_context = False
+
+        # Characteristics of the last emitted character:
+        #  - current position.
+        #  - is it a whitespace?
+        #  - is it an indention character
+        #    (indentation space, '-', '?', or ':')?
+        self.line = 0
+        self.column = 0
+        self.whitespace = True
+        self.indention = True
+
+        # Whether the document requires an explicit document indicator
+        self.open_ended = False
+
+        # Formatting details.
+        self.canonical = canonical
+        self.allow_unicode = allow_unicode
+        self.best_indent = 2
+        if indent and 1 < indent < 10:
+            self.best_indent = indent
+        self.best_width = 80
+        if width and width > self.best_indent*2:
+            self.best_width = width
+        self.best_line_break = '\n'
+        if line_break in ['\r', '\n', '\r\n']:
+            self.best_line_break = line_break
+
+        # Tag prefixes.
+        self.tag_prefixes = None
+
+        # Prepared anchor and tag.
+        self.prepared_anchor = None
+        self.prepared_tag = None
+
+        # Scalar analysis and style.
+        self.analysis = None
+        self.style = None
+
+    def dispose(self):
+        # Reset the state attributes (to clear self-references)
+        self.states = []
+        self.state = None
+
+    def emit(self, event):
+        self.events.append(event)
+        while not self.need_more_events():
+            self.event = self.events.pop(0)
+            self.state()
+            self.event = None
+
+    # In some cases, we wait for a few next events before emitting.
+
+    def need_more_events(self):
+        if not self.events:
+            return True
+        event = self.events[0]
+        if isinstance(event, DocumentStartEvent):
+            return self.need_events(1)
+        elif isinstance(event, SequenceStartEvent):
+            return self.need_events(2)
+        elif isinstance(event, MappingStartEvent):
+            return self.need_events(3)
+        else:
+            return False
+
+    def need_events(self, count):
+        level = 0
+        for event in self.events[1:]:
+            if isinstance(event, (DocumentStartEvent, CollectionStartEvent)):
+                level += 1
+            elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)):
+                level -= 1
+            elif isinstance(event, StreamEndEvent):
+                level = -1
+            if level < 0:
+                return False
+        return (len(self.events) < count+1)
+
+    def increase_indent(self, flow=False, indentless=False):
+        self.indents.append(self.indent)
+        if self.indent is None:
+            if flow:
+                self.indent = self.best_indent
+            else:
+                self.indent = 0
+        elif not indentless:
+            self.indent += self.best_indent
+
+    # States.
+
+    # Stream handlers.
+
+    def expect_stream_start(self):
+        if isinstance(self.event, StreamStartEvent):
+            if self.event.encoding and not hasattr(self.stream, 'encoding'):
+                self.encoding = self.event.encoding
+            self.write_stream_start()
+            self.state = self.expect_first_document_start
+        else:
+            raise EmitterError("expected StreamStartEvent, but got %s"
+                    % self.event)
+
+    def expect_nothing(self):
+        raise EmitterError("expected nothing, but got %s" % self.event)
+
+    # Document handlers.
+
+    def expect_first_document_start(self):
+        return self.expect_document_start(first=True)
+
+    def expect_document_start(self, first=False):
+        if isinstance(self.event, DocumentStartEvent):
+            if (self.event.version or self.event.tags) and self.open_ended:
+                self.write_indicator('...', True)
+                self.write_indent()
+            if self.event.version:
+                version_text = self.prepare_version(self.event.version)
+                self.write_version_directive(version_text)
+            self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
+            if self.event.tags:
+                handles = sorted(self.event.tags.keys())
+                for handle in handles:
+                    prefix = self.event.tags[handle]
+                    self.tag_prefixes[prefix] = handle
+                    handle_text = self.prepare_tag_handle(handle)
+                    prefix_text = self.prepare_tag_prefix(prefix)
+                    self.write_tag_directive(handle_text, prefix_text)
+            implicit = (first and not self.event.explicit and not self.canonical
+                    and not self.event.version and not self.event.tags
+                    and not self.check_empty_document())
+            if not implicit:
+                self.write_indent()
+                self.write_indicator('---', True)
+                if self.canonical:
+                    self.write_indent()
+            self.state = self.expect_document_root
+        elif isinstance(self.event, StreamEndEvent):
+            if self.open_ended:
+                self.write_indicator('...', True)
+                self.write_indent()
+            self.write_stream_end()
+            self.state = self.expect_nothing
+        else:
+            raise EmitterError("expected DocumentStartEvent, but got %s"
+                    % self.event)
+
+    def expect_document_end(self):
+        if isinstance(self.event, DocumentEndEvent):
+            self.write_indent()
+            if self.event.explicit:
+                self.write_indicator('...', True)
+                self.write_indent()
+            self.flush_stream()
+            self.state = self.expect_document_start
+        else:
+            raise EmitterError("expected DocumentEndEvent, but got %s"
+                    % self.event)
+
+    def expect_document_root(self):
+        self.states.append(self.expect_document_end)
+        self.expect_node(root=True)
+
+    # Node handlers.
+
+    def expect_node(self, root=False, sequence=False, mapping=False,
+            simple_key=False):
+        self.root_context = root
+        self.sequence_context = sequence
+        self.mapping_context = mapping
+        self.simple_key_context = simple_key
+        if isinstance(self.event, AliasEvent):
+            self.expect_alias()
+        elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)):
+            self.process_anchor('&')
+            self.process_tag()
+            if isinstance(self.event, ScalarEvent):
+                self.expect_scalar()
+            elif isinstance(self.event, SequenceStartEvent):
+                if self.flow_level or self.canonical or self.event.flow_style   \
+                        or self.check_empty_sequence():
+                    self.expect_flow_sequence()
+                else:
+                    self.expect_block_sequence()
+            elif isinstance(self.event, MappingStartEvent):
+                if self.flow_level or self.canonical or self.event.flow_style   \
+                        or self.check_empty_mapping():
+                    self.expect_flow_mapping()
+                else:
+                    self.expect_block_mapping()
+        else:
+            raise EmitterError("expected NodeEvent, but got %s" % self.event)
+
+    def expect_alias(self):
+        if self.event.anchor is None:
+            raise EmitterError("anchor is not specified for alias")
+        self.process_anchor('*')
+        self.state = self.states.pop()
+
+    def expect_scalar(self):
+        self.increase_indent(flow=True)
+        self.process_scalar()
+        self.indent = self.indents.pop()
+        self.state = self.states.pop()
+
+    # Flow sequence handlers.
+
+    def expect_flow_sequence(self):
+        self.write_indicator('[', True, whitespace=True)
+        self.flow_level += 1
+        self.increase_indent(flow=True)
+        self.state = self.expect_first_flow_sequence_item
+
+    def expect_first_flow_sequence_item(self):
+        if isinstance(self.event, SequenceEndEvent):
+            self.indent = self.indents.pop()
+            self.flow_level -= 1
+            self.write_indicator(']', False)
+            self.state = self.states.pop()
+        else:
+            if self.canonical or self.column > self.best_width:
+                self.write_indent()
+            self.states.append(self.expect_flow_sequence_item)
+            self.expect_node(sequence=True)
+
+    def expect_flow_sequence_item(self):
+        if isinstance(self.event, SequenceEndEvent):
+            self.indent = self.indents.pop()
+            self.flow_level -= 1
+            if self.canonical:
+                self.write_indicator(',', False)
+                self.write_indent()
+            self.write_indicator(']', False)
+            self.state = self.states.pop()
+        else:
+            self.write_indicator(',', False)
+            if self.canonical or self.column > self.best_width:
+                self.write_indent()
+            self.states.append(self.expect_flow_sequence_item)
+            self.expect_node(sequence=True)
+
+    # Flow mapping handlers.
+
+    def expect_flow_mapping(self):
+        self.write_indicator('{', True, whitespace=True)
+        self.flow_level += 1
+        self.increase_indent(flow=True)
+        self.state = self.expect_first_flow_mapping_key
+
+    def expect_first_flow_mapping_key(self):
+        if isinstance(self.event, MappingEndEvent):
+            self.indent = self.indents.pop()
+            self.flow_level -= 1
+            self.write_indicator('}', False)
+            self.state = self.states.pop()
+        else:
+            if self.canonical or self.column > self.best_width:
+                self.write_indent()
+            if not self.canonical and self.check_simple_key():
+                self.states.append(self.expect_flow_mapping_simple_value)
+                self.expect_node(mapping=True, simple_key=True)
+            else:
+                self.write_indicator('?', True)
+                self.states.append(self.expect_flow_mapping_value)
+                self.expect_node(mapping=True)
+
+    def expect_flow_mapping_key(self):
+        if isinstance(self.event, MappingEndEvent):
+            self.indent = self.indents.pop()
+            self.flow_level -= 1
+            if self.canonical:
+                self.write_indicator(',', False)
+                self.write_indent()
+            self.write_indicator('}', False)
+            self.state = self.states.pop()
+        else:
+            self.write_indicator(',', False)
+            if self.canonical or self.column > self.best_width:
+                self.write_indent()
+            if not self.canonical and self.check_simple_key():
+                self.states.append(self.expect_flow_mapping_simple_value)
+                self.expect_node(mapping=True, simple_key=True)
+            else:
+                self.write_indicator('?', True)
+                self.states.append(self.expect_flow_mapping_value)
+                self.expect_node(mapping=True)
+
+    def expect_flow_mapping_simple_value(self):
+        self.write_indicator(':', False)
+        self.states.append(self.expect_flow_mapping_key)
+        self.expect_node(mapping=True)
+
+    def expect_flow_mapping_value(self):
+        if self.canonical or self.column > self.best_width:
+            self.write_indent()
+        self.write_indicator(':', True)
+        self.states.append(self.expect_flow_mapping_key)
+        self.expect_node(mapping=True)
+
+    # Block sequence handlers.
+
+    def expect_block_sequence(self):
+        indentless = (self.mapping_context and not self.indention)
+        self.increase_indent(flow=False, indentless=indentless)
+        self.state = self.expect_first_block_sequence_item
+
+    def expect_first_block_sequence_item(self):
+        return self.expect_block_sequence_item(first=True)
+
+    def expect_block_sequence_item(self, first=False):
+        if not first and isinstance(self.event, SequenceEndEvent):
+            self.indent = self.indents.pop()
+            self.state = self.states.pop()
+        else:
+            self.write_indent()
+            self.write_indicator('-', True, indention=True)
+            self.states.append(self.expect_block_sequence_item)
+            self.expect_node(sequence=True)
+
+    # Block mapping handlers.
+
+    def expect_block_mapping(self):
+        self.increase_indent(flow=False)
+        self.state = self.expect_first_block_mapping_key
+
+    def expect_first_block_mapping_key(self):
+        return self.expect_block_mapping_key(first=True)
+
+    def expect_block_mapping_key(self, first=False):
+        if not first and isinstance(self.event, MappingEndEvent):
+            self.indent = self.indents.pop()
+            self.state = self.states.pop()
+        else:
+            self.write_indent()
+            if self.check_simple_key():
+                self.states.append(self.expect_block_mapping_simple_value)
+                self.expect_node(mapping=True, simple_key=True)
+            else:
+                self.write_indicator('?', True, indention=True)
+                self.states.append(self.expect_block_mapping_value)
+                self.expect_node(mapping=True)
+
+    def expect_block_mapping_simple_value(self):
+        self.write_indicator(':', False)
+        self.states.append(self.expect_block_mapping_key)
+        self.expect_node(mapping=True)
+
+    def expect_block_mapping_value(self):
+        self.write_indent()
+        self.write_indicator(':', True, indention=True)
+        self.states.append(self.expect_block_mapping_key)
+        self.expect_node(mapping=True)
+
+    # Checkers.
+
+    def check_empty_sequence(self):
+        return (isinstance(self.event, SequenceStartEvent) and self.events
+                and isinstance(self.events[0], SequenceEndEvent))
+
+    def check_empty_mapping(self):
+        return (isinstance(self.event, MappingStartEvent) and self.events
+                and isinstance(self.events[0], MappingEndEvent))
+
+    def check_empty_document(self):
+        if not isinstance(self.event, DocumentStartEvent) or not self.events:
+            return False
+        event = self.events[0]
+        return (isinstance(event, ScalarEvent) and event.anchor is None
+                and event.tag is None and event.implicit and event.value == '')
+
+    def check_simple_key(self):
+        length = 0
+        if isinstance(self.event, NodeEvent) and self.event.anchor is not None:
+            if self.prepared_anchor is None:
+                self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+            length += len(self.prepared_anchor)
+        if isinstance(self.event, (ScalarEvent, CollectionStartEvent))  \
+                and self.event.tag is not None:
+            if self.prepared_tag is None:
+                self.prepared_tag = self.prepare_tag(self.event.tag)
+            length += len(self.prepared_tag)
+        if isinstance(self.event, ScalarEvent):
+            if self.analysis is None:
+                self.analysis = self.analyze_scalar(self.event.value)
+            length += len(self.analysis.scalar)
+        return (length < 128 and (isinstance(self.event, AliasEvent)
+            or (isinstance(self.event, ScalarEvent)
+                    and not self.analysis.empty and not self.analysis.multiline)
+            or self.check_empty_sequence() or self.check_empty_mapping()))
+
+    # Anchor, Tag, and Scalar processors.
+
+    def process_anchor(self, indicator):
+        if self.event.anchor is None:
+            self.prepared_anchor = None
+            return
+        if self.prepared_anchor is None:
+            self.prepared_anchor = self.prepare_anchor(self.event.anchor)
+        if self.prepared_anchor:
+            self.write_indicator(indicator+self.prepared_anchor, True)
+        self.prepared_anchor = None
+
+    def process_tag(self):
+        tag = self.event.tag
+        if isinstance(self.event, ScalarEvent):
+            if self.style is None:
+                self.style = self.choose_scalar_style()
+            if ((not self.canonical or tag is None) and
+                ((self.style == '' and self.event.implicit[0])
+                        or (self.style != '' and self.event.implicit[1]))):
+                self.prepared_tag = None
+                return
+            if self.event.implicit[0] and tag is None:
+                tag = '!'
+                self.prepared_tag = None
+        else:
+            if (not self.canonical or tag is None) and self.event.implicit:
+                self.prepared_tag = None
+                return
+        if tag is None:
+            raise EmitterError("tag is not specified")
+        if self.prepared_tag is None:
+            self.prepared_tag = self.prepare_tag(tag)
+        if self.prepared_tag:
+            self.write_indicator(self.prepared_tag, True)
+        self.prepared_tag = None
+
+    def choose_scalar_style(self):
+        if self.analysis is None:
+            self.analysis = self.analyze_scalar(self.event.value)
+        if self.event.style == '"' or self.canonical:
+            return '"'
+        if not self.event.style and self.event.implicit[0]:
+            if (not (self.simple_key_context and
+                    (self.analysis.empty or self.analysis.multiline))
+                and (self.flow_level and self.analysis.allow_flow_plain
+                    or (not self.flow_level and self.analysis.allow_block_plain))):
+                return ''
+        if self.event.style and self.event.style in '|>':
+            if (not self.flow_level and not self.simple_key_context
+                    and self.analysis.allow_block):
+                return self.event.style
+        if not self.event.style or self.event.style == '\'':
+            if (self.analysis.allow_single_quoted and
+                    not (self.simple_key_context and self.analysis.multiline)):
+                return '\''
+        return '"'
+
+    def process_scalar(self):
+        if self.analysis is None:
+            self.analysis = self.analyze_scalar(self.event.value)
+        if self.style is None:
+            self.style = self.choose_scalar_style()
+        split = (not self.simple_key_context)
+        #if self.analysis.multiline and split    \
+        #        and (not self.style or self.style in '\'\"'):
+        #    self.write_indent()
+        if self.style == '"':
+            self.write_double_quoted(self.analysis.scalar, split)
+        elif self.style == '\'':
+            self.write_single_quoted(self.analysis.scalar, split)
+        elif self.style == '>':
+            self.write_folded(self.analysis.scalar)
+        elif self.style == '|':
+            self.write_literal(self.analysis.scalar)
+        else:
+            self.write_plain(self.analysis.scalar, split)
+        self.analysis = None
+        self.style = None
+
+    # Analyzers.
+
+    def prepare_version(self, version):
+        major, minor = version
+        if major != 1:
+            raise EmitterError("unsupported YAML version: %d.%d" % (major, minor))
+        return '%d.%d' % (major, minor)
+
+    def prepare_tag_handle(self, handle):
+        if not handle:
+            raise EmitterError("tag handle must not be empty")
+        if handle[0] != '!' or handle[-1] != '!':
+            raise EmitterError("tag handle must start and end with '!': %r" % handle)
+        for ch in handle[1:-1]:
+            if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'    \
+                    or ch in '-_'):
+                raise EmitterError("invalid character %r in the tag handle: %r"
+                        % (ch, handle))
+        return handle
+
+    def prepare_tag_prefix(self, prefix):
+        if not prefix:
+            raise EmitterError("tag prefix must not be empty")
+        chunks = []
+        start = end = 0
+        if prefix[0] == '!':
+            end = 1
+        while end < len(prefix):
+            ch = prefix[end]
+            if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+                    or ch in '-;/?!:@&=+$,_.~*\'()[]':
+                end += 1
+            else:
+                if start < end:
+                    chunks.append(prefix[start:end])
+                start = end = end+1
+                data = ch.encode('utf-8')
+                for ch in data:
+                    chunks.append('%%%02X' % ord(ch))
+        if start < end:
+            chunks.append(prefix[start:end])
+        return ''.join(chunks)
+
+    def prepare_tag(self, tag):
+        if not tag:
+            raise EmitterError("tag must not be empty")
+        if tag == '!':
+            return tag
+        handle = None
+        suffix = tag
+        prefixes = sorted(self.tag_prefixes.keys())
+        for prefix in prefixes:
+            if tag.startswith(prefix)   \
+                    and (prefix == '!' or len(prefix) < len(tag)):
+                handle = self.tag_prefixes[prefix]
+                suffix = tag[len(prefix):]
+        chunks = []
+        start = end = 0
+        while end < len(suffix):
+            ch = suffix[end]
+            if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \
+                    or ch in '-;/?:@&=+$,_.~*\'()[]'   \
+                    or (ch == '!' and handle != '!'):
+                end += 1
+            else:
+                if start < end:
+                    chunks.append(suffix[start:end])
+                start = end = end+1
+                data = ch.encode('utf-8')
+                for ch in data:
+                    chunks.append('%%%02X' % ch)
+        if start < end:
+            chunks.append(suffix[start:end])
+        suffix_text = ''.join(chunks)
+        if handle:
+            return '%s%s' % (handle, suffix_text)
+        else:
+            return '!<%s>' % suffix_text
+
+    def prepare_anchor(self, anchor):
+        if not anchor:
+            raise EmitterError("anchor must not be empty")
+        for ch in anchor:
+            if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'    \
+                    or ch in '-_'):
+                raise EmitterError("invalid character %r in the anchor: %r"
+                        % (ch, anchor))
+        return anchor
+
+    def analyze_scalar(self, scalar):
+
+        # Empty scalar is a special case.
+        if not scalar:
+            return ScalarAnalysis(scalar=scalar, empty=True, multiline=False,
+                    allow_flow_plain=False, allow_block_plain=True,
+                    allow_single_quoted=True, allow_double_quoted=True,
+                    allow_block=False)
+
+        # Indicators and special characters.
+        block_indicators = False
+        flow_indicators = False
+        line_breaks = False
+        special_characters = False
+
+        # Important whitespace combinations.
+        leading_space = False
+        leading_break = False
+        trailing_space = False
+        trailing_break = False
+        break_space = False
+        space_break = False
+
+        # Check document indicators.
+        if scalar.startswith('---') or scalar.startswith('...'):
+            block_indicators = True
+            flow_indicators = True
+
+        # First character or preceded by a whitespace.
+        preceded_by_whitespace = True
+
+        # Last character or followed by a whitespace.
+        followed_by_whitespace = (len(scalar) == 1 or
+                scalar[1] in '\0 \t\r\n\x85\u2028\u2029')
+
+        # The previous character is a space.
+        previous_space = False
+
+        # The previous character is a break.
+        previous_break = False
+
+        index = 0
+        while index < len(scalar):
+            ch = scalar[index]
+
+            # Check for indicators.
+            if index == 0:
+                # Leading indicators are special characters.
+                if ch in '#,[]{}&*!|>\'\"%@`':
+                    flow_indicators = True
+                    block_indicators = True
+                if ch in '?:':
+                    flow_indicators = True
+                    if followed_by_whitespace:
+                        block_indicators = True
+                if ch == '-' and followed_by_whitespace:
+                    flow_indicators = True
+                    block_indicators = True
+            else:
+                # Some indicators cannot appear within a scalar as well.
+                if ch in ',?[]{}':
+                    flow_indicators = True
+                if ch == ':':
+                    flow_indicators = True
+                    if followed_by_whitespace:
+                        block_indicators = True
+                if ch == '#' and preceded_by_whitespace:
+                    flow_indicators = True
+                    block_indicators = True
+
+            # Check for line breaks, special, and unicode characters.
+            if ch in '\n\x85\u2028\u2029':
+                line_breaks = True
+            if not (ch == '\n' or '\x20' <= ch <= '\x7E'):
+                if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF'
+                        or '\uE000' <= ch <= '\uFFFD'
+                        or '\U00010000' <= ch < '\U0010ffff') and ch != '\uFEFF':
+                    unicode_characters = True
+                    if not self.allow_unicode:
+                        special_characters = True
+                else:
+                    special_characters = True
+
+            # Detect important whitespace combinations.
+            if ch == ' ':
+                if index == 0:
+                    leading_space = True
+                if index == len(scalar)-1:
+                    trailing_space = True
+                if previous_break:
+                    break_space = True
+                previous_space = True
+                previous_break = False
+            elif ch in '\n\x85\u2028\u2029':
+                if index == 0:
+                    leading_break = True
+                if index == len(scalar)-1:
+                    trailing_break = True
+                if previous_space:
+                    space_break = True
+                previous_space = False
+                previous_break = True
+            else:
+                previous_space = False
+                previous_break = False
+
+            # Prepare for the next character.
+            index += 1
+            preceded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029')
+            followed_by_whitespace = (index+1 >= len(scalar) or
+                    scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029')
+
+        # Let's decide what styles are allowed.
+        allow_flow_plain = True
+        allow_block_plain = True
+        allow_single_quoted = True
+        allow_double_quoted = True
+        allow_block = True
+
+        # Leading and trailing whitespaces are bad for plain scalars.
+        if (leading_space or leading_break
+                or trailing_space or trailing_break):
+            allow_flow_plain = allow_block_plain = False
+
+        # We do not permit trailing spaces for block scalars.
+        if trailing_space:
+            allow_block = False
+
+        # Spaces at the beginning of a new line are only acceptable for block
+        # scalars.
+        if break_space:
+            allow_flow_plain = allow_block_plain = allow_single_quoted = False
+
+        # Spaces followed by breaks, as well as special character are only
+        # allowed for double quoted scalars.
+        if space_break or special_characters:
+            allow_flow_plain = allow_block_plain =  \
+            allow_single_quoted = allow_block = False
+
+        # Although the plain scalar writer supports breaks, we never emit
+        # multiline plain scalars.
+        if line_breaks:
+            allow_flow_plain = allow_block_plain = False
+
+        # Flow indicators are forbidden for flow plain scalars.
+        if flow_indicators:
+            allow_flow_plain = False
+
+        # Block indicators are forbidden for block plain scalars.
+        if block_indicators:
+            allow_block_plain = False
+
+        return ScalarAnalysis(scalar=scalar,
+                empty=False, multiline=line_breaks,
+                allow_flow_plain=allow_flow_plain,
+                allow_block_plain=allow_block_plain,
+                allow_single_quoted=allow_single_quoted,
+                allow_double_quoted=allow_double_quoted,
+                allow_block=allow_block)
+
+    # Writers.
+
+    def flush_stream(self):
+        if hasattr(self.stream, 'flush'):
+            self.stream.flush()
+
+    def write_stream_start(self):
+        # Write BOM if needed.
+        if self.encoding and self.encoding.startswith('utf-16'):
+            self.stream.write('\uFEFF'.encode(self.encoding))
+
+    def write_stream_end(self):
+        self.flush_stream()
+
+    def write_indicator(self, indicator, need_whitespace,
+            whitespace=False, indention=False):
+        if self.whitespace or not need_whitespace:
+            data = indicator
+        else:
+            data = ' '+indicator
+        self.whitespace = whitespace
+        self.indention = self.indention and indention
+        self.column += len(data)
+        self.open_ended = False
+        if self.encoding:
+            data = data.encode(self.encoding)
+        self.stream.write(data)
+
+    def write_indent(self):
+        indent = self.indent or 0
+        if not self.indention or self.column > indent   \
+                or (self.column == indent and not self.whitespace):
+            self.write_line_break()
+        if self.column < indent:
+            self.whitespace = True
+            data = ' '*(indent-self.column)
+            self.column = indent
+            if self.encoding:
+                data = data.encode(self.encoding)
+            self.stream.write(data)
+
+    def write_line_break(self, data=None):
+        if data is None:
+            data = self.best_line_break
+        self.whitespace = True
+        self.indention = True
+        self.line += 1
+        self.column = 0
+        if self.encoding:
+            data = data.encode(self.encoding)
+        self.stream.write(data)
+
+    def write_version_directive(self, version_text):
+        data = '%%YAML %s' % version_text
+        if self.encoding:
+            data = data.encode(self.encoding)
+        self.stream.write(data)
+        self.write_line_break()
+
+    def write_tag_directive(self, handle_text, prefix_text):
+        data = '%%TAG %s %s' % (handle_text, prefix_text)
+        if self.encoding:
+            data = data.encode(self.encoding)
+        self.stream.write(data)
+        self.write_line_break()
+
+    # Scalar streams.
+
+    def write_single_quoted(self, text, split=True):
+        self.write_indicator('\'', True)
+        spaces = False
+        breaks = False
+        start = end = 0
+        while end <= len(text):
+            ch = None
+            if end < len(text):
+                ch = text[end]
+            if spaces:
+                if ch is None or ch != ' ':
+                    if start+1 == end and self.column > self.best_width and split   \
+                            and start != 0 and end != len(text):
+                        self.write_indent()
+                    else:
+                        data = text[start:end]
+                        self.column += len(data)
+                        if self.encoding:
+                            data = data.encode(self.encoding)
+                        self.stream.write(data)
+                    start = end
+            elif breaks:
+                if ch is None or ch not in '\n\x85\u2028\u2029':
+                    if text[start] == '\n':
+                        self.write_line_break()
+                    for br in text[start:end]:
+                        if br == '\n':
+                            self.write_line_break()
+                        else:
+                            self.write_line_break(br)
+                    self.write_indent()
+                    start = end
+            else:
+                if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'':
+                    if start < end:
+                        data = text[start:end]
+                        self.column += len(data)
+                        if self.encoding:
+                            data = data.encode(self.encoding)
+                        self.stream.write(data)
+                        start = end
+            if ch == '\'':
+                data = '\'\''
+                self.column += 2
+                if self.encoding:
+                    data = data.encode(self.encoding)
+                self.stream.write(data)
+                start = end + 1
+            if ch is not None:
+                spaces = (ch == ' ')
+                breaks = (ch in '\n\x85\u2028\u2029')
+            end += 1
+        self.write_indicator('\'', False)
+
+    ESCAPE_REPLACEMENTS = {
+        '\0':       '0',
+        '\x07':     'a',
+        '\x08':     'b',
+        '\x09':     't',
+        '\x0A':     'n',
+        '\x0B':     'v',
+        '\x0C':     'f',
+        '\x0D':     'r',
+        '\x1B':     'e',
+        '\"':       '\"',
+        '\\':       '\\',
+        '\x85':     'N',
+        '\xA0':     '_',
+        '\u2028':   'L',
+        '\u2029':   'P',
+    }
+
+    def write_double_quoted(self, text, split=True):
+        self.write_indicator('"', True)
+        start = end = 0
+        while end <= len(text):
+            ch = None
+            if end < len(text):
+                ch = text[end]
+            if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \
+                    or not ('\x20' <= ch <= '\x7E'
+                        or (self.allow_unicode
+                            and ('\xA0' <= ch <= '\uD7FF'
+                                or '\uE000' <= ch <= '\uFFFD'))):
+                if start < end:
+                    data = text[start:end]
+                    self.column += len(data)
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+                    start = end
+                if ch is not None:
+                    if ch in self.ESCAPE_REPLACEMENTS:
+                        data = '\\'+self.ESCAPE_REPLACEMENTS[ch]
+                    elif ch <= '\xFF':
+                        data = '\\x%02X' % ord(ch)
+                    elif ch <= '\uFFFF':
+                        data = '\\u%04X' % ord(ch)
+                    else:
+                        data = '\\U%08X' % ord(ch)
+                    self.column += len(data)
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+                    start = end+1
+            if 0 < end < len(text)-1 and (ch == ' ' or start >= end)    \
+                    and self.column+(end-start) > self.best_width and split:
+                data = text[start:end]+'\\'
+                if start < end:
+                    start = end
+                self.column += len(data)
+                if self.encoding:
+                    data = data.encode(self.encoding)
+                self.stream.write(data)
+                self.write_indent()
+                self.whitespace = False
+                self.indention = False
+                if text[start] == ' ':
+                    data = '\\'
+                    self.column += len(data)
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+            end += 1
+        self.write_indicator('"', False)
+
+    def determine_block_hints(self, text):
+        hints = ''
+        if text:
+            if text[0] in ' \n\x85\u2028\u2029':
+                hints += str(self.best_indent)
+            if text[-1] not in '\n\x85\u2028\u2029':
+                hints += '-'
+            elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029':
+                hints += '+'
+        return hints
+
+    def write_folded(self, text):
+        hints = self.determine_block_hints(text)
+        self.write_indicator('>'+hints, True)
+        if hints[-1:] == '+':
+            self.open_ended = True
+        self.write_line_break()
+        leading_space = True
+        spaces = False
+        breaks = True
+        start = end = 0
+        while end <= len(text):
+            ch = None
+            if end < len(text):
+                ch = text[end]
+            if breaks:
+                if ch is None or ch not in '\n\x85\u2028\u2029':
+                    if not leading_space and ch is not None and ch != ' '   \
+                            and text[start] == '\n':
+                        self.write_line_break()
+                    leading_space = (ch == ' ')
+                    for br in text[start:end]:
+                        if br == '\n':
+                            self.write_line_break()
+                        else:
+                            self.write_line_break(br)
+                    if ch is not None:
+                        self.write_indent()
+                    start = end
+            elif spaces:
+                if ch != ' ':
+                    if start+1 == end and self.column > self.best_width:
+                        self.write_indent()
+                    else:
+                        data = text[start:end]
+                        self.column += len(data)
+                        if self.encoding:
+                            data = data.encode(self.encoding)
+                        self.stream.write(data)
+                    start = end
+            else:
+                if ch is None or ch in ' \n\x85\u2028\u2029':
+                    data = text[start:end]
+                    self.column += len(data)
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+                    if ch is None:
+                        self.write_line_break()
+                    start = end
+            if ch is not None:
+                breaks = (ch in '\n\x85\u2028\u2029')
+                spaces = (ch == ' ')
+            end += 1
+
+    def write_literal(self, text):
+        hints = self.determine_block_hints(text)
+        self.write_indicator('|'+hints, True)
+        if hints[-1:] == '+':
+            self.open_ended = True
+        self.write_line_break()
+        breaks = True
+        start = end = 0
+        while end <= len(text):
+            ch = None
+            if end < len(text):
+                ch = text[end]
+            if breaks:
+                if ch is None or ch not in '\n\x85\u2028\u2029':
+                    for br in text[start:end]:
+                        if br == '\n':
+                            self.write_line_break()
+                        else:
+                            self.write_line_break(br)
+                    if ch is not None:
+                        self.write_indent()
+                    start = end
+            else:
+                if ch is None or ch in '\n\x85\u2028\u2029':
+                    data = text[start:end]
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+                    if ch is None:
+                        self.write_line_break()
+                    start = end
+            if ch is not None:
+                breaks = (ch in '\n\x85\u2028\u2029')
+            end += 1
+
+    def write_plain(self, text, split=True):
+        if self.root_context:
+            self.open_ended = True
+        if not text:
+            return
+        if not self.whitespace:
+            data = ' '
+            self.column += len(data)
+            if self.encoding:
+                data = data.encode(self.encoding)
+            self.stream.write(data)
+        self.whitespace = False
+        self.indention = False
+        spaces = False
+        breaks = False
+        start = end = 0
+        while end <= len(text):
+            ch = None
+            if end < len(text):
+                ch = text[end]
+            if spaces:
+                if ch != ' ':
+                    if start+1 == end and self.column > self.best_width and split:
+                        self.write_indent()
+                        self.whitespace = False
+                        self.indention = False
+                    else:
+                        data = text[start:end]
+                        self.column += len(data)
+                        if self.encoding:
+                            data = data.encode(self.encoding)
+                        self.stream.write(data)
+                    start = end
+            elif breaks:
+                if ch not in '\n\x85\u2028\u2029':
+                    if text[start] == '\n':
+                        self.write_line_break()
+                    for br in text[start:end]:
+                        if br == '\n':
+                            self.write_line_break()
+                        else:
+                            self.write_line_break(br)
+                    self.write_indent()
+                    self.whitespace = False
+                    self.indention = False
+                    start = end
+            else:
+                if ch is None or ch in ' \n\x85\u2028\u2029':
+                    data = text[start:end]
+                    self.column += len(data)
+                    if self.encoding:
+                        data = data.encode(self.encoding)
+                    self.stream.write(data)
+                    start = end
+            if ch is not None:
+                spaces = (ch == ' ')
+                breaks = (ch in '\n\x85\u2028\u2029')
+            end += 1
diff --git a/ubuntu/venv/yaml/error.py b/ubuntu/venv/yaml/error.py
new file mode 100644
index 0000000..b796b4d
--- /dev/null
+++ b/ubuntu/venv/yaml/error.py
@@ -0,0 +1,75 @@
+
+__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError']
+
+class Mark:
+
+    def __init__(self, name, index, line, column, buffer, pointer):
+        self.name = name
+        self.index = index
+        self.line = line
+        self.column = column
+        self.buffer = buffer
+        self.pointer = pointer
+
+    def get_snippet(self, indent=4, max_length=75):
+        if self.buffer is None:
+            return None
+        head = ''
+        start = self.pointer
+        while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029':
+            start -= 1
+            if self.pointer-start > max_length/2-1:
+                head = ' ... '
+                start += 5
+                break
+        tail = ''
+        end = self.pointer
+        while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029':
+            end += 1
+            if end-self.pointer > max_length/2-1:
+                tail = ' ... '
+                end -= 5
+                break
+        snippet = self.buffer[start:end]
+        return ' '*indent + head + snippet + tail + '\n'  \
+                + ' '*(indent+self.pointer-start+len(head)) + '^'
+
+    def __str__(self):
+        snippet = self.get_snippet()
+        where = "  in \"%s\", line %d, column %d"   \
+                % (self.name, self.line+1, self.column+1)
+        if snippet is not None:
+            where += ":\n"+snippet
+        return where
+
+class YAMLError(Exception):
+    pass
+
+class MarkedYAMLError(YAMLError):
+
+    def __init__(self, context=None, context_mark=None,
+            problem=None, problem_mark=None, note=None):
+        self.context = context
+        self.context_mark = context_mark
+        self.problem = problem
+        self.problem_mark = problem_mark
+        self.note = note
+
+    def __str__(self):
+        lines = []
+        if self.context is not None:
+            lines.append(self.context)
+        if self.context_mark is not None  \
+            and (self.problem is None or self.problem_mark is None
+                    or self.context_mark.name != self.problem_mark.name
+                    or self.context_mark.line != self.problem_mark.line
+                    or self.context_mark.column != self.problem_mark.column):
+            lines.append(str(self.context_mark))
+        if self.problem is not None:
+            lines.append(self.problem)
+        if self.problem_mark is not None:
+            lines.append(str(self.problem_mark))
+        if self.note is not None:
+            lines.append(self.note)
+        return '\n'.join(lines)
+
diff --git a/ubuntu/venv/yaml/events.py b/ubuntu/venv/yaml/events.py
new file mode 100644
index 0000000..f79ad38
--- /dev/null
+++ b/ubuntu/venv/yaml/events.py
@@ -0,0 +1,86 @@
+
+# Abstract classes.
+
+class Event(object):
+    def __init__(self, start_mark=None, end_mark=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+    def __repr__(self):
+        attributes = [key for key in ['anchor', 'tag', 'implicit', 'value']
+                if hasattr(self, key)]
+        arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+                for key in attributes])
+        return '%s(%s)' % (self.__class__.__name__, arguments)
+
+class NodeEvent(Event):
+    def __init__(self, anchor, start_mark=None, end_mark=None):
+        self.anchor = anchor
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class CollectionStartEvent(NodeEvent):
+    def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None,
+            flow_style=None):
+        self.anchor = anchor
+        self.tag = tag
+        self.implicit = implicit
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.flow_style = flow_style
+
+class CollectionEndEvent(Event):
+    pass
+
+# Implementations.
+
+class StreamStartEvent(Event):
+    def __init__(self, start_mark=None, end_mark=None, encoding=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.encoding = encoding
+
+class StreamEndEvent(Event):
+    pass
+
+class DocumentStartEvent(Event):
+    def __init__(self, start_mark=None, end_mark=None,
+            explicit=None, version=None, tags=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.explicit = explicit
+        self.version = version
+        self.tags = tags
+
+class DocumentEndEvent(Event):
+    def __init__(self, start_mark=None, end_mark=None,
+            explicit=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.explicit = explicit
+
+class AliasEvent(NodeEvent):
+    pass
+
+class ScalarEvent(NodeEvent):
+    def __init__(self, anchor, tag, implicit, value,
+            start_mark=None, end_mark=None, style=None):
+        self.anchor = anchor
+        self.tag = tag
+        self.implicit = implicit
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.style = style
+
+class SequenceStartEvent(CollectionStartEvent):
+    pass
+
+class SequenceEndEvent(CollectionEndEvent):
+    pass
+
+class MappingStartEvent(CollectionStartEvent):
+    pass
+
+class MappingEndEvent(CollectionEndEvent):
+    pass
+
diff --git a/ubuntu/venv/yaml/loader.py b/ubuntu/venv/yaml/loader.py
new file mode 100644
index 0000000..e90c112
--- /dev/null
+++ b/ubuntu/venv/yaml/loader.py
@@ -0,0 +1,63 @@
+
+__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader']
+
+from .reader import *
+from .scanner import *
+from .parser import *
+from .composer import *
+from .constructor import *
+from .resolver import *
+
+class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver):
+
+    def __init__(self, stream):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        BaseConstructor.__init__(self)
+        BaseResolver.__init__(self)
+
+class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver):
+
+    def __init__(self, stream):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        FullConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver):
+
+    def __init__(self, stream):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        SafeConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+    def __init__(self, stream):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        Constructor.__init__(self)
+        Resolver.__init__(self)
+
+# UnsafeLoader is the same as Loader (which is and was always unsafe on
+# untrusted input). Use of either Loader or UnsafeLoader should be rare, since
+# FullLoad should be able to load almost all YAML safely. Loader is left intact
+# to ensure backwards compatibility.
+class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver):
+
+    def __init__(self, stream):
+        Reader.__init__(self, stream)
+        Scanner.__init__(self)
+        Parser.__init__(self)
+        Composer.__init__(self)
+        Constructor.__init__(self)
+        Resolver.__init__(self)
diff --git a/ubuntu/venv/yaml/nodes.py b/ubuntu/venv/yaml/nodes.py
new file mode 100644
index 0000000..c4f070c
--- /dev/null
+++ b/ubuntu/venv/yaml/nodes.py
@@ -0,0 +1,49 @@
+
+class Node(object):
+    def __init__(self, tag, value, start_mark, end_mark):
+        self.tag = tag
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+    def __repr__(self):
+        value = self.value
+        #if isinstance(value, list):
+        #    if len(value) == 0:
+        #        value = ''
+        #    elif len(value) == 1:
+        #        value = '<1 item>'
+        #    else:
+        #        value = '<%d items>' % len(value)
+        #else:
+        #    if len(value) > 75:
+        #        value = repr(value[:70]+u' ... ')
+        #    else:
+        #        value = repr(value)
+        value = repr(value)
+        return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value)
+
+class ScalarNode(Node):
+    id = 'scalar'
+    def __init__(self, tag, value,
+            start_mark=None, end_mark=None, style=None):
+        self.tag = tag
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.style = style
+
+class CollectionNode(Node):
+    def __init__(self, tag, value,
+            start_mark=None, end_mark=None, flow_style=None):
+        self.tag = tag
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.flow_style = flow_style
+
+class SequenceNode(CollectionNode):
+    id = 'sequence'
+
+class MappingNode(CollectionNode):
+    id = 'mapping'
+
diff --git a/ubuntu/venv/yaml/parser.py b/ubuntu/venv/yaml/parser.py
new file mode 100644
index 0000000..13a5995
--- /dev/null
+++ b/ubuntu/venv/yaml/parser.py
@@ -0,0 +1,589 @@
+
+# The following YAML grammar is LL(1) and is parsed by a recursive descent
+# parser.
+#
+# stream            ::= STREAM-START implicit_document? explicit_document* STREAM-END
+# implicit_document ::= block_node DOCUMENT-END*
+# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+# block_node_or_indentless_sequence ::=
+#                       ALIAS
+#                       | properties (block_content | indentless_block_sequence)?
+#                       | block_content
+#                       | indentless_block_sequence
+# block_node        ::= ALIAS
+#                       | properties block_content?
+#                       | block_content
+# flow_node         ::= ALIAS
+#                       | properties flow_content?
+#                       | flow_content
+# properties        ::= TAG ANCHOR? | ANCHOR TAG?
+# block_content     ::= block_collection | flow_collection | SCALAR
+# flow_content      ::= flow_collection | SCALAR
+# block_collection  ::= block_sequence | block_mapping
+# flow_collection   ::= flow_sequence | flow_mapping
+# block_sequence    ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+# indentless_sequence   ::= (BLOCK-ENTRY block_node?)+
+# block_mapping     ::= BLOCK-MAPPING_START
+#                       ((KEY block_node_or_indentless_sequence?)?
+#                       (VALUE block_node_or_indentless_sequence?)?)*
+#                       BLOCK-END
+# flow_sequence     ::= FLOW-SEQUENCE-START
+#                       (flow_sequence_entry FLOW-ENTRY)*
+#                       flow_sequence_entry?
+#                       FLOW-SEQUENCE-END
+# flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+# flow_mapping      ::= FLOW-MAPPING-START
+#                       (flow_mapping_entry FLOW-ENTRY)*
+#                       flow_mapping_entry?
+#                       FLOW-MAPPING-END
+# flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+#
+# FIRST sets:
+#
+# stream: { STREAM-START }
+# explicit_document: { DIRECTIVE DOCUMENT-START }
+# implicit_document: FIRST(block_node)
+# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
+# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# block_sequence: { BLOCK-SEQUENCE-START }
+# block_mapping: { BLOCK-MAPPING-START }
+# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY }
+# indentless_sequence: { ENTRY }
+# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
+# flow_sequence: { FLOW-SEQUENCE-START }
+# flow_mapping: { FLOW-MAPPING-START }
+# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY }
+
+__all__ = ['Parser', 'ParserError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+from .events import *
+from .scanner import *
+
+class ParserError(MarkedYAMLError):
+    pass
+
+class Parser:
+    # Since writing a recursive-descendant parser is a straightforward task, we
+    # do not give many comments here.
+
+    DEFAULT_TAGS = {
+        '!':   '!',
+        '!!':  'tag:yaml.org,2002:',
+    }
+
+    def __init__(self):
+        self.current_event = None
+        self.yaml_version = None
+        self.tag_handles = {}
+        self.states = []
+        self.marks = []
+        self.state = self.parse_stream_start
+
+    def dispose(self):
+        # Reset the state attributes (to clear self-references)
+        self.states = []
+        self.state = None
+
+    def check_event(self, *choices):
+        # Check the type of the next event.
+        if self.current_event is None:
+            if self.state:
+                self.current_event = self.state()
+        if self.current_event is not None:
+            if not choices:
+                return True
+            for choice in choices:
+                if isinstance(self.current_event, choice):
+                    return True
+        return False
+
+    def peek_event(self):
+        # Get the next event.
+        if self.current_event is None:
+            if self.state:
+                self.current_event = self.state()
+        return self.current_event
+
+    def get_event(self):
+        # Get the next event and proceed further.
+        if self.current_event is None:
+            if self.state:
+                self.current_event = self.state()
+        value = self.current_event
+        self.current_event = None
+        return value
+
+    # stream    ::= STREAM-START implicit_document? explicit_document* STREAM-END
+    # implicit_document ::= block_node DOCUMENT-END*
+    # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
+
+    def parse_stream_start(self):
+
+        # Parse the stream start.
+        token = self.get_token()
+        event = StreamStartEvent(token.start_mark, token.end_mark,
+                encoding=token.encoding)
+
+        # Prepare the next state.
+        self.state = self.parse_implicit_document_start
+
+        return event
+
+    def parse_implicit_document_start(self):
+
+        # Parse an implicit document.
+        if not self.check_token(DirectiveToken, DocumentStartToken,
+                StreamEndToken):
+            self.tag_handles = self.DEFAULT_TAGS
+            token = self.peek_token()
+            start_mark = end_mark = token.start_mark
+            event = DocumentStartEvent(start_mark, end_mark,
+                    explicit=False)
+
+            # Prepare the next state.
+            self.states.append(self.parse_document_end)
+            self.state = self.parse_block_node
+
+            return event
+
+        else:
+            return self.parse_document_start()
+
+    def parse_document_start(self):
+
+        # Parse any extra document end indicators.
+        while self.check_token(DocumentEndToken):
+            self.get_token()
+
+        # Parse an explicit document.
+        if not self.check_token(StreamEndToken):
+            token = self.peek_token()
+            start_mark = token.start_mark
+            version, tags = self.process_directives()
+            if not self.check_token(DocumentStartToken):
+                raise ParserError(None, None,
+                        "expected '', but found %r"
+                        % self.peek_token().id,
+                        self.peek_token().start_mark)
+            token = self.get_token()
+            end_mark = token.end_mark
+            event = DocumentStartEvent(start_mark, end_mark,
+                    explicit=True, version=version, tags=tags)
+            self.states.append(self.parse_document_end)
+            self.state = self.parse_document_content
+        else:
+            # Parse the end of the stream.
+            token = self.get_token()
+            event = StreamEndEvent(token.start_mark, token.end_mark)
+            assert not self.states
+            assert not self.marks
+            self.state = None
+        return event
+
+    def parse_document_end(self):
+
+        # Parse the document end.
+        token = self.peek_token()
+        start_mark = end_mark = token.start_mark
+        explicit = False
+        if self.check_token(DocumentEndToken):
+            token = self.get_token()
+            end_mark = token.end_mark
+            explicit = True
+        event = DocumentEndEvent(start_mark, end_mark,
+                explicit=explicit)
+
+        # Prepare the next state.
+        self.state = self.parse_document_start
+
+        return event
+
+    def parse_document_content(self):
+        if self.check_token(DirectiveToken,
+                DocumentStartToken, DocumentEndToken, StreamEndToken):
+            event = self.process_empty_scalar(self.peek_token().start_mark)
+            self.state = self.states.pop()
+            return event
+        else:
+            return self.parse_block_node()
+
+    def process_directives(self):
+        self.yaml_version = None
+        self.tag_handles = {}
+        while self.check_token(DirectiveToken):
+            token = self.get_token()
+            if token.name == 'YAML':
+                if self.yaml_version is not None:
+                    raise ParserError(None, None,
+                            "found duplicate YAML directive", token.start_mark)
+                major, minor = token.value
+                if major != 1:
+                    raise ParserError(None, None,
+                            "found incompatible YAML document (version 1.* is required)",
+                            token.start_mark)
+                self.yaml_version = token.value
+            elif token.name == 'TAG':
+                handle, prefix = token.value
+                if handle in self.tag_handles:
+                    raise ParserError(None, None,
+                            "duplicate tag handle %r" % handle,
+                            token.start_mark)
+                self.tag_handles[handle] = prefix
+        if self.tag_handles:
+            value = self.yaml_version, self.tag_handles.copy()
+        else:
+            value = self.yaml_version, None
+        for key in self.DEFAULT_TAGS:
+            if key not in self.tag_handles:
+                self.tag_handles[key] = self.DEFAULT_TAGS[key]
+        return value
+
+    # block_node_or_indentless_sequence ::= ALIAS
+    #               | properties (block_content | indentless_block_sequence)?
+    #               | block_content
+    #               | indentless_block_sequence
+    # block_node    ::= ALIAS
+    #                   | properties block_content?
+    #                   | block_content
+    # flow_node     ::= ALIAS
+    #                   | properties flow_content?
+    #                   | flow_content
+    # properties    ::= TAG ANCHOR? | ANCHOR TAG?
+    # block_content     ::= block_collection | flow_collection | SCALAR
+    # flow_content      ::= flow_collection | SCALAR
+    # block_collection  ::= block_sequence | block_mapping
+    # flow_collection   ::= flow_sequence | flow_mapping
+
+    def parse_block_node(self):
+        return self.parse_node(block=True)
+
+    def parse_flow_node(self):
+        return self.parse_node()
+
+    def parse_block_node_or_indentless_sequence(self):
+        return self.parse_node(block=True, indentless_sequence=True)
+
+    def parse_node(self, block=False, indentless_sequence=False):
+        if self.check_token(AliasToken):
+            token = self.get_token()
+            event = AliasEvent(token.value, token.start_mark, token.end_mark)
+            self.state = self.states.pop()
+        else:
+            anchor = None
+            tag = None
+            start_mark = end_mark = tag_mark = None
+            if self.check_token(AnchorToken):
+                token = self.get_token()
+                start_mark = token.start_mark
+                end_mark = token.end_mark
+                anchor = token.value
+                if self.check_token(TagToken):
+                    token = self.get_token()
+                    tag_mark = token.start_mark
+                    end_mark = token.end_mark
+                    tag = token.value
+            elif self.check_token(TagToken):
+                token = self.get_token()
+                start_mark = tag_mark = token.start_mark
+                end_mark = token.end_mark
+                tag = token.value
+                if self.check_token(AnchorToken):
+                    token = self.get_token()
+                    end_mark = token.end_mark
+                    anchor = token.value
+            if tag is not None:
+                handle, suffix = tag
+                if handle is not None:
+                    if handle not in self.tag_handles:
+                        raise ParserError("while parsing a node", start_mark,
+                                "found undefined tag handle %r" % handle,
+                                tag_mark)
+                    tag = self.tag_handles[handle]+suffix
+                else:
+                    tag = suffix
+            #if tag == '!':
+            #    raise ParserError("while parsing a node", start_mark,
+            #            "found non-specific tag '!'", tag_mark,
+            #            "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.")
+            if start_mark is None:
+                start_mark = end_mark = self.peek_token().start_mark
+            event = None
+            implicit = (tag is None or tag == '!')
+            if indentless_sequence and self.check_token(BlockEntryToken):
+                end_mark = self.peek_token().end_mark
+                event = SequenceStartEvent(anchor, tag, implicit,
+                        start_mark, end_mark)
+                self.state = self.parse_indentless_sequence_entry
+            else:
+                if self.check_token(ScalarToken):
+                    token = self.get_token()
+                    end_mark = token.end_mark
+                    if (token.plain and tag is None) or tag == '!':
+                        implicit = (True, False)
+                    elif tag is None:
+                        implicit = (False, True)
+                    else:
+                        implicit = (False, False)
+                    event = ScalarEvent(anchor, tag, implicit, token.value,
+                            start_mark, end_mark, style=token.style)
+                    self.state = self.states.pop()
+                elif self.check_token(FlowSequenceStartToken):
+                    end_mark = self.peek_token().end_mark
+                    event = SequenceStartEvent(anchor, tag, implicit,
+                            start_mark, end_mark, flow_style=True)
+                    self.state = self.parse_flow_sequence_first_entry
+                elif self.check_token(FlowMappingStartToken):
+                    end_mark = self.peek_token().end_mark
+                    event = MappingStartEvent(anchor, tag, implicit,
+                            start_mark, end_mark, flow_style=True)
+                    self.state = self.parse_flow_mapping_first_key
+                elif block and self.check_token(BlockSequenceStartToken):
+                    end_mark = self.peek_token().start_mark
+                    event = SequenceStartEvent(anchor, tag, implicit,
+                            start_mark, end_mark, flow_style=False)
+                    self.state = self.parse_block_sequence_first_entry
+                elif block and self.check_token(BlockMappingStartToken):
+                    end_mark = self.peek_token().start_mark
+                    event = MappingStartEvent(anchor, tag, implicit,
+                            start_mark, end_mark, flow_style=False)
+                    self.state = self.parse_block_mapping_first_key
+                elif anchor is not None or tag is not None:
+                    # Empty scalars are allowed even if a tag or an anchor is
+                    # specified.
+                    event = ScalarEvent(anchor, tag, (implicit, False), '',
+                            start_mark, end_mark)
+                    self.state = self.states.pop()
+                else:
+                    if block:
+                        node = 'block'
+                    else:
+                        node = 'flow'
+                    token = self.peek_token()
+                    raise ParserError("while parsing a %s node" % node, start_mark,
+                            "expected the node content, but found %r" % token.id,
+                            token.start_mark)
+        return event
+
+    # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END
+
+    def parse_block_sequence_first_entry(self):
+        token = self.get_token()
+        self.marks.append(token.start_mark)
+        return self.parse_block_sequence_entry()
+
+    def parse_block_sequence_entry(self):
+        if self.check_token(BlockEntryToken):
+            token = self.get_token()
+            if not self.check_token(BlockEntryToken, BlockEndToken):
+                self.states.append(self.parse_block_sequence_entry)
+                return self.parse_block_node()
+            else:
+                self.state = self.parse_block_sequence_entry
+                return self.process_empty_scalar(token.end_mark)
+        if not self.check_token(BlockEndToken):
+            token = self.peek_token()
+            raise ParserError("while parsing a block collection", self.marks[-1],
+                    "expected , but found %r" % token.id, token.start_mark)
+        token = self.get_token()
+        event = SequenceEndEvent(token.start_mark, token.end_mark)
+        self.state = self.states.pop()
+        self.marks.pop()
+        return event
+
+    # indentless_sequence ::= (BLOCK-ENTRY block_node?)+
+
+    def parse_indentless_sequence_entry(self):
+        if self.check_token(BlockEntryToken):
+            token = self.get_token()
+            if not self.check_token(BlockEntryToken,
+                    KeyToken, ValueToken, BlockEndToken):
+                self.states.append(self.parse_indentless_sequence_entry)
+                return self.parse_block_node()
+            else:
+                self.state = self.parse_indentless_sequence_entry
+                return self.process_empty_scalar(token.end_mark)
+        token = self.peek_token()
+        event = SequenceEndEvent(token.start_mark, token.start_mark)
+        self.state = self.states.pop()
+        return event
+
+    # block_mapping     ::= BLOCK-MAPPING_START
+    #                       ((KEY block_node_or_indentless_sequence?)?
+    #                       (VALUE block_node_or_indentless_sequence?)?)*
+    #                       BLOCK-END
+
+    def parse_block_mapping_first_key(self):
+        token = self.get_token()
+        self.marks.append(token.start_mark)
+        return self.parse_block_mapping_key()
+
+    def parse_block_mapping_key(self):
+        if self.check_token(KeyToken):
+            token = self.get_token()
+            if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+                self.states.append(self.parse_block_mapping_value)
+                return self.parse_block_node_or_indentless_sequence()
+            else:
+                self.state = self.parse_block_mapping_value
+                return self.process_empty_scalar(token.end_mark)
+        if not self.check_token(BlockEndToken):
+            token = self.peek_token()
+            raise ParserError("while parsing a block mapping", self.marks[-1],
+                    "expected , but found %r" % token.id, token.start_mark)
+        token = self.get_token()
+        event = MappingEndEvent(token.start_mark, token.end_mark)
+        self.state = self.states.pop()
+        self.marks.pop()
+        return event
+
+    def parse_block_mapping_value(self):
+        if self.check_token(ValueToken):
+            token = self.get_token()
+            if not self.check_token(KeyToken, ValueToken, BlockEndToken):
+                self.states.append(self.parse_block_mapping_key)
+                return self.parse_block_node_or_indentless_sequence()
+            else:
+                self.state = self.parse_block_mapping_key
+                return self.process_empty_scalar(token.end_mark)
+        else:
+            self.state = self.parse_block_mapping_key
+            token = self.peek_token()
+            return self.process_empty_scalar(token.start_mark)
+
+    # flow_sequence     ::= FLOW-SEQUENCE-START
+    #                       (flow_sequence_entry FLOW-ENTRY)*
+    #                       flow_sequence_entry?
+    #                       FLOW-SEQUENCE-END
+    # flow_sequence_entry   ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+    #
+    # Note that while production rules for both flow_sequence_entry and
+    # flow_mapping_entry are equal, their interpretations are different.
+    # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
+    # generate an inline mapping (set syntax).
+
+    def parse_flow_sequence_first_entry(self):
+        token = self.get_token()
+        self.marks.append(token.start_mark)
+        return self.parse_flow_sequence_entry(first=True)
+
+    def parse_flow_sequence_entry(self, first=False):
+        if not self.check_token(FlowSequenceEndToken):
+            if not first:
+                if self.check_token(FlowEntryToken):
+                    self.get_token()
+                else:
+                    token = self.peek_token()
+                    raise ParserError("while parsing a flow sequence", self.marks[-1],
+                            "expected ',' or ']', but got %r" % token.id, token.start_mark)
+            
+            if self.check_token(KeyToken):
+                token = self.peek_token()
+                event = MappingStartEvent(None, None, True,
+                        token.start_mark, token.end_mark,
+                        flow_style=True)
+                self.state = self.parse_flow_sequence_entry_mapping_key
+                return event
+            elif not self.check_token(FlowSequenceEndToken):
+                self.states.append(self.parse_flow_sequence_entry)
+                return self.parse_flow_node()
+        token = self.get_token()
+        event = SequenceEndEvent(token.start_mark, token.end_mark)
+        self.state = self.states.pop()
+        self.marks.pop()
+        return event
+
+    def parse_flow_sequence_entry_mapping_key(self):
+        token = self.get_token()
+        if not self.check_token(ValueToken,
+                FlowEntryToken, FlowSequenceEndToken):
+            self.states.append(self.parse_flow_sequence_entry_mapping_value)
+            return self.parse_flow_node()
+        else:
+            self.state = self.parse_flow_sequence_entry_mapping_value
+            return self.process_empty_scalar(token.end_mark)
+
+    def parse_flow_sequence_entry_mapping_value(self):
+        if self.check_token(ValueToken):
+            token = self.get_token()
+            if not self.check_token(FlowEntryToken, FlowSequenceEndToken):
+                self.states.append(self.parse_flow_sequence_entry_mapping_end)
+                return self.parse_flow_node()
+            else:
+                self.state = self.parse_flow_sequence_entry_mapping_end
+                return self.process_empty_scalar(token.end_mark)
+        else:
+            self.state = self.parse_flow_sequence_entry_mapping_end
+            token = self.peek_token()
+            return self.process_empty_scalar(token.start_mark)
+
+    def parse_flow_sequence_entry_mapping_end(self):
+        self.state = self.parse_flow_sequence_entry
+        token = self.peek_token()
+        return MappingEndEvent(token.start_mark, token.start_mark)
+
+    # flow_mapping  ::= FLOW-MAPPING-START
+    #                   (flow_mapping_entry FLOW-ENTRY)*
+    #                   flow_mapping_entry?
+    #                   FLOW-MAPPING-END
+    # flow_mapping_entry    ::= flow_node | KEY flow_node? (VALUE flow_node?)?
+
+    def parse_flow_mapping_first_key(self):
+        token = self.get_token()
+        self.marks.append(token.start_mark)
+        return self.parse_flow_mapping_key(first=True)
+
+    def parse_flow_mapping_key(self, first=False):
+        if not self.check_token(FlowMappingEndToken):
+            if not first:
+                if self.check_token(FlowEntryToken):
+                    self.get_token()
+                else:
+                    token = self.peek_token()
+                    raise ParserError("while parsing a flow mapping", self.marks[-1],
+                            "expected ',' or '}', but got %r" % token.id, token.start_mark)
+            if self.check_token(KeyToken):
+                token = self.get_token()
+                if not self.check_token(ValueToken,
+                        FlowEntryToken, FlowMappingEndToken):
+                    self.states.append(self.parse_flow_mapping_value)
+                    return self.parse_flow_node()
+                else:
+                    self.state = self.parse_flow_mapping_value
+                    return self.process_empty_scalar(token.end_mark)
+            elif not self.check_token(FlowMappingEndToken):
+                self.states.append(self.parse_flow_mapping_empty_value)
+                return self.parse_flow_node()
+        token = self.get_token()
+        event = MappingEndEvent(token.start_mark, token.end_mark)
+        self.state = self.states.pop()
+        self.marks.pop()
+        return event
+
+    def parse_flow_mapping_value(self):
+        if self.check_token(ValueToken):
+            token = self.get_token()
+            if not self.check_token(FlowEntryToken, FlowMappingEndToken):
+                self.states.append(self.parse_flow_mapping_key)
+                return self.parse_flow_node()
+            else:
+                self.state = self.parse_flow_mapping_key
+                return self.process_empty_scalar(token.end_mark)
+        else:
+            self.state = self.parse_flow_mapping_key
+            token = self.peek_token()
+            return self.process_empty_scalar(token.start_mark)
+
+    def parse_flow_mapping_empty_value(self):
+        self.state = self.parse_flow_mapping_key
+        return self.process_empty_scalar(self.peek_token().start_mark)
+
+    def process_empty_scalar(self, mark):
+        return ScalarEvent(None, None, (True, False), '', mark, mark)
+
diff --git a/ubuntu/venv/yaml/reader.py b/ubuntu/venv/yaml/reader.py
new file mode 100644
index 0000000..774b021
--- /dev/null
+++ b/ubuntu/venv/yaml/reader.py
@@ -0,0 +1,185 @@
+# This module contains abstractions for the input stream. You don't have to
+# looks further, there are no pretty code.
+#
+# We define two classes here.
+#
+#   Mark(source, line, column)
+# It's just a record and its only use is producing nice error messages.
+# Parser does not use it for any other purposes.
+#
+#   Reader(source, data)
+# Reader determines the encoding of `data` and converts it to unicode.
+# Reader provides the following methods and attributes:
+#   reader.peek(length=1) - return the next `length` characters
+#   reader.forward(length=1) - move the current position to `length` characters.
+#   reader.index - the number of the current character.
+#   reader.line, stream.column - the line and the column of the current character.
+
+__all__ = ['Reader', 'ReaderError']
+
+from .error import YAMLError, Mark
+
+import codecs, re
+
+class ReaderError(YAMLError):
+
+    def __init__(self, name, position, character, encoding, reason):
+        self.name = name
+        self.character = character
+        self.position = position
+        self.encoding = encoding
+        self.reason = reason
+
+    def __str__(self):
+        if isinstance(self.character, bytes):
+            return "'%s' codec can't decode byte #x%02x: %s\n"  \
+                    "  in \"%s\", position %d"    \
+                    % (self.encoding, ord(self.character), self.reason,
+                            self.name, self.position)
+        else:
+            return "unacceptable character #x%04x: %s\n"    \
+                    "  in \"%s\", position %d"    \
+                    % (self.character, self.reason,
+                            self.name, self.position)
+
+class Reader(object):
+    # Reader:
+    # - determines the data encoding and converts it to a unicode string,
+    # - checks if characters are in allowed range,
+    # - adds '\0' to the end.
+
+    # Reader accepts
+    #  - a `bytes` object,
+    #  - a `str` object,
+    #  - a file-like object with its `read` method returning `str`,
+    #  - a file-like object with its `read` method returning `unicode`.
+
+    # Yeah, it's ugly and slow.
+
+    def __init__(self, stream):
+        self.name = None
+        self.stream = None
+        self.stream_pointer = 0
+        self.eof = True
+        self.buffer = ''
+        self.pointer = 0
+        self.raw_buffer = None
+        self.raw_decode = None
+        self.encoding = None
+        self.index = 0
+        self.line = 0
+        self.column = 0
+        if isinstance(stream, str):
+            self.name = ""
+            self.check_printable(stream)
+            self.buffer = stream+'\0'
+        elif isinstance(stream, bytes):
+            self.name = ""
+            self.raw_buffer = stream
+            self.determine_encoding()
+        else:
+            self.stream = stream
+            self.name = getattr(stream, 'name', "")
+            self.eof = False
+            self.raw_buffer = None
+            self.determine_encoding()
+
+    def peek(self, index=0):
+        try:
+            return self.buffer[self.pointer+index]
+        except IndexError:
+            self.update(index+1)
+            return self.buffer[self.pointer+index]
+
+    def prefix(self, length=1):
+        if self.pointer+length >= len(self.buffer):
+            self.update(length)
+        return self.buffer[self.pointer:self.pointer+length]
+
+    def forward(self, length=1):
+        if self.pointer+length+1 >= len(self.buffer):
+            self.update(length+1)
+        while length:
+            ch = self.buffer[self.pointer]
+            self.pointer += 1
+            self.index += 1
+            if ch in '\n\x85\u2028\u2029'  \
+                    or (ch == '\r' and self.buffer[self.pointer] != '\n'):
+                self.line += 1
+                self.column = 0
+            elif ch != '\uFEFF':
+                self.column += 1
+            length -= 1
+
+    def get_mark(self):
+        if self.stream is None:
+            return Mark(self.name, self.index, self.line, self.column,
+                    self.buffer, self.pointer)
+        else:
+            return Mark(self.name, self.index, self.line, self.column,
+                    None, None)
+
+    def determine_encoding(self):
+        while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
+            self.update_raw()
+        if isinstance(self.raw_buffer, bytes):
+            if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
+                self.raw_decode = codecs.utf_16_le_decode
+                self.encoding = 'utf-16-le'
+            elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
+                self.raw_decode = codecs.utf_16_be_decode
+                self.encoding = 'utf-16-be'
+            else:
+                self.raw_decode = codecs.utf_8_decode
+                self.encoding = 'utf-8'
+        self.update(1)
+
+    NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]')
+    def check_printable(self, data):
+        match = self.NON_PRINTABLE.search(data)
+        if match:
+            character = match.group()
+            position = self.index+(len(self.buffer)-self.pointer)+match.start()
+            raise ReaderError(self.name, position, ord(character),
+                    'unicode', "special characters are not allowed")
+
+    def update(self, length):
+        if self.raw_buffer is None:
+            return
+        self.buffer = self.buffer[self.pointer:]
+        self.pointer = 0
+        while len(self.buffer) < length:
+            if not self.eof:
+                self.update_raw()
+            if self.raw_decode is not None:
+                try:
+                    data, converted = self.raw_decode(self.raw_buffer,
+                            'strict', self.eof)
+                except UnicodeDecodeError as exc:
+                    character = self.raw_buffer[exc.start]
+                    if self.stream is not None:
+                        position = self.stream_pointer-len(self.raw_buffer)+exc.start
+                    else:
+                        position = exc.start
+                    raise ReaderError(self.name, position, character,
+                            exc.encoding, exc.reason)
+            else:
+                data = self.raw_buffer
+                converted = len(data)
+            self.check_printable(data)
+            self.buffer += data
+            self.raw_buffer = self.raw_buffer[converted:]
+            if self.eof:
+                self.buffer += '\0'
+                self.raw_buffer = None
+                break
+
+    def update_raw(self, size=4096):
+        data = self.stream.read(size)
+        if self.raw_buffer is None:
+            self.raw_buffer = data
+        else:
+            self.raw_buffer += data
+        self.stream_pointer += len(data)
+        if not data:
+            self.eof = True
diff --git a/ubuntu/venv/yaml/representer.py b/ubuntu/venv/yaml/representer.py
new file mode 100644
index 0000000..808ca06
--- /dev/null
+++ b/ubuntu/venv/yaml/representer.py
@@ -0,0 +1,389 @@
+
+__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer',
+    'RepresenterError']
+
+from .error import *
+from .nodes import *
+
+import datetime, copyreg, types, base64, collections
+
+class RepresenterError(YAMLError):
+    pass
+
+class BaseRepresenter:
+
+    yaml_representers = {}
+    yaml_multi_representers = {}
+
+    def __init__(self, default_style=None, default_flow_style=False, sort_keys=True):
+        self.default_style = default_style
+        self.sort_keys = sort_keys
+        self.default_flow_style = default_flow_style
+        self.represented_objects = {}
+        self.object_keeper = []
+        self.alias_key = None
+
+    def represent(self, data):
+        node = self.represent_data(data)
+        self.serialize(node)
+        self.represented_objects = {}
+        self.object_keeper = []
+        self.alias_key = None
+
+    def represent_data(self, data):
+        if self.ignore_aliases(data):
+            self.alias_key = None
+        else:
+            self.alias_key = id(data)
+        if self.alias_key is not None:
+            if self.alias_key in self.represented_objects:
+                node = self.represented_objects[self.alias_key]
+                #if node is None:
+                #    raise RepresenterError("recursive objects are not allowed: %r" % data)
+                return node
+            #self.represented_objects[alias_key] = None
+            self.object_keeper.append(data)
+        data_types = type(data).__mro__
+        if data_types[0] in self.yaml_representers:
+            node = self.yaml_representers[data_types[0]](self, data)
+        else:
+            for data_type in data_types:
+                if data_type in self.yaml_multi_representers:
+                    node = self.yaml_multi_representers[data_type](self, data)
+                    break
+            else:
+                if None in self.yaml_multi_representers:
+                    node = self.yaml_multi_representers[None](self, data)
+                elif None in self.yaml_representers:
+                    node = self.yaml_representers[None](self, data)
+                else:
+                    node = ScalarNode(None, str(data))
+        #if alias_key is not None:
+        #    self.represented_objects[alias_key] = node
+        return node
+
+    @classmethod
+    def add_representer(cls, data_type, representer):
+        if not 'yaml_representers' in cls.__dict__:
+            cls.yaml_representers = cls.yaml_representers.copy()
+        cls.yaml_representers[data_type] = representer
+
+    @classmethod
+    def add_multi_representer(cls, data_type, representer):
+        if not 'yaml_multi_representers' in cls.__dict__:
+            cls.yaml_multi_representers = cls.yaml_multi_representers.copy()
+        cls.yaml_multi_representers[data_type] = representer
+
+    def represent_scalar(self, tag, value, style=None):
+        if style is None:
+            style = self.default_style
+        node = ScalarNode(tag, value, style=style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        return node
+
+    def represent_sequence(self, tag, sequence, flow_style=None):
+        value = []
+        node = SequenceNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        for item in sequence:
+            node_item = self.represent_data(item)
+            if not (isinstance(node_item, ScalarNode) and not node_item.style):
+                best_style = False
+            value.append(node_item)
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node
+
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        value = []
+        node = MappingNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        if hasattr(mapping, 'items'):
+            mapping = list(mapping.items())
+            if self.sort_keys:
+                try:
+                    mapping = sorted(mapping)
+                except TypeError:
+                    pass
+        for item_key, item_value in mapping:
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, ScalarNode) and not node_key.style):
+                best_style = False
+            if not (isinstance(node_value, ScalarNode) and not node_value.style):
+                best_style = False
+            value.append((node_key, node_value))
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node
+
+    def ignore_aliases(self, data):
+        return False
+
+class SafeRepresenter(BaseRepresenter):
+
+    def ignore_aliases(self, data):
+        if data is None:
+            return True
+        if isinstance(data, tuple) and data == ():
+            return True
+        if isinstance(data, (str, bytes, bool, int, float)):
+            return True
+
+    def represent_none(self, data):
+        return self.represent_scalar('tag:yaml.org,2002:null', 'null')
+
+    def represent_str(self, data):
+        return self.represent_scalar('tag:yaml.org,2002:str', data)
+
+    def represent_binary(self, data):
+        if hasattr(base64, 'encodebytes'):
+            data = base64.encodebytes(data).decode('ascii')
+        else:
+            data = base64.encodestring(data).decode('ascii')
+        return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|')
+
+    def represent_bool(self, data):
+        if data:
+            value = 'true'
+        else:
+            value = 'false'
+        return self.represent_scalar('tag:yaml.org,2002:bool', value)
+
+    def represent_int(self, data):
+        return self.represent_scalar('tag:yaml.org,2002:int', str(data))
+
+    inf_value = 1e300
+    while repr(inf_value) != repr(inf_value*inf_value):
+        inf_value *= inf_value
+
+    def represent_float(self, data):
+        if data != data or (data == 0.0 and data == 1.0):
+            value = '.nan'
+        elif data == self.inf_value:
+            value = '.inf'
+        elif data == -self.inf_value:
+            value = '-.inf'
+        else:
+            value = repr(data).lower()
+            # Note that in some cases `repr(data)` represents a float number
+            # without the decimal parts.  For instance:
+            #   >>> repr(1e17)
+            #   '1e17'
+            # Unfortunately, this is not a valid float representation according
+            # to the definition of the `!!float` tag.  We fix this by adding
+            # '.0' before the 'e' symbol.
+            if '.' not in value and 'e' in value:
+                value = value.replace('e', '.0e', 1)
+        return self.represent_scalar('tag:yaml.org,2002:float', value)
+
+    def represent_list(self, data):
+        #pairs = (len(data) > 0 and isinstance(data, list))
+        #if pairs:
+        #    for item in data:
+        #        if not isinstance(item, tuple) or len(item) != 2:
+        #            pairs = False
+        #            break
+        #if not pairs:
+            return self.represent_sequence('tag:yaml.org,2002:seq', data)
+        #value = []
+        #for item_key, item_value in data:
+        #    value.append(self.represent_mapping(u'tag:yaml.org,2002:map',
+        #        [(item_key, item_value)]))
+        #return SequenceNode(u'tag:yaml.org,2002:pairs', value)
+
+    def represent_dict(self, data):
+        return self.represent_mapping('tag:yaml.org,2002:map', data)
+
+    def represent_set(self, data):
+        value = {}
+        for key in data:
+            value[key] = None
+        return self.represent_mapping('tag:yaml.org,2002:set', value)
+
+    def represent_date(self, data):
+        value = data.isoformat()
+        return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+    def represent_datetime(self, data):
+        value = data.isoformat(' ')
+        return self.represent_scalar('tag:yaml.org,2002:timestamp', value)
+
+    def represent_yaml_object(self, tag, data, cls, flow_style=None):
+        if hasattr(data, '__getstate__'):
+            state = data.__getstate__()
+        else:
+            state = data.__dict__.copy()
+        return self.represent_mapping(tag, state, flow_style=flow_style)
+
+    def represent_undefined(self, data):
+        raise RepresenterError("cannot represent an object", data)
+
+SafeRepresenter.add_representer(type(None),
+        SafeRepresenter.represent_none)
+
+SafeRepresenter.add_representer(str,
+        SafeRepresenter.represent_str)
+
+SafeRepresenter.add_representer(bytes,
+        SafeRepresenter.represent_binary)
+
+SafeRepresenter.add_representer(bool,
+        SafeRepresenter.represent_bool)
+
+SafeRepresenter.add_representer(int,
+        SafeRepresenter.represent_int)
+
+SafeRepresenter.add_representer(float,
+        SafeRepresenter.represent_float)
+
+SafeRepresenter.add_representer(list,
+        SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(tuple,
+        SafeRepresenter.represent_list)
+
+SafeRepresenter.add_representer(dict,
+        SafeRepresenter.represent_dict)
+
+SafeRepresenter.add_representer(set,
+        SafeRepresenter.represent_set)
+
+SafeRepresenter.add_representer(datetime.date,
+        SafeRepresenter.represent_date)
+
+SafeRepresenter.add_representer(datetime.datetime,
+        SafeRepresenter.represent_datetime)
+
+SafeRepresenter.add_representer(None,
+        SafeRepresenter.represent_undefined)
+
+class Representer(SafeRepresenter):
+
+    def represent_complex(self, data):
+        if data.imag == 0.0:
+            data = '%r' % data.real
+        elif data.real == 0.0:
+            data = '%rj' % data.imag
+        elif data.imag > 0:
+            data = '%r+%rj' % (data.real, data.imag)
+        else:
+            data = '%r%rj' % (data.real, data.imag)
+        return self.represent_scalar('tag:yaml.org,2002:python/complex', data)
+
+    def represent_tuple(self, data):
+        return self.represent_sequence('tag:yaml.org,2002:python/tuple', data)
+
+    def represent_name(self, data):
+        name = '%s.%s' % (data.__module__, data.__name__)
+        return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '')
+
+    def represent_module(self, data):
+        return self.represent_scalar(
+                'tag:yaml.org,2002:python/module:'+data.__name__, '')
+
+    def represent_object(self, data):
+        # We use __reduce__ API to save the data. data.__reduce__ returns
+        # a tuple of length 2-5:
+        #   (function, args, state, listitems, dictitems)
+
+        # For reconstructing, we calls function(*args), then set its state,
+        # listitems, and dictitems if they are not None.
+
+        # A special case is when function.__name__ == '__newobj__'. In this
+        # case we create the object with args[0].__new__(*args).
+
+        # Another special case is when __reduce__ returns a string - we don't
+        # support it.
+
+        # We produce a !!python/object, !!python/object/new or
+        # !!python/object/apply node.
+
+        cls = type(data)
+        if cls in copyreg.dispatch_table:
+            reduce = copyreg.dispatch_table[cls](data)
+        elif hasattr(data, '__reduce_ex__'):
+            reduce = data.__reduce_ex__(2)
+        elif hasattr(data, '__reduce__'):
+            reduce = data.__reduce__()
+        else:
+            raise RepresenterError("cannot represent an object", data)
+        reduce = (list(reduce)+[None]*5)[:5]
+        function, args, state, listitems, dictitems = reduce
+        args = list(args)
+        if state is None:
+            state = {}
+        if listitems is not None:
+            listitems = list(listitems)
+        if dictitems is not None:
+            dictitems = dict(dictitems)
+        if function.__name__ == '__newobj__':
+            function = args[0]
+            args = args[1:]
+            tag = 'tag:yaml.org,2002:python/object/new:'
+            newobj = True
+        else:
+            tag = 'tag:yaml.org,2002:python/object/apply:'
+            newobj = False
+        function_name = '%s.%s' % (function.__module__, function.__name__)
+        if not args and not listitems and not dictitems \
+                and isinstance(state, dict) and newobj:
+            return self.represent_mapping(
+                    'tag:yaml.org,2002:python/object:'+function_name, state)
+        if not listitems and not dictitems  \
+                and isinstance(state, dict) and not state:
+            return self.represent_sequence(tag+function_name, args)
+        value = {}
+        if args:
+            value['args'] = args
+        if state or not isinstance(state, dict):
+            value['state'] = state
+        if listitems:
+            value['listitems'] = listitems
+        if dictitems:
+            value['dictitems'] = dictitems
+        return self.represent_mapping(tag+function_name, value)
+
+    def represent_ordered_dict(self, data):
+        # Provide uniform representation across different Python versions.
+        data_type = type(data)
+        tag = 'tag:yaml.org,2002:python/object/apply:%s.%s' \
+                % (data_type.__module__, data_type.__name__)
+        items = [[key, value] for key, value in data.items()]
+        return self.represent_sequence(tag, [items])
+
+Representer.add_representer(complex,
+        Representer.represent_complex)
+
+Representer.add_representer(tuple,
+        Representer.represent_tuple)
+
+Representer.add_multi_representer(type,
+        Representer.represent_name)
+
+Representer.add_representer(collections.OrderedDict,
+        Representer.represent_ordered_dict)
+
+Representer.add_representer(types.FunctionType,
+        Representer.represent_name)
+
+Representer.add_representer(types.BuiltinFunctionType,
+        Representer.represent_name)
+
+Representer.add_representer(types.ModuleType,
+        Representer.represent_module)
+
+Representer.add_multi_representer(object,
+        Representer.represent_object)
+
diff --git a/ubuntu/venv/yaml/resolver.py b/ubuntu/venv/yaml/resolver.py
new file mode 100644
index 0000000..3522bda
--- /dev/null
+++ b/ubuntu/venv/yaml/resolver.py
@@ -0,0 +1,227 @@
+
+__all__ = ['BaseResolver', 'Resolver']
+
+from .error import *
+from .nodes import *
+
+import re
+
+class ResolverError(YAMLError):
+    pass
+
+class BaseResolver:
+
+    DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str'
+    DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq'
+    DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
+
+    yaml_implicit_resolvers = {}
+    yaml_path_resolvers = {}
+
+    def __init__(self):
+        self.resolver_exact_paths = []
+        self.resolver_prefix_paths = []
+
+    @classmethod
+    def add_implicit_resolver(cls, tag, regexp, first):
+        if not 'yaml_implicit_resolvers' in cls.__dict__:
+            implicit_resolvers = {}
+            for key in cls.yaml_implicit_resolvers:
+                implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:]
+            cls.yaml_implicit_resolvers = implicit_resolvers
+        if first is None:
+            first = [None]
+        for ch in first:
+            cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
+
+    @classmethod
+    def add_path_resolver(cls, tag, path, kind=None):
+        # Note: `add_path_resolver` is experimental.  The API could be changed.
+        # `new_path` is a pattern that is matched against the path from the
+        # root to the node that is being considered.  `node_path` elements are
+        # tuples `(node_check, index_check)`.  `node_check` is a node class:
+        # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`.  `None`
+        # matches any kind of a node.  `index_check` could be `None`, a boolean
+        # value, a string value, or a number.  `None` and `False` match against
+        # any _value_ of sequence and mapping nodes.  `True` matches against
+        # any _key_ of a mapping node.  A string `index_check` matches against
+        # a mapping value that corresponds to a scalar key which content is
+        # equal to the `index_check` value.  An integer `index_check` matches
+        # against a sequence value with the index equal to `index_check`.
+        if not 'yaml_path_resolvers' in cls.__dict__:
+            cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
+        new_path = []
+        for element in path:
+            if isinstance(element, (list, tuple)):
+                if len(element) == 2:
+                    node_check, index_check = element
+                elif len(element) == 1:
+                    node_check = element[0]
+                    index_check = True
+                else:
+                    raise ResolverError("Invalid path element: %s" % element)
+            else:
+                node_check = None
+                index_check = element
+            if node_check is str:
+                node_check = ScalarNode
+            elif node_check is list:
+                node_check = SequenceNode
+            elif node_check is dict:
+                node_check = MappingNode
+            elif node_check not in [ScalarNode, SequenceNode, MappingNode]  \
+                    and not isinstance(node_check, str) \
+                    and node_check is not None:
+                raise ResolverError("Invalid node checker: %s" % node_check)
+            if not isinstance(index_check, (str, int))  \
+                    and index_check is not None:
+                raise ResolverError("Invalid index checker: %s" % index_check)
+            new_path.append((node_check, index_check))
+        if kind is str:
+            kind = ScalarNode
+        elif kind is list:
+            kind = SequenceNode
+        elif kind is dict:
+            kind = MappingNode
+        elif kind not in [ScalarNode, SequenceNode, MappingNode]    \
+                and kind is not None:
+            raise ResolverError("Invalid node kind: %s" % kind)
+        cls.yaml_path_resolvers[tuple(new_path), kind] = tag
+
+    def descend_resolver(self, current_node, current_index):
+        if not self.yaml_path_resolvers:
+            return
+        exact_paths = {}
+        prefix_paths = []
+        if current_node:
+            depth = len(self.resolver_prefix_paths)
+            for path, kind in self.resolver_prefix_paths[-1]:
+                if self.check_resolver_prefix(depth, path, kind,
+                        current_node, current_index):
+                    if len(path) > depth:
+                        prefix_paths.append((path, kind))
+                    else:
+                        exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+        else:
+            for path, kind in self.yaml_path_resolvers:
+                if not path:
+                    exact_paths[kind] = self.yaml_path_resolvers[path, kind]
+                else:
+                    prefix_paths.append((path, kind))
+        self.resolver_exact_paths.append(exact_paths)
+        self.resolver_prefix_paths.append(prefix_paths)
+
+    def ascend_resolver(self):
+        if not self.yaml_path_resolvers:
+            return
+        self.resolver_exact_paths.pop()
+        self.resolver_prefix_paths.pop()
+
+    def check_resolver_prefix(self, depth, path, kind,
+            current_node, current_index):
+        node_check, index_check = path[depth-1]
+        if isinstance(node_check, str):
+            if current_node.tag != node_check:
+                return
+        elif node_check is not None:
+            if not isinstance(current_node, node_check):
+                return
+        if index_check is True and current_index is not None:
+            return
+        if (index_check is False or index_check is None)    \
+                and current_index is None:
+            return
+        if isinstance(index_check, str):
+            if not (isinstance(current_index, ScalarNode)
+                    and index_check == current_index.value):
+                return
+        elif isinstance(index_check, int) and not isinstance(index_check, bool):
+            if index_check != current_index:
+                return
+        return True
+
+    def resolve(self, kind, value, implicit):
+        if kind is ScalarNode and implicit[0]:
+            if value == '':
+                resolvers = self.yaml_implicit_resolvers.get('', [])
+            else:
+                resolvers = self.yaml_implicit_resolvers.get(value[0], [])
+            wildcard_resolvers = self.yaml_implicit_resolvers.get(None, [])
+            for tag, regexp in resolvers + wildcard_resolvers:
+                if regexp.match(value):
+                    return tag
+            implicit = implicit[1]
+        if self.yaml_path_resolvers:
+            exact_paths = self.resolver_exact_paths[-1]
+            if kind in exact_paths:
+                return exact_paths[kind]
+            if None in exact_paths:
+                return exact_paths[None]
+        if kind is ScalarNode:
+            return self.DEFAULT_SCALAR_TAG
+        elif kind is SequenceNode:
+            return self.DEFAULT_SEQUENCE_TAG
+        elif kind is MappingNode:
+            return self.DEFAULT_MAPPING_TAG
+
+class Resolver(BaseResolver):
+    pass
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:bool',
+        re.compile(r'''^(?:yes|Yes|YES|no|No|NO
+                    |true|True|TRUE|false|False|FALSE
+                    |on|On|ON|off|Off|OFF)$''', re.X),
+        list('yYnNtTfFoO'))
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:float',
+        re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?
+                    |\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?
+                    |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
+                    |[-+]?\.(?:inf|Inf|INF)
+                    |\.(?:nan|NaN|NAN))$''', re.X),
+        list('-+0123456789.'))
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:int',
+        re.compile(r'''^(?:[-+]?0b[0-1_]+
+                    |[-+]?0[0-7_]+
+                    |[-+]?(?:0|[1-9][0-9_]*)
+                    |[-+]?0x[0-9a-fA-F_]+
+                    |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
+        list('-+0123456789'))
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:merge',
+        re.compile(r'^(?:<<)$'),
+        ['<'])
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:null',
+        re.compile(r'''^(?: ~
+                    |null|Null|NULL
+                    | )$''', re.X),
+        ['~', 'n', 'N', ''])
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:timestamp',
+        re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
+                    |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
+                     (?:[Tt]|[ \t]+)[0-9][0-9]?
+                     :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)?
+                     (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
+        list('0123456789'))
+
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:value',
+        re.compile(r'^(?:=)$'),
+        ['='])
+
+# The following resolver is only for documentation purposes. It cannot work
+# because plain scalars cannot start with '!', '&', or '*'.
+Resolver.add_implicit_resolver(
+        'tag:yaml.org,2002:yaml',
+        re.compile(r'^(?:!|&|\*)$'),
+        list('!&*'))
+
diff --git a/ubuntu/venv/yaml/scanner.py b/ubuntu/venv/yaml/scanner.py
new file mode 100644
index 0000000..de925b0
--- /dev/null
+++ b/ubuntu/venv/yaml/scanner.py
@@ -0,0 +1,1435 @@
+
+# Scanner produces tokens of the following types:
+# STREAM-START
+# STREAM-END
+# DIRECTIVE(name, value)
+# DOCUMENT-START
+# DOCUMENT-END
+# BLOCK-SEQUENCE-START
+# BLOCK-MAPPING-START
+# BLOCK-END
+# FLOW-SEQUENCE-START
+# FLOW-MAPPING-START
+# FLOW-SEQUENCE-END
+# FLOW-MAPPING-END
+# BLOCK-ENTRY
+# FLOW-ENTRY
+# KEY
+# VALUE
+# ALIAS(value)
+# ANCHOR(value)
+# TAG(value)
+# SCALAR(value, plain, style)
+#
+# Read comments in the Scanner code for more details.
+#
+
+__all__ = ['Scanner', 'ScannerError']
+
+from .error import MarkedYAMLError
+from .tokens import *
+
+class ScannerError(MarkedYAMLError):
+    pass
+
+class SimpleKey:
+    # See below simple keys treatment.
+
+    def __init__(self, token_number, required, index, line, column, mark):
+        self.token_number = token_number
+        self.required = required
+        self.index = index
+        self.line = line
+        self.column = column
+        self.mark = mark
+
+class Scanner:
+
+    def __init__(self):
+        """Initialize the scanner."""
+        # It is assumed that Scanner and Reader will have a common descendant.
+        # Reader do the dirty work of checking for BOM and converting the
+        # input data to Unicode. It also adds NUL to the end.
+        #
+        # Reader supports the following methods
+        #   self.peek(i=0)       # peek the next i-th character
+        #   self.prefix(l=1)     # peek the next l characters
+        #   self.forward(l=1)    # read the next l characters and move the pointer.
+
+        # Had we reached the end of the stream?
+        self.done = False
+
+        # The number of unclosed '{' and '['. `flow_level == 0` means block
+        # context.
+        self.flow_level = 0
+
+        # List of processed tokens that are not yet emitted.
+        self.tokens = []
+
+        # Add the STREAM-START token.
+        self.fetch_stream_start()
+
+        # Number of tokens that were emitted through the `get_token` method.
+        self.tokens_taken = 0
+
+        # The current indentation level.
+        self.indent = -1
+
+        # Past indentation levels.
+        self.indents = []
+
+        # Variables related to simple keys treatment.
+
+        # A simple key is a key that is not denoted by the '?' indicator.
+        # Example of simple keys:
+        #   ---
+        #   block simple key: value
+        #   ? not a simple key:
+        #   : { flow simple key: value }
+        # We emit the KEY token before all keys, so when we find a potential
+        # simple key, we try to locate the corresponding ':' indicator.
+        # Simple keys should be limited to a single line and 1024 characters.
+
+        # Can a simple key start at the current position? A simple key may
+        # start:
+        # - at the beginning of the line, not counting indentation spaces
+        #       (in block context),
+        # - after '{', '[', ',' (in the flow context),
+        # - after '?', ':', '-' (in the block context).
+        # In the block context, this flag also signifies if a block collection
+        # may start at the current position.
+        self.allow_simple_key = True
+
+        # Keep track of possible simple keys. This is a dictionary. The key
+        # is `flow_level`; there can be no more that one possible simple key
+        # for each level. The value is a SimpleKey record:
+        #   (token_number, required, index, line, column, mark)
+        # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow),
+        # '[', or '{' tokens.
+        self.possible_simple_keys = {}
+
+    # Public methods.
+
+    def check_token(self, *choices):
+        # Check if the next token is one of the given types.
+        while self.need_more_tokens():
+            self.fetch_more_tokens()
+        if self.tokens:
+            if not choices:
+                return True
+            for choice in choices:
+                if isinstance(self.tokens[0], choice):
+                    return True
+        return False
+
+    def peek_token(self):
+        # Return the next token, but do not delete if from the queue.
+        # Return None if no more tokens.
+        while self.need_more_tokens():
+            self.fetch_more_tokens()
+        if self.tokens:
+            return self.tokens[0]
+        else:
+            return None
+
+    def get_token(self):
+        # Return the next token.
+        while self.need_more_tokens():
+            self.fetch_more_tokens()
+        if self.tokens:
+            self.tokens_taken += 1
+            return self.tokens.pop(0)
+
+    # Private methods.
+
+    def need_more_tokens(self):
+        if self.done:
+            return False
+        if not self.tokens:
+            return True
+        # The current token may be a potential simple key, so we
+        # need to look further.
+        self.stale_possible_simple_keys()
+        if self.next_possible_simple_key() == self.tokens_taken:
+            return True
+
+    def fetch_more_tokens(self):
+
+        # Eat whitespaces and comments until we reach the next token.
+        self.scan_to_next_token()
+
+        # Remove obsolete possible simple keys.
+        self.stale_possible_simple_keys()
+
+        # Compare the current indentation and column. It may add some tokens
+        # and decrease the current indentation level.
+        self.unwind_indent(self.column)
+
+        # Peek the next character.
+        ch = self.peek()
+
+        # Is it the end of stream?
+        if ch == '\0':
+            return self.fetch_stream_end()
+
+        # Is it a directive?
+        if ch == '%' and self.check_directive():
+            return self.fetch_directive()
+
+        # Is it the document start?
+        if ch == '-' and self.check_document_start():
+            return self.fetch_document_start()
+
+        # Is it the document end?
+        if ch == '.' and self.check_document_end():
+            return self.fetch_document_end()
+
+        # TODO: support for BOM within a stream.
+        #if ch == '\uFEFF':
+        #    return self.fetch_bom()    <-- issue BOMToken
+
+        # Note: the order of the following checks is NOT significant.
+
+        # Is it the flow sequence start indicator?
+        if ch == '[':
+            return self.fetch_flow_sequence_start()
+
+        # Is it the flow mapping start indicator?
+        if ch == '{':
+            return self.fetch_flow_mapping_start()
+
+        # Is it the flow sequence end indicator?
+        if ch == ']':
+            return self.fetch_flow_sequence_end()
+
+        # Is it the flow mapping end indicator?
+        if ch == '}':
+            return self.fetch_flow_mapping_end()
+
+        # Is it the flow entry indicator?
+        if ch == ',':
+            return self.fetch_flow_entry()
+
+        # Is it the block entry indicator?
+        if ch == '-' and self.check_block_entry():
+            return self.fetch_block_entry()
+
+        # Is it the key indicator?
+        if ch == '?' and self.check_key():
+            return self.fetch_key()
+
+        # Is it the value indicator?
+        if ch == ':' and self.check_value():
+            return self.fetch_value()
+
+        # Is it an alias?
+        if ch == '*':
+            return self.fetch_alias()
+
+        # Is it an anchor?
+        if ch == '&':
+            return self.fetch_anchor()
+
+        # Is it a tag?
+        if ch == '!':
+            return self.fetch_tag()
+
+        # Is it a literal scalar?
+        if ch == '|' and not self.flow_level:
+            return self.fetch_literal()
+
+        # Is it a folded scalar?
+        if ch == '>' and not self.flow_level:
+            return self.fetch_folded()
+
+        # Is it a single quoted scalar?
+        if ch == '\'':
+            return self.fetch_single()
+
+        # Is it a double quoted scalar?
+        if ch == '\"':
+            return self.fetch_double()
+
+        # It must be a plain scalar then.
+        if self.check_plain():
+            return self.fetch_plain()
+
+        # No? It's an error. Let's produce a nice error message.
+        raise ScannerError("while scanning for the next token", None,
+                "found character %r that cannot start any token" % ch,
+                self.get_mark())
+
+    # Simple keys treatment.
+
+    def next_possible_simple_key(self):
+        # Return the number of the nearest possible simple key. Actually we
+        # don't need to loop through the whole dictionary. We may replace it
+        # with the following code:
+        #   if not self.possible_simple_keys:
+        #       return None
+        #   return self.possible_simple_keys[
+        #           min(self.possible_simple_keys.keys())].token_number
+        min_token_number = None
+        for level in self.possible_simple_keys:
+            key = self.possible_simple_keys[level]
+            if min_token_number is None or key.token_number < min_token_number:
+                min_token_number = key.token_number
+        return min_token_number
+
+    def stale_possible_simple_keys(self):
+        # Remove entries that are no longer possible simple keys. According to
+        # the YAML specification, simple keys
+        # - should be limited to a single line,
+        # - should be no longer than 1024 characters.
+        # Disabling this procedure will allow simple keys of any length and
+        # height (may cause problems if indentation is broken though).
+        for level in list(self.possible_simple_keys):
+            key = self.possible_simple_keys[level]
+            if key.line != self.line  \
+                    or self.index-key.index > 1024:
+                if key.required:
+                    raise ScannerError("while scanning a simple key", key.mark,
+                            "could not find expected ':'", self.get_mark())
+                del self.possible_simple_keys[level]
+
+    def save_possible_simple_key(self):
+        # The next token may start a simple key. We check if it's possible
+        # and save its position. This function is called for
+        #   ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
+
+        # Check if a simple key is required at the current position.
+        required = not self.flow_level and self.indent == self.column
+
+        # The next token might be a simple key. Let's save it's number and
+        # position.
+        if self.allow_simple_key:
+            self.remove_possible_simple_key()
+            token_number = self.tokens_taken+len(self.tokens)
+            key = SimpleKey(token_number, required,
+                    self.index, self.line, self.column, self.get_mark())
+            self.possible_simple_keys[self.flow_level] = key
+
+    def remove_possible_simple_key(self):
+        # Remove the saved possible key position at the current flow level.
+        if self.flow_level in self.possible_simple_keys:
+            key = self.possible_simple_keys[self.flow_level]
+            
+            if key.required:
+                raise ScannerError("while scanning a simple key", key.mark,
+                        "could not find expected ':'", self.get_mark())
+
+            del self.possible_simple_keys[self.flow_level]
+
+    # Indentation functions.
+
+    def unwind_indent(self, column):
+
+        ## In flow context, tokens should respect indentation.
+        ## Actually the condition should be `self.indent >= column` according to
+        ## the spec. But this condition will prohibit intuitively correct
+        ## constructions such as
+        ## key : {
+        ## }
+        #if self.flow_level and self.indent > column:
+        #    raise ScannerError(None, None,
+        #            "invalid indentation or unclosed '[' or '{'",
+        #            self.get_mark())
+
+        # In the flow context, indentation is ignored. We make the scanner less
+        # restrictive then specification requires.
+        if self.flow_level:
+            return
+
+        # In block context, we may need to issue the BLOCK-END tokens.
+        while self.indent > column:
+            mark = self.get_mark()
+            self.indent = self.indents.pop()
+            self.tokens.append(BlockEndToken(mark, mark))
+
+    def add_indent(self, column):
+        # Check if we need to increase indentation.
+        if self.indent < column:
+            self.indents.append(self.indent)
+            self.indent = column
+            return True
+        return False
+
+    # Fetchers.
+
+    def fetch_stream_start(self):
+        # We always add STREAM-START as the first token and STREAM-END as the
+        # last token.
+
+        # Read the token.
+        mark = self.get_mark()
+        
+        # Add STREAM-START.
+        self.tokens.append(StreamStartToken(mark, mark,
+            encoding=self.encoding))
+        
+
+    def fetch_stream_end(self):
+
+        # Set the current indentation to -1.
+        self.unwind_indent(-1)
+
+        # Reset simple keys.
+        self.remove_possible_simple_key()
+        self.allow_simple_key = False
+        self.possible_simple_keys = {}
+
+        # Read the token.
+        mark = self.get_mark()
+        
+        # Add STREAM-END.
+        self.tokens.append(StreamEndToken(mark, mark))
+
+        # The steam is finished.
+        self.done = True
+
+    def fetch_directive(self):
+        
+        # Set the current indentation to -1.
+        self.unwind_indent(-1)
+
+        # Reset simple keys.
+        self.remove_possible_simple_key()
+        self.allow_simple_key = False
+
+        # Scan and add DIRECTIVE.
+        self.tokens.append(self.scan_directive())
+
+    def fetch_document_start(self):
+        self.fetch_document_indicator(DocumentStartToken)
+
+    def fetch_document_end(self):
+        self.fetch_document_indicator(DocumentEndToken)
+
+    def fetch_document_indicator(self, TokenClass):
+
+        # Set the current indentation to -1.
+        self.unwind_indent(-1)
+
+        # Reset simple keys. Note that there could not be a block collection
+        # after '---'.
+        self.remove_possible_simple_key()
+        self.allow_simple_key = False
+
+        # Add DOCUMENT-START or DOCUMENT-END.
+        start_mark = self.get_mark()
+        self.forward(3)
+        end_mark = self.get_mark()
+        self.tokens.append(TokenClass(start_mark, end_mark))
+
+    def fetch_flow_sequence_start(self):
+        self.fetch_flow_collection_start(FlowSequenceStartToken)
+
+    def fetch_flow_mapping_start(self):
+        self.fetch_flow_collection_start(FlowMappingStartToken)
+
+    def fetch_flow_collection_start(self, TokenClass):
+
+        # '[' and '{' may start a simple key.
+        self.save_possible_simple_key()
+
+        # Increase the flow level.
+        self.flow_level += 1
+
+        # Simple keys are allowed after '[' and '{'.
+        self.allow_simple_key = True
+
+        # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(TokenClass(start_mark, end_mark))
+
+    def fetch_flow_sequence_end(self):
+        self.fetch_flow_collection_end(FlowSequenceEndToken)
+
+    def fetch_flow_mapping_end(self):
+        self.fetch_flow_collection_end(FlowMappingEndToken)
+
+    def fetch_flow_collection_end(self, TokenClass):
+
+        # Reset possible simple key on the current level.
+        self.remove_possible_simple_key()
+
+        # Decrease the flow level.
+        self.flow_level -= 1
+
+        # No simple keys after ']' or '}'.
+        self.allow_simple_key = False
+
+        # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(TokenClass(start_mark, end_mark))
+
+    def fetch_flow_entry(self):
+
+        # Simple keys are allowed after ','.
+        self.allow_simple_key = True
+
+        # Reset possible simple key on the current level.
+        self.remove_possible_simple_key()
+
+        # Add FLOW-ENTRY.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(FlowEntryToken(start_mark, end_mark))
+
+    def fetch_block_entry(self):
+
+        # Block context needs additional checks.
+        if not self.flow_level:
+
+            # Are we allowed to start a new entry?
+            if not self.allow_simple_key:
+                raise ScannerError(None, None,
+                        "sequence entries are not allowed here",
+                        self.get_mark())
+
+            # We may need to add BLOCK-SEQUENCE-START.
+            if self.add_indent(self.column):
+                mark = self.get_mark()
+                self.tokens.append(BlockSequenceStartToken(mark, mark))
+
+        # It's an error for the block entry to occur in the flow context,
+        # but we let the parser detect this.
+        else:
+            pass
+
+        # Simple keys are allowed after '-'.
+        self.allow_simple_key = True
+
+        # Reset possible simple key on the current level.
+        self.remove_possible_simple_key()
+
+        # Add BLOCK-ENTRY.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(BlockEntryToken(start_mark, end_mark))
+
+    def fetch_key(self):
+        
+        # Block context needs additional checks.
+        if not self.flow_level:
+
+            # Are we allowed to start a key (not necessary a simple)?
+            if not self.allow_simple_key:
+                raise ScannerError(None, None,
+                        "mapping keys are not allowed here",
+                        self.get_mark())
+
+            # We may need to add BLOCK-MAPPING-START.
+            if self.add_indent(self.column):
+                mark = self.get_mark()
+                self.tokens.append(BlockMappingStartToken(mark, mark))
+
+        # Simple keys are allowed after '?' in the block context.
+        self.allow_simple_key = not self.flow_level
+
+        # Reset possible simple key on the current level.
+        self.remove_possible_simple_key()
+
+        # Add KEY.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(KeyToken(start_mark, end_mark))
+
+    def fetch_value(self):
+
+        # Do we determine a simple key?
+        if self.flow_level in self.possible_simple_keys:
+
+            # Add KEY.
+            key = self.possible_simple_keys[self.flow_level]
+            del self.possible_simple_keys[self.flow_level]
+            self.tokens.insert(key.token_number-self.tokens_taken,
+                    KeyToken(key.mark, key.mark))
+
+            # If this key starts a new block mapping, we need to add
+            # BLOCK-MAPPING-START.
+            if not self.flow_level:
+                if self.add_indent(key.column):
+                    self.tokens.insert(key.token_number-self.tokens_taken,
+                            BlockMappingStartToken(key.mark, key.mark))
+
+            # There cannot be two simple keys one after another.
+            self.allow_simple_key = False
+
+        # It must be a part of a complex key.
+        else:
+            
+            # Block context needs additional checks.
+            # (Do we really need them? They will be caught by the parser
+            # anyway.)
+            if not self.flow_level:
+
+                # We are allowed to start a complex value if and only if
+                # we can start a simple key.
+                if not self.allow_simple_key:
+                    raise ScannerError(None, None,
+                            "mapping values are not allowed here",
+                            self.get_mark())
+
+            # If this value starts a new block mapping, we need to add
+            # BLOCK-MAPPING-START.  It will be detected as an error later by
+            # the parser.
+            if not self.flow_level:
+                if self.add_indent(self.column):
+                    mark = self.get_mark()
+                    self.tokens.append(BlockMappingStartToken(mark, mark))
+
+            # Simple keys are allowed after ':' in the block context.
+            self.allow_simple_key = not self.flow_level
+
+            # Reset possible simple key on the current level.
+            self.remove_possible_simple_key()
+
+        # Add VALUE.
+        start_mark = self.get_mark()
+        self.forward()
+        end_mark = self.get_mark()
+        self.tokens.append(ValueToken(start_mark, end_mark))
+
+    def fetch_alias(self):
+
+        # ALIAS could be a simple key.
+        self.save_possible_simple_key()
+
+        # No simple keys after ALIAS.
+        self.allow_simple_key = False
+
+        # Scan and add ALIAS.
+        self.tokens.append(self.scan_anchor(AliasToken))
+
+    def fetch_anchor(self):
+
+        # ANCHOR could start a simple key.
+        self.save_possible_simple_key()
+
+        # No simple keys after ANCHOR.
+        self.allow_simple_key = False
+
+        # Scan and add ANCHOR.
+        self.tokens.append(self.scan_anchor(AnchorToken))
+
+    def fetch_tag(self):
+
+        # TAG could start a simple key.
+        self.save_possible_simple_key()
+
+        # No simple keys after TAG.
+        self.allow_simple_key = False
+
+        # Scan and add TAG.
+        self.tokens.append(self.scan_tag())
+
+    def fetch_literal(self):
+        self.fetch_block_scalar(style='|')
+
+    def fetch_folded(self):
+        self.fetch_block_scalar(style='>')
+
+    def fetch_block_scalar(self, style):
+
+        # A simple key may follow a block scalar.
+        self.allow_simple_key = True
+
+        # Reset possible simple key on the current level.
+        self.remove_possible_simple_key()
+
+        # Scan and add SCALAR.
+        self.tokens.append(self.scan_block_scalar(style))
+
+    def fetch_single(self):
+        self.fetch_flow_scalar(style='\'')
+
+    def fetch_double(self):
+        self.fetch_flow_scalar(style='"')
+
+    def fetch_flow_scalar(self, style):
+
+        # A flow scalar could be a simple key.
+        self.save_possible_simple_key()
+
+        # No simple keys after flow scalars.
+        self.allow_simple_key = False
+
+        # Scan and add SCALAR.
+        self.tokens.append(self.scan_flow_scalar(style))
+
+    def fetch_plain(self):
+
+        # A plain scalar could be a simple key.
+        self.save_possible_simple_key()
+
+        # No simple keys after plain scalars. But note that `scan_plain` will
+        # change this flag if the scan is finished at the beginning of the
+        # line.
+        self.allow_simple_key = False
+
+        # Scan and add SCALAR. May change `allow_simple_key`.
+        self.tokens.append(self.scan_plain())
+
+    # Checkers.
+
+    def check_directive(self):
+
+        # DIRECTIVE:        ^ '%' ...
+        # The '%' indicator is already checked.
+        if self.column == 0:
+            return True
+
+    def check_document_start(self):
+
+        # DOCUMENT-START:   ^ '---' (' '|'\n')
+        if self.column == 0:
+            if self.prefix(3) == '---'  \
+                    and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+                return True
+
+    def check_document_end(self):
+
+        # DOCUMENT-END:     ^ '...' (' '|'\n')
+        if self.column == 0:
+            if self.prefix(3) == '...'  \
+                    and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+                return True
+
+    def check_block_entry(self):
+
+        # BLOCK-ENTRY:      '-' (' '|'\n')
+        return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+    def check_key(self):
+
+        # KEY(flow context):    '?'
+        if self.flow_level:
+            return True
+
+        # KEY(block context):   '?' (' '|'\n')
+        else:
+            return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+    def check_value(self):
+
+        # VALUE(flow context):  ':'
+        if self.flow_level:
+            return True
+
+        # VALUE(block context): ':' (' '|'\n')
+        else:
+            return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029'
+
+    def check_plain(self):
+
+        # A plain scalar may start with any non-space character except:
+        #   '-', '?', ':', ',', '[', ']', '{', '}',
+        #   '#', '&', '*', '!', '|', '>', '\'', '\"',
+        #   '%', '@', '`'.
+        #
+        # It may also start with
+        #   '-', '?', ':'
+        # if it is followed by a non-space character.
+        #
+        # Note that we limit the last rule to the block context (except the
+        # '-' character) because we want the flow context to be space
+        # independent.
+        ch = self.peek()
+        return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`'  \
+                or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029'
+                        and (ch == '-' or (not self.flow_level and ch in '?:')))
+
+    # Scanners.
+
+    def scan_to_next_token(self):
+        # We ignore spaces, line breaks and comments.
+        # If we find a line break in the block context, we set the flag
+        # `allow_simple_key` on.
+        # The byte order mark is stripped if it's the first character in the
+        # stream. We do not yet support BOM inside the stream as the
+        # specification requires. Any such mark will be considered as a part
+        # of the document.
+        #
+        # TODO: We need to make tab handling rules more sane. A good rule is
+        #   Tabs cannot precede tokens
+        #   BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
+        #   KEY(block), VALUE(block), BLOCK-ENTRY
+        # So the checking code is
+        #   if :
+        #       self.allow_simple_keys = False
+        # We also need to add the check for `allow_simple_keys == True` to
+        # `unwind_indent` before issuing BLOCK-END.
+        # Scanners for block, flow, and plain scalars need to be modified.
+
+        if self.index == 0 and self.peek() == '\uFEFF':
+            self.forward()
+        found = False
+        while not found:
+            while self.peek() == ' ':
+                self.forward()
+            if self.peek() == '#':
+                while self.peek() not in '\0\r\n\x85\u2028\u2029':
+                    self.forward()
+            if self.scan_line_break():
+                if not self.flow_level:
+                    self.allow_simple_key = True
+            else:
+                found = True
+
+    def scan_directive(self):
+        # See the specification for details.
+        start_mark = self.get_mark()
+        self.forward()
+        name = self.scan_directive_name(start_mark)
+        value = None
+        if name == 'YAML':
+            value = self.scan_yaml_directive_value(start_mark)
+            end_mark = self.get_mark()
+        elif name == 'TAG':
+            value = self.scan_tag_directive_value(start_mark)
+            end_mark = self.get_mark()
+        else:
+            end_mark = self.get_mark()
+            while self.peek() not in '\0\r\n\x85\u2028\u2029':
+                self.forward()
+        self.scan_directive_ignored_line(start_mark)
+        return DirectiveToken(name, value, start_mark, end_mark)
+
+    def scan_directive_name(self, start_mark):
+        # See the specification for details.
+        length = 0
+        ch = self.peek(length)
+        while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'  \
+                or ch in '-_':
+            length += 1
+            ch = self.peek(length)
+        if not length:
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected alphabetic or numeric character, but found %r"
+                    % ch, self.get_mark())
+        value = self.prefix(length)
+        self.forward(length)
+        ch = self.peek()
+        if ch not in '\0 \r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected alphabetic or numeric character, but found %r"
+                    % ch, self.get_mark())
+        return value
+
+    def scan_yaml_directive_value(self, start_mark):
+        # See the specification for details.
+        while self.peek() == ' ':
+            self.forward()
+        major = self.scan_yaml_directive_number(start_mark)
+        if self.peek() != '.':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected a digit or '.', but found %r" % self.peek(),
+                    self.get_mark())
+        self.forward()
+        minor = self.scan_yaml_directive_number(start_mark)
+        if self.peek() not in '\0 \r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected a digit or ' ', but found %r" % self.peek(),
+                    self.get_mark())
+        return (major, minor)
+
+    def scan_yaml_directive_number(self, start_mark):
+        # See the specification for details.
+        ch = self.peek()
+        if not ('0' <= ch <= '9'):
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected a digit, but found %r" % ch, self.get_mark())
+        length = 0
+        while '0' <= self.peek(length) <= '9':
+            length += 1
+        value = int(self.prefix(length))
+        self.forward(length)
+        return value
+
+    def scan_tag_directive_value(self, start_mark):
+        # See the specification for details.
+        while self.peek() == ' ':
+            self.forward()
+        handle = self.scan_tag_directive_handle(start_mark)
+        while self.peek() == ' ':
+            self.forward()
+        prefix = self.scan_tag_directive_prefix(start_mark)
+        return (handle, prefix)
+
+    def scan_tag_directive_handle(self, start_mark):
+        # See the specification for details.
+        value = self.scan_tag_handle('directive', start_mark)
+        ch = self.peek()
+        if ch != ' ':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected ' ', but found %r" % ch, self.get_mark())
+        return value
+
+    def scan_tag_directive_prefix(self, start_mark):
+        # See the specification for details.
+        value = self.scan_tag_uri('directive', start_mark)
+        ch = self.peek()
+        if ch not in '\0 \r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected ' ', but found %r" % ch, self.get_mark())
+        return value
+
+    def scan_directive_ignored_line(self, start_mark):
+        # See the specification for details.
+        while self.peek() == ' ':
+            self.forward()
+        if self.peek() == '#':
+            while self.peek() not in '\0\r\n\x85\u2028\u2029':
+                self.forward()
+        ch = self.peek()
+        if ch not in '\0\r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a directive", start_mark,
+                    "expected a comment or a line break, but found %r"
+                        % ch, self.get_mark())
+        self.scan_line_break()
+
+    def scan_anchor(self, TokenClass):
+        # The specification does not restrict characters for anchors and
+        # aliases. This may lead to problems, for instance, the document:
+        #   [ *alias, value ]
+        # can be interpreted in two ways, as
+        #   [ "value" ]
+        # and
+        #   [ *alias , "value" ]
+        # Therefore we restrict aliases to numbers and ASCII letters.
+        start_mark = self.get_mark()
+        indicator = self.peek()
+        if indicator == '*':
+            name = 'alias'
+        else:
+            name = 'anchor'
+        self.forward()
+        length = 0
+        ch = self.peek(length)
+        while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'  \
+                or ch in '-_':
+            length += 1
+            ch = self.peek(length)
+        if not length:
+            raise ScannerError("while scanning an %s" % name, start_mark,
+                    "expected alphabetic or numeric character, but found %r"
+                    % ch, self.get_mark())
+        value = self.prefix(length)
+        self.forward(length)
+        ch = self.peek()
+        if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`':
+            raise ScannerError("while scanning an %s" % name, start_mark,
+                    "expected alphabetic or numeric character, but found %r"
+                    % ch, self.get_mark())
+        end_mark = self.get_mark()
+        return TokenClass(value, start_mark, end_mark)
+
+    def scan_tag(self):
+        # See the specification for details.
+        start_mark = self.get_mark()
+        ch = self.peek(1)
+        if ch == '<':
+            handle = None
+            self.forward(2)
+            suffix = self.scan_tag_uri('tag', start_mark)
+            if self.peek() != '>':
+                raise ScannerError("while parsing a tag", start_mark,
+                        "expected '>', but found %r" % self.peek(),
+                        self.get_mark())
+            self.forward()
+        elif ch in '\0 \t\r\n\x85\u2028\u2029':
+            handle = None
+            suffix = '!'
+            self.forward()
+        else:
+            length = 1
+            use_handle = False
+            while ch not in '\0 \r\n\x85\u2028\u2029':
+                if ch == '!':
+                    use_handle = True
+                    break
+                length += 1
+                ch = self.peek(length)
+            handle = '!'
+            if use_handle:
+                handle = self.scan_tag_handle('tag', start_mark)
+            else:
+                handle = '!'
+                self.forward()
+            suffix = self.scan_tag_uri('tag', start_mark)
+        ch = self.peek()
+        if ch not in '\0 \r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a tag", start_mark,
+                    "expected ' ', but found %r" % ch, self.get_mark())
+        value = (handle, suffix)
+        end_mark = self.get_mark()
+        return TagToken(value, start_mark, end_mark)
+
+    def scan_block_scalar(self, style):
+        # See the specification for details.
+
+        if style == '>':
+            folded = True
+        else:
+            folded = False
+
+        chunks = []
+        start_mark = self.get_mark()
+
+        # Scan the header.
+        self.forward()
+        chomping, increment = self.scan_block_scalar_indicators(start_mark)
+        self.scan_block_scalar_ignored_line(start_mark)
+
+        # Determine the indentation level and go to the first non-empty line.
+        min_indent = self.indent+1
+        if min_indent < 1:
+            min_indent = 1
+        if increment is None:
+            breaks, max_indent, end_mark = self.scan_block_scalar_indentation()
+            indent = max(min_indent, max_indent)
+        else:
+            indent = min_indent+increment-1
+            breaks, end_mark = self.scan_block_scalar_breaks(indent)
+        line_break = ''
+
+        # Scan the inner part of the block scalar.
+        while self.column == indent and self.peek() != '\0':
+            chunks.extend(breaks)
+            leading_non_space = self.peek() not in ' \t'
+            length = 0
+            while self.peek(length) not in '\0\r\n\x85\u2028\u2029':
+                length += 1
+            chunks.append(self.prefix(length))
+            self.forward(length)
+            line_break = self.scan_line_break()
+            breaks, end_mark = self.scan_block_scalar_breaks(indent)
+            if self.column == indent and self.peek() != '\0':
+
+                # Unfortunately, folding rules are ambiguous.
+                #
+                # This is the folding according to the specification:
+                
+                if folded and line_break == '\n'    \
+                        and leading_non_space and self.peek() not in ' \t':
+                    if not breaks:
+                        chunks.append(' ')
+                else:
+                    chunks.append(line_break)
+                
+                # This is Clark Evans's interpretation (also in the spec
+                # examples):
+                #
+                #if folded and line_break == '\n':
+                #    if not breaks:
+                #        if self.peek() not in ' \t':
+                #            chunks.append(' ')
+                #        else:
+                #            chunks.append(line_break)
+                #else:
+                #    chunks.append(line_break)
+            else:
+                break
+
+        # Chomp the tail.
+        if chomping is not False:
+            chunks.append(line_break)
+        if chomping is True:
+            chunks.extend(breaks)
+
+        # We are done.
+        return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+                style)
+
+    def scan_block_scalar_indicators(self, start_mark):
+        # See the specification for details.
+        chomping = None
+        increment = None
+        ch = self.peek()
+        if ch in '+-':
+            if ch == '+':
+                chomping = True
+            else:
+                chomping = False
+            self.forward()
+            ch = self.peek()
+            if ch in '0123456789':
+                increment = int(ch)
+                if increment == 0:
+                    raise ScannerError("while scanning a block scalar", start_mark,
+                            "expected indentation indicator in the range 1-9, but found 0",
+                            self.get_mark())
+                self.forward()
+        elif ch in '0123456789':
+            increment = int(ch)
+            if increment == 0:
+                raise ScannerError("while scanning a block scalar", start_mark,
+                        "expected indentation indicator in the range 1-9, but found 0",
+                        self.get_mark())
+            self.forward()
+            ch = self.peek()
+            if ch in '+-':
+                if ch == '+':
+                    chomping = True
+                else:
+                    chomping = False
+                self.forward()
+        ch = self.peek()
+        if ch not in '\0 \r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a block scalar", start_mark,
+                    "expected chomping or indentation indicators, but found %r"
+                    % ch, self.get_mark())
+        return chomping, increment
+
+    def scan_block_scalar_ignored_line(self, start_mark):
+        # See the specification for details.
+        while self.peek() == ' ':
+            self.forward()
+        if self.peek() == '#':
+            while self.peek() not in '\0\r\n\x85\u2028\u2029':
+                self.forward()
+        ch = self.peek()
+        if ch not in '\0\r\n\x85\u2028\u2029':
+            raise ScannerError("while scanning a block scalar", start_mark,
+                    "expected a comment or a line break, but found %r" % ch,
+                    self.get_mark())
+        self.scan_line_break()
+
+    def scan_block_scalar_indentation(self):
+        # See the specification for details.
+        chunks = []
+        max_indent = 0
+        end_mark = self.get_mark()
+        while self.peek() in ' \r\n\x85\u2028\u2029':
+            if self.peek() != ' ':
+                chunks.append(self.scan_line_break())
+                end_mark = self.get_mark()
+            else:
+                self.forward()
+                if self.column > max_indent:
+                    max_indent = self.column
+        return chunks, max_indent, end_mark
+
+    def scan_block_scalar_breaks(self, indent):
+        # See the specification for details.
+        chunks = []
+        end_mark = self.get_mark()
+        while self.column < indent and self.peek() == ' ':
+            self.forward()
+        while self.peek() in '\r\n\x85\u2028\u2029':
+            chunks.append(self.scan_line_break())
+            end_mark = self.get_mark()
+            while self.column < indent and self.peek() == ' ':
+                self.forward()
+        return chunks, end_mark
+
+    def scan_flow_scalar(self, style):
+        # See the specification for details.
+        # Note that we loose indentation rules for quoted scalars. Quoted
+        # scalars don't need to adhere indentation because " and ' clearly
+        # mark the beginning and the end of them. Therefore we are less
+        # restrictive then the specification requires. We only need to check
+        # that document separators are not included in scalars.
+        if style == '"':
+            double = True
+        else:
+            double = False
+        chunks = []
+        start_mark = self.get_mark()
+        quote = self.peek()
+        self.forward()
+        chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+        while self.peek() != quote:
+            chunks.extend(self.scan_flow_scalar_spaces(double, start_mark))
+            chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark))
+        self.forward()
+        end_mark = self.get_mark()
+        return ScalarToken(''.join(chunks), False, start_mark, end_mark,
+                style)
+
+    ESCAPE_REPLACEMENTS = {
+        '0':    '\0',
+        'a':    '\x07',
+        'b':    '\x08',
+        't':    '\x09',
+        '\t':   '\x09',
+        'n':    '\x0A',
+        'v':    '\x0B',
+        'f':    '\x0C',
+        'r':    '\x0D',
+        'e':    '\x1B',
+        ' ':    '\x20',
+        '\"':   '\"',
+        '\\':   '\\',
+        '/':    '/',
+        'N':    '\x85',
+        '_':    '\xA0',
+        'L':    '\u2028',
+        'P':    '\u2029',
+    }
+
+    ESCAPE_CODES = {
+        'x':    2,
+        'u':    4,
+        'U':    8,
+    }
+
+    def scan_flow_scalar_non_spaces(self, double, start_mark):
+        # See the specification for details.
+        chunks = []
+        while True:
+            length = 0
+            while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029':
+                length += 1
+            if length:
+                chunks.append(self.prefix(length))
+                self.forward(length)
+            ch = self.peek()
+            if not double and ch == '\'' and self.peek(1) == '\'':
+                chunks.append('\'')
+                self.forward(2)
+            elif (double and ch == '\'') or (not double and ch in '\"\\'):
+                chunks.append(ch)
+                self.forward()
+            elif double and ch == '\\':
+                self.forward()
+                ch = self.peek()
+                if ch in self.ESCAPE_REPLACEMENTS:
+                    chunks.append(self.ESCAPE_REPLACEMENTS[ch])
+                    self.forward()
+                elif ch in self.ESCAPE_CODES:
+                    length = self.ESCAPE_CODES[ch]
+                    self.forward()
+                    for k in range(length):
+                        if self.peek(k) not in '0123456789ABCDEFabcdef':
+                            raise ScannerError("while scanning a double-quoted scalar", start_mark,
+                                    "expected escape sequence of %d hexadecimal numbers, but found %r" %
+                                        (length, self.peek(k)), self.get_mark())
+                    code = int(self.prefix(length), 16)
+                    chunks.append(chr(code))
+                    self.forward(length)
+                elif ch in '\r\n\x85\u2028\u2029':
+                    self.scan_line_break()
+                    chunks.extend(self.scan_flow_scalar_breaks(double, start_mark))
+                else:
+                    raise ScannerError("while scanning a double-quoted scalar", start_mark,
+                            "found unknown escape character %r" % ch, self.get_mark())
+            else:
+                return chunks
+
+    def scan_flow_scalar_spaces(self, double, start_mark):
+        # See the specification for details.
+        chunks = []
+        length = 0
+        while self.peek(length) in ' \t':
+            length += 1
+        whitespaces = self.prefix(length)
+        self.forward(length)
+        ch = self.peek()
+        if ch == '\0':
+            raise ScannerError("while scanning a quoted scalar", start_mark,
+                    "found unexpected end of stream", self.get_mark())
+        elif ch in '\r\n\x85\u2028\u2029':
+            line_break = self.scan_line_break()
+            breaks = self.scan_flow_scalar_breaks(double, start_mark)
+            if line_break != '\n':
+                chunks.append(line_break)
+            elif not breaks:
+                chunks.append(' ')
+            chunks.extend(breaks)
+        else:
+            chunks.append(whitespaces)
+        return chunks
+
+    def scan_flow_scalar_breaks(self, double, start_mark):
+        # See the specification for details.
+        chunks = []
+        while True:
+            # Instead of checking indentation, we check for document
+            # separators.
+            prefix = self.prefix(3)
+            if (prefix == '---' or prefix == '...')   \
+                    and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+                raise ScannerError("while scanning a quoted scalar", start_mark,
+                        "found unexpected document separator", self.get_mark())
+            while self.peek() in ' \t':
+                self.forward()
+            if self.peek() in '\r\n\x85\u2028\u2029':
+                chunks.append(self.scan_line_break())
+            else:
+                return chunks
+
+    def scan_plain(self):
+        # See the specification for details.
+        # We add an additional restriction for the flow context:
+        #   plain scalars in the flow context cannot contain ',' or '?'.
+        # We also keep track of the `allow_simple_key` flag here.
+        # Indentation rules are loosed for the flow context.
+        chunks = []
+        start_mark = self.get_mark()
+        end_mark = start_mark
+        indent = self.indent+1
+        # We allow zero indentation for scalars, but then we need to check for
+        # document separators at the beginning of the line.
+        #if indent == 0:
+        #    indent = 1
+        spaces = []
+        while True:
+            length = 0
+            if self.peek() == '#':
+                break
+            while True:
+                ch = self.peek(length)
+                if ch in '\0 \t\r\n\x85\u2028\u2029'    \
+                        or (ch == ':' and
+                                self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029'
+                                      + (u',[]{}' if self.flow_level else u''))\
+                        or (self.flow_level and ch in ',?[]{}'):
+                    break
+                length += 1
+            if length == 0:
+                break
+            self.allow_simple_key = False
+            chunks.extend(spaces)
+            chunks.append(self.prefix(length))
+            self.forward(length)
+            end_mark = self.get_mark()
+            spaces = self.scan_plain_spaces(indent, start_mark)
+            if not spaces or self.peek() == '#' \
+                    or (not self.flow_level and self.column < indent):
+                break
+        return ScalarToken(''.join(chunks), True, start_mark, end_mark)
+
+    def scan_plain_spaces(self, indent, start_mark):
+        # See the specification for details.
+        # The specification is really confusing about tabs in plain scalars.
+        # We just forbid them completely. Do not use tabs in YAML!
+        chunks = []
+        length = 0
+        while self.peek(length) in ' ':
+            length += 1
+        whitespaces = self.prefix(length)
+        self.forward(length)
+        ch = self.peek()
+        if ch in '\r\n\x85\u2028\u2029':
+            line_break = self.scan_line_break()
+            self.allow_simple_key = True
+            prefix = self.prefix(3)
+            if (prefix == '---' or prefix == '...')   \
+                    and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+                return
+            breaks = []
+            while self.peek() in ' \r\n\x85\u2028\u2029':
+                if self.peek() == ' ':
+                    self.forward()
+                else:
+                    breaks.append(self.scan_line_break())
+                    prefix = self.prefix(3)
+                    if (prefix == '---' or prefix == '...')   \
+                            and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029':
+                        return
+            if line_break != '\n':
+                chunks.append(line_break)
+            elif not breaks:
+                chunks.append(' ')
+            chunks.extend(breaks)
+        elif whitespaces:
+            chunks.append(whitespaces)
+        return chunks
+
+    def scan_tag_handle(self, name, start_mark):
+        # See the specification for details.
+        # For some strange reasons, the specification does not allow '_' in
+        # tag handles. I have allowed it anyway.
+        ch = self.peek()
+        if ch != '!':
+            raise ScannerError("while scanning a %s" % name, start_mark,
+                    "expected '!', but found %r" % ch, self.get_mark())
+        length = 1
+        ch = self.peek(length)
+        if ch != ' ':
+            while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'  \
+                    or ch in '-_':
+                length += 1
+                ch = self.peek(length)
+            if ch != '!':
+                self.forward(length)
+                raise ScannerError("while scanning a %s" % name, start_mark,
+                        "expected '!', but found %r" % ch, self.get_mark())
+            length += 1
+        value = self.prefix(length)
+        self.forward(length)
+        return value
+
+    def scan_tag_uri(self, name, start_mark):
+        # See the specification for details.
+        # Note: we do not check if URI is well-formed.
+        chunks = []
+        length = 0
+        ch = self.peek(length)
+        while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z'  \
+                or ch in '-;/?:@&=+$,_.!~*\'()[]%':
+            if ch == '%':
+                chunks.append(self.prefix(length))
+                self.forward(length)
+                length = 0
+                chunks.append(self.scan_uri_escapes(name, start_mark))
+            else:
+                length += 1
+            ch = self.peek(length)
+        if length:
+            chunks.append(self.prefix(length))
+            self.forward(length)
+            length = 0
+        if not chunks:
+            raise ScannerError("while parsing a %s" % name, start_mark,
+                    "expected URI, but found %r" % ch, self.get_mark())
+        return ''.join(chunks)
+
+    def scan_uri_escapes(self, name, start_mark):
+        # See the specification for details.
+        codes = []
+        mark = self.get_mark()
+        while self.peek() == '%':
+            self.forward()
+            for k in range(2):
+                if self.peek(k) not in '0123456789ABCDEFabcdef':
+                    raise ScannerError("while scanning a %s" % name, start_mark,
+                            "expected URI escape sequence of 2 hexadecimal numbers, but found %r"
+                            % self.peek(k), self.get_mark())
+            codes.append(int(self.prefix(2), 16))
+            self.forward(2)
+        try:
+            value = bytes(codes).decode('utf-8')
+        except UnicodeDecodeError as exc:
+            raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark)
+        return value
+
+    def scan_line_break(self):
+        # Transforms:
+        #   '\r\n'      :   '\n'
+        #   '\r'        :   '\n'
+        #   '\n'        :   '\n'
+        #   '\x85'      :   '\n'
+        #   '\u2028'    :   '\u2028'
+        #   '\u2029     :   '\u2029'
+        #   default     :   ''
+        ch = self.peek()
+        if ch in '\r\n\x85':
+            if self.prefix(2) == '\r\n':
+                self.forward(2)
+            else:
+                self.forward()
+            return '\n'
+        elif ch in '\u2028\u2029':
+            self.forward()
+            return ch
+        return ''
diff --git a/ubuntu/venv/yaml/serializer.py b/ubuntu/venv/yaml/serializer.py
new file mode 100644
index 0000000..fe911e6
--- /dev/null
+++ b/ubuntu/venv/yaml/serializer.py
@@ -0,0 +1,111 @@
+
+__all__ = ['Serializer', 'SerializerError']
+
+from .error import YAMLError
+from .events import *
+from .nodes import *
+
+class SerializerError(YAMLError):
+    pass
+
+class Serializer:
+
+    ANCHOR_TEMPLATE = 'id%03d'
+
+    def __init__(self, encoding=None,
+            explicit_start=None, explicit_end=None, version=None, tags=None):
+        self.use_encoding = encoding
+        self.use_explicit_start = explicit_start
+        self.use_explicit_end = explicit_end
+        self.use_version = version
+        self.use_tags = tags
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+        self.closed = None
+
+    def open(self):
+        if self.closed is None:
+            self.emit(StreamStartEvent(encoding=self.use_encoding))
+            self.closed = False
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        else:
+            raise SerializerError("serializer is already opened")
+
+    def close(self):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif not self.closed:
+            self.emit(StreamEndEvent())
+            self.closed = True
+
+    #def __del__(self):
+    #    self.close()
+
+    def serialize(self, node):
+        if self.closed is None:
+            raise SerializerError("serializer is not opened")
+        elif self.closed:
+            raise SerializerError("serializer is closed")
+        self.emit(DocumentStartEvent(explicit=self.use_explicit_start,
+            version=self.use_version, tags=self.use_tags))
+        self.anchor_node(node)
+        self.serialize_node(node, None, None)
+        self.emit(DocumentEndEvent(explicit=self.use_explicit_end))
+        self.serialized_nodes = {}
+        self.anchors = {}
+        self.last_anchor_id = 0
+
+    def anchor_node(self, node):
+        if node in self.anchors:
+            if self.anchors[node] is None:
+                self.anchors[node] = self.generate_anchor(node)
+        else:
+            self.anchors[node] = None
+            if isinstance(node, SequenceNode):
+                for item in node.value:
+                    self.anchor_node(item)
+            elif isinstance(node, MappingNode):
+                for key, value in node.value:
+                    self.anchor_node(key)
+                    self.anchor_node(value)
+
+    def generate_anchor(self, node):
+        self.last_anchor_id += 1
+        return self.ANCHOR_TEMPLATE % self.last_anchor_id
+
+    def serialize_node(self, node, parent, index):
+        alias = self.anchors[node]
+        if node in self.serialized_nodes:
+            self.emit(AliasEvent(alias))
+        else:
+            self.serialized_nodes[node] = True
+            self.descend_resolver(parent, index)
+            if isinstance(node, ScalarNode):
+                detected_tag = self.resolve(ScalarNode, node.value, (True, False))
+                default_tag = self.resolve(ScalarNode, node.value, (False, True))
+                implicit = (node.tag == detected_tag), (node.tag == default_tag)
+                self.emit(ScalarEvent(alias, node.tag, implicit, node.value,
+                    style=node.style))
+            elif isinstance(node, SequenceNode):
+                implicit = (node.tag
+                            == self.resolve(SequenceNode, node.value, True))
+                self.emit(SequenceStartEvent(alias, node.tag, implicit,
+                    flow_style=node.flow_style))
+                index = 0
+                for item in node.value:
+                    self.serialize_node(item, node, index)
+                    index += 1
+                self.emit(SequenceEndEvent())
+            elif isinstance(node, MappingNode):
+                implicit = (node.tag
+                            == self.resolve(MappingNode, node.value, True))
+                self.emit(MappingStartEvent(alias, node.tag, implicit,
+                    flow_style=node.flow_style))
+                for key, value in node.value:
+                    self.serialize_node(key, node, None)
+                    self.serialize_node(value, node, key)
+                self.emit(MappingEndEvent())
+            self.ascend_resolver()
+
diff --git a/ubuntu/venv/yaml/tokens.py b/ubuntu/venv/yaml/tokens.py
new file mode 100644
index 0000000..4d0b48a
--- /dev/null
+++ b/ubuntu/venv/yaml/tokens.py
@@ -0,0 +1,104 @@
+
+class Token(object):
+    def __init__(self, start_mark, end_mark):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+    def __repr__(self):
+        attributes = [key for key in self.__dict__
+                if not key.endswith('_mark')]
+        attributes.sort()
+        arguments = ', '.join(['%s=%r' % (key, getattr(self, key))
+                for key in attributes])
+        return '%s(%s)' % (self.__class__.__name__, arguments)
+
+#class BOMToken(Token):
+#    id = ''
+
+class DirectiveToken(Token):
+    id = ''
+    def __init__(self, name, value, start_mark, end_mark):
+        self.name = name
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class DocumentStartToken(Token):
+    id = ''
+
+class DocumentEndToken(Token):
+    id = ''
+
+class StreamStartToken(Token):
+    id = ''
+    def __init__(self, start_mark=None, end_mark=None,
+            encoding=None):
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.encoding = encoding
+
+class StreamEndToken(Token):
+    id = ''
+
+class BlockSequenceStartToken(Token):
+    id = ''
+
+class BlockMappingStartToken(Token):
+    id = ''
+
+class BlockEndToken(Token):
+    id = ''
+
+class FlowSequenceStartToken(Token):
+    id = '['
+
+class FlowMappingStartToken(Token):
+    id = '{'
+
+class FlowSequenceEndToken(Token):
+    id = ']'
+
+class FlowMappingEndToken(Token):
+    id = '}'
+
+class KeyToken(Token):
+    id = '?'
+
+class ValueToken(Token):
+    id = ':'
+
+class BlockEntryToken(Token):
+    id = '-'
+
+class FlowEntryToken(Token):
+    id = ','
+
+class AliasToken(Token):
+    id = ''
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class AnchorToken(Token):
+    id = ''
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class TagToken(Token):
+    id = ''
+    def __init__(self, value, start_mark, end_mark):
+        self.value = value
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+
+class ScalarToken(Token):
+    id = ''
+    def __init__(self, value, plain, start_mark, end_mark, style=None):
+        self.value = value
+        self.plain = plain
+        self.start_mark = start_mark
+        self.end_mark = end_mark
+        self.style = style
+