summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAdam Williamson <awilliam@redhat.com>2015-08-18 00:55:42 -0700
committerAdam Williamson <awilliam@redhat.com>2015-08-18 00:55:42 -0700
commit9ed7afd955933a89d23d41b8d4d35090398c2a9a (patch)
tree942edf1c2ba1083b023243e30d30591d628b3636
parent5134af656b55ded99db23faa561ab4b9cc1e02e7 (diff)
add Release check and diff methods, fix a couple of bugs
This is a dirty commit (sorry) with some significant feature additions mixed with a couple of bug fixes / small enhancements. It's kind of a pain that you always had to specify the search terms for a Query() as an iterable even when you often only wanted to pass one term, so as a convenience I made Query() tolerate a single term passed as a string. It'll handle a single term passed as any instance of one of the standard string types (per six.string_types). No more ('term',)! Yay! Also a small bugfix in NightlyRelease.find_images(): it also needs to look for disk-type compose methods (in Koji) if we're searching for 'docker' or 'vagrant' imagetypes, not just 'disk'. vagrant is new, but the fact that it was missing 'docker' before was a bug. Onto the new features. Release.check_expected() checks if any of the 'expected' important images for a Release are missing. Release classes must implement the 'expected_images' property to back this method. Release.difference(other) returns a set of the images present in this instance but not in 'other'. In both cases, (payload, imagetype, arch) tuples are used for identifying images.
-rw-r--r--fedfind/release.py114
1 files changed, 113 insertions, 1 deletions
diff --git a/fedfind/release.py b/fedfind/release.py
index f524db7..4e51478 100644
--- a/fedfind/release.py
+++ b/fedfind/release.py
@@ -25,6 +25,7 @@ import abc
import datetime
import logging
import os
+import six
# Upstream docs recommend using subprocess32 if possible, but make
# this optional as it's not available in EPEL.
try:
@@ -157,6 +158,9 @@ class Query(object):
def __init__(self, attr, terms, exact=True, neg=False):
self.attr = attr
self.getter = attrgetter(attr)
+ # Handle a single search term passed as a string
+ if isinstance(terms, six.string_types):
+ terms = (terms,)
self.terms = terms
self.exact = exact
self.neg = neg
@@ -238,6 +242,59 @@ class Release(object):
"""
pass
+ @abc.abstractproperty
+ def expected_images(self):
+ """These are the most important images that should be expected
+ to exist for a release of the given type. Basically if any of
+ these images does not exist, we ought to be worried. Pretty
+ close to the concept of a 'release blocking' image, but I
+ didn't want to commit to this being exactly that. Must be an
+ iterable of (payload, imagetype, arch) tuples.
+ """
+ pass
+
+ def difference(self, other):
+ """Similarly to the behaviour of the set.difference() method,
+ this tells you which images exist for this release but not
+ for other. Returns a set of (payload, imagetype, arch) tuples
+ identifying the unique images. 'other' must be another Release
+ instance (you may often want to use self.previous_release).
+ """
+ ours = set(
+ (img.payload, img.imagetype, img.arch) for img in self.all_images)
+ theirs = set(
+ (img.payload, img.imagetype, img.arch) for img in other.all_images)
+ return ours.difference(theirs)
+
+ def check_expected(self):
+ """This checks whether all expected images are included in the
+ release. If the release doesn't exist, it will raise an
+ exception. If any expected images are missing, it will return
+ a set of (payload, imagetype, arch) tuples identifying the
+ missing images. If nothing is missing, it will return an empty
+ set.
+ """
+ if not self.exists:
+ raise ValueError('Release does not exist!')
+ missing = set()
+ for (payload, imagetype, arch) in self.expected_images:
+ queries = (
+ Query('payload', payload),
+ Query('imagetype', imagetype),
+ Query('arch', arch),
+ )
+ # This is a bit of a hack, but we happen to know that
+ # we'll need to check basically all the images, and for
+ # nightlies it's actually slower to use the 'optimised'
+ # method in this case because it doesn't cache all its
+ # results
+ res = self._query_images(queries)
+ if res:
+ logger.debug('check_expected: found {0}'.format(res))
+ else:
+ missing.add((payload, imagetype, arch))
+ return missing
+
def find_images(self, queries=None, orq=False):
"""This is a simple 'fallback' query method which simply finds
all images for the release and runs the query function. The
@@ -335,6 +392,29 @@ class Nightly(Release):
def exists(self):
return fedfind.helpers.url_exists(self.https_url_generic)
+ @property
+ def expected_images(self):
+ """See abstract class docstring for information on what this
+ is. For a nightly release we expect for the Intel arches a
+ generic boot.iso and Workstation and KDE live images, and for
+ ARM we expect a minimal and a KDE disk image. For Cloud we
+ expect, er, FIXME?
+ """
+ imgs = list()
+ intels = (arch.name for arch in fedfind.const.ARCHES if
+ arch.group == 'intel')
+ arms = (arch.name for arch in fedfind.const.ARCHES if
+ arch.group == 'arm')
+ for arch in intels:
+ imgs.append(('generic', 'boot', arch))
+ imgs.append(('workstation', 'live', arch))
+ imgs.append(('kde', 'live', arch))
+ for arch in arms:
+ imgs.append(('minimal', 'disk', arch))
+ imgs.append(('kde', 'disk', arch))
+ logger.debug("expected images: {0}".format(imgs))
+ return tuple(imgs)
+
def _get_boot_url(self, arch):
"""Get the expected URL for the boot.iso of a given arch."""
tmpl = '{0}/{1}/os/images/boot.iso'
@@ -408,7 +488,8 @@ class Nightly(Release):
needboot = False
if 'live' not in query.terms:
needlive = False
- if 'disk' not in query.terms:
+ if not any(term in query.terms
+ for term in ('disk', 'docker', 'vagrant')):
needdisk = False
if query.attr == 'arch' and not query.neg and query.exact:
arches = query.terms
@@ -560,6 +641,37 @@ class MirrorRelease(Release):
else:
return tmpl.format(fedfind.const.HTTPS, self._rsyncpath)
+ @property
+ def expected_images(self):
+ """See abstract class docstring for information on what this
+ is. Getting this right for all historic releases would be a
+ bit of a pain, so the stuff here is really only valid for F21+
+ - but that's all we're likely to want to check anyway. For
+ these releases, for the Intel arches we expect a Server DVD
+ and netinst and Workstation and KDE live images, and for ARM
+ (after it became a primary arch in F20) we expect a minimal
+ and a KDE disk image (for F23+) or an Xfce disk image (for
+ <F23). For Cloud we expect, er, FIXME?
+ """
+ imgs = list()
+ intels = (arch.name for arch in fedfind.const.ARCHES if
+ arch.group == 'intel')
+ arms = (arch.name for arch in fedfind.const.ARCHES if
+ arch.group == 'arm')
+ for arch in intels:
+ imgs.append(('server', 'netinst', arch))
+ imgs.append(('server', 'dvd', arch))
+ imgs.append(('workstation', 'live', arch))
+ imgs.append(('kde', 'live', arch))
+ if int(self.release) > 19:
+ for arch in arms:
+ imgs.append(('minimal', 'disk', arch))
+ if int(self.release) < 23:
+ imgs.append(('xfce', 'disk', arch))
+ else:
+ imgs.append(('kde', 'disk', arch))
+ return tuple(imgs)
+
def get_mirror_images(self, prefurl=''):
"""Find images in the main mirror tree by parsing rsync output.
This is about the simplest / most reliable method I could