[calm - Cygwin server-side packaging maintenance script] branch master, updated. ba688d6c07b1cfe410dd6063882ee757c549992f

[jturney at sourceware dot org]

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=ba688d6c07b1cfe410dd6063882ee757c549992f

commit ba688d6c07b1cfe410dd6063882ee757c549992f
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Jun 20 14:10:16 2016 +0100

    Check packages which share the same source package are all at the same version
    
    Report the offending packages if they aren't listed as past mistakes

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=c6c1b815b02daaffc9ff85cc6d8dd0d498cfe174

commit c6c1b815b02daaffc9ff85cc6d8dd0d498cfe174
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu Jun 16 22:29:49 2016 +0000

    Check that packages have some versions, and warn if no curr version
    
    Update tests appropriately:
    - Remove naim test package, it's no longer valid
    - Change to using arc package to test no sha512.sum behaviour

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=2bfb4654a43d765715a40e8b16d7e898e0a385e8

commit 2bfb4654a43d765715a40e8b16d7e898e0a385e8
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Mon Jun 20 14:18:59 2016 +0100

    Validate existing packages for all arches before stopping due to any error

https://sourceware.org/git/gitweb.cgi?p=cygwin-apps/calm.git;h=c859c2726d16002ec19fefa2f42f7754c77a0eda

commit c859c2726d16002ec19fefa2f42f7754c77a0eda
Author: Jon Turney <jon.turney@dronecode.org.uk>
Date:   Thu May 19 12:15:51 2016 +0000

    Removing setup.hint files isn't allowed
    
    Removing a package doesn't make much sense currently.
    
    We can't remove a package when it has been replaced by another one, all we
    can do is replace it with an empty, obsolete one which depends on it's
    replacement, so existing installs are updated to that.
    
    Removing a package which is no longer relevant (i.e. an old soversion of a
    shared library which has no users left) is permissible, but the files it
    contains will linger forever in any existing installs.


Diff:
---
 calm/calm.py                                      |    6 ++-
 calm/package.py                                   |   46 +++++++++++++++++++++
 calm/past_mistakes.py                             |   26 ++++++++++++
 test/test_calm.py                                 |    2 +-
 test/testdata/process_arch/rel_area.expected      |    3 +-
 test/testdata/relarea/x86/release/naim/setup.hint |    5 --
 6 files changed, 79 insertions(+), 9 deletions(-)

diff --git a/calm/calm.py b/calm/calm.py
index 97746fb..0f41223 100755
--- a/calm/calm.py
+++ b/calm/calm.py
@@ -82,6 +82,7 @@ def process(args):
             logging.warning("--dry-run is in effect, nothing will really be done")
 
         # for each arch
+        error = False
         packages = {}
         for arch in common_constants.ARCHES:
             logging.debug("reading existing packages for arch %s" % (arch))
@@ -92,7 +93,10 @@ def process(args):
             # validate the package set
             if not package.validate_packages(args, packages[arch]):
                 logging.error("existing %s package set has errors", arch)
-                return None
+                error = True
+
+        if error:
+            return None
 
         # read maintainer list
         mlist = maintainers.Maintainer.read(args)
diff --git a/calm/package.py b/calm/package.py
index 7cdd987..088da76 100755
--- a/calm/package.py
+++ b/calm/package.py
@@ -50,6 +50,7 @@ class Package(object):
         self.path = ''  # path to package, relative to release area
         self.tars = {}
         self.hints = {}
+        self.is_used_by = set()
 
     def __repr__(self):
         return "Package('%s', %s, %s)" % (self.path, pprint.pformat(self.tars),
@@ -397,6 +398,14 @@ def validate_packages(args, packages):
             if l in packages[p].hints:
                 packages[p].stability[l] = packages[p].hints[l]
 
+        # the package must have some versions
+        if not packages[p].stability:
+            logging.error("package '%s' doesn't have any versions" % (p))
+            error = True
+        # it's also probably a really good idea if a curr version exists
+        elif 'curr' not in packages[p].stability:
+            logging.warning("package '%s' doesn't have a curr version" % (p))
+
         # If, for every stability level, the install tarball is empty and there
         # is no source tarball, we should probably be marked obsolete
         if 'skip' not in packages[p].hints:
@@ -432,6 +441,7 @@ def validate_packages(args, packages):
             # mark the source tarfile as being used by an install tarfile
             if 'source' in packages[p].vermap[v]:
                 packages[p].tars[packages[p].vermap[v]['source']].is_used = True
+                packages[p].is_used_by.add(p)
                 continue
 
             if 'external-source' in packages[p].hints:
@@ -439,6 +449,7 @@ def validate_packages(args, packages):
                 if es_p in packages:
                     if 'source' in packages[es_p].vermap[v]:
                         packages[es_p].tars[packages[es_p].vermap[v]['source']].is_used = True
+                        packages[es_p].is_used_by.add(p)
                         continue
 
             # unless this package is marked as 'self-source'
@@ -462,6 +473,41 @@ def validate_packages(args, packages):
                 logging.error("package '%s' version '%s' source has no non-empty install tarfiles" % (p, v))
                 error = True
 
+    # do all the packages which use this source package have the same
+    # current version?
+    for source_p in sorted(packages.keys()):
+        versions = defaultdict(list)
+
+        for install_p in packages[source_p].is_used_by:
+            # ignore obsolete packages
+            if '_obsolete' in packages[install_p].hints['category']:
+                continue
+            # ignore runtime library packages, as we may keep old versions of
+            # those
+            if re.match(r'^lib.*\d', install_p):
+                continue
+
+            versions[packages[install_p].stability['curr']].append(install_p)
+
+        if len(versions) > 1:
+            out = []
+            most_common = True
+
+            for v in sorted(versions, key=lambda v: len(versions[v]), reverse=True):
+                # try to keep the output compact by not listing all the packages
+                # the most common current version has, unless it's only one.
+                if most_common and len(versions[v]) != 1:
+                    out.append("%s (%s others)" % (v, len(versions[v])))
+                else:
+                    out.append("%s (%s)" % (v, ','.join(versions[v])))
+                most_common = False
+
+            lvl = logging.INFO
+            if source_p not in past_mistakes.nonunique_versions:
+                lvl = logging.ERROR
+                error = True
+            logging.log(lvl, "install packages from source package '%s' have non-unique current versions %s" % (source_p, ', '.join(reversed(out))))
+
     # validate that all packages are in the package maintainers list
     validate_package_maintainers(args, packages)
 
diff --git a/calm/past_mistakes.py b/calm/past_mistakes.py
index 5e1132f..e4acfcb 100644
--- a/calm/past_mistakes.py
+++ b/calm/past_mistakes.py
@@ -95,3 +95,29 @@ self_source = [
     'tesseract-training-spa',
     'tesseract-training-vie',
 ]
+
+# these are source packages which currently have non-unique install versions
+nonunique_versions = [
+    'bzr',              # debuginfo from NMU needs to age out?
+    'cgdb',             # ditto
+    'e2fsprogs',
+    'fltk',
+    'gnome-panel',
+    'guile',
+    'libical',
+    'libjpeg-turbo',    # libturbojpeg isn't detected as an old soversion
+    'librsvg2',
+    'mutter',
+    'ocaml',            # ocaml-camlp4 removed from ocaml distribution after 4.01.0
+    'openmpi',          # libopenmpi isn't detected as an old soversion
+    'python-openssl',   # no debuginfo anymore due to pure python?
+    'python3-openssl',  # ditto?
+    'qt5-webkit',
+    'rsync',            # debuginfo from NMU needs to age out?
+    'ruby-gdk3',
+    'sng',
+    'socat',
+    'sqlite3',          # sqlite3-zlib removed in 3.8.10, use sqlite3-compress instead
+    'w3m',
+    'zlib',
+]
diff --git a/test/test_calm.py b/test/test_calm.py
index 60246d1..c872be6 100755
--- a/test/test_calm.py
+++ b/test/test_calm.py
@@ -285,7 +285,7 @@ class CalmTest(unittest.TestCase):
         # ensure sha512.sum files exist
         os.system("find testdata/relarea/x86 testdata/relarea/noarch -type d -exec sh -c 'cd {} ; sha512sum * >sha512.sum 2>/dev/null' \;")
         # should remove a sha512.sum file so that we test functioning when it's absent
-        os.unlink('testdata/relarea/x86/release/naim/sha512.sum')
+        os.unlink('testdata/relarea/x86/release/arc/sha512.sum')
         # remove !ready files
         os.system("find testdata/homes -name !ready -exec rm {} \;")
 
diff --git a/test/testdata/process_arch/rel_area.expected b/test/testdata/process_arch/rel_area.expected
index b147754..d697a10 100644
--- a/test/testdata/process_arch/rel_area.expected
+++ b/test/testdata/process_arch/rel_area.expected
@@ -9,7 +9,7 @@
                                       'sha512.sum'],
  'x86': ['sha512.sum'],
  'x86/release': ['sha512.sum'],
- 'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.tar.bz2', 'setup.hint', 'sha512.sum'],
+ 'x86/release/arc': ['arc-4.32.7-10-src.tar.bz2', 'arc-4.32.7-10.tar.bz2', 'setup.hint'],
  'x86/release/base-cygwin': ['base-cygwin-3.6-1.tar.xz', 'base-cygwin-3.8-1.tar.xz', 'setup.hint', 'sha512.sum'],
  'x86/release/corrupt': ['corrupt-2.0.0-1-src.tar.xz', 'corrupt-2.0.0-1.tar.xz', 'setup.hint', 'sha512.sum'],
  'x86/release/cygwin': ['.this-should-be-ignored',
@@ -52,7 +52,6 @@
  'x86/release/mDNSResponder/libdns_sd1': ['libdns_sd1-379.32.1-1.tar.bz2', 'setup.hint', 'sha512.sum'],
  'x86/release/mingw64-i686-binutils': ['setup.hint', 'sha512.sum'],
  'x86/release/mingw64-i686-binutils/mingw64-i686-binutils-debuginfo': ['setup.hint', 'sha512.sum'],
- 'x86/release/naim': ['setup.hint'],
  'x86/release/openssh': ['openssh-7.2p2-1-src.tar.xz', 'openssh-7.2p2-1.tar.xz', 'setup.hint', 'sha512.sum'],
  'x86/release/proj': ['setup.hint', 'sha512.sum'],
  'x86/release/proj/libproj-devel': ['setup.hint', 'sha512.sum'],
diff --git a/test/testdata/relarea/x86/release/naim/setup.hint b/test/testdata/relarea/x86/release/naim/setup.hint
deleted file mode 100644
index 7de6e7b..0000000
--- a/test/testdata/relarea/x86/release/naim/setup.hint
+++ /dev/null
@@ -1,5 +0,0 @@
-category: Net
-requires:
-sdesc: "Console AIM, ICQ, IRC, and Lily client"
-ldesc: " naim is a console client for AOL Instant Messenger (AIM),
-AOL I Seek You (ICQ), Internet Relay Chat (IRC), and The lily CMC."