Merge branch 'doc/testing-example' of github.com:dib-lab/khmer into d…

…oc/testing-example

doc/dev/release.rst

@@ -67,12 +67,9 @@ release makers, following this checklist by MRC.
 #. (Optional) Check for updates to versioneer::
 
         pip install --upgrade versioneer
-        versioneer-installer
+        versioneer install
+        git diff --staged
 
-        git diff
-
-        ./setup.py versioneer
-        git diff
         git commit -m -a "new version of versioneer.py"
         # or
         git checkout -- versioneer.py khmer/_version.py khmer/__init__.py MANIFEST.in
@@ -128,6 +125,7 @@ release makers, following this checklist by MRC.
         pip uninstall -y khmer; pip uninstall -y khmer; make install
         mkdir ../not-khmer # make sure py.test executes tests
                            # from the installed khmer module
+        # you might want to add 'and not huge' to the test selection
         pushd ../not-khmer; pytest --pyargs khmer -m 'not known_failing'; popd
 
 

khmer/_version.py

@@ -6,7 +6,7 @@
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.15+dev (https://github.com/warner/python-versioneer)
+# versioneer-0.18 (https://github.com/warner/python-versioneer)
 
 """Git implementation of _version.py."""
 
@@ -25,12 +25,12 @@ def get_keywords():
     # get_keywords().
     git_refnames = "$Format:%d$"
     git_full = "$Format:%H$"
-    keywords = {"refnames": git_refnames, "full": git_full}
+    git_date = "$Format:%ci$"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
     return keywords
 
 
 class VersioneerConfig:
-
     """Container for Versioneer configuration parameters."""
 
 
@@ -49,7 +49,6 @@ def get_config():
 
 
 class NotThisMethod(Exception):
-
     """Exception raised if a method is not valid for the current scenario."""
 
 
@@ -68,15 +67,17 @@ def decorate(f):
     return decorate
 
 
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
     """Call the given command(s)."""
     assert isinstance(commands, list)
     p = None
     for c in commands:
         try:
             dispcmd = str([c] + args)
             # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
                                  stderr=(subprocess.PIPE if hide_stderr
                                          else None))
             break
@@ -87,36 +88,45 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
             if verbose:
                 print("unable to run %s" % dispcmd)
                 print(e)
-            return None
+            return None, None
     else:
         if verbose:
             print("unable to find command, tried %s" % (commands,))
-        return None
+        return None, None
     stdout = p.communicate()[0].strip()
     if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
             print("unable to run %s (error)" % dispcmd)
-        return None
-    return stdout
+            print("stdout was %s" % stdout)
+        return None, p.returncode
+    return stdout, p.returncode
 
 
 def versions_from_parentdir(parentdir_prefix, root, verbose):
     """Try to determine the version from the parent directory name.
 
-    Source tarballs conventionally unpack into a directory that includes
-    both the project name and a version string.
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
     """
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with "
-                  "prefix '%s'" % (root, dirname, parentdir_prefix))
-        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
-    return {"version": dirname[len(parentdir_prefix):],
-            "full-revisionid": None,
-            "dirty": False, "error": None}
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %s but none started with prefix %s" %
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
 
 
 @register_vcs_handler("git", "get_keywords")
@@ -138,6 +148,10 @@ def git_get_keywords(versionfile_abs):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
                     keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
@@ -149,6 +163,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
     """Get version information from git keywords."""
     if not keywords:
         raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
     refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
@@ -180,14 +203,14 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
                 print("picking %s" % r)
             return {"version": r,
                     "full-revisionid": keywords["full"].strip(),
-                    "dirty": False, "error": None
-                    }
+                    "dirty": False, "error": None,
+                    "date": date}
     # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
         print("no suitable tags, using unknown + full revision id")
     return {"version": "0+unknown",
             "full-revisionid": keywords["full"].strip(),
-            "dirty": False, "error": "no suitable tags"}
+            "dirty": False, "error": "no suitable tags", "date": None}
 
 
 @register_vcs_handler("git", "pieces_from_vcs")
@@ -198,25 +221,28 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     expanded, and _version.py hasn't already been rewritten with a short
     version string, meaning we're inside a checked out source tree.
     """
-    if not os.path.exists(os.path.join(root, ".git")):
-        if verbose:
-            print("no .git in %s" % root)
-        raise NotThisMethod("no .git directory")
-
     GITS = ["git"]
     if sys.platform == "win32":
         GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
+        if verbose:
+            print("Directory %s not under git control" % root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
     # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
     # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
-                                      "--always", "--long",
-                                      "--match", "%s*" % tag_prefix],
-                               cwd=root)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%s*" % tag_prefix],
+                                   cwd=root)
     # --long was added in git-1.5.5
     if describe_out is None:
         raise NotThisMethod("'git describe' failed")
     describe_out = describe_out.strip()
-    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
     if full_out is None:
         raise NotThisMethod("'git rev-parse' failed")
     full_out = full_out.strip()
@@ -267,10 +293,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     else:
         # HEX: no tags
         pieces["closest-tag"] = None
-        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                cwd=root)
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
         pieces["distance"] = int(count_out)  # total number of commits
 
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
     return pieces
 
 
@@ -417,7 +448,8 @@ def render(pieces, style):
         return {"version": "unknown",
                 "full-revisionid": pieces.get("long"),
                 "dirty": None,
-                "error": pieces["error"]}
+                "error": pieces["error"],
+                "date": None}
 
     if not style or style == "default":
         style = "pep440"  # the default
@@ -438,7 +470,8 @@ def render(pieces, style):
         raise ValueError("unknown style '%s'" % style)
 
     return {"version": rendered, "full-revisionid": pieces["long"],
-            "dirty": pieces["dirty"], "error": None}
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
 
 
 def get_versions():
@@ -467,7 +500,8 @@ def get_versions():
     except NameError:
         return {"version": "0+unknown", "full-revisionid": None,
                 "dirty": None,
-                "error": "unable to find root of source tree"}
+                "error": "unable to find root of source tree",
+                "date": None}
 
     try:
         pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
@@ -483,4 +517,4 @@ def get_versions():
 
     return {"version": "0+unknown", "full-revisionid": None,
             "dirty": None,
-            "error": "unable to compute version"}
+            "error": "unable to compute version", "date": None}

khmer/khmer_args.py

@@ -83,7 +83,9 @@ class CitationAction(argparse.Action):
 
     def __init__(self, *args, **kwargs):
         self.citations = kwargs.pop('citations')
-        super(CitationAction, self).__init__(*args, nargs=0, **kwargs)
+        super(CitationAction, self).__init__(*args, nargs=0,
+                                             default=argparse.SUPPRESS,
+                                             **kwargs)
 
     def __call__(self, parser, namespace, values, option_string=None):
         info(parser.prog, self.citations)
@@ -137,6 +139,7 @@ def __init__(self, citations=None, formatter_class=ComboFormatter,
         self.add_argument('--version', action=_VersionStdErrAction,
                           version='khmer {v}'.format(v=__version__))
         self.add_argument('--info', action=CitationAction,
+                          help='print citation information',
                           citations=self._citations)
         self.add_argument('-h', '--help', action=_HelpAction,
                           default=argparse.SUPPRESS,
@@ -439,7 +442,7 @@ def build_graph_args(descr=None, epilog=None, parser=None, citations=None):
         parser = KhmerArgumentParser(description=descr, epilog=epilog,
                                      citations=citations)
 
-    parser.add_argument('--ksize', '-k', type=int, default=DEFAULT_K,
+    parser.add_argument('-k', '--ksize', type=int, default=DEFAULT_K,
                         help='k-mer size to use')
 
     help = ('number of tables to use in k-mer countgraph' if expert_help
@@ -615,7 +618,7 @@ def report_on_config(args, graphtype='countgraph'):
 
 def add_threading_args(parser):
     """Add option for threading to options parser."""
-    parser.add_argument('--threads', '-T', default=DEFAULT_N_THREADS, type=int,
+    parser.add_argument('-T', '--threads', default=DEFAULT_N_THREADS, type=int,
                         help='Number of simultaneous threads to execute')
 
 

scripts/abundance-dist-single.py

@@ -94,11 +94,11 @@ def get_parser():
     parser.add_argument('-s', '--squash', dest='squash_output', default=False,
                         action='store_true',
                         help='Overwrite output file if it exists')
-    parser.add_argument('--savegraph', default='', metavar="filename",
+    parser.add_argument('--savegraph', metavar="filename",
                         help="Save the k-mer countgraph to the specified "
                         "filename.")
     parser.add_argument('-f', '--force', default=False, action='store_true',
-                        help='Overwrite output file if it exists')
+                        help='Override sanity checks')
     parser.add_argument('-q', '--quiet', dest='quiet', default=False,
                         action='store_true')
     return parser
@@ -112,7 +112,7 @@ def main():  # pylint: disable=too-many-locals,too-many-branches
     report_on_config(args, graph_type)
 
     check_input_files(args.input_sequence_filename, args.force)
-    if args.savegraph:
+    if args.savegraph is not None:
         graphsize = calculate_graphsize(args, graph_type)
         check_space_for_graph(args.savegraph, graphsize, args.force)
     if (not args.squash_output and
@@ -213,7 +213,7 @@ def __do_abundance_dist__(read_parser):
         if sofar == total:
             break
 
-    if args.savegraph:
+    if args.savegraph is not None:
         log_info('Saving k-mer countgraph to {savegraph}',
                  savegraph=args.savegraph)
         countgraph.save(args.savegraph)

scripts/annotate-partitions.py

@@ -73,7 +73,7 @@ def get_parser():
         description="Annotate sequences with partition IDs.",
         epilog=textwrap.dedent(epilog))
 
-    parser.add_argument('--ksize', '-k', type=int, default=DEFAULT_K,
+    parser.add_argument('-k', '--ksize', type=int, default=DEFAULT_K,
                         help="k-mer size (default: %d)" % DEFAULT_K)
     parser.add_argument('graphbase', help='basename for input and output '
                         'files')

scripts/do-partition.py

@@ -48,7 +48,6 @@
 import threading
 import os.path
 import os
-import gc
 import textwrap
 from khmer import khmer_args
 from khmer.khmer_args import (build_nodegraph_args, report_on_config,
@@ -88,15 +87,14 @@ def get_parser():
         descr='Load, partition, and annotate FAST[AQ] sequences',
         epilog=textwrap.dedent(epilog), citations=['graph'])
     add_threading_args(parser)
-    parser.add_argument('--subset-size', '-s', default=DEFAULT_SUBSET_SIZE,
+    parser.add_argument('-s', '--subset-size', default=DEFAULT_SUBSET_SIZE,
                         dest='subset_size', type=float,
                         help='Set subset size (usually 1e5-1e6 is good)')
     parser.add_argument('--no-big-traverse', dest='no_big_traverse',
                         action='store_true', default=False,
                         help='Truncate graph joins at big traversals')
-    parser.add_argument('--keep-subsets', dest='remove_subsets',
-                        default=True, action='store_false',
-                        help='Keep individual subsets (default: False)')
+    parser.add_argument('--keep-subsets', default=False, action='store_true',
+                        help='Keep individual subsets')
     parser.add_argument('graphbase', help="base name for output files")
     parser.add_argument('input_filenames', metavar='input_sequence_filename',
                         nargs='+', help='input FAST[AQ] sequence filenames')
@@ -212,7 +210,7 @@ def main():  # pylint: disable=too-many-locals,too-many-statements
         print('merging', pmap_file, file=sys.stderr)
         nodegraph.merge_subset_from_disk(pmap_file)
 
-    if args.remove_subsets:
+    if not args.keep_subsets:
         print('removing pmap files', file=sys.stderr)
         for pmap_file in pmap_files:
             os.unlink(pmap_file)