From c8c34705653bdf6434603b94e98a2b8fc5501976 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 10:56:27 +0200 Subject: [PATCH 01/10] Modify CLI help for abundance-dist-single.py --- scripts/abundance-dist-single.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/abundance-dist-single.py b/scripts/abundance-dist-single.py index 71547c2c9d..5d6270e049 100755 --- a/scripts/abundance-dist-single.py +++ b/scripts/abundance-dist-single.py @@ -98,7 +98,7 @@ def get_parser(): help="Save the k-mer countgraph to the specified " "filename.") parser.add_argument('-f', '--force', default=False, action='store_true', - help='Overwrite output file if it exists') + help='Override sanity checks') parser.add_argument('-q', '--quiet', dest='quiet', default=False, action='store_true') return parser From a2736db4208e11f583ea6310c074c8f62780f8b8 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:10:13 +0200 Subject: [PATCH 02/10] Change --force documentation --- scripts/abundance-dist-single.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/abundance-dist-single.py b/scripts/abundance-dist-single.py index 5d6270e049..3cf77e8c3c 100755 --- a/scripts/abundance-dist-single.py +++ b/scripts/abundance-dist-single.py @@ -94,7 +94,7 @@ def get_parser(): parser.add_argument('-s', '--squash', dest='squash_output', default=False, action='store_true', help='Overwrite output file if it exists') - parser.add_argument('--savegraph', default='', metavar="filename", + parser.add_argument('--savegraph', metavar="filename", help="Save the k-mer countgraph to the specified " "filename.") parser.add_argument('-f', '--force', default=False, action='store_true', @@ -112,7 +112,7 @@ def main(): # pylint: disable=too-many-locals,too-many-branches report_on_config(args, graph_type) check_input_files(args.input_sequence_filename, args.force) - if args.savegraph: + if args.savegraph is not None: graphsize = calculate_graphsize(args, graph_type) check_space_for_graph(args.savegraph, graphsize, args.force) if (not args.squash_output and @@ -213,7 +213,7 @@ def __do_abundance_dist__(read_parser): if sofar == total: break - if args.savegraph: + if args.savegraph is not None: log_info('Saving k-mer countgraph to {savegraph}', savegraph=args.savegraph) countgraph.save(args.savegraph) From 498164f3d67676650f28ca53b798573d913cfdaf Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:13:42 +0200 Subject: [PATCH 03/10] Change internal name of keep-subsets CLI argument --- scripts/do-partition.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/do-partition.py b/scripts/do-partition.py index f3275f693e..e049a05a19 100755 --- a/scripts/do-partition.py +++ b/scripts/do-partition.py @@ -48,7 +48,6 @@ import threading import os.path import os -import gc import textwrap from khmer import khmer_args from khmer.khmer_args import (build_nodegraph_args, report_on_config, @@ -94,9 +93,8 @@ def get_parser(): parser.add_argument('--no-big-traverse', dest='no_big_traverse', action='store_true', default=False, help='Truncate graph joins at big traversals') - parser.add_argument('--keep-subsets', dest='remove_subsets', - default=True, action='store_false', - help='Keep individual subsets (default: False)') + parser.add_argument('--keep-subsets', default=True, action='store_false', + help='Keep individual subsets') parser.add_argument('graphbase', help="base name for output files") parser.add_argument('input_filenames', metavar='input_sequence_filename', nargs='+', help='input FAST[AQ] sequence filenames') @@ -212,7 +210,7 @@ def main(): # pylint: disable=too-many-locals,too-many-statements print('merging', pmap_file, file=sys.stderr) nodegraph.merge_subset_from_disk(pmap_file) - if args.remove_subsets: + if not args.keep_subsets: print('removing pmap files', file=sys.stderr) for pmap_file in pmap_files: os.unlink(pmap_file) From cbb093a5cfd4795a48c050f032e4649d84b5af02 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:20:46 +0200 Subject: [PATCH 04/10] Change order of short and long CLI argument --- scripts/extract-paired-reads.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/extract-paired-reads.py b/scripts/extract-paired-reads.py index 1e82441234..7ce24ece01 100755 --- a/scripts/extract-paired-reads.py +++ b/scripts/extract-paired-reads.py @@ -94,11 +94,11 @@ def get_parser(): parser.add_argument('-d', '--output-dir', default='', help='Output ' 'split reads to specified directory. Creates ' 'directory if necessary') - parser.add_argument('--output-paired', '-p', metavar="filename", + parser.add_argument('-p', '--output-paired', metavar="filename", type=khFileType('wb'), default=None, help='Output paired reads to this ' 'file') - parser.add_argument('--output-single', '-s', metavar="filename", + parser.add_argument('-s', '--output-single', metavar="filename", type=khFileType('wb'), default=None, help='Output orphaned reads to this file') parser.add_argument('-f', '--force', default=False, action='store_true', From d232351aebae1f9738edc21092b30ea6f58be971 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:34:03 +0200 Subject: [PATCH 05/10] Add help message for report CLI argument --- scripts/normalize-by-median.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/normalize-by-median.py b/scripts/normalize-by-median.py index 8b1d7baaea..2567549f20 100755 --- a/scripts/normalize-by-median.py +++ b/scripts/normalize-by-median.py @@ -55,13 +55,13 @@ from khmer import khmer_args from contextlib import contextmanager from khmer.khmer_args import (build_counting_args, add_loadgraph_args, - report_on_config, info, calculate_graphsize, + report_on_config, calculate_graphsize, sanitize_help, check_argument_range) from khmer.khmer_args import FileType as khFileType import argparse from khmer.kfile import (check_space, check_space_for_graph, check_valid_file_exists, add_output_compression_type, - get_file_writer, is_block, describe_file_handle) + get_file_writer, describe_file_handle) from khmer.utils import (write_record, broken_paired_reader, ReadBundle, clean_input_reads) from khmer.khmer_logger import (configure_logging, log_info, log_error) @@ -273,10 +273,11 @@ def get_parser(): metavar="unpaired_reads_filename", help='include a file of unpaired reads to which ' '-p/--paired does not apply.') - parser.add_argument('-s', '--savegraph', metavar="filename", default='', + parser.add_argument('-s', '--savegraph', metavar="filename", default=None, help='save the k-mer countgraph to disk after all ' 'reads are loaded.') parser.add_argument('-R', '--report', + help='write progress report to report_filename', metavar='report_filename', type=argparse.FileType('w')) parser.add_argument('--report-frequency', metavar='report_frequency', type=int, @@ -329,7 +330,7 @@ def main(): # pylint: disable=too-many-branches,too-many-statements # check that files exist and there is sufficient output disk space. check_valid_file_exists(args.input_filenames) check_space(args.input_filenames, args.force) - if args.savegraph: + if args.savegraph is not None: graphsize = calculate_graphsize(args, 'countgraph') check_space_for_graph(args.savegraph, graphsize, args.force) @@ -399,7 +400,7 @@ def main(): # pylint: disable=too-many-branches,too-many-statements log_info('Total number of unique k-mers: {umers}', umers=countgraph.n_unique_kmers()) - if args.savegraph: + if args.savegraph is not None: log_info('...saving to {name}', name=args.savegraph) countgraph.save(args.savegraph) From 5e6313f4bbc617780460ec8522573bc5f8a2afc7 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:37:09 +0200 Subject: [PATCH 06/10] Add help for report frequency --- scripts/normalize-by-median.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/normalize-by-median.py b/scripts/normalize-by-median.py index 2567549f20..0900829869 100755 --- a/scripts/normalize-by-median.py +++ b/scripts/normalize-by-median.py @@ -280,8 +280,8 @@ def get_parser(): help='write progress report to report_filename', metavar='report_filename', type=argparse.FileType('w')) parser.add_argument('--report-frequency', - metavar='report_frequency', type=int, - default=100000) + metavar='report_frequency', type=int, default=100000, + help='report progress every report_frequency reads') parser.add_argument('-f', '--force', dest='force', help='continue past file reading errors', action='store_true') From 5a5f1d11194f69533aae8a9e64dca6e523879478 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Thu, 27 Apr 2017 11:59:31 +0200 Subject: [PATCH 07/10] Add ignore-pairs CLI information --- khmer/khmer_args.py | 5 ++++- scripts/readstats.py | 1 - scripts/trim-low-abund.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/khmer/khmer_args.py b/khmer/khmer_args.py index 98fc8a78ac..f1e881a6c2 100644 --- a/khmer/khmer_args.py +++ b/khmer/khmer_args.py @@ -83,7 +83,9 @@ class CitationAction(argparse.Action): def __init__(self, *args, **kwargs): self.citations = kwargs.pop('citations') - super(CitationAction, self).__init__(*args, nargs=0, **kwargs) + super(CitationAction, self).__init__(*args, nargs=0, + default=argparse.SUPPRESS, + **kwargs) def __call__(self, parser, namespace, values, option_string=None): info(parser.prog, self.citations) @@ -137,6 +139,7 @@ def __init__(self, citations=None, formatter_class=ComboFormatter, self.add_argument('--version', action=_VersionStdErrAction, version='khmer {v}'.format(v=__version__)) self.add_argument('--info', action=CitationAction, + help='print citation information', citations=self._citations) self.add_argument('-h', '--help', action=_HelpAction, default=argparse.SUPPRESS, diff --git a/scripts/readstats.py b/scripts/readstats.py index f3887d0373..b42cb30556 100755 --- a/scripts/readstats.py +++ b/scripts/readstats.py @@ -49,7 +49,6 @@ import screed import textwrap -from khmer import __version__ from khmer.khmer_args import sanitize_help, KhmerArgumentParser diff --git a/scripts/trim-low-abund.py b/scripts/trim-low-abund.py index ac3e231c12..b4a0533a56 100755 --- a/scripts/trim-low-abund.py +++ b/scripts/trim-low-abund.py @@ -140,7 +140,8 @@ def get_parser(): # expert options parser.add_argument('--force', default=False, action='store_true') - parser.add_argument('--ignore-pairs', default=False, action='store_true') + parser.add_argument('--ignore-pairs', default=False, action='store_true', + help='treat all reads as if they were singletons') parser.add_argument('--tempdir', '-T', type=str, default='./', help="Set location of temporary directory for " "second pass") From a2e480db28fb4049d96b392bfc885fe93e67aa33 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 28 Apr 2017 20:45:56 +0200 Subject: [PATCH 08/10] Fix default --keep-subsets argument value --- scripts/do-partition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/do-partition.py b/scripts/do-partition.py index e049a05a19..350b63f18e 100755 --- a/scripts/do-partition.py +++ b/scripts/do-partition.py @@ -93,7 +93,7 @@ def get_parser(): parser.add_argument('--no-big-traverse', dest='no_big_traverse', action='store_true', default=False, help='Truncate graph joins at big traversals') - parser.add_argument('--keep-subsets', default=True, action='store_false', + parser.add_argument('--keep-subsets', default=False, action='store_true', help='Keep individual subsets') parser.add_argument('graphbase', help="base name for output files") parser.add_argument('input_filenames', metavar='input_sequence_filename', From 8d5abe8f03a8254de009da51fc8d7785c4c1ca03 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Fri, 28 Apr 2017 21:12:14 +0200 Subject: [PATCH 09/10] Unify the order of short and long command-line flags --- khmer/khmer_args.py | 4 ++-- scripts/annotate-partitions.py | 2 +- scripts/do-partition.py | 2 +- scripts/extract-partitions.py | 8 ++++---- scripts/filter-abund-single.py | 6 +++--- scripts/filter-abund.py | 6 +++--- scripts/filter-stoptags.py | 4 ++-- scripts/load-into-counting.py | 2 +- scripts/make-initial-stoptags.py | 4 ++-- scripts/merge-partitions.py | 2 +- scripts/partition-graph.py | 8 +++----- scripts/trim-low-abund.py | 8 ++++---- scripts/unique-kmers.py | 8 ++++---- 13 files changed, 31 insertions(+), 33 deletions(-) diff --git a/khmer/khmer_args.py b/khmer/khmer_args.py index f1e881a6c2..e9b8d1c37d 100644 --- a/khmer/khmer_args.py +++ b/khmer/khmer_args.py @@ -442,7 +442,7 @@ def build_graph_args(descr=None, epilog=None, parser=None, citations=None): parser = KhmerArgumentParser(description=descr, epilog=epilog, citations=citations) - parser.add_argument('--ksize', '-k', type=int, default=DEFAULT_K, + parser.add_argument('-k', '--ksize', type=int, default=DEFAULT_K, help='k-mer size to use') help = ('number of tables to use in k-mer countgraph' if expert_help @@ -618,7 +618,7 @@ def report_on_config(args, graphtype='countgraph'): def add_threading_args(parser): """Add option for threading to options parser.""" - parser.add_argument('--threads', '-T', default=DEFAULT_N_THREADS, type=int, + parser.add_argument('-T', '--threads', default=DEFAULT_N_THREADS, type=int, help='Number of simultaneous threads to execute') diff --git a/scripts/annotate-partitions.py b/scripts/annotate-partitions.py index cb20ab96f5..fd2b41e3ed 100755 --- a/scripts/annotate-partitions.py +++ b/scripts/annotate-partitions.py @@ -73,7 +73,7 @@ def get_parser(): description="Annotate sequences with partition IDs.", epilog=textwrap.dedent(epilog)) - parser.add_argument('--ksize', '-k', type=int, default=DEFAULT_K, + parser.add_argument('-k', '--ksize', type=int, default=DEFAULT_K, help="k-mer size (default: %d)" % DEFAULT_K) parser.add_argument('graphbase', help='basename for input and output ' 'files') diff --git a/scripts/do-partition.py b/scripts/do-partition.py index 350b63f18e..4dfa7e846d 100755 --- a/scripts/do-partition.py +++ b/scripts/do-partition.py @@ -87,7 +87,7 @@ def get_parser(): descr='Load, partition, and annotate FAST[AQ] sequences', epilog=textwrap.dedent(epilog), citations=['graph']) add_threading_args(parser) - parser.add_argument('--subset-size', '-s', default=DEFAULT_SUBSET_SIZE, + parser.add_argument('-s', '--subset-size', default=DEFAULT_SUBSET_SIZE, dest='subset_size', type=float, help='Set subset size (usually 1e5-1e6 is good)') parser.add_argument('--no-big-traverse', dest='no_big_traverse', diff --git a/scripts/extract-partitions.py b/scripts/extract-partitions.py index a404b41832..a9184896de 100755 --- a/scripts/extract-partitions.py +++ b/scripts/extract-partitions.py @@ -94,16 +94,16 @@ def get_parser(): parser.add_argument('prefix', metavar='output_filename_prefix') parser.add_argument('part_filenames', metavar='input_partition_filename', nargs='+') - parser.add_argument('--max-size', '-X', dest='max_size', + parser.add_argument('-X', '--max-size', dest='max_size', default=DEFAULT_MAX_SIZE, type=int, help='Max group size (n sequences)') - parser.add_argument('--min-partition-size', '-m', dest='min_part_size', + parser.add_argument('-m', '--min-partition-size', dest='min_part_size', default=DEFAULT_THRESHOLD, type=int, help='Minimum partition size worth keeping') - parser.add_argument('--no-output-groups', '-n', dest='output_groups', + parser.add_argument('-n', '--no-output-groups', dest='output_groups', default=True, action='store_false', help='Do not actually output groups files.') - parser.add_argument('--output-unassigned', '-U', default=False, + parser.add_argument('-U', '--output-unassigned', default=False, action='store_true', help='Output unassigned sequences, too') parser.add_argument('-f', '--force', default=False, action='store_true', diff --git a/scripts/filter-abund-single.py b/scripts/filter-abund-single.py index 4dcc966b92..efc3547529 100755 --- a/scripts/filter-abund-single.py +++ b/scripts/filter-abund-single.py @@ -90,14 +90,14 @@ def get_parser(): citations=['counting', 'SeqAn']) add_threading_args(parser) - parser.add_argument('--cutoff', '-C', default=DEFAULT_CUTOFF, + parser.add_argument('-C', '--cutoff', default=DEFAULT_CUTOFF, type=check_argument_range(0, 256, "cutoff"), help="Trim at k-mers below this abundance.") - parser.add_argument('--variable-coverage', '-V', action='store_true', + parser.add_argument('-V', '--variable-coverage', action='store_true', dest='variable_coverage', default=False, help='Only trim low-abundance k-mers from sequences ' 'that have high coverage.') - parser.add_argument('--normalize-to', '-Z', type=int, dest='normalize_to', + parser.add_argument('-Z', '--normalize-to', type=int, dest='normalize_to', help='Base the variable-coverage cutoff on this median' ' k-mer abundance.', default=DEFAULT_NORMALIZE_LIMIT) diff --git a/scripts/filter-abund.py b/scripts/filter-abund.py index a400a5b6f6..483ff2c4ed 100755 --- a/scripts/filter-abund.py +++ b/scripts/filter-abund.py @@ -87,15 +87,15 @@ def get_parser(): parser.add_argument('input_filename', metavar='input_sequence_filename', help='Input FAST[AQ] sequence filename', nargs='+') add_threading_args(parser) - parser.add_argument('--cutoff', '-C', dest='cutoff', + parser.add_argument('-C', '--cutoff', dest='cutoff', default=DEFAULT_CUTOFF, type=check_argument_range(0, 256, 'cutoff'), help="Trim at k-mers below this abundance.") - parser.add_argument('--variable-coverage', '-V', action='store_true', + parser.add_argument('-V', '--variable-coverage', action='store_true', dest='variable_coverage', default=False, help='Only trim low-abundance k-mers from sequences ' 'that have high coverage.') - parser.add_argument('--normalize-to', '-Z', type=int, dest='normalize_to', + parser.add_argument('-Z', '--normalize-to', type=int, dest='normalize_to', help='Base the variable-coverage cutoff on this median' ' k-mer abundance.', default=DEFAULT_NORMALIZE_LIMIT) diff --git a/scripts/filter-stoptags.py b/scripts/filter-stoptags.py index 85689bbf7f..ee37feaf93 100755 --- a/scripts/filter-stoptags.py +++ b/scripts/filter-stoptags.py @@ -49,7 +49,7 @@ import os import textwrap import sys -from khmer import __version__, Nodegraph +from khmer import Nodegraph from khmer.thread_utils import ThreadedSequenceProcessor, verbose_loader from khmer.kfile import check_input_files, check_space from khmer.khmer_args import sanitize_help, KhmerArgumentParser @@ -67,7 +67,7 @@ def get_parser(): parser = KhmerArgumentParser( description="Trim sequences at stoptags.", epilog=textwrap.dedent(epilog), citations=['graph']) - parser.add_argument('--ksize', '-k', default=DEFAULT_K, type=int, + parser.add_argument('-k', '--ksize', default=DEFAULT_K, type=int, help='k-mer size') parser.add_argument('stoptags_file', metavar='input_stoptags_filename') parser.add_argument('input_filenames', metavar='input_sequence_filename', diff --git a/scripts/load-into-counting.py b/scripts/load-into-counting.py index 4797aec3b4..3e3502bf0c 100755 --- a/scripts/load-into-counting.py +++ b/scripts/load-into-counting.py @@ -93,7 +93,7 @@ def get_parser(): action='store_false', help="The default behaviour is " "to count past 255 using bigcount. This flag turns " "bigcount off, limiting counts to 255.") - parser.add_argument('--summary-info', '-s', type=str, default=None, + parser.add_argument('-s', '--summary-info', type=str, default=None, metavar="FORMAT", choices=[str('json'), str('tsv')], help="What format should the machine readable run " "summary be in? (`json` or `tsv`, disabled by" diff --git a/scripts/make-initial-stoptags.py b/scripts/make-initial-stoptags.py index 765ffe5a18..2e6351751b 100755 --- a/scripts/make-initial-stoptags.py +++ b/scripts/make-initial-stoptags.py @@ -87,10 +87,10 @@ def get_parser(): descr="Find an initial set of highly connected k-mers.", epilog=textwrap.dedent(epilog), citations=['graph']) - parser.add_argument('--subset-size', '-s', default=DEFAULT_SUBSET_SIZE, + parser.add_argument('-s', '--subset-size', default=DEFAULT_SUBSET_SIZE, dest='subset_size', type=float, help='Set subset size (default 1e4 is prob ok)') - parser.add_argument('--stoptags', '-S', metavar='filename', default='', + parser.add_argument('-S', '--stoptags', metavar='filename', default='', help="Use stoptags in this file during partitioning") parser.add_argument('graphbase', help='basename for input and output ' 'filenames') diff --git a/scripts/merge-partitions.py b/scripts/merge-partitions.py index 5d7ed72c49..efa1b237f2 100755 --- a/scripts/merge-partitions.py +++ b/scripts/merge-partitions.py @@ -65,7 +65,7 @@ def get_parser(): description="Merge partition map '.pmap' files.", epilog=textwrap.dedent(epilog), citations=['graph']) - parser.add_argument('--ksize', '-k', type=int, default=DEFAULT_K, + parser.add_argument('-k', '--ksize', type=int, default=DEFAULT_K, help="k-mer size (default: %d)" % DEFAULT_K) parser.add_argument('--keep-subsets', dest='remove_subsets', default=True, action='store_false', diff --git a/scripts/partition-graph.py b/scripts/partition-graph.py index fc14c7a5e4..2780a219ea 100755 --- a/scripts/partition-graph.py +++ b/scripts/partition-graph.py @@ -48,10 +48,8 @@ import threading import textwrap import sys -import gc -import os.path -from khmer import __version__, load_nodegraph +from khmer import load_nodegraph from khmer.khmer_args import (add_threading_args, sanitize_help, KhmerArgumentParser) from khmer.kfile import check_input_files @@ -79,9 +77,9 @@ def get_parser(): parser.add_argument('basename', help="basename of the input k-mer " "nodegraph + tagset files") - parser.add_argument('--stoptags', '-S', metavar='filename', default='', + parser.add_argument('-S', '--stoptags', metavar='filename', default='', help="Use stoptags in this file during partitioning") - parser.add_argument('--subset-size', '-s', default=DEFAULT_SUBSET_SIZE, + parser.add_argument('-s', '--subset-size', default=DEFAULT_SUBSET_SIZE, type=float, help='Set subset size (usually 1e5-1e6 is ' 'good)') parser.add_argument('--no-big-traverse', action='store_true', diff --git a/scripts/trim-low-abund.py b/scripts/trim-low-abund.py index b4a0533a56..f36a021669 100755 --- a/scripts/trim-low-abund.py +++ b/scripts/trim-low-abund.py @@ -105,11 +105,11 @@ def get_parser(): parser.add_argument('input_filenames', nargs='+') - parser.add_argument('--cutoff', '-C', type=int, + parser.add_argument('-C', '--cutoff', type=int, help='remove k-mers below this abundance', default=DEFAULT_CUTOFF) - parser.add_argument('--trim-at-coverage', '-Z', '--normalize-to', + parser.add_argument('-Z', '--trim-at-coverage', '--normalize-to', type=int, help='trim reads when entire read above this coverage', default=DEFAULT_TRIM_AT_COVERAGE) @@ -121,7 +121,7 @@ def get_parser(): 'specify that output should go to STDOUT (the ' 'terminal)') - parser.add_argument('--variable-coverage', '-V', action='store_true', + parser.add_argument('-V', '--variable-coverage', action='store_true', default=False, help='Only trim low-abundance k-mers from sequences ' 'that have high coverage.') @@ -142,7 +142,7 @@ def get_parser(): parser.add_argument('--force', default=False, action='store_true') parser.add_argument('--ignore-pairs', default=False, action='store_true', help='treat all reads as if they were singletons') - parser.add_argument('--tempdir', '-T', type=str, default='./', + parser.add_argument('-T', '--tempdir', type=str, default='./', help="Set location of temporary directory for " "second pass") add_output_compression_type(parser) diff --git a/scripts/unique-kmers.py b/scripts/unique-kmers.py index a41a4e4f03..263a31344c 100755 --- a/scripts/unique-kmers.py +++ b/scripts/unique-kmers.py @@ -102,18 +102,18 @@ def get_parser(): parser.add_argument('-q', '--quiet', dest='quiet', default=False, action='store_true') - parser.add_argument('--ksize', '-k', type=int, default=env_ksize, + parser.add_argument('-k', '--ksize', type=int, default=env_ksize, help='k-mer size to use') - parser.add_argument('--error-rate', '-e', type=float, default=0.01, + parser.add_argument('-e', '--error-rate', type=float, default=0.01, help='Acceptable error rate') - parser.add_argument('--report', '-R', + parser.add_argument('-R', '--report', metavar='filename', type=argparse.FileType('w'), help='generate informational report and write to' ' filename') - parser.add_argument('--stream-records', '-S', default=False, + parser.add_argument('-S', '--stream-records', default=False, action='store_true', help='write input sequences to STDOUT') From d98464314d3e98c7ab34831209862572a1a059c2 Mon Sep 17 00:00:00 2001 From: Tim Head Date: Sat, 6 May 2017 04:26:06 +0200 Subject: [PATCH 10/10] [MRG] Update release checklist (#1688) * Add comment about selecting small tests * Updated versioneer --- doc/dev/release.rst | 8 +- khmer/_version.py | 114 ++++++---- versioneer.py | 525 ++++++++++++++++++++++++-------------------- 3 files changed, 364 insertions(+), 283 deletions(-) diff --git a/doc/dev/release.rst b/doc/dev/release.rst index f8ac567ce6..d82f5a1c27 100644 --- a/doc/dev/release.rst +++ b/doc/dev/release.rst @@ -67,12 +67,9 @@ release makers, following this checklist by MRC. #. (Optional) Check for updates to versioneer:: pip install --upgrade versioneer - versioneer-installer + versioneer install + git diff --staged - git diff - - ./setup.py versioneer - git diff git commit -m -a "new version of versioneer.py" # or git checkout -- versioneer.py khmer/_version.py khmer/__init__.py MANIFEST.in @@ -128,6 +125,7 @@ release makers, following this checklist by MRC. pip uninstall -y khmer; pip uninstall -y khmer; make install mkdir ../not-khmer # make sure py.test executes tests # from the installed khmer module + # you might want to add 'and not huge' to the test selection pushd ../not-khmer; pytest --pyargs khmer -m 'not known_failing'; popd diff --git a/khmer/_version.py b/khmer/_version.py index 6ad1827e8b..adbd2e72ef 100644 --- a/khmer/_version.py +++ b/khmer/_version.py @@ -6,7 +6,7 @@ # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.15+dev (https://github.com/warner/python-versioneer) +# versioneer-0.18 (https://github.com/warner/python-versioneer) """Git implementation of _version.py.""" @@ -25,12 +25,12 @@ def get_keywords(): # get_keywords(). git_refnames = "$Format:%d$" git_full = "$Format:%H$" - keywords = {"refnames": git_refnames, "full": git_full} + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: - """Container for Versioneer configuration parameters.""" @@ -49,7 +49,6 @@ def get_config(): class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" @@ -68,7 +67,8 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -76,7 +76,8 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)) break @@ -87,36 +88,45 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if verbose: print("unable to run %s" % dispcmd) print(e) - return None + return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) - return None + return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) - return None - return stdout + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") @@ -138,6 +148,10 @@ def git_get_keywords(versionfile_abs): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) f.close() except EnvironmentError: pass @@ -149,6 +163,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if not keywords: raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: @@ -180,14 +203,14 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") @@ -198,25 +221,28 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -267,10 +293,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces @@ -417,7 +448,8 @@ def render(pieces, style): return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, - "error": pieces["error"]} + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -438,7 +470,8 @@ def render(pieces, style): raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): @@ -467,7 +500,8 @@ def get_versions(): except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to find root of source tree"} + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -483,4 +517,4 @@ def get_versions(): return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to compute version"} + "error": "unable to compute version", "date": None} diff --git a/versioneer.py b/versioneer.py index 57509b324b..64fea1c892 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.15+dev +# Version: 0.18 """The Versioneer - like a rocketeer, but for versions. @@ -10,7 +10,7 @@ * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy * [![Latest Version] (https://pypip.in/version/versioneer/badge.svg?style=flat) ](https://pypi.python.org/pypi/versioneer/) @@ -88,125 +88,7 @@ ## Installation -First, decide on values for the following configuration variables: - -* `VCS`: the version control system you use. Currently accepts "git". - -* `style`: the style of version string to be produced. See "Styles" below for - details. Defaults to "pep440", which looks like - `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. - -* `versionfile_source`: - - A project-relative pathname into which the generated version strings should - be written. This is usually a `_version.py` next to your project's main - `__init__.py` file, so it can be imported at runtime. If your project uses - `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. - This file should be checked in to your VCS as usual: the copy created below - by `setup.py setup_versioneer` will include code that parses expanded VCS - keywords in generated tarballs. The 'build' and 'sdist' commands will - replace it with a copy that has just the calculated version string. - - This must be set even if your project does not have any modules (and will - therefore never import `_version.py`), since "setup.py sdist" -based trees - still need somewhere to record the pre-calculated version strings. Anywhere - in the source tree should do. If there is a `__init__.py` next to your - `_version.py`, the `setup.py setup_versioneer` command (described below) - will append some `__version__`-setting assignments, if they aren't already - present. - -* `versionfile_build`: - - Like `versionfile_source`, but relative to the build directory instead of - the source directory. These will differ when your setup.py uses - 'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`, - then you will probably have `versionfile_build='myproject/_version.py'` and - `versionfile_source='src/myproject/_version.py'`. - - If this is set to None, then `setup.py build` will not attempt to rewrite - any `_version.py` in the built tree. If your project does not have any - libraries (e.g. if it only builds a script), then you should use - `versionfile_build = None` and override `distutils.command.build_scripts` - to explicitly insert a copy of `versioneer.get_version()` into your - generated script. - -* `tag_prefix`: - - a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. - If your tags look like 'myproject-1.2.0', then you should use - tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this - should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. - -* `parentdir_prefix`: - - a optional string, frequently the same as tag_prefix, which appears at the - start of all unpacked tarball filenames. If your tarball unpacks into - 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, - just omit the field from your `setup.cfg`. - -This tool provides one script, named `versioneer`. That script has one mode, -"install", which writes a copy of `versioneer.py` into the current directory -and runs `versioneer.py setup` to finish the installation. - -To versioneer-enable your project: - -* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and - populating it with the configuration values you decided earlier (note that - the option names are not case-sensitive): - - ```` - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - ```` - -* 2: Run `versioneer install`. This will do the following: - - * copy `versioneer.py` into the top of your source tree - * create `_version.py` in the right place (`versionfile_source`) - * modify your `__init__.py` (if one exists next to `_version.py`) to define - `__version__` (by calling a function from `_version.py`) - * modify your `MANIFEST.in` to include both `versioneer.py` and the - generated `_version.py` in sdist tarballs - - `versioneer install` will complain about any problems it finds with your - `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all - the problems. - -* 3: add a `import versioneer` to your setup.py, and add the following - arguments to the setup() call: - - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), - -* 4: commit these changes to your VCS. To make sure you won't forget, - `versioneer install` will mark everything it touched for addition using - `git add`. Don't forget to add `setup.py` and `setup.cfg` too. - -## Post-Installation Usage - -Once established, all uses of your tree from a VCS checkout should get the -current version string. All generated tarballs should include an embedded -version string (so users who unpack them will not need a VCS tool installed). - -If you distribute your project through PyPI, then the release process should -boil down to two steps: - -* 1: git tag 1.0 -* 2: python setup.py register sdist upload - -If you distribute it through github (i.e. users use github to generate -tarballs with `git archive`), the process is: - -* 1: git tag 1.0 -* 2: git push; git push --tags - -Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at -least one tag in its history. +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. ## Version-String Flavors @@ -227,6 +109,10 @@ * `['full-revisionid']`: detailed revision identifier. For Git, this is the full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. + * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that this is only accurate if run in a VCS checkout, otherwise it is likely to be False or None @@ -265,8 +151,8 @@ software (exactly equal to a known tag), the identifier will only contain the stripped tag, e.g. "0.11". -Other styles are available. See details.md in the Versioneer source tree for -descriptions. +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. ## Debugging @@ -276,47 +162,95 @@ display the full contents of `get_versions()` (including the `error` string, which may help identify what went wrong). -## Updating Versioneer +## Known Limitations -To upgrade your project to a new release of Versioneer, do the following: +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/warner/python-versioneer/issues). -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files +### Subprojects + +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: + +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other langauges) in subdirectories. + +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). + +`pip install --editable .` should work correctly. `setup.py install` might +work too. + +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. + +[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. + +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. + +### Editable installs with setuptools <= 18.5 -### Upgrading to 0.15 +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. -Starting with this version, Versioneer is configured with a `[versioneer]` -section in your `setup.cfg` file. Earlier versions required the `setup.py` to -set attributes on the `versioneer` module immediately after import. The new -version will refuse to run (raising an exception during import) until you -have provided the necessary `setup.cfg` section. +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. -In addition, the Versioneer package provides an executable named -`versioneer`, and the installation process is driven by running `versioneer -install`. In 0.14 and earlier, the executable was named -`versioneer-installer` and was run without an argument. +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. -### Upgrading to 0.14 +[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. -0.14 changes the format of the version string. 0.13 and earlier used -hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a -plus-separated "local version" section strings, with dot-separated -components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old -format, but should be ok with the new one. +### Unicode version strings -### Upgrading from 0.11 to 0.12 +While Versioneer works (and is continually tested) with both Python 2 and +Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. +Newer releases probably generate unicode version strings on py2. It's not +clear that this is wrong, but it may be surprising for applications when then +write these strings to a network connection or include them in bytes-oriented +APIs like cryptographic checksums. -Nothing special. +[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates +this question. -### Upgrading from 0.10 to 0.11 -You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running -`setup.py setup_versioneer`. This will enable the use of additional -version-control systems (SVN, etc) in the future. +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files ## Future Directions @@ -356,7 +290,6 @@ class VersioneerConfig: - """Container for Versioneer configuration parameters.""" @@ -389,7 +322,9 @@ def get_root(): # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. me = os.path.realpath(os.path.abspath(__file__)) - if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: print("Warning: build in %s is using versioneer.py from %s" % (os.path.dirname(me), versioneer_py)) except NameError: @@ -427,9 +362,9 @@ def get(parser, name): class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" + # these dictionaries contain VCS-specific tools LONG_VERSION_PY = {} HANDLERS = {} @@ -446,7 +381,8 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -454,7 +390,8 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)) break @@ -465,19 +402,22 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if verbose: print("unable to run %s" % dispcmd) print(e) - return None + return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) - return None + return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) - return None - return stdout + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag @@ -486,7 +426,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.15+dev (https://github.com/warner/python-versioneer) +# versioneer-0.18 (https://github.com/warner/python-versioneer) """Git implementation of _version.py.""" @@ -505,12 +445,12 @@ def get_keywords(): # get_keywords(). git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full} + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: - """Container for Versioneer configuration parameters.""" @@ -529,7 +469,6 @@ def get_config(): class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" @@ -548,7 +487,8 @@ def decorate(f): return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -556,7 +496,8 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None)) break @@ -567,36 +508,45 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if verbose: print("unable to run %%s" %% dispcmd) print(e) - return None + return None, None else: if verbose: print("unable to find command, tried %%s" %% (commands,)) - return None + return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) - return None - return stdout + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with " - "prefix '%%s'" %% (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") @@ -618,6 +568,10 @@ def git_get_keywords(versionfile_abs): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) f.close() except EnvironmentError: pass @@ -629,6 +583,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if not keywords: raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: @@ -649,7 +612,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%%s', no digits" %% ",".join(refs-tags)) + print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) for ref in sorted(tags): @@ -660,14 +623,14 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %%s" %% r) return {"version": r, "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") @@ -678,25 +641,28 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %%s" %% root) - raise NotThisMethod("no .git directory") - GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -747,10 +713,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces @@ -897,7 +868,8 @@ def render(pieces, style): return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, - "error": pieces["error"]} + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -918,7 +890,8 @@ def render(pieces, style): raise ValueError("unknown style '%%s'" %% style) return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} def get_versions(): @@ -947,7 +920,8 @@ def get_versions(): except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to find root of source tree"} + "error": "unable to find root of source tree", + "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -963,7 +937,7 @@ def get_versions(): return {"version": "0+unknown", "full-revisionid": None, "dirty": None, - "error": "unable to compute version"} + "error": "unable to compute version", "date": None} ''' @@ -986,6 +960,10 @@ def git_get_keywords(versionfile_abs): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) f.close() except EnvironmentError: pass @@ -997,6 +975,15 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if not keywords: raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: @@ -1017,7 +1004,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs-tags)) + print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -1028,14 +1015,14 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "dirty": False, "error": None, + "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags"} + "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") @@ -1046,25 +1033,28 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - raise NotThisMethod("no .git directory") - GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() - full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() @@ -1115,10 +1105,15 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) pieces["distance"] = int(count_out) # total number of commits + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + return pieces @@ -1126,7 +1121,7 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py - for export-time keyword substitution. + for export-subst keyword substitution. """ GITS = ["git"] if sys.platform == "win32": @@ -1163,27 +1158,35 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. - Source tarballs conventionally unpack into a directory that includes - both the project name and a version string. + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory """ - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with " - "prefix '%s'" % (root, dirname, parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None} + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.15+dev) from +# This file was generated by 'versioneer.py' (0.18) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. import json -import sys version_json = ''' %s @@ -1204,6 +1207,9 @@ def versions_from_file(filename): raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) + if not mo: + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) @@ -1363,7 +1369,8 @@ def render(pieces, style): return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, - "error": pieces["error"]} + "error": pieces["error"], + "date": None} if not style or style == "default": style = "pep440" # the default @@ -1384,11 +1391,11 @@ def render(pieces, style): raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" @@ -1464,7 +1471,8 @@ def get_versions(verbose=False): print("unable to compute version") return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version"} + "dirty": None, "error": "unable to compute version", + "date": None} def get_version(): @@ -1510,6 +1518,7 @@ def run(self): print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) cmds["version"] = cmd_version @@ -1523,6 +1532,11 @@ def run(self): # setuptools/bdist_egg -> distutils/install_lib -> build_py # setuptools/install -> bdist_egg ->.. # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? # we override different "build_py" commands for both environments if "setuptools" in sys.modules: @@ -1547,6 +1561,12 @@ def run(self): if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... class cmd_build_exe(_build_exe): def run(self): @@ -1571,6 +1591,34 @@ def run(self): cmds["build_exe"] = cmd_build_exe del cmds["build_py"] + if 'py2exe' in sys.modules: # py2exe enabled? + try: + from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + except ImportError: + from py2exe.build_exe import py2exe as _py2exe # py2 + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["py2exe"] = cmd_py2exe + # we override different "sdist" commands for both environments if "setuptools" in sys.modules: from setuptools.command.sdist import sdist as _sdist @@ -1722,7 +1770,7 @@ def do_setup(): print(" versionfile_source already in MANIFEST.in") # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-time keyword + # .gitattributes to mark _version.py for export-subst keyword # substitution. do_vcs_install(manifest_in, cfg.versionfile_source, ipy) return 0 @@ -1764,6 +1812,7 @@ def scan_setup_py(): errors += 1 return errors + if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup":