Merge pull request #2617 from CounterpartyXCP/develop

v10.6.1

.github/workflows/test_compose.sh

@@ -128,7 +128,7 @@ done
 # Run compare hashes test
 . "$HOME/.profile"
 cd counterparty-core
-# sudo python3 -m pytest counterpartycore/test/mainnet_test.py --testapidb --comparehashes
+sudo python3 -m pytest counterpartycore/test/mainnet_test.py --testapidb --comparehashes
 cd ..
 
 

counterparty-core/counterpartycore/lib/api/api_server.py

@@ -174,12 +174,15 @@ def prepare_args(route, **kwargs):
         if arg_name in function_args:
             continue
 
-        str_arg = request.args.get(arg_name)
-        if str_arg is not None and str_arg.lower() == "none":
+        str_arg = query_params().get(arg_name)
+        if str_arg is not None and isinstance(str_arg, str) and str_arg.lower() == "none":
             str_arg = None
         if str_arg is None and arg["required"]:
             raise ValueError(f"Missing required parameter: {arg_name}")
 
+        if arg["type"] != "list" and isinstance(str_arg, list):
+            str_arg = str_arg[0]  # we take the first argument
+
         if str_arg is None:
             function_args[arg_name] = arg["default"]
         elif arg["type"] == "bool":
@@ -194,6 +197,11 @@ def prepare_args(route, **kwargs):
                 function_args[arg_name] = float(str_arg)
             except ValueError as e:
                 raise ValueError(f"Invalid float: {arg_name}") from e
+        elif arg["type"] == "list":
+            if not isinstance(str_arg, list):
+                function_args[arg_name] = [str_arg]
+            else:
+                function_args[arg_name] = str_arg
         else:
             function_args[arg_name] = str_arg
 
@@ -245,13 +253,18 @@ def get_transaction_name(rule):
     return "".join([part.capitalize() for part in ROUTES[rule]["function"].__name__.split("_")])
 
 
+def query_params():
+    params = request.args.to_dict(flat=False)
+    return {key: value[0] if len(value) == 1 else value for key, value in params.items()}
+
+
 @auth.login_required
 def handle_route(**kwargs):
     if request.method == "OPTIONS":
         return handle_options()
 
     start_time = time.time()
-    query_args = request.args.to_dict() | kwargs
+    query_args = query_params() | kwargs
 
     logger.trace(f"API Request - {request.remote_addr} {request.method} {request.url}")
     logger.debug(get_log_prefix(query_args))
@@ -397,7 +410,7 @@ def run_api_server(args, server_ready_value, stop_event):
 
     # Initialize Sentry, logging, config, etc.
     sentry.init()
-    server.initialise_log_and_config(argparse.Namespace(**args))
+    server.initialise_log_and_config(argparse.Namespace(**args), api=True)
 
     watcher = api_watcher.APIWatcher()
     watcher.start()

counterparty-core/counterpartycore/lib/api/compose.py

@@ -131,6 +131,16 @@
         False,
         "(API v1 only) Returns a single hex-encoded string instead of an array",
     ),
+    "use_utxos_with_balances": (
+        bool,
+        False,
+        "Use UTXO with balances",
+    ),
+    "exclude_utxos_with_balances": (
+        bool,
+        False,
+        "Exclude silently UTXO with balances instead of raising an exception",
+    ),
 }
 
 
@@ -381,6 +391,8 @@ def compose_mpma(
     assets: str,
     destinations: str,
     quantities: str,
+    memos: list = None,
+    memos_are_hex: bool = None,
     memo: str = None,
     memo_is_hex: bool = False,
     **construct_args,
@@ -391,8 +403,10 @@ def compose_mpma(
     :param assets: comma-separated list of assets to send (e.g. XCP,$ASSET_5)
     :param destinations: comma-separated list of addresses to send to (e.g. $ADDRESS_1,$ADDRESS_2)
     :param quantities: comma-separated list of quantities to send (in satoshis, hence integer) (e.g. 1,2)
-    :param memo: The Memo associated with this transaction (e.g. "Hello, world!")
-    :param memo_is_hex: Whether the memo field is a hexadecimal string (e.g. False)
+    :param memos: One `memos` argument by send, if any
+    :param memos_are_hex: Whether the memos are in hexadecimal format
+    :param memo: The Memo associated with this transaction, used by default for all sends if no `memos` are specified
+    :param memo_is_hex: Whether the memo field is a hexadecimal string
     """
     asset_list = assets.split(",")
     destination_list = destinations.split(",")
@@ -407,6 +421,17 @@ def compose_mpma(
     quantity_list = [int(quantity) for quantity in quantity_list]
     asset_dest_quant_list = list(zip(asset_list, destination_list, quantity_list))
 
+    if memos:
+        if not isinstance(memos, list):
+            raise exceptions.ComposeError("Memos must be a list")
+        if len(memos) != len(asset_dest_quant_list):
+            raise exceptions.ComposeError(
+                "The number of memos must be equal to the number of sends"
+            )
+        for i, send_memo in enumerate(memos):
+            if send_memo:
+                asset_dest_quant_list[i] += (send_memo, memos_are_hex)
+
     params = {
         "source": address,
         "asset_dest_quant_list": asset_dest_quant_list,
@@ -736,8 +761,6 @@ def compose_movetoutxo(
     if change > 0:
         change_output_address = change_address or source_address
         outputs += [{change_output_address: str(change)}]
-    print("inputs", inputs)
-    print("outputs", outputs)
     rawtransaction = backend.bitcoind.createrawtransaction(inputs, outputs)
 
     return {

counterparty-core/counterpartycore/lib/api/queries.py

@@ -1562,6 +1562,25 @@ def get_address_balances(
     )
 
 
+def get_utxo_balances(db, utxo: str, cursor: str = None, limit: int = 100, offset: int = None):
+    """
+    Returns the balances of an utxo
+    :param str utxo: The utxo to return (e.g. $UTXO_WITH_BALANCE)
+    :param str cursor: The last index of the balances to return
+    :param int limit: The maximum number of balances to return (e.g. 5)
+    :param int offset: The number of lines to skip before returning results (overrides the `cursor` parameter)
+    """
+    return select_rows(
+        db,
+        "balances",
+        where={"utxo": utxo, "quantity__gt": 0},
+        last_cursor=cursor,
+        limit=limit,
+        offset=offset,
+        select="asset, quantity, utxo, utxo_address",
+    )
+
+
 def get_balances_by_addresses(
     db, addresses: str, cursor: str = None, limit: int = 100, offset: int = None, sort: str = None
 ):

counterparty-core/counterpartycore/lib/api/routes.py

@@ -76,6 +76,7 @@ def get_routes():
         "/v2/addresses/<address>/fairminters": queries.get_fairminters_by_address,
         "/v2/addresses/<address>/fairmints": queries.get_fairmints_by_address,
         "/v2/addresses/<address>/fairmints/<asset>": queries.get_fairmints_by_address_and_asset,
+        "/v2/utxos/<utxo>/balances": queries.get_utxo_balances,
         ### /addresses/<address>/compose/ ###
         "/v2/addresses/<address>/compose/bet": compose.compose_bet,
         "/v2/addresses/<address>/compose/broadcast": compose.compose_broadcast,

counterparty-core/counterpartycore/lib/api/util.py

@@ -57,7 +57,7 @@ def healthz_heavy(db):
             "quantity": 100000000,
         },
         allow_unconfirmed_inputs=True,
-        fee=1000,
+        exact_fee=1000,
     )
 
 

counterparty-core/counterpartycore/lib/api/wsgi.py

@@ -129,7 +129,7 @@ def spawn_worker(self):
         # Child process
         global logger  # noqa F811
         worker.pid = os.getpid()
-        logger = log.re_set_up(f".gunicorn.{worker.pid}")
+        logger = log.re_set_up(f".gunicorn.{worker.pid}", api=True)
         try:
             gunicorn_util._setproctitle(f"worker [{self.proc_name}]")
             logger.trace("Booting Gunicorn worker with pid: %s", worker.pid)

counterparty-core/counterpartycore/lib/config.py

@@ -5,7 +5,7 @@
 
 
 # Semantic Version
-__version__ = "10.6.0"  # for hatch
+__version__ = "10.6.1"  # for hatch
 VERSION_STRING = __version__
 version = VERSION_STRING.split("-")[0].split(".")
 VERSION_MAJOR = int(version[0])

counterparty-core/counterpartycore/lib/log.py

@@ -209,11 +209,11 @@ def handle_exception(exc_type, exc_value, exc_traceback):
     return logger
 
 
-def re_set_up(suffix=""):
+def re_set_up(suffix="", api=False):
     return set_up(
         verbose=config.VERBOSE,
         quiet=config.QUIET,
-        log_file=config.LOG + suffix,
+        log_file=(config.LOG if not api else config.API_LOG) + suffix,
         json_logs=config.JSON_LOGS,
         max_log_file_size=config.MAX_LOG_FILE_SIZE,
         max_log_file_rotations=config.MAX_LOG_FILE_ROTATIONS,

counterparty-core/counterpartycore/lib/messages/utxo.py

@@ -117,11 +117,15 @@ def compose(db, source, destination, asset, quantity):
     data += struct.pack(f">{len(data_content)}s", data_content)
 
     source_address = source
+    destinations = []
     # if source is a UTXO, we get the corresponding address
-    if util.is_utxo_format(source):
+    if util.is_utxo_format(source):  # detach from utxo
         source_address, _value = backend.bitcoind.get_utxo_address_and_value(source)
+    elif not destination:  # attach to utxo
+        # if no destination, we use the source address as the destination
+        destinations.append((source_address, None))
 
-    return (source_address, [], data)
+    return (source_address, destinations, data)
 
 
 def unpack(message, return_dict=False):

counterparty-core/counterpartycore/lib/messages/versions/mpma.py

@@ -98,9 +98,24 @@ def validate(db, source, asset_dest_quant_list, block_index):
     return problems
 
 
-def compose(db, source: str, asset_dest_quant_list: list, memo: str, memo_is_hex: bool):
+def compose(
+    db, source: str, asset_dest_quant_list: list, memo: str = None, memo_is_hex: bool = None
+):
+    """
+    Compose a MPMA send message.
+    :param db: sqlite3 database
+    :param source: source address
+    :param asset_dest_quant_list: list of tuples of the form (asset, destination, quantity, memo, is_hex), where memo and is_hex are optional; if not specified for a send, memo is used.
+    :param memo: optional memo for the entire send
+    :param memo_is_hex: optional boolean indicating if the memo is in hex format
+    """
     cursor = db.cursor()
 
+    if memo and not isinstance(memo, str):
+        raise exceptions.ComposeError("`memo` must be a string")
+    if memo_is_hex and not isinstance(memo_is_hex, bool):
+        raise exceptions.ComposeError("`memo_is_hex` must be a boolean")
+
     out_balances = util.accumulate([(t[0], t[2]) for t in asset_dest_quant_list])
     for asset, quantity in out_balances:
         if util.enabled("mpma_subasset_support"):

counterparty-core/counterpartycore/lib/transaction.py

@@ -191,6 +191,8 @@ def construct(
     p2sh_source_multisig_pubkeys=None,
     p2sh_source_multisig_pubkeys_required=None,
     p2sh_pretx_txid=None,
+    use_utxos_with_balances=False,
+    exclude_utxos_with_balances=False,
 ):
     # Extract tx_info
     (source, destinations, data) = tx_info
@@ -236,6 +238,7 @@ def construct(
     """Inputs"""
 
     (inputs, change_output, btc_in, final_fee) = transaction_inputs.prepare_inputs(
+        db,
         source,
         data,
         destination_outputs,
@@ -256,6 +259,8 @@ def construct(
         multisig_dust_size,
         disable_utxo_locks,
         exclude_utxos,
+        use_utxos_with_balances,
+        exclude_utxos_with_balances,
     )
 
     """Finish"""

counterparty-core/counterpartycore/lib/transaction_helper/transaction_inputs.py

@@ -7,7 +7,7 @@
 import bitcoin as bitcoinlib
 import cachetools
 
-from counterpartycore.lib import backend, config, exceptions, script, util
+from counterpartycore.lib import backend, config, exceptions, ledger, script, util
 from counterpartycore.lib.transaction_helper import p2sh_serializer, transaction_outputs
 
 logger = logging.getLogger(config.LOGGER_NAME)
@@ -169,6 +169,7 @@ def sort_unspent_txouts(unspent, dust_size=config.DEFAULT_REGULAR_DUST_SIZE):
 
 
 def construct_coin_selection(
+    db,
     size_for_fee,
     encoding,
     data_array,
@@ -186,6 +187,8 @@ def construct_coin_selection(
     multisig_dust_size,
     disable_utxo_locks,
     exclude_utxos,
+    use_utxos_with_balances,
+    exclude_utxos_with_balances,
 ):
     if inputs_set:
         if isinstance(inputs_set, str):
@@ -242,7 +245,22 @@ def construct_coin_selection(
         # self.logger.debug(f"Sorted candidate UTXOs: {[print_coin(coin) for coin in unspent]}")
         use_inputs = unspent
 
-    use_inputs = unspent = UTXOLocks().filter_unspents(source, unspent, exclude_utxos)
+    # remove locked UTXOs
+    unspent = UTXOLocks().filter_unspents(source, unspent, exclude_utxos)
+
+    # remove UTXOs with balances if not specified
+    if not use_utxos_with_balances:
+        filtered_unspent = []
+        for utxo in unspent:
+            str_input = f"{utxo['txid']}:{utxo['vout']}"
+            if len(ledger.get_utxo_balances(db, str_input)) > 0:
+                if not exclude_utxos_with_balances and inputs_set:
+                    raise exceptions.ComposeError(f"invalid UTXO: {str_input}")
+            else:
+                filtered_unspent.append(utxo)
+        use_inputs = unspent = filtered_unspent
+    else:
+        use_inputs = unspent
 
     # dont override fee_per_kb if specified
     estimate_fee_per_kb = None
@@ -347,6 +365,7 @@ def construct_coin_selection(
 
 
 def prepare_inputs(
+    db,
     source,
     data,
     destination_outputs,
@@ -367,6 +386,8 @@ def prepare_inputs(
     multisig_dust_size,
     disable_utxo_locks,
     exclude_utxos,
+    use_utxos_with_balances,
+    exclude_utxos_with_balances,
 ):
     btc_in = 0
     final_fee = 0
@@ -397,6 +418,7 @@ def prepare_inputs(
 
     if not (encoding == "p2sh" and p2sh_pretx_txid):
         inputs, change_quantity, n_btc_in, n_final_fee = construct_coin_selection(
+            db,
             size_for_fee,
             encoding,
             data_array,
@@ -414,6 +436,8 @@ def prepare_inputs(
             multisig_dust_size,
             disable_utxo_locks,
             exclude_utxos,
+            use_utxos_with_balances,
+            exclude_utxos_with_balances,
         )
         btc_in = n_btc_in
         final_fee = n_final_fee

counterparty-core/counterpartycore/server.py

@@ -605,7 +605,7 @@ def initialise_config(
     config.GUNICORN_WORKERS = gunicorn_workers
 
 
-def initialise_log_and_config(args):
+def initialise_log_and_config(args, api=False):
     # Configuration
     init_args = {
         "data_dir": args.data_dir,
@@ -671,7 +671,7 @@ def initialise_log_and_config(args):
     log.set_up(
         verbose=config.VERBOSE,
         quiet=config.QUIET,
-        log_file=config.LOG,
+        log_file=config.LOG if not api else config.API_LOG,
         json_logs=config.JSON_LOGS,
         max_log_file_size=config.MAX_LOG_FILE_SIZE,
         max_log_file_rotations=config.MAX_LOG_FILE_ROTATIONS,