Merge #8149: Segregated witness rebased

f852813 BIP9 parameters for testnet (Johnson Lau)
070dbc4 --- [SEGWIT] begin: deployment --- (Pieter Wuille)
fdb43df [qa] Add GetTransactionSigOpCost unit tests (Jonas Nick)
d846e02 [qa] script_tests: witness tests can specify tx amount (Suhas Daftuar)
330b0f3 [qa] p2p segwit tests (Suhas Daftuar)
4f7ff00 [qa] Add rpc test for segwit (Alex Morcos)
66cca79 [qa] Autogeneration support for witness in script_tests (Pieter Wuille)
06d3805 [qa] Add segwit support to script_tests (Pieter Wuille)
00f46cb [qa] Add transaction tests for segwit (NicolasDorier)
0aa9207 [qa] Witness version 0 signing unit tests (Pieter Wuille)
978e200 --- [SEGWIT] begin: tests --- (Pieter Wuille)
745eb67 [RPC] signrawtransaction can sign P2WSH (NicolasDorier)
f4691ab [RPC] Add wallet support for witness transactions (using P2SH) (Pieter Wuille)
605e847 BIP143: Signing logic (Pieter Wuille)
9757b57 --- [SEGWIT] begin: wallet --- (Pieter Wuille)
af87a67 Do not use compact blocks when segwit is enabled (Pieter Wuille)
6032f69 Add rewind logic to deal with post-fork software updates (Pieter Wuille)
b7dbeb2 [libconsensus] Script verification API with amounts (Thomas Kerin)
2b1f6f9 BIP141: Other consensus critical limits, and BIP145 (Pieter Wuille)
7c4bf77 [RPC] Return witness data in blockchain RPCs (Johnson Lau)
3dd4102 BIP143: Verification logic (Pieter Wuille)
0ef1dd3 Refactor script validation to observe amounts (Pieter Wuille)
b8a9749 BIP144: Handshake and relay (receiver side) (Pieter Wuille)
8b49040 BIP141: Commitment structure and deployment (Pieter Wuille)
449f9b8 BIP141: Witness program (Pieter Wuille)
7030d9e BIP144: Serialization, hashes, relay (sender side) (Pieter Wuille)
ecacfd9 --- [SEGWIT] begin: P2P/node/consensus --- (Pieter Wuille)

qa/pull-tester/rpc-tests.py

@@ -136,6 +136,8 @@ testScripts = [
     'invalidtxrequest.py',
     'abandonconflict.py',
     'p2p-versionbits-warning.py',
+    'p2p-segwit.py',
+    'segwit.py',
     'importprunedfunds.py',
     'signmessages.py',
 ]

qa/rpc-tests/maxuploadtarget.py

@@ -97,7 +97,7 @@ class MaxUploadTest(BitcoinTestFramework):
     def setup_network(self):
         # Start a node with maxuploadtarget of 200 MB (/24h)
         self.nodes = []
-        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-maxuploadtarget=200", "-blockmaxsize=999000"]))
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-debug", "-maxuploadtarget=800", "-blockmaxsize=999000"]))
 
     def mine_full_block(self, node, address):
         # Want to create a full block
@@ -175,13 +175,13 @@ class MaxUploadTest(BitcoinTestFramework):
         getdata_request = msg_getdata()
         getdata_request.inv.append(CInv(2, big_old_block))
 
-        max_bytes_per_day = 200*1024*1024
-        daily_buffer = 144 * MAX_BLOCK_SIZE
+        max_bytes_per_day = 800*1024*1024
+        daily_buffer = 144 * 4000000
         max_bytes_available = max_bytes_per_day - daily_buffer
         success_count = max_bytes_available // old_block_size
 
-        # 144MB will be reserved for relaying new blocks, so expect this to
-        # succeed for ~70 tries.
+        # 576MB will be reserved for relaying new blocks, so expect this to
+        # succeed for ~235 tries.
         for i in range(success_count):
             test_nodes[0].send_message(getdata_request)
             test_nodes[0].sync_with_ping()
@@ -198,9 +198,9 @@ class MaxUploadTest(BitcoinTestFramework):
 
         # Requesting the current block on test_nodes[1] should succeed indefinitely,
         # even when over the max upload target.
-        # We'll try 200 times
+        # We'll try 800 times
         getdata_request.inv = [CInv(2, big_new_block)]
-        for i in range(200):
+        for i in range(800):
             test_nodes[1].send_message(getdata_request)
             test_nodes[1].sync_with_ping()
             assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1)

qa/rpc-tests/segwit.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+# Copyright (c) 2016 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test the SegWit changeover logic
+#
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+from test_framework.mininode import sha256, ripemd160
+import os
+import shutil
+
+NODE_0 = 0
+NODE_1 = 1
+NODE_2 = 2
+WIT_V0 = 0
+WIT_V1 = 1
+
+def witness_script(version, pubkey):
+    if (version == 0):
+        pubkeyhash = bytes_to_hex_str(ripemd160(sha256(hex_str_to_bytes(pubkey))))
+        pkscript = "0014" + pubkeyhash
+    elif (version == 1):
+        # 1-of-1 multisig
+        scripthash = bytes_to_hex_str(sha256(hex_str_to_bytes("5121" + pubkey + "51ae")))
+        pkscript = "0020" + scripthash
+    else:
+        assert("Wrong version" == "0 or 1")
+    return pkscript
+
+def addlength(script):
+    scriptlen = format(len(script)//2, 'x')
+    assert(len(scriptlen) == 2)
+    return scriptlen + script
+
+def create_witnessprogram(version, node, utxo, pubkey, encode_p2sh, amount):
+    pkscript = witness_script(version, pubkey);
+    if (encode_p2sh):
+        p2sh_hash = bytes_to_hex_str(ripemd160(sha256(hex_str_to_bytes(pkscript))))
+        pkscript = "a914"+p2sh_hash+"87"
+    inputs = []
+    outputs = {}
+    inputs.append({ "txid" : utxo["txid"], "vout" : utxo["vout"]} )
+    DUMMY_P2SH = "2MySexEGVzZpRgNQ1JdjdP5bRETznm3roQ2" # P2SH of "OP_1 OP_DROP"
+    outputs[DUMMY_P2SH] = amount
+    tx_to_witness = node.createrawtransaction(inputs,outputs)
+    #replace dummy output with our own
+    tx_to_witness = tx_to_witness[0:110] + addlength(pkscript) + tx_to_witness[-8:]
+    return tx_to_witness
+
+def send_to_witness(version, node, utxo, pubkey, encode_p2sh, amount, sign=True, insert_redeem_script=""):
+    tx_to_witness = create_witnessprogram(version, node, utxo, pubkey, encode_p2sh, amount)
+    if (sign):
+        signed = node.signrawtransaction(tx_to_witness)
+        assert("errors" not in signed or len(["errors"]) == 0)
+        return node.sendrawtransaction(signed["hex"])
+    else:
+        if (insert_redeem_script):
+            tx_to_witness = tx_to_witness[0:82] + addlength(insert_redeem_script) + tx_to_witness[84:]
+
+    return node.sendrawtransaction(tx_to_witness)
+
+def getutxo(txid):
+    utxo = {}
+    utxo["vout"] = 0
+    utxo["txid"] = txid
+    return utxo
+
+class SegWitTest(BitcoinTestFramework):
+
+    def setup_chain(self):
+        print("Initializing test directory "+self.options.tmpdir)
+        initialize_chain_clean(self.options.tmpdir, 3)
+
+    def setup_network(self):
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-logtimemicros", "-debug", "-walletprematurewitness"]))
+        self.nodes.append(start_node(1, self.options.tmpdir, ["-logtimemicros", "-debug", "-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]))
+        self.nodes.append(start_node(2, self.options.tmpdir, ["-logtimemicros", "-debug", "-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]))
+        connect_nodes(self.nodes[1], 0)
+        connect_nodes(self.nodes[2], 1)
+        connect_nodes(self.nodes[0], 2)
+        self.is_network_split = False
+        self.sync_all()
+
+    def success_mine(self, node, txid, sign, redeem_script=""):
+        send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
+        block = node.generate(1)
+        assert_equal(len(node.getblock(block[0])["tx"]), 2)
+        sync_blocks(self.nodes)
+
+    def skip_mine(self, node, txid, sign, redeem_script=""):
+        send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
+        block = node.generate(1)
+        assert_equal(len(node.getblock(block[0])["tx"]), 1)
+        sync_blocks(self.nodes)
+
+    def fail_accept(self, node, txid, sign, redeem_script=""):
+        try:
+            send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
+        except JSONRPCException as exp:
+            assert(exp.error["code"] == -26)
+        else:
+            raise AssertionError("Tx should not have been accepted")
+
+    def fail_mine(self, node, txid, sign, redeem_script=""):
+        send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script)
+        try:
+            node.generate(1)
+        except JSONRPCException as exp:
+            assert(exp.error["code"] == -1)
+        else:
+            raise AssertionError("Created valid block when TestBlockValidity should have failed")
+        sync_blocks(self.nodes)
+
+    def run_test(self):
+        self.nodes[0].generate(160) #block 160
+
+        self.pubkey = []
+        p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh
+        wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness
+        for i in range(3):
+            newaddress = self.nodes[i].getnewaddress()
+            self.pubkey.append(self.nodes[i].validateaddress(newaddress)["pubkey"])
+            multiaddress = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]])
+            self.nodes[i].addwitnessaddress(newaddress)
+            self.nodes[i].addwitnessaddress(multiaddress)
+            p2sh_ids.append([])
+            wit_ids.append([])
+            for v in range(2):
+                p2sh_ids[i].append([])
+                wit_ids[i].append([])
+
+        for i in range(5):
+            for n in range(3):
+                for v in range(2):
+                    wit_ids[n][v].append(send_to_witness(v, self.nodes[0], self.nodes[0].listunspent()[0], self.pubkey[n], False, Decimal("49.999")))
+                    p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], self.nodes[0].listunspent()[0], self.pubkey[n], True, Decimal("49.999")))
+
+        self.nodes[0].generate(1) #block 161
+        sync_blocks(self.nodes)
+
+        # Make sure all nodes recognize the transactions as theirs
+        assert_equal(self.nodes[0].getbalance(), 60*50 - 60*50 + 20*Decimal("49.999") + 50)
+        assert_equal(self.nodes[1].getbalance(), 20*Decimal("49.999"))
+        assert_equal(self.nodes[2].getbalance(), 20*Decimal("49.999"))
+
+        self.nodes[0].generate(262) #block 423
+        sync_blocks(self.nodes)
+
+        print("Verify default node can't accept any witness format txs before fork")
+        # unsigned, no scriptsig
+        self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], False)
+        self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], False)
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], False)
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], False)
+        # unsigned with redeem script
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], False, addlength(witness_script(0, self.pubkey[0])))
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], False, addlength(witness_script(1, self.pubkey[0])))
+        # signed
+        self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True)
+        self.fail_accept(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True)
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True)
+        self.fail_accept(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True)
+
+        print("Verify witness txs are skipped for mining before the fork")
+        self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) #block 424
+        self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) #block 425
+        self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) #block 426
+        self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][0], True) #block 427
+
+        # TODO: An old node would see these txs without witnesses and be able to mine them
+
+        print("Verify unsigned bare witness txs in versionbits-setting blocks are valid before the fork")
+        self.success_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][1], False) #block 428
+        self.success_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][1], False) #block 429
+
+        print("Verify unsigned p2sh witness txs without a redeem script are invalid")
+        self.fail_accept(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][1], False)
+        self.fail_accept(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][1], False)
+
+        print("Verify unsigned p2sh witness txs with a redeem script in versionbits-settings blocks are valid before the fork")
+        self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][1], False, addlength(witness_script(0, self.pubkey[2]))) #block 430
+        self.success_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][1], False, addlength(witness_script(1, self.pubkey[2]))) #block 431
+
+        print("Verify previous witness txs skipped for mining can now be mined")
+        assert_equal(len(self.nodes[2].getrawmempool()), 4)
+        block = self.nodes[2].generate(1) #block 432 (first block with new rules; 432 = 144 * 3)
+        sync_blocks(self.nodes)
+        assert_equal(len(self.nodes[2].getrawmempool()), 0)
+        assert_equal(len(self.nodes[2].getblock(block[0])["tx"]), 5)
+
+        print("Verify witness txs without witness data are invalid after the fork")
+        self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][2], False)
+        self.fail_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][2], False)
+        self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][2], False, addlength(witness_script(0, self.pubkey[2])))
+        self.fail_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][2], False, addlength(witness_script(1, self.pubkey[2])))
+
+        print("Verify default node can now use witness txs")
+        self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) #block 432
+        self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) #block 433
+        self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) #block 434
+        self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) #block 435
+
+if __name__ == '__main__':
+    SegWitTest().main()

qa/rpc-tests/test_framework/blocktools.py

@@ -5,7 +5,7 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 from .mininode import *
-from .script import CScript, OP_TRUE, OP_CHECKSIG
+from .script import CScript, OP_TRUE, OP_CHECKSIG, OP_RETURN
 
 # Create a block (with regtest difficulty)
 def create_block(hashprev, coinbase, nTime=None):
@@ -22,6 +22,29 @@ def create_block(hashprev, coinbase, nTime=None):
     block.calc_sha256()
     return block
 
+# From BIP141
+WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed"
+
+# According to BIP141, blocks with witness rules active must commit to the
+# hash of all in-block transactions including witness.
+def add_witness_commitment(block, nonce=0):
+    # First calculate the merkle root of the block's
+    # transactions, with witnesses.
+    witness_nonce = nonce
+    witness_root = block.calc_witness_merkle_root()
+    witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce)))
+    # witness_nonce should go to coinbase witness.
+    block.vtx[0].wit.vtxinwit = [CTxinWitness()]
+    block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(witness_nonce)]
+
+    # witness commitment is the last OP_RETURN output in coinbase
+    output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment)
+    block.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, output_data])))
+    block.vtx[0].rehash()
+    block.hashMerkleRoot = block.calc_merkle_root()
+    block.rehash()
+
+
 def serialize_script_num(value):
     r = bytearray(0)
     if value == 0:

qa/rpc-tests/test_framework/mininode.py

@@ -28,7 +28,7 @@ import asyncore
 import time
 import sys
 import random
-from binascii import hexlify, unhexlify
+from .util import hex_str_to_bytes, bytes_to_hex_str
 from io import BytesIO
 from codecs import encode
 import hashlib
@@ -46,6 +46,11 @@ MAX_BLOCK_SIZE = 1000000
 
 COIN = 100000000 # 1 btc in satoshis
 
+NODE_NETWORK = (1 << 0)
+NODE_GETUTXO = (1 << 1)
+NODE_BLOOM = (1 << 2)
+NODE_WITNESS = (1 << 3)
+
 # Keep our own socket map for asyncore, so that we can track disconnects
 # ourselves (to workaround an issue with closing an asyncore socket when 
 # using select)
@@ -63,6 +68,8 @@ mininode_lock = RLock()
 def sha256(s):
     return hashlib.new('sha256', s).digest()
 
+def ripemd160(s):
+    return hashlib.new('ripemd160', s).digest()
 
 def hash256(s):
     return sha256(sha256(s))
@@ -133,7 +140,10 @@ def deser_vector(f, c):
     return r
 
 
-def ser_vector(l):
+# ser_function_name: Allow for an alternate serialization function on the
+# entries in the vector (we use this for serializing the vector of transactions
+# for a witness block).
+def ser_vector(l, ser_function_name=None):
     r = b""
     if len(l) < 253:
         r = struct.pack("B", len(l))
@@ -144,7 +154,10 @@ def ser_vector(l):
     else:
         r = struct.pack("<BQ", 255, len(l))
     for i in l:
-        r += i.serialize()
+        if ser_function_name:
+            r += getattr(i, ser_function_name)()
+        else:
+            r += i.serialize()
     return r
 
 
@@ -239,12 +252,12 @@ def ser_int_vector(l):
 
 # Deserialize from a hex string representation (eg from RPC)
 def FromHex(obj, hex_string):
-    obj.deserialize(BytesIO(unhexlify(hex_string.encode('ascii'))))
+    obj.deserialize(BytesIO(hex_str_to_bytes(hex_string)))
     return obj
 
 # Convert a binary-serializable object to hex (eg for submission via RPC)
 def ToHex(obj):
-    return hexlify(obj.serialize()).decode('ascii')
+    return bytes_to_hex_str(obj.serialize())
 
 # Objects that map to bitcoind objects, which can be serialized/deserialized
 
@@ -273,12 +286,16 @@ class CAddress(object):
         return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
                                                          self.ip, self.port)
 
+MSG_WITNESS_FLAG = 1<<30
 
 class CInv(object):
     typemap = {
         0: "Error",
         1: "TX",
-        2: "Block"}
+        2: "Block",
+        1|MSG_WITNESS_FLAG: "WitnessTx",
+        2|MSG_WITNESS_FLAG : "WitnessBlock"
+    }
 
     def __init__(self, t=0, h=0):
         self.type = t
@@ -362,7 +379,7 @@ class CTxIn(object):
 
     def __repr__(self):
         return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \
-            % (repr(self.prevout), hexlify(self.scriptSig),
+            % (repr(self.prevout), bytes_to_hex_str(self.scriptSig),
                self.nSequence)
 
 
@@ -384,7 +401,67 @@ class CTxOut(object):
     def __repr__(self):
         return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \
             % (self.nValue // COIN, self.nValue % COIN,
-               hexlify(self.scriptPubKey))
+               bytes_to_hex_str(self.scriptPubKey))
+
+
+class CScriptWitness(object):
+    def __init__(self):
+        # stack is a vector of strings
+        self.stack = []
+
+    def __repr__(self):
+        return "CScriptWitness(%s)" % \
+               (",".join([bytes_to_hex_str(x) for x in self.stack]))
+
+    def is_null(self):
+        if self.stack:
+            return False
+        return True
+
+
+class CTxinWitness(object):
+    def __init__(self):
+        self.scriptWitness = CScriptWitness()
+
+    def deserialize(self, f):
+        self.scriptWitness.stack = deser_string_vector(f)
+
+    def serialize(self):
+        return ser_string_vector(self.scriptWitness.stack)
+
+    def __repr__(self):
+        return repr(self.scriptWitness)
+
+    def is_null(self):
+        return self.scriptWitness.is_null()
+
+
+class CTxWitness(object):
+    def __init__(self):
+        self.vtxinwit = []
+
+    def deserialize(self, f):
+        for i in range(len(self.vtxinwit)):
+            self.vtxinwit[i].deserialize(f)
+
+    def serialize(self):
+        r = b""
+        # This is different than the usual vector serialization --
+        # we omit the length of the vector, which is required to be
+        # the same length as the transaction's vin vector.
+        for x in self.vtxinwit:
+            r += x.serialize()
+        return r
+
+    def __repr__(self):
+        return "CTxWitness(%s)" % \
+               (';'.join([repr(x) for x in self.vtxinwit]))
+
+    def is_null(self):
+        for x in self.vtxinwit:
+            if not x.is_null():
+                return False
+        return True
 
 
 class CTransaction(object):
@@ -393,6 +470,7 @@ class CTransaction(object):
             self.nVersion = 1
             self.vin = []
             self.vout = []
+            self.wit = CTxWitness()
             self.nLockTime = 0
             self.sha256 = None
             self.hash = None
@@ -401,18 +479,31 @@ class CTransaction(object):
             self.vin = copy.deepcopy(tx.vin)
             self.vout = copy.deepcopy(tx.vout)
             self.nLockTime = tx.nLockTime
-            self.sha256 = None
-            self.hash = None
+            self.sha256 = tx.sha256
+            self.hash = tx.hash
+            self.wit = copy.deepcopy(tx.wit)
 
     def deserialize(self, f):
         self.nVersion = struct.unpack("<i", f.read(4))[0]
         self.vin = deser_vector(f, CTxIn)
-        self.vout = deser_vector(f, CTxOut)
+        flags = 0
+        if len(self.vin) == 0:
+            flags = struct.unpack("<B", f.read(1))[0]
+            # Not sure why flags can't be zero, but this
+            # matches the implementation in bitcoind
+            if (flags != 0):
+                self.vin = deser_vector(f, CTxIn)
+                self.vout = deser_vector(f, CTxOut)
+        else:
+            self.vout = deser_vector(f, CTxOut)
+        if flags != 0:
+            self.wit.vtxinwit = [CTxinWitness()]*len(self.vin)
+            self.wit.deserialize(f)
         self.nLockTime = struct.unpack("<I", f.read(4))[0]
         self.sha256 = None
         self.hash = None
 
-    def serialize(self):
+    def serialize_without_witness(self):
         r = b""
         r += struct.pack("<i", self.nVersion)
         r += ser_vector(self.vin)
@@ -420,13 +511,48 @@ class CTransaction(object):
         r += struct.pack("<I", self.nLockTime)
         return r
 
+    # Only serialize with witness when explicitly called for
+    def serialize_with_witness(self):
+        flags = 0
+        if not self.wit.is_null():
+            flags |= 1
+        r = b""
+        r += struct.pack("<i", self.nVersion)
+        if flags:
+            dummy = []
+            r += ser_vector(dummy)
+            r += struct.pack("<B", flags)
+        r += ser_vector(self.vin)
+        r += ser_vector(self.vout)
+        if flags & 1:
+            if (len(self.wit.vtxinwit) != len(self.vin)):
+                # vtxinwit must have the same length as vin
+                self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)]
+                for i in range(len(self.wit.vtxinwit), len(self.vin)):
+                    self.wit.vtxinwit.append(CTxinWitness())
+            r += self.wit.serialize()
+        r += struct.pack("<I", self.nLockTime)
+        return r
+
+    # Regular serialization is without witness -- must explicitly
+    # call serialize_with_witness to include witness data.
+    def serialize(self):
+        return self.serialize_without_witness()
+
+    # Recalculate the txid (transaction hash without witness)
     def rehash(self):
         self.sha256 = None
         self.calc_sha256()
 
-    def calc_sha256(self):
+    # We will only cache the serialization without witness in
+    # self.sha256 and self.hash -- those are expected to be the txid.
+    def calc_sha256(self, with_witness=False):
+        if with_witness:
+            # Don't cache the result, just return it
+            return uint256_from_str(hash256(self.serialize_with_witness()))
+
         if self.sha256 is None:
-            self.sha256 = uint256_from_str(hash256(self.serialize()))
+            self.sha256 = uint256_from_str(hash256(self.serialize_without_witness()))
         self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii')
 
     def is_valid(self):
@@ -518,17 +644,17 @@ class CBlock(CBlockHeader):
         super(CBlock, self).deserialize(f)
         self.vtx = deser_vector(f, CTransaction)
 
-    def serialize(self):
+    def serialize(self, with_witness=False):
         r = b""
         r += super(CBlock, self).serialize()
-        r += ser_vector(self.vtx)
+        if with_witness:
+            r += ser_vector(self.vtx, "serialize_with_witness")
+        else:
+            r += ser_vector(self.vtx)
         return r
 
-    def calc_merkle_root(self):
-        hashes = []
-        for tx in self.vtx:
-            tx.calc_sha256()
-            hashes.append(ser_uint256(tx.sha256))
+    # Calculate the merkle root given a vector of transaction hashes
+    def get_merkle_root(self, hashes):
         while len(hashes) > 1:
             newhashes = []
             for i in range(0, len(hashes), 2):
@@ -537,6 +663,24 @@ class CBlock(CBlockHeader):
             hashes = newhashes
         return uint256_from_str(hashes[0])
 
+    def calc_merkle_root(self):
+        hashes = []
+        for tx in self.vtx:
+            tx.calc_sha256()
+            hashes.append(ser_uint256(tx.sha256))
+        return self.get_merkle_root(hashes)
+
+    def calc_witness_merkle_root(self):
+        # For witness root purposes, the hash of the
+        # coinbase, with witness, is defined to be 0...0
+        hashes = [ser_uint256(0)]
+
+        for tx in self.vtx[1:]:
+            # Calculate the hashes with witness data
+            hashes.append(ser_uint256(tx.calc_sha256(True)))
+
+        return self.get_merkle_root(hashes)
+
     def is_valid(self):
         self.calc_sha256()
         target = uint256_from_compact(self.nBits)
@@ -812,11 +956,16 @@ class msg_tx(object):
         self.tx.deserialize(f)
 
     def serialize(self):
-        return self.tx.serialize()
+        return self.tx.serialize_without_witness()
 
     def __repr__(self):
         return "msg_tx(tx=%s)" % (repr(self.tx))
 
+class msg_witness_tx(msg_tx):
+
+    def serialize(self):
+        return self.tx.serialize_with_witness()
+
 
 class msg_block(object):
     command = b"block"
@@ -849,6 +998,12 @@ class msg_generic(object):
     def __repr__(self):
         return "msg_generic()"
 
+class msg_witness_block(msg_block):
+
+    def serialize(self):
+        r = self.block.serialize(with_witness=True)
+        return r
+
 class msg_getaddr(object):
     command = b"getaddr"
 
@@ -947,6 +1102,7 @@ class msg_sendheaders(object):
     def __repr__(self):
         return "msg_sendheaders()"
 
+
 # getheaders message has
 # number of entries
 # vector of hashes
@@ -1068,6 +1224,8 @@ class NodeConnCB(object):
         # tests; it causes message delivery to sleep for the specified time
         # before acquiring the global lock and delivering the next message.
         self.deliver_sleep_time = None
+        # Remember the services our peer has advertised
+        self.peer_services = None
 
     def set_deliver_sleep_time(self, value):
         with mininode_lock:
@@ -1105,6 +1263,7 @@ class NodeConnCB(object):
         conn.ver_send = min(MY_VERSION, message.nVersion)
         if message.nVersion < 209:
             conn.ver_recv = conn.ver_send
+        conn.nServices = message.nServices
 
     def on_verack(self, conn, message):
         conn.ver_recv = conn.ver_send
@@ -1135,6 +1294,7 @@ class NodeConnCB(object):
     def on_mempool(self, conn): pass
     def on_pong(self, conn, message): pass
     def on_feefilter(self, conn, message): pass
+    def on_sendheaders(self, conn, message): pass
 
 # More useful callbacks and functions for NodeConnCB's which have a single NodeConn
 class SingleNodeConnCB(NodeConnCB):
@@ -1183,15 +1343,16 @@ class NodeConn(asyncore.dispatcher):
         b"getheaders": msg_getheaders,
         b"reject": msg_reject,
         b"mempool": msg_mempool,
-        b"feefilter": msg_feefilter
+        b"feefilter": msg_feefilter,
+        b"sendheaders": msg_sendheaders
     }
     MAGIC_BYTES = {
         "mainnet": b"\xf9\xbe\xb4\xd9",   # mainnet
         "testnet3": b"\x0b\x11\x09\x07",  # testnet3
-        "regtest": b"\xfa\xbf\xb5\xda"    # regtest
+        "regtest": b"\xfa\xbf\xb5\xda",   # regtest
     }
 
-    def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=1):
+    def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK):
         asyncore.dispatcher.__init__(self, map=mininode_socket_map)
         self.log = logging.getLogger("NodeConn(%s:%d)" % (dstaddr, dstport))
         self.dstaddr = dstaddr
@@ -1206,6 +1367,7 @@ class NodeConn(asyncore.dispatcher):
         self.network = net
         self.cb = callback
         self.disconnect = False
+        self.nServices = 0
 
         # stuff version msg into sendbuf
         vt = msg_version()

qa/rpc-tests/test_framework/script.py

@@ -15,8 +15,9 @@ Functionality to build scripts, as well as SignatureHash().
 """
 
 
-from .mininode import CTransaction, CTxOut, hash256
+from .mininode import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string
 from binascii import hexlify
+import hashlib
 
 import sys
 bchr = chr
@@ -36,6 +37,10 @@ MAX_SCRIPT_OPCODES = 201
 
 OPCODE_NAMES = {}
 
+def hash160(s):
+    return hashlib.new('ripemd160', sha256(s)).digest()
+
+
 _opcode_instances = []
 class CScriptOp(int):
     """A single script opcode"""
@@ -895,3 +900,48 @@ def SignatureHash(script, txTo, inIdx, hashtype):
     hash = hash256(s)
 
     return (hash, None)
+
+# TODO: Allow cached hashPrevouts/hashSequence/hashOutputs to be provided.
+# Performance optimization probably not necessary for python tests, however.
+# Note that this corresponds to sigversion == 1 in EvalScript, which is used
+# for version 0 witnesses.
+def SegwitVersion1SignatureHash(script, txTo, inIdx, hashtype, amount):
+
+    hashPrevouts = 0
+    hashSequence = 0
+    hashOutputs = 0
+
+    if not (hashtype & SIGHASH_ANYONECANPAY):
+        serialize_prevouts = bytes()
+        for i in txTo.vin:
+            serialize_prevouts += i.prevout.serialize()
+        hashPrevouts = uint256_from_str(hash256(serialize_prevouts))
+
+    if (not (hashtype & SIGHASH_ANYONECANPAY) and (hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+        serialize_sequence = bytes()
+        for i in txTo.vin:
+            serialize_sequence += struct.pack("<I", i.nSequence)
+        hashSequence = uint256_from_str(hash256(serialize_sequence))
+
+    if ((hashtype & 0x1f) != SIGHASH_SINGLE and (hashtype & 0x1f) != SIGHASH_NONE):
+        serialize_outputs = bytes()
+        for o in txTo.vout:
+            serialize_outputs += o.serialize()
+        hashOutputs = uint256_from_str(hash256(serialize_outputs))
+    elif ((hashtype & 0x1f) == SIGHASH_SINGLE and inIdx < len(txTo.vout)):
+        serialize_outputs = txTo.vout[inIdx].serialize()
+        hashOutputs = uint256_from_str(hash256(serialize_outputs))
+
+    ss = bytes()
+    ss += struct.pack("<i", txTo.nVersion)
+    ss += ser_uint256(hashPrevouts)
+    ss += ser_uint256(hashSequence)
+    ss += txTo.vin[inIdx].prevout.serialize()
+    ss += ser_string(script)
+    ss += struct.pack("<q", amount)
+    ss += struct.pack("<I", txTo.vin[inIdx].nSequence)
+    ss += ser_uint256(hashOutputs)
+    ss += struct.pack("<i", txTo.nLockTime)
+    ss += struct.pack("<I", hashtype)
+
+    return hash256(ss)