rpc: check fopen return code in dumptxoutset

This change improves the usability of the `dumptxoutset` RPC in two ways,
in the case that an invalid path is passed:
  1. return from the RPC immediately, rather then when the file is first
     tried to be written (which is _after_ calculating the UTXO set hash)
  2. return a proper return code and error message instead of the cryptic
     "CAutoFile::operator<<: file handle is nullptr: unspecified
      iostream_category error" (-1)

test/functional/rpc_dumptxoutset.py

@@ -49,9 +49,13 @@ class DumptxoutsetTest(BitcoinTestFramework):
             out['txoutset_hash'], '1f7e3befd45dc13ae198dfbb22869a9c5c4196f8e9ef9735831af1288033f890')
         assert_equal(out['nchaintx'], 101)
 
-        # Specifying a path to an existing file will fail.
+        # Specifying a path to an existing or invalid file will fail.
         assert_raises_rpc_error(
             -8, '{} already exists'.format(FILENAME),  node.dumptxoutset, FILENAME)
+        invalid_path = str(Path(node.datadir) / "invalid" / "path")
+        assert_raises_rpc_error(
+            -8, "Couldn't open file {}.incomplete for writing".format(invalid_path), node.dumptxoutset, invalid_path)
+
 
 if __name__ == '__main__':
     DumptxoutsetTest().main()