Merge #15935: Add <datadir>/settings.json persistent settings storage

9c69cfe4c5 Add <datadir>/settings.json persistent settings storage. (Russell Yanofsky)
eb682c5700 util: Add ReadSettings and WriteSettings functions (Russell Yanofsky)

Pull request description:

  Persistent settings are used in followup PRs #15936 to unify gui settings between bitcoin-qt and bitcoind, and #15937 to add a load_on_startup flag to the loadwallet RPC and maintain a dynamic list of wallets that should be loaded on startup that also can be shared between bitcoind and bitcoin-qt.

ACKs for top commit:
  MarcoFalke:
    Approach re-ACK 9c69cfe4c5 🌾
  jnewbery:
    utACK 9c69cfe4c5

Tree-SHA512: 39fcc6051717117c9141e934de1d0d3f739484be4685cdf97d54de967c8c816502b4fd0de12114433beaa5c5b7060c810fd8ae4e2b3ce7c371eb729ac01ba2e1

src/util/settings.cpp

@@ -4,6 +4,7 @@
 
 #include <util/settings.h>
 
+#include <tinyformat.h>
 #include <univalue.h>
 
 namespace util {
@@ -12,12 +13,13 @@ namespace {
 enum class Source {
    FORCED,
    COMMAND_LINE,
+   RW_SETTINGS,
    CONFIG_FILE_NETWORK_SECTION,
    CONFIG_FILE_DEFAULT_SECTION
 };
 
 //! Merge settings from multiple sources in precedence order:
-//! Forced config > command line > config file network-specific section > config file default section
+//! Forced config > command line > read-write settings file > config file network-specific section > config file default section
 //!
 //! This function is provided with a callback function fn that contains
 //! specific logic for how to merge the sources.
@@ -32,6 +34,10 @@ static void MergeSettings(const Settings& settings, const std::string& section,
     if (auto* values = FindKey(settings.command_line_options, name)) {
         fn(SettingsSpan(*values), Source::COMMAND_LINE);
     }
+    // Merge in the read-write settings
+    if (const SettingsValue* value = FindKey(settings.rw_settings, name)) {
+        fn(SettingsSpan(*value), Source::RW_SETTINGS);
+    }
     // Merge in the network-specific section of the config file
     if (!section.empty()) {
         if (auto* map = FindKey(settings.ro_config, section)) {
@@ -49,6 +55,62 @@ static void MergeSettings(const Settings& settings, const std::string& section,
 }
 } // namespace
 
+bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& values, std::vector<std::string>& errors)
+{
+    values.clear();
+    errors.clear();
+
+    fsbridge::ifstream file;
+    file.open(path);
+    if (!file.is_open()) return true; // Ok for file not to exist.
+
+    SettingsValue in;
+    if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) {
+        errors.emplace_back(strprintf("Unable to parse settings file %s", path.string()));
+        return false;
+    }
+
+    if (file.fail()) {
+        errors.emplace_back(strprintf("Failed reading settings file %s", path.string()));
+        return false;
+    }
+    file.close(); // Done with file descriptor. Release while copying data.
+
+    if (!in.isObject()) {
+        errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), path.string()));
+        return false;
+    }
+
+    const std::vector<std::string>& in_keys = in.getKeys();
+    const std::vector<SettingsValue>& in_values = in.getValues();
+    for (size_t i = 0; i < in_keys.size(); ++i) {
+        auto inserted = values.emplace(in_keys[i], in_values[i]);
+        if (!inserted.second) {
+            errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], path.string()));
+        }
+    }
+    return errors.empty();
+}
+
+bool WriteSettings(const fs::path& path,
+    const std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors)
+{
+    SettingsValue out(SettingsValue::VOBJ);
+    for (const auto& value : values) {
+        out.__pushKV(value.first, value.second);
+    }
+    fsbridge::ofstream file;
+    file.open(path);
+    if (file.fail()) {
+        errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", path.string()));
+        return false;
+    }
+    file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl;
+    file.close();
+    return true;
+}
+
 SettingsValue GetSetting(const Settings& settings,
     const std::string& section,
     const std::string& name,

src/util/settings.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_UTIL_SETTINGS_H
 #define BITCOIN_UTIL_SETTINGS_H
 
+#include <fs.h>
+
 #include <map>
 #include <string>
 #include <vector>
@@ -24,19 +26,31 @@ namespace util {
 //!       https://github.com/bitcoin/bitcoin/pull/15934/files#r337691812)
 using SettingsValue = UniValue;
 
-//! Stored bitcoin settings. This struct combines settings from the command line
-//! and a read-only configuration file.
+//! Stored settings. This struct combines settings from the command line, a
+//! read-only configuration file, and a read-write runtime settings file.
 struct Settings {
     //! Map of setting name to forced setting value.
     std::map<std::string, SettingsValue> forced_settings;
     //! Map of setting name to list of command line values.
     std::map<std::string, std::vector<SettingsValue>> command_line_options;
+    //! Map of setting name to read-write file setting value.
+    std::map<std::string, SettingsValue> rw_settings;
     //! Map of config section name and setting name to list of config file values.
     std::map<std::string, std::map<std::string, std::vector<SettingsValue>>> ro_config;
 };
 
+//! Read settings file.
+bool ReadSettings(const fs::path& path,
+    std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors);
+
+//! Write settings file.
+bool WriteSettings(const fs::path& path,
+    const std::map<std::string, SettingsValue>& values,
+    std::vector<std::string>& errors);
+
 //! Get settings value from combined sources: forced settings, command line
-//! arguments and the read-only config file.
+//! arguments, runtime read-write settings, and the read-only config file.
 //!
 //! @param ignore_default_section_config - ignore values in the default section
 //!                                        of the config file (part before any

src/util/system.cpp

@@ -73,6 +73,7 @@
 const int64_t nStartupTime = GetTime();
 
 const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf";
+const char * const BITCOIN_SETTINGS_FILENAME = "settings.json";
 
 ArgsManager gArgs;
 
@@ -372,6 +373,84 @@ bool ArgsManager::IsArgSet(const std::string& strArg) const
     return !GetSetting(strArg).isNull();
 }
 
+bool ArgsManager::InitSettings(std::string& error)
+{
+    if (!GetSettingsPath()) {
+        return true; // Do nothing if settings file disabled.
+    }
+
+    std::vector<std::string> errors;
+    if (!ReadSettingsFile(&errors)) {
+        error = strprintf("Failed loading settings file:\n- %s\n", Join(errors, "\n- "));
+        return false;
+    }
+    if (!WriteSettingsFile(&errors)) {
+        error = strprintf("Failed saving settings file:\n- %s\n", Join(errors, "\n- "));
+        return false;
+    }
+    return true;
+}
+
+bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const
+{
+    if (IsArgNegated("-settings")) {
+        return false;
+    }
+    if (filepath) {
+        std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME);
+        *filepath = fs::absolute(temp ? settings + ".tmp" : settings, GetDataDir(/* net_specific= */ true));
+    }
+    return true;
+}
+
+static void SaveErrors(const std::vector<std::string> errors, std::vector<std::string>* error_out)
+{
+    for (const auto& error : errors) {
+        if (error_out) {
+            error_out->emplace_back(error);
+        } else {
+            LogPrintf("%s\n", error);
+        }
+    }
+}
+
+bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors)
+{
+    fs::path path;
+    if (!GetSettingsPath(&path, /* temp= */ false)) {
+        return true; // Do nothing if settings file disabled.
+    }
+
+    LOCK(cs_args);
+    m_settings.rw_settings.clear();
+    std::vector<std::string> read_errors;
+    if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) {
+        SaveErrors(read_errors, errors);
+        return false;
+    }
+    return true;
+}
+
+bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors) const
+{
+    fs::path path, path_tmp;
+    if (!GetSettingsPath(&path, /* temp= */ false) || !GetSettingsPath(&path_tmp, /* temp= */ true)) {
+        throw std::logic_error("Attempt to write settings file when dynamic settings are disabled.");
+    }
+
+    LOCK(cs_args);
+    std::vector<std::string> write_errors;
+    if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) {
+        SaveErrors(write_errors, errors);
+        return false;
+    }
+    if (!RenameOver(path_tmp, path)) {
+        SaveErrors({strprintf("Failed renaming settings file %s to %s\n", path_tmp.string(), path.string())}, errors);
+        return false;
+    }
+    return true;
+}
+
 bool ArgsManager::IsArgNegated(const std::string& strArg) const
 {
     return GetSetting(strArg).isFalse();
@@ -893,6 +972,9 @@ void ArgsManager::LogArgs() const
     for (const auto& section : m_settings.ro_config) {
         logArgsPrefix("Config file arg:", section.first, section.second);
     }
+    for (const auto& setting : m_settings.rw_settings) {
+        LogPrintf("Setting file arg: %s = %s\n", setting.first, setting.second.write());
+    }
     logArgsPrefix("Command-line arg:", "", m_settings.command_line_options);
 }
 

src/util/system.h

@@ -41,6 +41,7 @@
 int64_t GetStartupTime();
 
 extern const char * const BITCOIN_CONF_FILENAME;
+extern const char * const BITCOIN_SETTINGS_FILENAME;
 
 void SetupEnvironment();
 bool SetupNetworking();
@@ -333,6 +334,39 @@ public:
      */
     Optional<unsigned int> GetArgFlags(const std::string& name) const;
 
+    /**
+     * Read and update settings file with saved settings. This needs to be
+     * called after SelectParams() because the settings file location is
+     * network-specific.
+     */
+    bool InitSettings(std::string& error);
+
+    /**
+     * Get settings file path, or return false if read-write settings were
+     * disabled with -nosettings.
+     */
+    bool GetSettingsPath(fs::path* filepath = nullptr, bool temp = false) const;
+
+    /**
+     * Read settings file. Push errors to vector, or log them if null.
+     */
+    bool ReadSettingsFile(std::vector<std::string>* errors = nullptr);
+
+    /**
+     * Write settings file. Push errors to vector, or log them if null.
+     */
+    bool WriteSettingsFile(std::vector<std::string>* errors = nullptr) const;
+
+    /**
+     * Access settings with lock held.
+     */
+    template <typename Fn>
+    void LockSettings(Fn&& fn)
+    {
+        LOCK(cs_args);
+        fn(m_settings);
+    }
+
     /**
      * Log the config file options and the command line arguments,
      * useful for troubleshooting.