From ad2aad05e00ca4457ea008684bfbfb0852fcbfd5 Mon Sep 17 00:00:00 2001 From: calle <93376500+callebtc@users.noreply.github.com> Date: Sun, 17 Jul 2022 13:11:13 +0200 Subject: [PATCH] CI: Migration SQLite to PostgreSQL (#719) * migrate all extensions * errors if migration is missing --- .dockerignore | 2 +- .github/workflows/migrations.yml | 49 ++++++ docs/guide/installation.md | 4 +- lnbits/extensions/lnurldevice/migrations.py | 2 +- .../templates/lnurldevice/_api_docs.html | 2 +- conv.py => tools/conv.py | 139 ++++++++++++++---- 6 files changed, 167 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/migrations.yml rename conv.py => tools/conv.py (84%) diff --git a/.dockerignore b/.dockerignore index c8872f9ed..51cee13c3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ docker docs tests venv +tools *.md *.log @@ -12,7 +13,6 @@ venv .gitignore .prettierrc -conv.py LICENSE Makefile mypy.ini diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml new file mode 100644 index 000000000..45de97277 --- /dev/null +++ b/.github/workflows/migrations.yml @@ -0,0 +1,49 @@ +name: migrations + +on: [pull_request] + +jobs: + sqlite-to-postgres: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + # maps tcp port 5432 on service container to the host + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: + matrix: + python-version: [3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + env: + VIRTUAL_ENV: ./venv + PATH: ${{ env.VIRTUAL_ENV }}/bin:${{ env.PATH }} + run: | + python -m venv ${{ env.VIRTUAL_ENV }} + ./venv/bin/python -m pip install --upgrade pip + ./venv/bin/pip install -r requirements.txt + ./venv/bin/pip install pytest pytest-asyncio pytest-cov requests mock + - name: Run migrations + run: | + rm -rf ./data + mkdir -p ./data + export LNBITS_DATA_FOLDER="./data" + timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres" + timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + ./venv/bin/python tools/conv.py --dont-ignore-missing diff --git a/docs/guide/installation.md b/docs/guide/installation.md index de391b884..daa9ae131 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -142,8 +142,8 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits" # START LNbits # STOP LNbits -# on the LNBits folder, locate and edit 'conv.py' with the relevant credentials -python3 conv.py +# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials +python3 tools/conv.py ``` Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly. diff --git a/lnbits/extensions/lnurldevice/migrations.py b/lnbits/extensions/lnurldevice/migrations.py index 67065347c..625949f5c 100644 --- a/lnbits/extensions/lnurldevice/migrations.py +++ b/lnbits/extensions/lnurldevice/migrations.py @@ -42,7 +42,7 @@ async def m002_redux(db): """ try: for row in [ - list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlposs") + list(row) for row in await db2.fetchall("SELECT * FROM lnurlpos.lnurlpos") ]: await db.execute( """ diff --git a/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html b/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html index d5b4b5b8e..940d4691e 100644 --- a/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html +++ b/lnbits/extensions/lnurldevice/templates/lnurldevice/_api_docs.html @@ -120,7 +120,7 @@ GET - /lnurldevice/api/v1/lnurlposs
Headers
{"X-Api-Key": <invoice_key>}
diff --git a/conv.py b/tools/conv.py similarity index 84% rename from conv.py rename to tools/conv.py index aa66a998d..24414da15 100644 --- a/conv.py +++ b/tools/conv.py @@ -1,6 +1,13 @@ import psycopg2 import sqlite3 import os +import argparse + + +from environs import Env # type: ignore + +env = Env() +env.read_env() # Python script to migrate an LNbits SQLite DB to Postgres # All credits to @Fritz446 for the awesome work @@ -11,13 +18,27 @@ import os # Change these values as needed + sqfolder = "data/" -pgdb = "lnbits" -pguser = "postgres" -pgpswd = "yourpassword" -pghost = "localhost" -pgport = "5432" -pgschema = "" + +LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None) +if LNBITS_DATABASE_URL is None: + pgdb = "lnbits" + pguser = "lnbits" + pgpswd = "postgres" + pghost = "localhost" + pgport = "5432" + pgschema = "" +else: + # parse postgres://lnbits:postgres@localhost:5432/lnbits + pgdb = LNBITS_DATABASE_URL.split("/")[-1] + pguser = LNBITS_DATABASE_URL.split("@")[0].split(":")[-2][2:] + pgpswd = LNBITS_DATABASE_URL.split("@")[0].split(":")[-1] + pghost = LNBITS_DATABASE_URL.split("@")[1].split(":")[0] + pgport = LNBITS_DATABASE_URL.split("@")[1].split(":")[1].split("/")[0] + pgschema = "" + +print(pgdb, pguser, pgpswd, pghost, pgport, pgschema) def get_sqlite_cursor(sqdb) -> sqlite3: @@ -35,8 +56,6 @@ def get_postgres_cursor(): def check_db_versions(sqdb): sqlite = get_sqlite_cursor(sqdb) dblite = dict(sqlite.execute("SELECT * FROM dbversions;").fetchall()) - if "lnurlpos" in dblite: - del dblite["lnurlpos"] sqlite.close() postgres = get_postgres_cursor() @@ -128,9 +147,14 @@ def migrate_core(sqlite_db_file): print("Migrated: core") -def migrate_ext(sqlite_db_file, schema): - sq = get_sqlite_cursor(sqlite_db_file) +def migrate_ext(sqlite_db_file, schema, ignore_missing=True): + # skip this file it has been moved to ext_lnurldevices.sqlite3 + if sqlite_db_file == "data/ext_lnurlpos.sqlite3": + return + + print(f"Migrating {sqlite_db_file}.{schema}") + sq = get_sqlite_cursor(sqlite_db_file) if schema == "bleskomat": # BLESKOMAT LNURLS res = sq.execute("SELECT * FROM bleskomat_lnurls;") @@ -515,19 +539,19 @@ def migrate_ext(sqlite_db_file, schema): items = res.fetchall() insert_to_pg(q, items) fix_id("offlineshop.items_id_seq", items) - elif schema == "lnurlpos": - # LNURLPOSS - res = sq.execute("SELECT * FROM lnurlposs;") + elif schema == "lnurlpos" or schema == "lnurldevice": + # lnurldevice + res = sq.execute("SELECT * FROM lnurldevices;") q = f""" - INSERT INTO lnurlpos.lnurlposs (id, key, title, wallet, currency, timestamp) - VALUES (%s, %s, %s, %s, %s, to_timestamp(%s)); + INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit) + VALUES (%s, %s, %s, %s, %s, %s, %s); """ insert_to_pg(q, res.fetchall()) - # LNURLPOS PAYMENT - res = sq.execute("SELECT * FROM lnurlpospayment;") + # lnurldevice PAYMENT + res = sq.execute("SELECT * FROM lnurldevicepayment;") q = f""" - INSERT INTO lnurlpos.lnurlpospayment (id, posid, payhash, payload, pin, sats, timestamp) - VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s)); + INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats) + VALUES (%s, %s, %s, %s, %s, %s); """ insert_to_pg(q, res.fetchall()) elif schema == "lnurlp": @@ -546,9 +570,10 @@ def migrate_ext(sqlite_db_file, schema): success_url, currency, comment_chars, - max + max, + fiat_base_multiplier ) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); """ pay_links = res.fetchall() insert_to_pg(q, pay_links) @@ -637,17 +662,79 @@ def migrate_ext(sqlite_db_file, schema): tracks = res.fetchall() insert_to_pg(q, tracks) fix_id("livestream.tracks_id_seq", tracks) + elif schema == "lnaddress": + # DOMAINS + res = sq.execute("SELECT * FROM domain;") + q = f""" + INSERT INTO lnaddress.domain( + id, wallet, domain, webhook, cf_token, cf_zone_id, cost, "time") + VALUES (%s, %s, %s, %s, %s, %s, %s, to_timestamp(%s)); + """ + insert_to_pg(q, res.fetchall()) + # ADDRESSES + res = sq.execute("SELECT * FROM address;") + q = f""" + INSERT INTO lnaddress.address( + id, wallet, domain, email, username, wallet_key, wallet_endpoint, sats, duration, paid, "time") + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s::boolean, to_timestamp(%s)); + """ + insert_to_pg(q, res.fetchall()) + elif schema == "discordbot": + # USERS + res = sq.execute("SELECT * FROM users;") + q = f""" + INSERT INTO discordbot.users( + id, name, admin, discord_id) + VALUES (%s, %s, %s, %s); + """ + insert_to_pg(q, res.fetchall()) + # WALLETS + res = sq.execute("SELECT * FROM wallets;") + q = f""" + INSERT INTO discordbot.wallets( + id, admin, name, "user", adminkey, inkey) + VALUES (%s, %s, %s, %s, %s, %s); + """ + insert_to_pg(q, res.fetchall()) else: - print(f"Not implemented: {schema}") + print(f"❌ Not implemented: {schema}") sq.close() + + if ignore_missing == False: + raise Exception( + f"Not implemented: {schema}. Use --ignore-missing to skip missing extensions." + ) return - print(f"Migrated: {schema}") + print(f"✅ Migrated: {schema}") sq.close() -check_db_versions("data/database.sqlite3") -migrate_core("data/database.sqlite3") +parser = argparse.ArgumentParser(description="Migrate data from SQLite to PostgreSQL") +parser.add_argument( + dest="sqlite_file", + const=True, + nargs="?", + help="SQLite DB to migrate from", + default="data/database.sqlite3", + type=str, +) +parser.add_argument( + "-i", + "--dont-ignore-missing", + help="Error if migration is missing for an extension.", + required=False, + default=False, + const=True, + nargs="?", + type=bool, +) +args = parser.parse_args() + +print(args) + +check_db_versions(args.sqlite_file) +migrate_core(args.sqlite_file) files = os.listdir(sqfolder) for file in files: @@ -655,4 +742,4 @@ for file in files: if file.startswith("ext_"): schema = file.replace("ext_", "").split(".")[0] print(f"Migrating: {schema}") - migrate_ext(path, schema) + migrate_ext(path, schema, ignore_missing=not args.dont_ignore_missing)