Merge pull request #721 from MatthewCroughan/matthewcroughan/nixify

Add Nix and Poetry to the project

.gitignore

@@ -35,3 +35,6 @@ coverage.xml
 node_modules
 lnbits/static/bundle.*
 docker
+
+# Nix
+*result*

build.py

@@ -0,0 +1,110 @@
+import warnings
+import subprocess
+import glob
+import os
+from os import path
+from typing import Any, List, NamedTuple, Optional
+from pathlib import Path
+
+LNBITS_PATH = path.dirname(path.realpath(__file__)) + "/lnbits"
+
+def get_js_vendored(prefer_minified: bool = False) -> List[str]:
+    paths = get_vendored(".js", prefer_minified)
+
+    def sorter(key: str):
+        if "moment@" in key:
+            return 1
+        if "vue@" in key:
+            return 2
+        if "vue-router@" in key:
+            return 3
+        if "polyfills" in key:
+            return 4
+        return 9
+
+    return sorted(paths, key=sorter)
+
+
+def get_css_vendored(prefer_minified: bool = False) -> List[str]:
+    paths = get_vendored(".css", prefer_minified)
+
+    def sorter(key: str):
+        if "quasar@" in key:
+            return 1
+        if "vue@" in key:
+            return 2
+        if "chart.js@" in key:
+            return 100
+        return 9
+
+    return sorted(paths, key=sorter)
+
+
+def get_vendored(ext: str, prefer_minified: bool = False) -> List[str]:
+    paths: List[str] = []
+    for path in glob.glob(
+        os.path.join(LNBITS_PATH, "static/vendor/**"), recursive=True
+    ):
+        if path.endswith(".min" + ext):
+            # path is minified
+            unminified = path.replace(".min" + ext, ext)
+            if prefer_minified:
+                paths.append(path)
+                if unminified in paths:
+                    paths.remove(unminified)
+            elif unminified not in paths:
+                paths.append(path)
+
+        elif path.endswith(ext):
+            # path is not minified
+            minified = path.replace(ext, ".min" + ext)
+            if not prefer_minified:
+                paths.append(path)
+                if minified in paths:
+                    paths.remove(minified)
+            elif minified not in paths:
+                paths.append(path)
+
+    return sorted(paths)
+
+
+def url_for_vendored(abspath: str) -> str:
+    return "/" + os.path.relpath(abspath, LNBITS_PATH)
+
+def transpile_scss():
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        from scss.compiler import compile_string  # type: ignore
+
+        with open(os.path.join(LNBITS_PATH, "static/scss/base.scss")) as scss:
+            with open(os.path.join(LNBITS_PATH, "static/css/base.css"), "w") as css:
+                css.write(compile_string(scss.read()))
+
+def bundle_vendored():
+    for getfiles, outputpath in [
+        (get_js_vendored, os.path.join(LNBITS_PATH, "static/bundle.js")),
+        (get_css_vendored, os.path.join(LNBITS_PATH, "static/bundle.css")),
+    ]:
+        output = ""
+        for path in getfiles():
+            with open(path) as f:
+                output += "/* " + url_for_vendored(path) + " */\n" + f.read() + ";\n"
+        with open(outputpath, "w") as f:
+            f.write(output)
+
+
+def build():
+    transpile_scss()
+    bundle_vendored()
+#    root = Path("lnbits/static/foo")
+#    root.mkdir(parents=True)
+#    root.joinpath("example.css").write_text("")
+
+if __name__ == "__main__":
+   build()
+
+#def build(setup_kwargs):
+#    """Build """
+#    transpile_scss()
+#    bundle_vendored()
+#    subprocess.run(["ls", "-la", "./lnbits/static"])

flake.lock

@@ -0,0 +1,77 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1656928814,
+        "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1657114324,
+        "narHash": "sha256-fWuaUNXrHcz/ciHRHlcSO92dvV3EVS0GJQUSBO5JIB4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "a5c867d9fe9e4380452628e8f171c26b69fa9d3d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1657261001,
+        "narHash": "sha256-sUZeuRYfhG59uD6xafM07bc7bAIkpcGq84Vj4B+cyms=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "0be91cefefde5701f8fa957904618a13e3bb51d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "poetry2nix": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1657149754,
+        "narHash": "sha256-iSnZoqwNDDVoO175whSuvl4sS9lAb/2zZ3Sa4ywo970=",
+        "owner": "nix-community",
+        "repo": "poetry2nix",
+        "rev": "fc1930e011dea149db81863aac22fe701f36f1b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "poetry2nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "poetry2nix": "poetry2nix"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,55 @@
+{
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    poetry2nix.url = "github:nix-community/poetry2nix";
+  };
+  outputs = { self, nixpkgs, poetry2nix }@inputs:
+    let
+      supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
+      forSystems = systems: f:
+        nixpkgs.lib.genAttrs systems
+        (system: f system (import nixpkgs { inherit system; overlays = [ poetry2nix.overlay self.overlays.default ]; }));
+      forAllSystems = forSystems supportedSystems;
+      projectName = "lnbits";
+    in
+    {
+      devShells = forAllSystems (system: pkgs: {
+        default = pkgs.mkShell {
+          buildInputs = with pkgs; [
+            nodePackages.prettier
+          ];
+        };
+      });
+      overlays = {
+        default = final: prev: {
+          ${projectName} = self.packages.${final.hostPlatform.system}.${projectName};
+        };
+      };
+      packages = forAllSystems (system: pkgs: {
+        default = self.packages.${system}.${projectName};
+        ${projectName} = pkgs.poetry2nix.mkPoetryApplication {
+          projectDir = ./.;
+          python = pkgs.python39;
+        };
+      });
+      nixosModules = {
+        default = { pkgs, lib, config, ... }: {
+          imports = [ "${./nix/modules/${projectName}-service.nix}" ];
+          nixpkgs.overlays = [ self.overlays.default ];
+        };
+      };
+      checks = forAllSystems (system: pkgs:
+        let
+          vmTests = import ./nix/tests {
+            makeTest = (import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }).makeTest;
+            inherit inputs pkgs;
+          };
+        in
+        pkgs.lib.optionalAttrs pkgs.stdenv.isLinux vmTests # vmTests can only be ran on Linux, so append them only if on Linux.
+        //
+        {
+          # Other checks here...
+        }
+      );
+    };
+}

lnbits/__main__.py

@@ -4,7 +4,7 @@ import uvloop
 from loguru import logger
 from starlette.requests import Request
 
-from .commands import bundle_vendored, migrate_databases, transpile_scss
+from .commands import migrate_databases
 from .settings import (
     DEBUG,
     HOST,
@@ -19,8 +19,6 @@ from .settings import (
 uvloop.install()
 
 asyncio.create_task(migrate_databases())
-transpile_scss()
-bundle_vendored()
 
 from .app import create_app
 

lnbits/app.py

@@ -46,9 +46,9 @@ def create_app(config_object="lnbits.settings") -> FastAPI:
     configure_logger()
 
     app = FastAPI()
-    app.mount("/static", StaticFiles(directory="lnbits/static"), name="static")
+    app.mount("/static", StaticFiles(packages=[("lnbits", "static")]), name="static")
     app.mount(
-        "/core/static", StaticFiles(directory="lnbits/core/static"), name="core_static"
+        "/core/static", StaticFiles(packages=[("lnbits.core", "static")]), name="core_static"
     )
 
     origins = ["*"]

lnbits/extensions/bleskomat/__init__.py

@@ -9,7 +9,7 @@ db = Database("ext_bleskomat")
 bleskomat_static_files = [
     {
         "path": "/bleskomat/static",
-        "app": StaticFiles(directory="lnbits/extensions/bleskomat/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/bleskomat/static")]),
         "name": "bleskomat_static",
     }
 ]

lnbits/extensions/copilot/__init__.py

@@ -12,7 +12,7 @@ db = Database("ext_copilot")
 copilot_static_files = [
     {
         "path": "/copilot/static",
-        "app": StaticFiles(directory="lnbits/extensions/copilot/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/copilot/static")]),
         "name": "copilot_static",
     }
 ]

lnbits/extensions/discordbot/__init__.py

@@ -9,7 +9,7 @@ db = Database("ext_discordbot")
 discordbot_static_files = [
     {
         "path": "/discordbot/static",
-        "app": StaticFiles(directory="lnbits/extensions/discordbot/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/discordbot/static")]),
         "name": "discordbot_static",
     }
 ]

lnbits/extensions/jukebox/__init__.py

@@ -12,7 +12,7 @@ db = Database("ext_jukebox")
 jukebox_static_files = [
     {
         "path": "/jukebox/static",
-        "app": StaticFiles(directory="lnbits/extensions/jukebox/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/jukebox/static")]),
         "name": "jukebox_static",
     }
 ]

lnbits/extensions/livestream/__init__.py

@@ -12,7 +12,7 @@ db = Database("ext_livestream")
 livestream_static_files = [
     {
         "path": "/livestream/static",
-        "app": StaticFiles(directory="lnbits/extensions/livestream/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/livestream/static")]),
         "name": "livestream_static",
     }
 ]

lnbits/extensions/lnurlp/__init__.py

@@ -12,7 +12,7 @@ db = Database("ext_lnurlp")
 lnurlp_static_files = [
     {
         "path": "/lnurlp/static",
-        "app": StaticFiles(directory="lnbits/extensions/lnurlp/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/lnurlp/static")]),
         "name": "lnurlp_static",
     }
 ]

lnbits/extensions/offlineshop/__init__.py

@@ -9,7 +9,7 @@ db = Database("ext_offlineshop")
 offlineshop_static_files = [
     {
         "path": "/offlineshop/static",
-        "app": StaticFiles(directory="lnbits/extensions/offlineshop/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/offlineshop/static")]),
         "name": "offlineshop_static",
     }
 ]
@@ -21,6 +21,7 @@ def offlineshop_renderer():
     return template_renderer(["lnbits/extensions/offlineshop/templates"])
 
 
+
 from .lnurl import *  # noqa
 from .views import *  # noqa
 from .views_api import *  # noqa

lnbits/extensions/splitpayments/__init__.py

@@ -12,7 +12,7 @@ db = Database("ext_splitpayments")
 splitpayments_static_files = [
     {
         "path": "/splitpayments/static",
-        "app": StaticFiles(directory="lnbits/extensions/splitpayments/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/splitpayments/static")]),
         "name": "splitpayments_static",
     }
 ]

lnbits/extensions/withdraw/__init__.py

@@ -9,7 +9,7 @@ db = Database("ext_withdraw")
 withdraw_static_files = [
     {
         "path": "/withdraw/static",
-        "app": StaticFiles(directory="lnbits/extensions/withdraw/static"),
+        "app": StaticFiles(packages=[("lnbits", "extensions/withdraw/static")]),
         "name": "withdraw_static",
     }
 ]

lnbits/server.py

@@ -0,0 +1,17 @@
+import uvicorn
+
+import click
+
+@click.command()
+@click.option('--port', default="5000", help='Port to run LNBits on')
+@click.option('--host', default="127.0.0.1", help='Host to run LNBits on')
+def main(port, host):
+    """Launched with `poetry run lnbits` at root level"""
+    uvicorn.run("lnbits.__main__:app", port=port, host=host)
+
+if __name__ == '__main__':
+    main()
+
+#def main():
+#    """Launched with `poetry run start` at root level"""
+#    uvicorn.run("lnbits.__main__:app")

nix/modules/lnbits-service.nix

@@ -0,0 +1,106 @@
+{ config, pkgs, lib, ... }:
+
+let
+  defaultUser = "lnbits";
+  cfg = config.services.lnbits;
+  inherit (lib) mkOption mkIf types optionalAttrs;
+in
+
+{
+  options = {
+    services.lnbits = {
+      enable = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          Whether to enable the lnbits service
+        '';
+      };
+      openFirewall = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to open the ports used by lnbits in the firewall for the server
+        '';
+      };
+      package = mkOption {
+        type = types.package;
+        default = pkgs.lnbits;
+        description = ''
+          The lnbits package to use.
+        '';
+      };
+      stateDir = mkOption {
+        type = types.path;
+        default = "/var/lib/lnbits";
+        description = ''
+          The lnbits state directory which LNBITS_DATA_FOLDER will be set to
+        '';
+      };
+      host = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = ''
+          The host to bind to
+        '';
+      };
+      port = mkOption {
+        type = types.port;
+        default = 8231;
+        description = ''
+          The port to run on
+        '';
+      };
+      user = mkOption {
+        type = types.str;
+        default = "lnbits";
+        description = "user to run lnbits as";
+      };
+      group = mkOption {
+        type = types.str;
+        default = "lnbits";
+        description = "group to run lnbits as";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.users = optionalAttrs (cfg.user == defaultUser) {
+      ${defaultUser} = {
+        isSystemUser = true;
+        group = defaultUser;
+      };
+    };
+
+    users.groups = optionalAttrs (cfg.group == defaultUser) {
+      ${defaultUser} = { };
+    };
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.stateDir}                            0700 ${cfg.user} ${cfg.group} - -"
+    ];
+
+    systemd.services.lnbits = {
+      enable = true;
+      description = "lnbits";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      environment = {
+        LNBITS_DATA_FOLDER = "${cfg.stateDir}";
+      };
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = "${cfg.package.src}";
+        StateDirectory = "${cfg.stateDir}";
+        ExecStart = "${lib.getExe cfg.package} --port ${toString cfg.port} --host ${cfg.host}";
+        Restart = "always";
+        PrivateTmp = true;
+      };
+    };
+    networking.firewall = mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
+  };
+}
+

nix/tests/default.nix

@@ -0,0 +1,4 @@
+{ pkgs, makeTest, inputs }:
+{
+  vmTest = import ./nixos-module { inherit pkgs makeTest inputs; };
+}

nix/tests/nixos-module/default.nix

@@ -0,0 +1,25 @@
+{ pkgs, makeTest, inputs }:
+makeTest {
+  nodes = {
+    client = { config, pkgs, ... }: {
+      environment.systemPackages = [ pkgs.curl ];
+    };
+    lnbits = { ... }: {
+      imports = [ inputs.self.nixosModules.default ];
+      services.lnbits = {
+        enable = true;
+        openFirewall = true;
+        host = "0.0.0.0";
+      };
+    };
+  };
+  testScript = { nodes, ... }: ''
+    start_all()
+    lnbits.wait_for_open_port(${toString nodes.lnbits.config.services.lnbits.port})
+    client.wait_for_unit("multi-user.target")
+    with subtest("Check that the lnbits webserver can be reached."):
+        assert "<title>LNbits</title>" in client.succeed(
+            "curl -sSf http:/lnbits:8231/ | grep title"
+        )
+  '';
+}

pyproject.toml

@@ -0,0 +1,70 @@
+[tool.poetry]
+name = "lnbits"
+version = "0.1.0"
+description = ""
+authors = ["matthewcroughan <matt@croughan.sh>"]
+
+[tool.poetry.build]
+generate-setup-file = false
+script = "build.py"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+aiofiles = "0.7.0"
+asgiref = "3.4.1"
+attrs = "21.2.0"
+bech32 = "1.2.0"
+bitstring = "3.1.9"
+cerberus = "1.3.4"
+certifi = "2021.5.30"
+charset-normalizer = "2.0.6"
+click = "8.0.1"
+ecdsa = "0.17.0"
+embit = "0.4.9"
+environs = "9.3.3"
+fastapi = "0.78.0"
+h11 = "0.12.0"
+httpcore = "0.13.7"
+httptools = "0.2.0"
+httpx = "0.19.0"
+idna = "3.2"
+importlib-metadata = "4.8.1"
+jinja2 = "3.0.1"
+lnurl = "0.3.6"
+markupsafe = "2.0.1"
+marshmallow = "3.13.0"
+outcome = "1.1.0"
+psycopg2-binary = "2.9.1"
+pycryptodomex = "3.14.1"
+pydantic = "1.8.2"
+pypng = "0.0.21"
+pyqrcode = "1.2.1"
+pyscss = "1.3.7"
+python-dotenv = "0.19.0"
+pyyaml = "5.4.1"
+represent = "1.6.0.post0"
+rfc3986 = "1.5.0"
+secp256k1 = "0.14.0"
+shortuuid = "1.0.1"
+six = "1.16.0"
+sniffio = "1.2.0"
+sqlalchemy = "1.3.23"
+sqlalchemy-aio = "0.16.0"
+sse-starlette = "0.6.2"
+typing-extensions = "3.10.0.2"
+uvicorn = "0.18.1"
+uvloop = "0.16.0"
+watchgod = "0.7"
+websockets = "10.0"
+zipp = "3.5.0"
+loguru = "0.5.3"
+cffi = "1.15.0"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+lnbits = "lnbits.server:main"

requirements.txt

@@ -1,52 +1,52 @@
-aiofiles==0.7.0
-anyio==3.3.1
-asgiref==3.4.1
+aiofiles==0.8.0
+anyio==3.6.1
 asyncio==3.4.3
-attrs==21.2.0
+attrs==21.4.0
 bech32==1.2.0
 bitstring==3.1.9
 cerberus==1.3.4
-certifi==2021.5.30
-charset-normalizer==2.0.6
-click==8.0.1
-ecdsa==0.17.0
-embit==0.4.9
-environs==9.3.3
-fastapi==0.68.1
+certifi==2022.6.15
+cffi==1.15.0
+click==8.1.3
+ecdsa==0.18.0
+embit==0.5.0
+environs==9.5.0
+fastapi==0.79.0
 h11==0.12.0
 httpcore==0.15.0
-httptools==0.2.0
+httptools==0.4.0
 httpx==0.23.0
-idna==3.2
-importlib-metadata==4.8.1
+idna==3.3
 jinja2==3.0.1
 lnurl==0.3.6
 loguru==0.6.0
-markupsafe==2.0.1
-marshmallow==3.13.0
-outcome==1.1.0
-psycopg2-binary==2.9.1
-pycryptodomex==3.14.1
-pydantic==1.8.2
-pypng==0.0.21
+markupsafe==2.1.1
+marshmallow==3.17.0
+outcome==1.2.0
+packaging==21.3
+psycopg2-binary==2.9.3
+pycparser==2.21
+pycryptodomex==3.15.0
+pydantic==1.9.1
+pyngrok==5.1.0
+pyparsing==3.0.9
+pypng==0.20220715.0
 pyqrcode==1.2.1
-pyscss==1.3.7
-python-dotenv==0.19.0
-pyyaml==5.4.1
+pyscss==1.4.0
+python-dotenv==0.20.0
+pyyaml==6.0
 represent==1.6.0.post0
 rfc3986==1.5.0
 secp256k1==0.14.0
-cffi==1.15.0
-shortuuid==1.0.1
+shortuuid==1.0.9
 six==1.16.0
 sniffio==1.2.0
+sqlalchemy-aio==0.17.0
 sqlalchemy==1.3.23
-sqlalchemy-aio==0.16.0
-sse-starlette==0.6.2
-starlette==0.14.2
-typing-extensions==3.10.0.2
-uvicorn==0.15.0
+sse-starlette==0.10.3
+starlette==0.19.1
+typing-extensions==4.3.0
+uvicorn==0.18.2
 uvloop==0.16.0
-watchgod==0.7
-websockets==10.0
-zipp==3.5.0
+watchfiles==0.16.0
+websockets==10.3