refactor configs to be loaded using callback in a singleton

This commit is contained in:
antonleviathan 2023-02-18 22:29:31 -05:00
parent da963fe772
commit e5c070b66b
No known key found for this signature in database
GPG Key ID: 44A86CFF1FDF0E85
13 changed files with 540 additions and 178 deletions

View File

@ -10,7 +10,7 @@ payments:
feeSchedules:
admission:
- enabled: false
descripton: Admission fee charged per public key in msats (1000 msats = 1 satoshi)
description: Admission fee charged per public key in msats (1000 msats = 1 satoshi)
amount: 1000000
whitelists:
pubkeys:

368
seeds/configs.json Normal file
View File

@ -0,0 +1,368 @@
[
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "relay_url",
"value": "wss://nostream.your-domain.com",
"category": "info"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "name",
"value": "nostream.your-domain.com",
"category": "info"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "description",
"value": "A nostr relay written in Typescript.",
"category": "info"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "pubkey",
"value": "replace-with-your-pubkey-in-hex",
"category": "info"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "contact",
"value": "operator@your-domain.com",
"category": "info"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "enabled",
"value": false,
"category": "authentication"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "enabled",
"value": false,
"category": "payments"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "processor",
"value": "zebedee",
"category": "payments"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "feeSchedules",
"value": {
"admission": [
{
"enabled": false,
"description": "Admission fee charged per public key in msats (1000 msats = 1 satoshi)",
"amount": 1000000,
"whitelists": {
"pubkeys": [
"replace-with-your-pubkey-in-hex"
]
}
}
],
"publication": [
{
"enabled": false,
"description": "Publication fee charged per event in msats (1000 msats = 1 satoshi)",
"amount": 10,
"whitelists": {
"pubkeys": [
"replace-with-your-pubkey-in-hex"
]
}
}
]
},
"category": "payments"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "zebedee",
"value": {
"baseURL": "https://api.zebedee.io/",
"callbackBaseURL": "https://nostream.your-domain.com/callbacks/zebedee",
"ipWhitelist": [
"3.225.112.64",
"::ffff:3.225.112.64"
]
},
"category": "paymentsProcessors"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "maxPayloadSize",
"value": 131072,
"category": "network"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "remoteIpHeader",
"value": "x-forwarded-for",
"category": "network"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "idleTimeout",
"value": 60,
"category": "network"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "count",
"value": 0,
"category": "workers"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "static",
"value": [],
"category": "mirroring"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "invoice",
"value": {
"rateLimits": [
{
"period": 60000,
"rate": 3
},
{
"period": 3600000,
"rate": 10
},
{
"period": 86400000,
"rate": 20
}
],
"ipWhitelist": [
"::1",
"10.10.10.1",
"::ffff:10.10.10.1"
]
},
"category": "limits"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "connection",
"value": {
"rateLimits": [
{
"period": 1000,
"rate": 6
},
{
"period": 60000,
"rate": 30
},
{
"period": 3600000,
"rate": 300
},
{
"period": 86400000,
"rate": 1440
}
],
"ipWhitelist": [
"::1",
"10.10.10.1",
"::ffff:10.10.10.1"
]
},
"category": "limits"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "event",
"value": {
"eventId": {
"minLeadingZeroBits": 0
},
"kind": {
"whitelist": [],
"blacklist": []
},
"pubkey": {
"minBalance": 0,
"minLeadingZeroBits": 0,
"whitelist": [],
"blacklist": []
},
"createdAt": {
"maxPositiveDelta": 900,
"maxNegativeDelta": 0
},
"content": [
{
"description": "64 KB for event kind ranges 0-10 and 40-49",
"kinds": [
[
0,
10
],
[
40,
49
]
],
"maxLength": 65536
},
{
"description": "96 KB for event kind ranges 11-39 and 50-max",
"kinds": [
[
11,
39
],
[
50,
9007199254740991
]
],
"maxLength": 98304
}
],
"rateLimits": [
{
"description": "6 events/min for event kinds 0, 3, 40 and 41",
"kinds": [
0,
3,
40,
41
],
"period": 60000,
"rate": 6
},
{
"description": "12 events/min for event kinds 1, 2, 4 and 42",
"kinds": [
1,
2,
4,
42
],
"period": 60000,
"rate": 12
},
{
"description": "360 events/hour for event kinds 1, 2, 4 and 42",
"kinds": [
1,
2,
4,
42
],
"period": 3600000,
"rate": 360
},
{
"description": "30 events/min for event kind ranges 5-7 and 43-49",
"kinds": [
[
5,
7
],
[
43,
49
]
],
"period": 60000,
"rate": 30
},
{
"description": "24 events/min for replaceable events and parameterized replaceable events",
"kinds": [
[
10000,
19999
],
[
30000,
39999
]
],
"period": 60000,
"rate": 24
},
{
"description": "60 events/min for ephemeral events",
"kinds": [
[
20000,
29999
]
],
"period": 60000,
"rate": 60
},
{
"description": "720 events/hour for all events",
"period": 3600000,
"rate": 720
},
{
"description": "2880 events/day for all events",
"period": 86400000,
"rate": 2880
}
],
"whitelists": {
"pubkeys": [],
"ipAddresses": [
"::1",
"10.10.10.1",
"::ffff:10.10.10.1"
]
}
},
"category": "limits"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "client",
"value": {
"subscription": {
"maxSubscriptions": 10,
"maxFilters": 10
}
},
"category": "limits"
},
{
"id": "b3b5fb49-4e95-567b-ab64-2d92b7957b13",
"key": "message",
"value": {
"rateLimits": [
{
"description": "120 raw messages/min",
"period": 60000,
"rate": 120
},
{
"description": "3600 raw messages/hour",
"period": 3600000,
"rate": 3600
},
{
"description": "86400 raw messages/day",
"period": 86400000,
"rate": 86400
}
],
"ipWhitelist": [
"::1",
"10.10.10.1",
"::ffff:10.10.10.1"
]
},
"category": "limits"
}
]

View File

@ -1,43 +1,46 @@
/* eslint-disable @typescript-eslint/no-var-requires */
import { Category } from '../@types/category'
const { extname, join } = require('path')
const fs = require('fs')
const yaml = require('js-yaml')
const { v5: uuidv5 } = require('uuid')
const SettingsFileTypes = {
yaml: 'yaml',
json: 'json',
}
const NAMESPACE = 'c646b451-db73-47fb-9a70-ea24ce8a225a'
exports.seed = async function (knex) {
const { v5: uuidv5 } = require('uuid')
const settingsFilePath = `${process.cwd()}/seeds/configs.json`
const defaultConfigs = fs.readFileSync(settingsFilePath)
// await knew.batchInsert('configs', defaultConfigs, 10)
const rawConfigs = getConfigs()
const categories = Object.keys(Category)
const parsedConfigs = parseAll(rawConfigs)
// TODO: Finish logic
// Do we want to flatten settings so that we can look them up by key more easily
// Or do we organize by category? If by category
const configsByCategory = categories.map(category => {
return {
id: uuidv5(event.id, NAMESPACE),
value: rawConfigs[category],
category,
}
})
await knex.batchInsert('configs', configsByCategory, 10)
if (parsedConfigs) {
// await knex.batchInsert('configs', configsByCategory, 10)
}
}
const getConfigs = () => {
const settingsFilePath = process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr')
const files = fs.readdirSync(settingsFilePath)
const settingsFilesTotal = files.filter(file => file.match(/settings/))
if (settingsFilesTotal.length > 1) {
throw new Error('There are more than 1 settings file, please delete all files that contain the word settings in their name, and restart the relay')
}
const filteredFile = files.find(fn => fn.startsWith('settings'))
let settingsFile
if (filteredFile) {
const extension = extname(filteredFile).substring(1)
if (SettingsFileTypes[extension]) {
const extension = SettingsFileTypes[extension]
settingsFileNamePath = `${settingsFilePath}/settings.${extension}`
if (extension === SettingsFileTypes.json) {
settingsFile = loadAndParseJsonFile(settingsFileNamePath)
@ -45,9 +48,9 @@ const getConfigs = () => {
settingsFile = loadAndParseYamlFile(settingsFileNamePath)
}
}
} else {
settingsFile = loadAndParseYamlFile('')
}
return settingsFile
}
const loadAndParseJsonFile = path => {
@ -63,4 +66,31 @@ const loadAndParseYamlFile = path => {
const defaultSettingsFileContent = fs.readFileSync(path, { encoding: 'utf-8' })
const defaults = yaml.load(defaultSettingsFileContent)
return defaults
}
}
const parseAll = (jsonConfigs) => {
if (!jsonConfigs) return
const keys = Object.keys(jsonConfigs)
const configs = keys.map(key => {
return parseOneLevelDeepConfigs(jsonConfigs[key], key)
})
return configs.flat()
}
const parseOneLevelDeepConfigs = (configs, category) => {
const keys = Object.keys(configs)
console.log(keys)
const flattenedConfigs = Object.keys(configs).map(key => {
return {
id: uuidv5('id', NAMESPACE),
key,
value: configs[key],
category
}
})
return flattenedConfigs
}

View File

@ -2,7 +2,7 @@ import { PassThrough } from 'stream'
import { DatabaseClient, EventId, Pubkey } from './base'
import { DBEvent, Event } from './event'
import { Config } from './config'
import { Setting } from './setting'
import { Invoice } from './invoice'
import { SubscriptionFilter } from './subscription'
import { User } from './user'
@ -43,7 +43,8 @@ export interface IUserRepository {
getBalanceByPubkey(pubkey: Pubkey, client?: DatabaseClient): Promise<bigint>
}
export interface IConfigRepository {
getConfig(key: string, client?: DatabaseClient): Promise<Config | undefined>
upsert(config: Partial<Config>, client?: DatabaseClient): Promise<number>
export interface ISettingRepository {
getSetting(category: string, key: string, client?: DatabaseClient): Promise<Setting | undefined>
getSettings(): Promise<Setting[] | undefined>
upsertSetting(config: Partial<Setting>, client?: DatabaseClient): Promise<number>
}

View File

@ -1,4 +1,4 @@
export interface Config {
export interface Setting {
key: string,
value: object,
category: Category,
@ -21,7 +21,7 @@ enum Category {
// message
}
export interface DBConfig {
export interface DBSetting {
key: string,
value: object,
category: Category,

View File

@ -38,7 +38,7 @@ export class App implements IRunnable {
public run(): void {
const settings = this.settings()
this.watchers = SettingsStatic.watchSettings()
// this.watchers = SettingsStatic.watchSettings()
console.log(`

View File

@ -3,7 +3,6 @@ import { IWebSocketServerAdapter } from '../@types/adapters'
import { createLogger } from '../factories/logger-factory'
import { FSWatcher } from 'fs'
import { SettingsStatic } from '../utils/settings'
const debug = createLogger('app-worker')
export class AppWorker implements IRunnable {
@ -23,8 +22,6 @@ export class AppWorker implements IRunnable {
}
public run(): void {
this.watchers = SettingsStatic.watchSettings()
const port = process.env.PORT || process.env.RELAY_PORT || 8008
this.adapter.listen(typeof port === 'number' ? port : Number(port))
}

View File

@ -6,24 +6,39 @@ import { appFactory } from './factories/app-factory'
import { maintenanceWorkerFactory } from './factories/maintenance-worker-factory'
import { staticMirroringWorkerFactory } from './factories/static-mirroring.worker-factory'
import { workerFactory } from './factories/worker-factory'
import { getMasterDbClient } from './database/client'
import { SettingsStatic } from './utils/settings'
export const getRunner = () => {
if (cluster.isPrimary) {
return appFactory()
} else {
switch (process.env.WORKER_TYPE) {
case 'worker':
return workerFactory()
case 'maintenance':
return maintenanceWorkerFactory()
case 'static-mirroring':
return staticMirroringWorkerFactory()
default:
throw new Error(`Unknown worker: ${process.env.WORKER_TYPE}`)
}
}
export const getRunner = (): any => {
const dbClient = getMasterDbClient()
const initializeSettings = new SettingsStatic(dbClient).init()
console.log('here1i')
initializeSettings
.then(() => {
if (cluster.isPrimary) {
appFactory().run()
} else {
switch (process.env.WORKER_TYPE) {
case 'worker':
workerFactory().run()
return
case 'maintenance':
maintenanceWorkerFactory().run()
return
case 'static-mirroring':
staticMirroringWorkerFactory().run()
return
default:
throw new Error(`Unknown worker: ${process.env.WORKER_TYPE}`)
}
}
})
.catch(error => {
throw new Error('Failed to load settings', error)
})
}
if (require.main === module) {
getRunner().run()
getRunner()
}

View File

@ -1,41 +1,57 @@
import { always, applySpec, omit, prop } from 'ramda'
import { Config, DBConfig } from '../@types/config'
import { Setting, DBSetting } from '../@types/setting'
import { createLogger } from '../factories/logger-factory'
import { DatabaseClient } from '../@types/base'
import { fromDBConfig } from '../utils/transform'
import { IConfigRepository } from '../@types/repositories'
import { fromDBSetting } from '../utils/transform'
import { ISettingRepository } from '../@types/repositories'
const debug = createLogger('config-repository')
export class ConfigRepository implements IConfigRepository {
export class SettingRepository implements ISettingRepository {
public constructor(private readonly dbClient: DatabaseClient) { }
public async getConfig(
public async getSetting(
category: string,
key: string,
client: DatabaseClient = this.dbClient
): Promise<Config | undefined> {
debug('find config by key: %s', key)
const [dbconfig] = await client<DBConfig>('configs')
): Promise<Setting | undefined> {
debug('find config by key: %s and category %s', category, key);
const [dbsetting] = await client<DBSetting>('configs')
.where('key', key)
.where('category', category)
.select()
if (!dbconfig) {
if (!dbsetting) {
return
}
return fromDBConfig(dbconfig)
return fromDBSetting(dbsetting)
}
public async upsert(
config: Config,
public async getSettings(
client: DatabaseClient = this.dbClient
): Promise<Setting[] | undefined> {
debug('get all configs');
const settings = await client<Setting>('configs')
.select()
if (!settings) {
return
}
return settings
}
public async upsertSetting(
config: Setting,
client: DatabaseClient = this.dbClient,
): Promise<number> {
debug('upsert: %o', config)
const date = new Date()
const row = applySpec<DBConfig>({
const row = applySpec<DBSetting>({
key: prop('key'),
value: prop('value'),
category: prop('category'),
@ -43,7 +59,7 @@ export class ConfigRepository implements IConfigRepository {
created_at: always(date),
})(config)
const query = client<DBConfig>('configs')
const query = client<DBSetting>('configs')
.insert(row)
.onConflict('key')
.merge(

View File

@ -289,7 +289,7 @@ export const getEventExpiration = (event: Event): number | undefined => {
const expirationTime = Number(rawExpirationTime)
if ((Number.isSafeInteger(expirationTime) && Math.log10(expirationTime))) {
return expirationTime
}
}
}
export const getEventProofOfWork = (eventId: EventId): number => {

View File

@ -1,138 +1,73 @@
import fs from 'fs'
import yaml from 'js-yaml'
import { extname, join } from 'path'
import { mergeDeepRight } from 'ramda'
import { createLogger } from '../factories/logger-factory'
import { Settings } from '../@types/settings'
import { SettingRepository } from '../repositories/settings-repository'
import { Setting } from '../@types/setting'
import { DatabaseClient } from '../@types/base'
const debug = createLogger('settings')
export enum SettingsFileTypes {
yaml = 'yaml',
json = 'json',
}
export class SettingsStatic {
private static _instance: SettingsStatic
private static dbClient: DatabaseClient
static _settings: Settings | undefined
static settingsRepository: SettingRepository | undefined
public static getSettingsFileBasePath(): string {
return process.env.NOSTR_CONFIG_DIR ?? join(process.cwd(), '.nostr')
constructor(dbClient: DatabaseClient) {
SettingsStatic.dbClient = dbClient
SettingsStatic.settingsRepository = new SettingRepository(dbClient)
if (SettingsStatic._instance)
throw new Error("Use Singleton.instance instead of new.")
SettingsStatic._instance = this;
}
public static getDefaultSettingsFilePath(): string {
return join(process.cwd(), 'resources', 'default-settings.yaml')
}
public static loadAndParseYamlFile(path: string): Settings {
const defaultSettingsFileContent = fs.readFileSync(path, { encoding: 'utf-8' })
const defaults = yaml.load(defaultSettingsFileContent) as Settings
return defaults
}
public static loadAndParseJsonFile(path: string) {
return JSON.parse(
fs.readFileSync(
path,
{ encoding: 'utf-8' }
)
)
}
public static settingsFileType(path: string): SettingsFileTypes | undefined {
const files: string[] = fs.readdirSync(path)
const filteredFile = files.find(fn => fn.startsWith('settings'))
if (filteredFile) {
const extension = extname(filteredFile).substring(1)
if (SettingsFileTypes[extension]) {
return SettingsFileTypes[extension]
public init() {
debug('SettingsStatic.init()')
return new Promise((resolve, reject) => {
const settingsPromise = SettingsStatic.loadSettingsFromDb(SettingsStatic.constructSettingsJsonBlob)
if (settingsPromise) {
resolve('success')
}
}
return SettingsFileTypes.yaml
reject('Failed to initialize settings')
})
}
public static loadSettings(path: string, fileType: SettingsFileTypes) {
debug('loading settings from %s', path)
switch (fileType) {
case SettingsFileTypes.json: {
console.warn('settings.json is deprecated, please use a yaml file based on resources/default-settings.yaml')
return SettingsStatic.loadAndParseJsonFile(path)
}
case SettingsFileTypes.yaml: {
return SettingsStatic.loadAndParseYamlFile(path)
}
default: {
throw new Error('settings file was missing or did not contain .yaml or .json extensions.')
}
}
static get instance() {
return SettingsStatic._instance ?? (SettingsStatic._instance = new SettingsStatic(this.dbClient));
}
private static loadSettingsFromDb(callback) {
debug('SettingsStatic.loadSettingsFromDb()')
const promise = SettingsStatic.settingsRepository.getSettings()
return promise.then(rawSettingsFromDb => {
const settingsJsonBlob = callback(rawSettingsFromDb);
this._settings = settingsJsonBlob
});
}
public static createSettings(): Settings {
if (SettingsStatic._settings) {
return SettingsStatic._settings
}
debug('creating settings')
const basePath = SettingsStatic.getSettingsFileBasePath()
if (!fs.existsSync(basePath)) {
fs.mkdirSync(basePath)
}
const defaultsFilePath = SettingsStatic.getDefaultSettingsFilePath()
const fileType = SettingsStatic.settingsFileType(basePath)
const settingsFilePath = join(basePath, `settings.${fileType}`)
const defaults = SettingsStatic.loadSettings(defaultsFilePath, SettingsFileTypes.yaml)
try {
if (fileType) {
SettingsStatic._settings = mergeDeepRight(
defaults,
SettingsStatic.loadSettings(settingsFilePath, fileType)
)
} else {
SettingsStatic.saveSettings(basePath, defaults)
SettingsStatic._settings = mergeDeepRight({}, defaults)
}
if (typeof SettingsStatic._settings === 'undefined') {
throw new Error('Unable to set settings')
}
return SettingsStatic._settings
} catch (error) {
debug('error reading config file at %s: %o', settingsFilePath, error)
return defaults
}
return this._settings
}
public static saveSettings(path: string, settings: Settings) {
debug('saving settings to %s: %o', path, settings)
return fs.writeFileSync(
join(path, 'settings.yaml'),
yaml.dump(settings),
{ encoding: 'utf-8' },
)
public static async updateSetting(config: Setting) {
await SettingsStatic.settingsRepository.upsertSetting(config)
this.updateSingletonSettings(config)
}
public static watchSettings() {
const basePath = SettingsStatic.getSettingsFileBasePath()
const defaultsFilePath = SettingsStatic.getDefaultSettingsFilePath()
const fileType = SettingsStatic.settingsFileType(basePath)
const settingsFilePath = join(basePath, `settings.${fileType}`)
private static updateSingletonSettings(setting) {
const updateSettings = this._settings
updateSettings[setting.category][setting.key] = setting.value
}
const reload = () => {
console.log('reloading settings')
SettingsStatic._settings = undefined
SettingsStatic.createSettings()
}
private static constructSettingsJsonBlob(rawSettingsFromDb): any {
const settings = {}
rawSettingsFromDb.map(setting => {
settings[setting.category][setting.key] = setting.value
})
return [
fs.watch(defaultsFilePath, 'utf8', reload),
fs.watch(settingsFilePath, 'utf8', reload),
]
return settings
}
}

View File

@ -1,9 +1,9 @@
import { always, applySpec, ifElse, is, isNil, path, pipe, prop, propSatisfies } from 'ramda'
import { bech32 } from 'bech32'
import { Config } from '../@types/config'
import { Invoice } from '../@types/invoice'
import { User } from '../@types/user'
import { Setting } from '../@types/setting'
export const toJSON = (input: any) => JSON.stringify(input)
@ -42,7 +42,7 @@ export const fromDBUser = applySpec<User>({
updatedAt: prop('updated_at'),
})
export const fromDBConfig = applySpec<Config>({
export const fromDBSetting = applySpec<Setting>({
key: prop('key'),
value: prop('value'),
category: prop('category'),

View File

@ -39,7 +39,7 @@ BeforeAll({ timeout: 1000 }, async function () {
dbClient = getMasterDbClient()
rrDbClient = getReadReplicaDbClient()
await dbClient.raw('SELECT 1=1')
Sinon.stub(SettingsStatic, 'watchSettings')
// Sinon.stub(SettingsStatic, 'watchSettings')
const settings = SettingsStatic.createSettings()
SettingsStatic._settings = pipe(