diff --git a/.mocharc.js b/.mocharc.js index ccbb8b8..628e8fb 100644 --- a/.mocharc.js +++ b/.mocharc.js @@ -1,6 +1,6 @@ module.exports = { extension: ['ts'], - require: ['ts-node/register/transpile-only', 'source-map-support/register'], + require: ['ts-node/register', 'source-map-support/register'], reporter: 'mochawesome', slow: 75, sorted: true, diff --git a/src/@types/repositories.ts b/src/@types/repositories.ts index 47a0747..d2dd6fb 100644 --- a/src/@types/repositories.ts +++ b/src/@types/repositories.ts @@ -5,7 +5,7 @@ import { SubscriptionFilter } from './subscription' export type ExposedPromiseKeys = 'then' | 'catch' | 'finally' export interface IQueryResult extends Pick, keyof Promise & ExposedPromiseKeys> { - stream(): PassThrough & AsyncIterable + stream(options?: Record): PassThrough & AsyncIterable } export interface IEventRepository { diff --git a/src/@types/subscription.ts b/src/@types/subscription.ts index ae8e1a0..e5a7c49 100644 --- a/src/@types/subscription.ts +++ b/src/@types/subscription.ts @@ -1,3 +1,4 @@ +import { EventKinds } from '../constants/base' import { Pubkey } from './base' import { EventId } from './event' @@ -5,7 +6,7 @@ export type SubscriptionId = string export interface SubscriptionFilter { ids?: EventId[] - kinds?: number[] + kinds?: EventKinds[] since?: number until?: number authors?: Pubkey[] diff --git a/src/repositories/event-repository.ts b/src/repositories/event-repository.ts index 1b41854..3f38564 100644 --- a/src/repositories/event-repository.ts +++ b/src/repositories/event-repository.ts @@ -1,6 +1,5 @@ import { Knex } from 'knex' -import { applySpec, omit, pipe, prop } from 'ramda' -import { PassThrough } from 'stream' +import { __, applySpec, equals, modulo, omit, pipe, prop, cond, always, groupBy, T, evolve, forEach, isEmpty, forEachObjIndexed, isNil, complement, toPairs, filter, nth, ifElse, invoker } from 'ramda' import { DBEvent, Event } from '../@types/event' import { IEventRepository, IQueryResult } from '../@types/repositories' @@ -8,8 +7,18 @@ import { SubscriptionFilter } from '../@types/subscription' import { isGenericTagQuery } from '../utils/filter' import { toBuffer, toJSON } from '../utils/transform' +const even = pipe(modulo(__, 2), equals(0)) -const evenLengthTruncate = (input: string) => input.substring(0, input.length >> 1 << 1) +const groupByLengthSpec = groupBy( + pipe( + prop('length'), + cond([ + [equals(64), always('exact')], + [even, always('even')], + [T, always('odd')], + ]) + ) +) export class EventRepository implements IEventRepository { public constructor(private readonly dbClient: Knex) {} @@ -18,69 +27,82 @@ export class EventRepository implements IEventRepository { if (!Array.isArray(filters) || !filters.length) { throw new Error('Filters cannot be empty') } - const queries = filters.map((filter) => { + const queries = filters.map((currentFilter) => { const builder = this.dbClient('events') - if (Array.isArray(filter.authors)) { - builder.andWhere(function (bd) { - bd.whereIn( - 'event_pubkey', - filter.authors.filter((author) => author.length === 64).map(toBuffer) - ) - - for (const author of filter.authors.filter((author) => author.length < 64)) { - const prefix = evenLengthTruncate(author) - if (prefix.length) { - bd.orWhereRaw('substring("event_pubkey" from 1 for ?) = ?', [prefix.length >> 1, toBuffer(prefix)]) - } - - } + forEachObjIndexed((tableField: string, filterName: string) => { + builder.andWhere((bd) => { + cond([ + [isEmpty, () => void bd.whereRaw('1 = 0')], + [ + complement(isNil), + pipe( + groupByLengthSpec, + evolve({ + exact: (pubkeys: string[]) => void bd.whereIn(tableField, pubkeys.map(toBuffer)), + even: forEach((prefix: string) => void bd.orWhereRaw( + `substring("${tableField}" from 1 for ?) = ?`, + [prefix.length >> 1, toBuffer(prefix)] + )), + odd: forEach((prefix: string) => void bd.orWhereRaw( + `substring("${tableField}" from 1 for ?) BETWEEN ? AND ?`, + [ + (prefix.length >> 1) + 1, + `\\x${prefix}0`, + `\\x${prefix}f` + ], + )), + }), + ), + ], + ])(currentFilter[filterName] as string[]) }) + })({ + authors: 'event_pubkey', + ids: 'event_id', + }) + + if (Array.isArray(currentFilter.kinds)) { + builder.whereIn('event_kind', currentFilter.kinds) } - if (Array.isArray(filter.ids)) { - builder.andWhere(function (bd) { - bd.whereIn( - 'event_id', - filter.ids.filter((id) => id.length === 64).map(toBuffer) - ) - - for (const id of filter.ids.filter((id) => id.length < 64)) { - const prefix = evenLengthTruncate(id) - if (prefix.length) { - bd.orWhereRaw('substring("event_id" from 1 for ?) = ?', [prefix.length >> 1, toBuffer(prefix)]) - } - } - }) + if (typeof currentFilter.since === 'number') { + builder.where('event_created_at', '>=', currentFilter.since) } - if (Array.isArray(filter.kinds)) { - builder.whereIn('event_kind', filter.kinds) + if (typeof currentFilter.until === 'number') { + builder.where('event_created_at', '<=', currentFilter.until) } - if (typeof filter.since === 'number') { - builder.where('event_created_at', '>=', filter.since) - } - - if (typeof filter.until === 'number') { - builder.where('event_created_at', '<=', filter.until) - } - - if (typeof filter.limit === 'number') { - builder.limit(filter.limit).orderBy('event_created_at', 'DESC') + if (typeof currentFilter.limit === 'number') { + builder.limit(currentFilter.limit).orderBy('event_created_at', 'DESC') } else { builder.orderBy('event_created_at', 'asc') } - Object.entries(filter) - .filter(([key, criteria]) => isGenericTagQuery(key) && Array.isArray(criteria)) - .forEach(([key, criteria]) => { - builder.andWhere(function (bd) { - criteria.forEach((criterion) => { - bd.orWhereRaw('"event_tags" @> ?', [JSON.stringify([[key[1], criterion]])]) - }) + const andWhereRaw = invoker(1, 'andWhereRaw') + const orWhereRaw = invoker(2, 'orWhereRaw') + + + pipe( + toPairs, + filter(pipe(nth(0), isGenericTagQuery)) as any, + forEach(([filterName, criteria]: [string, string[]]) => { + builder.andWhere((bd) => { + ifElse( + isEmpty, + () => andWhereRaw('1 = 0', bd), + forEach((criterion: string[]) => void orWhereRaw( + '"event_tags" @> ?', + [ + JSON.stringify([[filterName[1], criterion]]) as any + ], + bd, + )), + )(criteria) }) - }) + }), + )(currentFilter as any) return builder }) @@ -90,8 +112,6 @@ export class EventRepository implements IEventRepository { query.union(subqueries, true) } - console.log('query', query.toString()) - return query } diff --git a/test/unit/repositories/event-repository.spec.ts b/test/unit/repositories/event-repository.spec.ts new file mode 100644 index 0000000..0d4d94d --- /dev/null +++ b/test/unit/repositories/event-repository.spec.ts @@ -0,0 +1,362 @@ +import * as chai from 'chai' +import knex from 'knex' +import * as sinon from 'sinon' +import sinonChai from 'sinon-chai' +import { IEventRepository } from '../../../src/@types/repositories' +import { SubscriptionFilter } from '../../../src/@types/subscription' + +chai.use(sinonChai) + +const { expect } = chai + +import { EventRepository } from '../../../src/repositories/event-repository' + +describe.only('EventRepository', () => { + let repository: IEventRepository + let sandbox: sinon.SinonSandbox + + beforeEach(() => { + sandbox = sinon.createSandbox() + + repository = new EventRepository(knex({ + client: 'pg' + })) + }) + + afterEach(() => { + sandbox.restore() + }) + + describe('findByFilters', () => { + it('returns a function with stream and then', () => { + expect(repository.findByFilters([{}])).to.have.property('stream') + expect(repository.findByFilters([{}])).to.have.property('then') + expect(repository.findByFilters([{}])).to.have.property('catch') + expect(repository.findByFilters([{}])).to.have.property('finally') + }) + + it('throws error if filters is not an array', () => { + expect(() => repository.findByFilters(null)).to.throw(Error, 'Filters cannot be empty') + }) + + it('throws error if filters is empty', () => { + expect(() => repository.findByFilters([])).to.throw(Error, 'Filters cannot be empty') + }) + + describe('1 filter', () => { + it('selects all events', () => { + const filters = [{}] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" order by "event_created_at" asc') + }) + + describe('authors', () => { + it('selects no events given empty list of authors', () => { + const filters: SubscriptionFilter[] = [{ authors: [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc') + }) + + it('selects events by one author', () => { + const filters = [{ authors: ['22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_pubkey" in (X\'22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793\')) order by "event_created_at" asc') + }) + + it('selects events by two authors', () => { + const filters = [ + { + authors: [ + '22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793', + '32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245' + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_pubkey" in (X\'22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793\', X\'32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\')) order by "event_created_at" asc') + }) + + it('selects events by one author prefix (even length)', () => { + const filters = [ + { + authors: [ + '22e804', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 3) = X\'22e804\') order by "event_created_at" asc') + }) + + it('selects events by one author prefix (odd length)', () => { + const filters = [ + { + authors: [ + '22e804f', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 4) BETWEEN E\'\\\\x22e804f0\' AND E\'\\\\x22e804ff\') order by "event_created_at" asc') + }) + + it('selects events by two author prefix (first even, second odd)', () => { + const filters = [ + { + authors: [ + '22e804', + '32e1827', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 3) = X\'22e804\' or substring("event_pubkey" from 1 for 4) BETWEEN E\'\\\\x32e18270\' AND E\'\\\\x32e1827f\') order by "event_created_at" asc') + }) + }) + + describe('ids', () => { + it('selects no events given empty list of ids', () => { + const filters: SubscriptionFilter[] = [{ ids: [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc') + }) + + it('selects events by one id', () => { + const filters = [{ ids: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_id" in (X\'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\')) order by "event_created_at" asc') + }) + + it('selects events by two ids', () => { + const filters = [ + { + ids: [ + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_id" in (X\'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\', X\'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\')) order by "event_created_at" asc') + }) + + it('selects events by one id prefix (even length)', () => { + const filters = [ + { + ids: [ + 'abcd', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 2) = X\'abcd\') order by "event_created_at" asc') + }) + + it('selects events by one id prefix (odd length)', () => { + const filters = [ + { + ids: [ + 'abc', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 2) BETWEEN E\'\\\\xabc0\' AND E\'\\\\xabcf\') order by "event_created_at" asc') + }) + + it('selects events by two id prefix (first even, second odd)', () => { + const filters = [ + { + ids: [ + 'abcdef', + 'abc', + ] + } + ] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 3) = X\'abcdef\' or substring("event_id" from 1 for 2) BETWEEN E\'\\\\xabc0\' AND E\'\\\\xabcf\') order by "event_created_at" asc') + }) + }) + + describe('kinds', () => { + it('selects no events given empty list of kinds', () => { + const filters: SubscriptionFilter[] = [{ kinds: [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where 1 = 0 order by "event_created_at" asc') + }) + + it('selects events by one kind', () => { + const filters = [{ kinds: [1] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where "event_kind" in (1) order by "event_created_at" asc') + }) + + it('selects events by two kinds', () => { + const filters = [{ kinds: [1, 2] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where "event_kind" in (1, 2) order by "event_created_at" asc') + }) + }) + + describe('since', () => { + it('selects events since given timestamp', () => { + const filters = [{ since: 1000 }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where "event_created_at" >= 1000 order by "event_created_at" asc') + }) + }) + + describe('until', () => { + it('selects events until given timestamp', () => { + const filters = [{ until: 1000 }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where "event_created_at" <= 1000 order by "event_created_at" asc') + }) + }) + + describe('limit', () => { + it('selects 1000 events', () => { + const filters = [{ limit: 1000 }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" order by "event_created_at" DESC limit 1000') + }) + }) + + describe('#e', () => { + it('selects no events given empty list of #e tags', () => { + const filters = [{ '#e': [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc') + }) + + it('selects events by one #e tag', () => { + const filters = [{ '#e': ['aaaaaa'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["e","aaaaaa"]]\') order by "event_created_at" asc') + }) + + it('selects events by two #e tag', () => { + const filters = [{ '#e': ['aaaaaa', 'bbbbbb'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["e","aaaaaa"]]\' or "event_tags" @> \'[["e","bbbbbb"]]\') order by "event_created_at" asc') + }) + }) + + describe('#p', () => { + it('selects no events given empty list of #p tags', () => { + const filters = [{ '#p': [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc') + }) + + it('selects events by one #p tag', () => { + const filters = [{ '#p': ['aaaaaa'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["p","aaaaaa"]]\') order by "event_created_at" asc') + }) + + it('selects events by two #p tag', () => { + const filters = [{ '#p': ['aaaaaa', 'bbbbbb'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["p","aaaaaa"]]\' or "event_tags" @> \'[["p","bbbbbb"]]\') order by "event_created_at" asc') + }) + }) + + describe('#r', () => { + it('selects no events given empty list of #r tags', () => { + const filters = [{ '#r': [] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc') + }) + + it('selects events by one #r tag', () => { + const filters = [{ '#r': ['aaaaaa'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["r","aaaaaa"]]\') order by "event_created_at" asc') + }) + + it('selects events by two #r tag', () => { + const filters = [{ '#r': ['aaaaaa', 'bbbbbb'] }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["r","aaaaaa"]]\' or "event_tags" @> \'[["r","bbbbbb"]]\') order by "event_created_at" asc') + }) + }) + }) + + describe('2 filters', () => { + it('selects union of both filters', () => { + const filters = [{}, {}] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('(select * from "events") union (select * from "events" order by "event_created_at" asc) order by "event_created_at" asc') + }) + }) + + describe('many filters', () => { + it('selects union of all filters', () => { + const filters = [{ kinds: [1] }, { ids: ['aaaaa'] }, { authors: ['bbbbb'] }, { since: 1000 }, { until: 1000 }, { limit: 1000 }] + + const query = repository.findByFilters(filters).toString() + + expect(query).to.equal('(select * from "events" where "event_kind" in (1)) union (select * from "events" where (substring("event_id" from 1 for 3) BETWEEN E\'\\\\xaaaaa0\' AND E\'\\\\xaaaaaf\') order by "event_created_at" asc) union (select * from "events" where (substring("event_pubkey" from 1 for 3) BETWEEN E\'\\\\xbbbbb0\' AND E\'\\\\xbbbbbf\') order by "event_created_at" asc) union (select * from "events" where "event_created_at" >= 1000 order by "event_created_at" asc) union (select * from "events" where "event_created_at" <= 1000 order by "event_created_at" asc) union (select * from "events" order by "event_created_at" DESC limit 1000) order by "event_created_at" asc') + }) + }) + }) +})