fix: prefix search & add tests

This commit is contained in:
Ricardo Arturo Cabral Mejia 2022-08-16 03:38:07 +00:00
parent 86f1382ed4
commit 246e472fc4
No known key found for this signature in database
GPG Key ID: 5931EBF43A650245
5 changed files with 440 additions and 57 deletions

View File

@ -1,6 +1,6 @@
module.exports = {
extension: ['ts'],
require: ['ts-node/register/transpile-only', 'source-map-support/register'],
require: ['ts-node/register', 'source-map-support/register'],
reporter: 'mochawesome',
slow: 75,
sorted: true,

View File

@ -5,7 +5,7 @@ import { SubscriptionFilter } from './subscription'
export type ExposedPromiseKeys = 'then' | 'catch' | 'finally'
export interface IQueryResult<T> extends Pick<Promise<T>, keyof Promise<T> & ExposedPromiseKeys> {
stream(): PassThrough & AsyncIterable<T>
stream(options?: Record<string, any>): PassThrough & AsyncIterable<T>
}
export interface IEventRepository {

View File

@ -1,3 +1,4 @@
import { EventKinds } from '../constants/base'
import { Pubkey } from './base'
import { EventId } from './event'
@ -5,7 +6,7 @@ export type SubscriptionId = string
export interface SubscriptionFilter {
ids?: EventId[]
kinds?: number[]
kinds?: EventKinds[]
since?: number
until?: number
authors?: Pubkey[]

View File

@ -1,6 +1,5 @@
import { Knex } from 'knex'
import { applySpec, omit, pipe, prop } from 'ramda'
import { PassThrough } from 'stream'
import { __, applySpec, equals, modulo, omit, pipe, prop, cond, always, groupBy, T, evolve, forEach, isEmpty, forEachObjIndexed, isNil, complement, toPairs, filter, nth, ifElse, invoker } from 'ramda'
import { DBEvent, Event } from '../@types/event'
import { IEventRepository, IQueryResult } from '../@types/repositories'
@ -8,8 +7,18 @@ import { SubscriptionFilter } from '../@types/subscription'
import { isGenericTagQuery } from '../utils/filter'
import { toBuffer, toJSON } from '../utils/transform'
const even = pipe(modulo(__, 2), equals(0))
const evenLengthTruncate = (input: string) => input.substring(0, input.length >> 1 << 1)
const groupByLengthSpec = groupBy(
pipe(
prop('length'),
cond([
[equals(64), always('exact')],
[even, always('even')],
[T, always('odd')],
])
)
)
export class EventRepository implements IEventRepository {
public constructor(private readonly dbClient: Knex) {}
@ -18,69 +27,82 @@ export class EventRepository implements IEventRepository {
if (!Array.isArray(filters) || !filters.length) {
throw new Error('Filters cannot be empty')
}
const queries = filters.map((filter) => {
const queries = filters.map((currentFilter) => {
const builder = this.dbClient<DBEvent>('events')
if (Array.isArray(filter.authors)) {
builder.andWhere(function (bd) {
bd.whereIn(
'event_pubkey',
filter.authors.filter((author) => author.length === 64).map(toBuffer)
)
for (const author of filter.authors.filter((author) => author.length < 64)) {
const prefix = evenLengthTruncate(author)
if (prefix.length) {
bd.orWhereRaw('substring("event_pubkey" from 1 for ?) = ?', [prefix.length >> 1, toBuffer(prefix)])
}
}
forEachObjIndexed((tableField: string, filterName: string) => {
builder.andWhere((bd) => {
cond([
[isEmpty, () => void bd.whereRaw('1 = 0')],
[
complement(isNil),
pipe(
groupByLengthSpec,
evolve({
exact: (pubkeys: string[]) => void bd.whereIn(tableField, pubkeys.map(toBuffer)),
even: forEach((prefix: string) => void bd.orWhereRaw(
`substring("${tableField}" from 1 for ?) = ?`,
[prefix.length >> 1, toBuffer(prefix)]
)),
odd: forEach((prefix: string) => void bd.orWhereRaw(
`substring("${tableField}" from 1 for ?) BETWEEN ? AND ?`,
[
(prefix.length >> 1) + 1,
`\\x${prefix}0`,
`\\x${prefix}f`
],
)),
}),
),
],
])(currentFilter[filterName] as string[])
})
})({
authors: 'event_pubkey',
ids: 'event_id',
})
if (Array.isArray(currentFilter.kinds)) {
builder.whereIn('event_kind', currentFilter.kinds)
}
if (Array.isArray(filter.ids)) {
builder.andWhere(function (bd) {
bd.whereIn(
'event_id',
filter.ids.filter((id) => id.length === 64).map(toBuffer)
)
for (const id of filter.ids.filter((id) => id.length < 64)) {
const prefix = evenLengthTruncate(id)
if (prefix.length) {
bd.orWhereRaw('substring("event_id" from 1 for ?) = ?', [prefix.length >> 1, toBuffer(prefix)])
}
}
})
if (typeof currentFilter.since === 'number') {
builder.where('event_created_at', '>=', currentFilter.since)
}
if (Array.isArray(filter.kinds)) {
builder.whereIn('event_kind', filter.kinds)
if (typeof currentFilter.until === 'number') {
builder.where('event_created_at', '<=', currentFilter.until)
}
if (typeof filter.since === 'number') {
builder.where('event_created_at', '>=', filter.since)
}
if (typeof filter.until === 'number') {
builder.where('event_created_at', '<=', filter.until)
}
if (typeof filter.limit === 'number') {
builder.limit(filter.limit).orderBy('event_created_at', 'DESC')
if (typeof currentFilter.limit === 'number') {
builder.limit(currentFilter.limit).orderBy('event_created_at', 'DESC')
} else {
builder.orderBy('event_created_at', 'asc')
}
Object.entries(filter)
.filter(([key, criteria]) => isGenericTagQuery(key) && Array.isArray(criteria))
.forEach(([key, criteria]) => {
builder.andWhere(function (bd) {
criteria.forEach((criterion) => {
bd.orWhereRaw('"event_tags" @> ?', [JSON.stringify([[key[1], criterion]])])
})
const andWhereRaw = invoker(1, 'andWhereRaw')
const orWhereRaw = invoker(2, 'orWhereRaw')
pipe(
toPairs,
filter(pipe(nth(0), isGenericTagQuery)) as any,
forEach(([filterName, criteria]: [string, string[]]) => {
builder.andWhere((bd) => {
ifElse(
isEmpty,
() => andWhereRaw('1 = 0', bd),
forEach((criterion: string[]) => void orWhereRaw(
'"event_tags" @> ?',
[
JSON.stringify([[filterName[1], criterion]]) as any
],
bd,
)),
)(criteria)
})
})
}),
)(currentFilter as any)
return builder
})
@ -90,8 +112,6 @@ export class EventRepository implements IEventRepository {
query.union(subqueries, true)
}
console.log('query', query.toString())
return query
}

View File

@ -0,0 +1,362 @@
import * as chai from 'chai'
import knex from 'knex'
import * as sinon from 'sinon'
import sinonChai from 'sinon-chai'
import { IEventRepository } from '../../../src/@types/repositories'
import { SubscriptionFilter } from '../../../src/@types/subscription'
chai.use(sinonChai)
const { expect } = chai
import { EventRepository } from '../../../src/repositories/event-repository'
describe.only('EventRepository', () => {
let repository: IEventRepository
let sandbox: sinon.SinonSandbox
beforeEach(() => {
sandbox = sinon.createSandbox()
repository = new EventRepository(knex({
client: 'pg'
}))
})
afterEach(() => {
sandbox.restore()
})
describe('findByFilters', () => {
it('returns a function with stream and then', () => {
expect(repository.findByFilters([{}])).to.have.property('stream')
expect(repository.findByFilters([{}])).to.have.property('then')
expect(repository.findByFilters([{}])).to.have.property('catch')
expect(repository.findByFilters([{}])).to.have.property('finally')
})
it('throws error if filters is not an array', () => {
expect(() => repository.findByFilters(null)).to.throw(Error, 'Filters cannot be empty')
})
it('throws error if filters is empty', () => {
expect(() => repository.findByFilters([])).to.throw(Error, 'Filters cannot be empty')
})
describe('1 filter', () => {
it('selects all events', () => {
const filters = [{}]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" order by "event_created_at" asc')
})
describe('authors', () => {
it('selects no events given empty list of authors', () => {
const filters: SubscriptionFilter[] = [{ authors: [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc')
})
it('selects events by one author', () => {
const filters = [{ authors: ['22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_pubkey" in (X\'22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793\')) order by "event_created_at" asc')
})
it('selects events by two authors', () => {
const filters = [
{
authors: [
'22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793',
'32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245'
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_pubkey" in (X\'22e804d26ed16b68db5259e78449e96dab5d464c8f470bda3eb1a70467f2c793\', X\'32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245\')) order by "event_created_at" asc')
})
it('selects events by one author prefix (even length)', () => {
const filters = [
{
authors: [
'22e804',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 3) = X\'22e804\') order by "event_created_at" asc')
})
it('selects events by one author prefix (odd length)', () => {
const filters = [
{
authors: [
'22e804f',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 4) BETWEEN E\'\\\\x22e804f0\' AND E\'\\\\x22e804ff\') order by "event_created_at" asc')
})
it('selects events by two author prefix (first even, second odd)', () => {
const filters = [
{
authors: [
'22e804',
'32e1827',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_pubkey" from 1 for 3) = X\'22e804\' or substring("event_pubkey" from 1 for 4) BETWEEN E\'\\\\x32e18270\' AND E\'\\\\x32e1827f\') order by "event_created_at" asc')
})
})
describe('ids', () => {
it('selects no events given empty list of ids', () => {
const filters: SubscriptionFilter[] = [{ ids: [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc')
})
it('selects events by one id', () => {
const filters = [{ ids: ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_id" in (X\'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\')) order by "event_created_at" asc')
})
it('selects events by two ids', () => {
const filters = [
{
ids: [
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_id" in (X\'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\', X\'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\')) order by "event_created_at" asc')
})
it('selects events by one id prefix (even length)', () => {
const filters = [
{
ids: [
'abcd',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 2) = X\'abcd\') order by "event_created_at" asc')
})
it('selects events by one id prefix (odd length)', () => {
const filters = [
{
ids: [
'abc',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 2) BETWEEN E\'\\\\xabc0\' AND E\'\\\\xabcf\') order by "event_created_at" asc')
})
it('selects events by two id prefix (first even, second odd)', () => {
const filters = [
{
ids: [
'abcdef',
'abc',
]
}
]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (substring("event_id" from 1 for 3) = X\'abcdef\' or substring("event_id" from 1 for 2) BETWEEN E\'\\\\xabc0\' AND E\'\\\\xabcf\') order by "event_created_at" asc')
})
})
describe('kinds', () => {
it('selects no events given empty list of kinds', () => {
const filters: SubscriptionFilter[] = [{ kinds: [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where 1 = 0 order by "event_created_at" asc')
})
it('selects events by one kind', () => {
const filters = [{ kinds: [1] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where "event_kind" in (1) order by "event_created_at" asc')
})
it('selects events by two kinds', () => {
const filters = [{ kinds: [1, 2] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where "event_kind" in (1, 2) order by "event_created_at" asc')
})
})
describe('since', () => {
it('selects events since given timestamp', () => {
const filters = [{ since: 1000 }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where "event_created_at" >= 1000 order by "event_created_at" asc')
})
})
describe('until', () => {
it('selects events until given timestamp', () => {
const filters = [{ until: 1000 }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where "event_created_at" <= 1000 order by "event_created_at" asc')
})
})
describe('limit', () => {
it('selects 1000 events', () => {
const filters = [{ limit: 1000 }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" order by "event_created_at" DESC limit 1000')
})
})
describe('#e', () => {
it('selects no events given empty list of #e tags', () => {
const filters = [{ '#e': [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc')
})
it('selects events by one #e tag', () => {
const filters = [{ '#e': ['aaaaaa'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["e","aaaaaa"]]\') order by "event_created_at" asc')
})
it('selects events by two #e tag', () => {
const filters = [{ '#e': ['aaaaaa', 'bbbbbb'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["e","aaaaaa"]]\' or "event_tags" @> \'[["e","bbbbbb"]]\') order by "event_created_at" asc')
})
})
describe('#p', () => {
it('selects no events given empty list of #p tags', () => {
const filters = [{ '#p': [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc')
})
it('selects events by one #p tag', () => {
const filters = [{ '#p': ['aaaaaa'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["p","aaaaaa"]]\') order by "event_created_at" asc')
})
it('selects events by two #p tag', () => {
const filters = [{ '#p': ['aaaaaa', 'bbbbbb'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["p","aaaaaa"]]\' or "event_tags" @> \'[["p","bbbbbb"]]\') order by "event_created_at" asc')
})
})
describe('#r', () => {
it('selects no events given empty list of #r tags', () => {
const filters = [{ '#r': [] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where (1 = 0) order by "event_created_at" asc')
})
it('selects events by one #r tag', () => {
const filters = [{ '#r': ['aaaaaa'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["r","aaaaaa"]]\') order by "event_created_at" asc')
})
it('selects events by two #r tag', () => {
const filters = [{ '#r': ['aaaaaa', 'bbbbbb'] }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('select * from "events" where ("event_tags" @> \'[["r","aaaaaa"]]\' or "event_tags" @> \'[["r","bbbbbb"]]\') order by "event_created_at" asc')
})
})
})
describe('2 filters', () => {
it('selects union of both filters', () => {
const filters = [{}, {}]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('(select * from "events") union (select * from "events" order by "event_created_at" asc) order by "event_created_at" asc')
})
})
describe('many filters', () => {
it('selects union of all filters', () => {
const filters = [{ kinds: [1] }, { ids: ['aaaaa'] }, { authors: ['bbbbb'] }, { since: 1000 }, { until: 1000 }, { limit: 1000 }]
const query = repository.findByFilters(filters).toString()
expect(query).to.equal('(select * from "events" where "event_kind" in (1)) union (select * from "events" where (substring("event_id" from 1 for 3) BETWEEN E\'\\\\xaaaaa0\' AND E\'\\\\xaaaaaf\') order by "event_created_at" asc) union (select * from "events" where (substring("event_pubkey" from 1 for 3) BETWEEN E\'\\\\xbbbbb0\' AND E\'\\\\xbbbbbf\') order by "event_created_at" asc) union (select * from "events" where "event_created_at" >= 1000 order by "event_created_at" asc) union (select * from "events" where "event_created_at" <= 1000 order by "event_created_at" asc) union (select * from "events" order by "event_created_at" DESC limit 1000) order by "event_created_at" asc')
})
})
})
})