From f315b5b0d11bd500b0968447234494c07063347a Mon Sep 17 00:00:00 2001 From: Wilmer Paulino Date: Fri, 10 Aug 2018 20:24:04 -0700 Subject: [PATCH] channeldb: support querying for invoices within a specific time range In this commit, we introduce support for querying the database for invoices that occurred within a specific add index range. The query format includes an index to start with and a limit on the number of returned results. Co-authored-by: Valentine Wallace --- channeldb/invoice_test.go | 125 ++++++++++++++++++++++++++++++++++++ channeldb/invoices.go | 132 +++++++++++++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 3 deletions(-) diff --git a/channeldb/invoice_test.go b/channeldb/invoice_test.go index e752c30b5..73c58270d 100644 --- a/channeldb/invoice_test.go +++ b/channeldb/invoice_test.go @@ -375,3 +375,128 @@ func TestDuplicateSettleInvoice(t *testing.T) { spew.Sdump(invoice), spew.Sdump(dbInvoice)) } } + +// TestQueryInvoices ensures that we can properly query the invoice database for +// invoices between specific time intervals. +func TestQueryInvoices(t *testing.T) { + t.Parallel() + + db, cleanUp, err := makeTestDB() + defer cleanUp() + if err != nil { + t.Fatalf("unable to make test db: %v", err) + } + + // To begin the test, we'll add 100 invoices to the database. We'll + // assume that the index of the invoice within the database is the same + // as the amount of the invoice itself. + const numInvoices = 100 + for i := lnwire.MilliSatoshi(0); i < numInvoices; i++ { + invoice, err := randInvoice(i) + if err != nil { + t.Fatalf("unable to create invoice: %v", err) + } + + if _, err := db.AddInvoice(invoice); err != nil { + t.Fatalf("unable to add invoice: %v", err) + } + + // We'll only settle half of all invoices created. + if i%2 == 0 { + paymentHash := sha256.Sum256(invoice.Terms.PaymentPreimage[:]) + if _, err := db.SettleInvoice(paymentHash, i); err != nil { + t.Fatalf("unable to settle invoice: %v", err) + } + } + } + + // With the invoices created, we can begin querying the database. We'll + // start with a simple query to retrieve all invoices. + query := InvoiceQuery{ + NumMaxInvoices: numInvoices, + } + res, err := db.QueryInvoices(query) + if err != nil { + t.Fatalf("unable to query invoices: %v", err) + } + if len(res.Invoices) != numInvoices { + t.Fatalf("expected %d invoices, got %d", numInvoices, + len(res.Invoices)) + } + + // Now, we'll limit the query to only return the latest 30 invoices. + query.IndexOffset = 70 + res, err = db.QueryInvoices(query) + if err != nil { + t.Fatalf("unable to query invoices: %v", err) + } + if uint32(len(res.Invoices)) != numInvoices-query.IndexOffset { + t.Fatalf("expected %d invoices, got %d", + numInvoices-query.IndexOffset, len(res.Invoices)) + } + for _, invoice := range res.Invoices { + if uint32(invoice.Terms.Value) < query.IndexOffset { + t.Fatalf("found invoice with index %v before offset %v", + invoice.Terms.Value, query.IndexOffset) + } + } + + // Limit the query from above to return 25 invoices max. + query.NumMaxInvoices = 25 + res, err = db.QueryInvoices(query) + if err != nil { + t.Fatalf("unable to query invoices: %v", err) + } + if uint32(len(res.Invoices)) != query.NumMaxInvoices { + t.Fatalf("expected %d invoices, got %d", query.NumMaxInvoices, + len(res.Invoices)) + } + + // Reset the query to fetch all unsettled invoices within the time + // slice. + query = InvoiceQuery{ + PendingOnly: true, + NumMaxInvoices: numInvoices, + } + res, err = db.QueryInvoices(query) + if err != nil { + t.Fatalf("unable to query invoices: %v", err) + } + // Since only invoices with even amounts were settled, we should see + // that there are 50 invoices within the response. + if len(res.Invoices) != numInvoices/2 { + t.Fatalf("expected %d pending invoices, got %d", numInvoices/2, + len(res.Invoices)) + } + for _, invoice := range res.Invoices { + if invoice.Terms.Value%2 == 0 { + t.Fatal("retrieved unexpected settled invoice") + } + } + + // Finally, we'll skip the first 10 invoices from the set of unsettled + // invoices. + query.IndexOffset = 10 + res, err = db.QueryInvoices(query) + if err != nil { + t.Fatalf("unable to query invoices: %v", err) + } + if uint32(len(res.Invoices)) != (numInvoices/2)-query.IndexOffset { + t.Fatalf("expected %d invoices, got %d", + (numInvoices/2)-query.IndexOffset, len(res.Invoices)) + } + // To ensure the correct invoices were returned, we'll make sure each + // invoice has an odd value (meaning unsettled). Since the 10 invoices + // skipped should be unsettled, the value of the invoice must be at + // least the index of the 11th unsettled invoice. + for _, invoice := range res.Invoices { + if uint32(invoice.Terms.Value) < query.IndexOffset*2 { + t.Fatalf("found invoice with index %v before offset %v", + invoice.Terms.Value, query.IndexOffset*2) + } + if invoice.Terms.Value%2 == 0 { + t.Fatalf("found unexpected settled invoice with index %v", + invoice.Terms.Value) + } + } +} diff --git a/channeldb/invoices.go b/channeldb/invoices.go index feb074420..141f79067 100644 --- a/channeldb/invoices.go +++ b/channeldb/invoices.go @@ -27,7 +27,7 @@ var ( // for looking up incoming HTLCs to determine if we're able to settle // them fully. // - // maps: payHash => invoiceIndex + // maps: payHash => invoiceKey invoiceIndexBucket = []byte("paymenthashes") // numInvoicesKey is the name of key which houses the auto-incrementing @@ -44,7 +44,7 @@ var ( // // In addition to this sequence number, we map: // - // addIndexNo => invoiceIndex + // addIndexNo => invoiceKey addIndexBucket = []byte("invoice-add-index") // settleIndexBucket is an index bucket that we'll use to create a @@ -54,7 +54,7 @@ var ( // // In addition to this sequence number, we map: // - // settleIndexNo => invoiceIndex + // settleIndexNo => invoiceKey settleIndexBucket = []byte("invoice-settle-index") ) @@ -396,6 +396,132 @@ func (d *DB) FetchAllInvoices(pendingOnly bool) ([]Invoice, error) { return invoices, nil } +// InvoiceQuery represents a query to the invoice database. The query allows a +// caller to retrieve all invoices starting from a particular add index and +// limit the number of results returned. +type InvoiceQuery struct { + // IndexOffset is the offset within the add indices to start at. This + // can be used to start the response at a particular invoice. + IndexOffset uint32 + + // NumMaxInvoices is the maximum number of invoices that should be + // starting from the add index. + NumMaxInvoices uint32 + + // PendingOnly, if set, returns unsettled invoices starting from the + // add index. + PendingOnly bool +} + +// InvoiceSlice is the response to a invoice query. It includes the original +// query, the set of invoices that match the query, and an integer which +// represents the offset index of the last item in the set of returned invoices. +// This integer allows callers to resume their query using this offset in the +// event that the query's response exceeds the maximum number of returnable +// invoices. +type InvoiceSlice struct { + InvoiceQuery + + // Invoices is the set of invoices that matched the query above. + Invoices []*Invoice + + // LastIndexOffset is the index of the last element in the set of + // returned Invoices above. Callers can use this to resume their query + // in the event that the time slice has too many events to fit into a + // single response. + LastIndexOffset uint32 +} + +// QueryInvoices allows a caller to query the invoice database for invoices +// within the specified add index range. +func (d *DB) QueryInvoices(q InvoiceQuery) (InvoiceSlice, error) { + resp := InvoiceSlice{ + InvoiceQuery: q, + } + + // If the caller provided an index offset, then we'll not know how many + // records we need to skip. We'll also keep track of the record offset + // as that's part of the final return value. + invoicesToSkip := q.IndexOffset + invoiceOffset := q.IndexOffset + + err := d.View(func(tx *bolt.Tx) error { + // If the bucket wasn't found, then there aren't any invoices + // within the database yet, so we can simply exit. + invoices := tx.Bucket(invoiceBucket) + if invoices == nil { + return ErrNoInvoicesCreated + } + invoiceAddedIndex := invoices.Bucket(addIndexBucket) + if invoiceAddedIndex == nil { + return ErrNoInvoicesCreated + } + + // We'll be using a cursor to seek into the database, so we'll + // populate byte slices that represent the start of the key + // space we're interested in. + var startIndex [8]byte + switch q.PendingOnly { + case true: + // We have to start from the beginning so we know + // how many pending invoices we're skipping. + byteOrder.PutUint64(startIndex[:], uint64(1)) + default: + // We can seek right to the invoice offset we want + // to start with. + invoicesToSkip = 0 + byteOrder.PutUint64(startIndex[:], uint64(invoiceOffset+1)) + } + + // If we know that a set of invoices exists, then we'll begin + // our seek through the bucket in order to satisfy the query. + // We'll continue until either we reach the end of the range, + // or reach our max number of events. + cursor := invoiceAddedIndex.Cursor() + _, invoiceKey := cursor.Seek(startIndex[:]) + for ; invoiceKey != nil; _, invoiceKey = cursor.Next() { + // If our current return payload exceeds the max number + // of invoices, then we'll exit now. + if uint32(len(resp.Invoices)) >= q.NumMaxInvoices { + return nil + } + + invoice, err := fetchInvoice(invoiceKey, invoices) + if err != nil { + return err + } + + // Skip any settled invoices if the caller is only + // interested in unsettled. + if q.PendingOnly && invoice.Terms.Settled { + continue + } + // If we're not yet past the user defined offset, then + // we'll continue to seek forward. + if invoicesToSkip > 0 { + invoicesToSkip-- + continue + } + + // At this point, we've exhausted the offset, so we'll + // begin collecting invoices found within the range. + resp.Invoices = append(resp.Invoices, &invoice) + invoiceOffset++ + } + + return nil + }) + if err != nil && err != ErrNoInvoicesCreated { + return resp, err + } + + // Finally, record the index of the last invoice added so that the + // caller can resume from this point later on. + resp.LastIndexOffset = invoiceOffset + + return resp, nil +} + // SettleInvoice attempts to mark an invoice corresponding to the passed // payment hash as fully settled. If an invoice matching the passed payment // hash doesn't existing within the database, then the action will fail with a