highperfocused/P4wnP1_aloa
1
0
Fork 0
mirror of https://github.com/RoganDawes/P4wnP1_aloa.git synced 2025-03-17 21:31:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

DB backup/restore fixes; deletions of stored templates

Browse Source
This commit is contained in:
MaMe82 2018-10-24 19:11:09 +02:00
parent 7bb1dfa254
commit 85cc7a9291
2 changed files with 210 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
service/datastore/store.go
View File

@ -1,9 +1,14 @@
// +build linux,arm
package datastore
import (
"errors"
"fmt"
"github.com/dgraph-io/badger"
_ "github.com/dgraph-io/badger"
"github.com/dgraph-io/badger/options"
"io/ioutil"
"os"
"strings"
)
@ -14,18 +19,17 @@ https://github.com/dgraph-io/badger --> Apache 2 (compatible)
var (
ErrCreate = errors.New("Error creating store")
ErrOpen = errors.New("Error opening store")
ErrGet = errors.New("Error retrieving value from store")
ErrKeys = errors.New("Error retrieving keys from store")
ErrOpen = errors.New("Error opening store")
ErrGet = errors.New("Error retrieving value from store")
ErrKeys = errors.New("Error retrieving keys from store")
ErrDelete = errors.New("Error deleting value from store")
ErrPut = errors.New("Error putting value into store")
ErrPut = errors.New("Error putting value into store")
ErrExists = errors.New("Error key exists already")
)
type Store struct {
Path string
Db *badger.DB
Path string
Db *badger.DB
serializer Serializer
}
@ -34,8 +38,10 @@ func (s *Store) Open() (err error) {
badgerOpts.Dir = s.Path
badgerOpts.ValueDir = s.Path
badgerOpts.SyncWrites = true
badgerOpts.ManagedTxns = true //needed for DropAll()
s.Db,err = badger.Open(badgerOpts)
badgerOpts.ManagedTxns = false
badgerOpts.TableLoadingMode = options.FileIO
badgerOpts.ValueLogLoadingMode = options.FileIO
s.Db, err = badger.Open(badgerOpts)
if s.serializer == nil {
s.serializer = NewSerializerProtobuf(false)
}
@ -46,6 +52,17 @@ func (s *Store) Close() {
s.Db.Close()
}
func (s *Store) Clear() (err error) {
keys, err := s.Keys()
if err != nil {
return
}
// ToDo: Other transactions could add/delete keys meanwhile, which should be avoided (without locking the functions accessing badger)
err = s.DeleteMulti(keys)
return nil
}
// ToDo: Backup and restore could be synchronized to avoid concurrent transactions
func (s *Store) Backup(filePath string) (err error) {
f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664)
@ -53,21 +70,111 @@ func (s *Store) Backup(filePath string) (err error) {
return err
}
defer f.Close()
_,err = s.Db.Backup(f,0)
_, err = s.Db.Backup(f, 0)
return
}
func (s *Store) Restore(filePath string) (err error) {
func (s *Store) Restore(filePath string, replace bool) (err error) {
fmt.Println("Restoring DB...")
f, err := os.Open(filePath)
if err != nil {
return err
}
defer f.Close()
// DB should be cleared first
err = s.Db.DropAll()
if err != nil { return }
err = s.Db.Load(f)
return
if replace {
// As backups are version agnostic, we can't replace newer keys or keys deleted in a later version
// To solve this, replacement is handled like this
// - the backup is loaded into a temporary DB which hasn't got any (versioned) entries
// - all keys of the current DB are deleted (deletion has results in a newer version of the key as the one in the backup)
// - copy over all key-value-pairs of the temporary DB to the current DB via set (updates the version of the key)
fmt.Println("...Clear current DB")
err = s.Clear()
if err != nil {
return
}
//create temp db
fmt.Println("... create temp DB")
tmpDbDir, err := ioutil.TempDir("/tmp", "badger_backup")
if err != nil {
return err
}
fmt.Println("... temp DB created")
defer os.RemoveAll(tmpDbDir)
fmt.Println("... opening temp DB at", tmpDbDir)
badgerOpts := badger.DefaultOptions
badgerOpts.Dir = tmpDbDir
badgerOpts.ValueDir = tmpDbDir
badgerOpts.SyncWrites = true
badgerOpts.TableLoadingMode = options.FileIO
badgerOpts.ValueLogLoadingMode = options.FileIO
tmpDB, err := badger.Open(badgerOpts)
if err != nil {
fmt.Println("... opening of temp DB failed:", err)
return err
}
defer tmpDB.Close()
fmt.Println("... loading backup to temp DB")
tmpDB.Load(f)
fmt.Println("... retrieving keys of temp DB")
restoreKeys := make([][]byte, 0)
err = tmpDB.View(func(txn *badger.Txn) error {
iterOpts := badger.DefaultIteratorOptions
iterOpts.PrefetchValues = false
iter := txn.NewIterator(iterOpts)
defer iter.Close()
for iter.Rewind(); iter.Valid(); iter.Next() {
key := iter.Item().Key()
fmt.Println("... found key:", string(key))
resKey := make([]byte, len(key))
copy(resKey, key)
restoreKeys = append(restoreKeys, resKey)
}
return nil
})
if err != nil {
return err
}
keycount := len(restoreKeys)
fmt.Printf("... found %d keys", keycount)
txnReadSrc := tmpDB.NewTransaction(false)
defer txnReadSrc.Discard()
fmt.Printf("... restoring %d key-value-pairs\n", keycount)
for idx, restoreKey := range restoreKeys {
fmt.Printf("... copy over key %d of %d '%s' ...", idx+1, keycount, string(restoreKey))
item, err := txnReadSrc.Get(restoreKey)
if err != nil {
continue
}
s.Db.Update(func(txn *badger.Txn) error {
val, valErr := item.Value()
if valErr == nil {
// Ignore keys with empty vals, see https://github.com/dgraph-io/badger/issues/521
if len(val) > 0 {
txn.Set(restoreKey, val)
fmt.Println("added")
} else {
fmt.Println("ignored, empty value")
}
}
return nil
})
}
return nil
} else {
err = s.Db.Load(f)
return
}
}
func (s *Store) Put(key string, value interface{}, allowOverwrite bool) (err error) {
@ -77,14 +184,18 @@ func (s *Store) Put(key string, value interface{}, allowOverwrite bool) (err err
}
// serialize value
sv,err := s.serializer.Encode(value)
if err != nil { return }
sv, err := s.serializer.Encode(value)
if err != nil {
return
}
err = s.Db.Update(func(txn *badger.Txn) error {
err := txn.Set([]byte(key), []byte(sv))
return err
})
if err != nil { return ErrPut }
if err != nil {
return ErrPut
}
return
}
@ -97,7 +208,9 @@ func (s *Store) Exists(key string) (exists bool) {
}
return nil
})
if err != nil { return false }
if err != nil {
return false
}
return true
}
@ -115,57 +228,97 @@ func (s *Store) Get(key string, target interface{}) (err error) {
//fmt.Printf("The answer is: %s\n", val)
//return nil
})
if err != nil { return ErrGet }
if err != nil {
return ErrGet
}
return
}
func (s *Store) Keys() (keys []string, err error) {
err = s.Db.View(func(txn *badger.Txn) error {
iter := txn.NewIterator(badger.DefaultIteratorOptions)
iterOpts := badger.DefaultIteratorOptions
iterOpts.PrefetchValues = false
iter := txn.NewIterator(iterOpts)
defer iter.Close()
for iter.Rewind(); iter.Valid(); iter.Next() {
keys = append(keys, string(iter.Item().Key()))
key := iter.Item().Key()
resKey := make([]byte, len(key))
copy(resKey, key)
keys = append(keys, string(resKey))
}
return nil
})
if err != nil { return keys, ErrKeys }
if err != nil {
return keys, ErrKeys
}
return
}
func (s *Store) KeysPrefix(prefix string, trimPrefix bool) (keys []string, err error) {
bprefix := []byte(prefix)
err = s.Db.View(func(txn *badger.Txn) error {
iter := txn.NewIterator(badger.DefaultIteratorOptions)
iterOpts := badger.DefaultIteratorOptions
iterOpts.PrefetchValues = false
iter := txn.NewIterator(iterOpts)
defer iter.Close()
for iter.Seek(bprefix); iter.ValidForPrefix(bprefix); iter.Next() {
key := iter.Item().Key()
resKey := make([]byte, len(key))
copy(resKey, key)
if trimPrefix {
s := strings.TrimPrefix(string(iter.Item().Key()), prefix)
keys = append(keys,s)
s := strings.TrimPrefix(string(resKey), prefix)
keys = append(keys, s)
} else {
keys = append(keys, string(iter.Item().Key()))
keys = append(keys, string(resKey))
}
}
return nil
})
if err != nil { return keys, ErrKeys }
if err != nil {
return keys, ErrKeys
}
return
}
func (s *Store) Delete(key string) (err error) {
err = s.Db.Update(func(txn *badger.Txn) error {
return txn.Delete([]byte(key))
})
if err != nil { return ErrDelete }
if err != nil {
return ErrDelete
}
return
}
func (s *Store) DeleteMulti(keys []string) (err error) {
txn := s.Db.NewTransaction(true)
for _, key := range keys {
errDel := txn.Delete([]byte(key))
if errDel != nil {
if err == badger.ErrTxnTooBig {
txn.Commit(nil) // commit current transaction
txn = s.Db.NewTransaction(true) // replace with new transaction
txn.Delete([]byte(key)) // add Delete which produced error to new transaction
} else {
txn.Discard()
return ErrDelete
}
}
}
txn.Commit(nil)
txn.Discard()
return nil
}
func Open(path string) (store *Store, err error) {
store = &Store{
Path: path,
}
if err = store.Open(); err != nil {
return nil,err
return nil, err
}
return
}

27
service/rpc_server.go
View File

@ -52,6 +52,30 @@ type server struct {
listenAddrWeb string
}
func (s *server) DeleteStoredUSBSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) {
e = &pb.Empty{}
err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_USB_SETTINGS + name.Msg)
return
}
func (s *server) DeleteStoredWifiSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) {
e = &pb.Empty{}
err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_WIFI_SETTINGS + name.Msg)
return
}
func (s *server) DeleteStoredEthernetInterfaceSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) {
e = &pb.Empty{}
err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_ETHERNET_INTERFACE_SETTINGS + name.Msg)
return
}
func (s *server) DeleteStoredTriggerActionSet(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) {
e = &pb.Empty{}
err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_TRIGGER_ACTION_SET + name.Msg)
return
}
func (s *server) DBBackup(ctx context.Context, filename *pb.StringMessage) (e *pb.Empty, err error) {
e = &pb.Empty{}
fname := filename.Msg
@ -65,13 +89,14 @@ func (s *server) DBBackup(ctx context.Context, filename *pb.StringMessage) (e *p
}
func (s *server) DBRestore(ctx context.Context, filename *pb.StringMessage) (e *pb.Empty, err error) {
fmt.Println("DB restore: ", filename.Msg)
e = &pb.Empty{}
fname := filename.Msg
ext := filepath.Ext(fname)
if lext := strings.ToLower(ext); lext != ".db" {
fname = fname + ".db"
}
err = s.rootSvc.SubSysDataStore.Restore(PATH_DATA_STORE_BACKUP + "/" + fname)
err = s.rootSvc.SubSysDataStore.Restore(PATH_DATA_STORE_BACKUP + "/" + fname, true)
return
}