diff --git a/service/datastore/store.go b/service/datastore/store.go index 6922fd0..eabf32c 100644 --- a/service/datastore/store.go +++ b/service/datastore/store.go @@ -1,9 +1,14 @@ +// +build linux,arm + package datastore import ( "errors" + "fmt" "github.com/dgraph-io/badger" _ "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/options" + "io/ioutil" "os" "strings" ) @@ -14,18 +19,17 @@ https://github.com/dgraph-io/badger --> Apache 2 (compatible) var ( ErrCreate = errors.New("Error creating store") - ErrOpen = errors.New("Error opening store") - ErrGet = errors.New("Error retrieving value from store") - ErrKeys = errors.New("Error retrieving keys from store") + ErrOpen = errors.New("Error opening store") + ErrGet = errors.New("Error retrieving value from store") + ErrKeys = errors.New("Error retrieving keys from store") ErrDelete = errors.New("Error deleting value from store") - ErrPut = errors.New("Error putting value into store") + ErrPut = errors.New("Error putting value into store") ErrExists = errors.New("Error key exists already") ) - type Store struct { - Path string - Db *badger.DB + Path string + Db *badger.DB serializer Serializer } @@ -34,8 +38,10 @@ func (s *Store) Open() (err error) { badgerOpts.Dir = s.Path badgerOpts.ValueDir = s.Path badgerOpts.SyncWrites = true - badgerOpts.ManagedTxns = true //needed for DropAll() - s.Db,err = badger.Open(badgerOpts) + badgerOpts.ManagedTxns = false + badgerOpts.TableLoadingMode = options.FileIO + badgerOpts.ValueLogLoadingMode = options.FileIO + s.Db, err = badger.Open(badgerOpts) if s.serializer == nil { s.serializer = NewSerializerProtobuf(false) } @@ -46,6 +52,17 @@ func (s *Store) Close() { s.Db.Close() } +func (s *Store) Clear() (err error) { + keys, err := s.Keys() + if err != nil { + return + } + // ToDo: Other transactions could add/delete keys meanwhile, which should be avoided (without locking the functions accessing badger) + err = s.DeleteMulti(keys) + + return nil +} + // ToDo: Backup and restore could be synchronized to avoid concurrent transactions func (s *Store) Backup(filePath string) (err error) { f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0664) @@ -53,21 +70,111 @@ func (s *Store) Backup(filePath string) (err error) { return err } defer f.Close() - _,err = s.Db.Backup(f,0) + _, err = s.Db.Backup(f, 0) return } -func (s *Store) Restore(filePath string) (err error) { +func (s *Store) Restore(filePath string, replace bool) (err error) { + fmt.Println("Restoring DB...") + f, err := os.Open(filePath) if err != nil { return err } defer f.Close() - // DB should be cleared first - err = s.Db.DropAll() - if err != nil { return } - err = s.Db.Load(f) - return + if replace { + // As backups are version agnostic, we can't replace newer keys or keys deleted in a later version + // To solve this, replacement is handled like this + // - the backup is loaded into a temporary DB which hasn't got any (versioned) entries + // - all keys of the current DB are deleted (deletion has results in a newer version of the key as the one in the backup) + // - copy over all key-value-pairs of the temporary DB to the current DB via set (updates the version of the key) + fmt.Println("...Clear current DB") + err = s.Clear() + if err != nil { + return + } + + //create temp db + fmt.Println("... create temp DB") + tmpDbDir, err := ioutil.TempDir("/tmp", "badger_backup") + if err != nil { + return err + } + fmt.Println("... temp DB created") + defer os.RemoveAll(tmpDbDir) + + fmt.Println("... opening temp DB at", tmpDbDir) + badgerOpts := badger.DefaultOptions + badgerOpts.Dir = tmpDbDir + badgerOpts.ValueDir = tmpDbDir + badgerOpts.SyncWrites = true + badgerOpts.TableLoadingMode = options.FileIO + badgerOpts.ValueLogLoadingMode = options.FileIO + tmpDB, err := badger.Open(badgerOpts) + if err != nil { + fmt.Println("... opening of temp DB failed:", err) + return err + } + defer tmpDB.Close() + + fmt.Println("... loading backup to temp DB") + tmpDB.Load(f) + + fmt.Println("... retrieving keys of temp DB") + restoreKeys := make([][]byte, 0) + err = tmpDB.View(func(txn *badger.Txn) error { + iterOpts := badger.DefaultIteratorOptions + iterOpts.PrefetchValues = false + iter := txn.NewIterator(iterOpts) + defer iter.Close() + for iter.Rewind(); iter.Valid(); iter.Next() { + key := iter.Item().Key() + fmt.Println("... found key:", string(key)) + resKey := make([]byte, len(key)) + copy(resKey, key) + restoreKeys = append(restoreKeys, resKey) + } + return nil + }) + if err != nil { + return err + } + + keycount := len(restoreKeys) + fmt.Printf("... found %d keys", keycount) + + txnReadSrc := tmpDB.NewTransaction(false) + defer txnReadSrc.Discard() + + fmt.Printf("... restoring %d key-value-pairs\n", keycount) + for idx, restoreKey := range restoreKeys { + fmt.Printf("... copy over key %d of %d '%s' ...", idx+1, keycount, string(restoreKey)) + item, err := txnReadSrc.Get(restoreKey) + if err != nil { + continue + } + s.Db.Update(func(txn *badger.Txn) error { + val, valErr := item.Value() + if valErr == nil { + // Ignore keys with empty vals, see https://github.com/dgraph-io/badger/issues/521 + if len(val) > 0 { + txn.Set(restoreKey, val) + fmt.Println("added") + } else { + fmt.Println("ignored, empty value") + } + } + return nil + }) + } + + return nil + + } else { + err = s.Db.Load(f) + return + } + } func (s *Store) Put(key string, value interface{}, allowOverwrite bool) (err error) { @@ -77,14 +184,18 @@ func (s *Store) Put(key string, value interface{}, allowOverwrite bool) (err err } // serialize value - sv,err := s.serializer.Encode(value) - if err != nil { return } + sv, err := s.serializer.Encode(value) + if err != nil { + return + } err = s.Db.Update(func(txn *badger.Txn) error { err := txn.Set([]byte(key), []byte(sv)) return err }) - if err != nil { return ErrPut } + if err != nil { + return ErrPut + } return } @@ -97,7 +208,9 @@ func (s *Store) Exists(key string) (exists bool) { } return nil }) - if err != nil { return false } + if err != nil { + return false + } return true } @@ -115,57 +228,97 @@ func (s *Store) Get(key string, target interface{}) (err error) { //fmt.Printf("The answer is: %s\n", val) //return nil }) - if err != nil { return ErrGet } + if err != nil { + return ErrGet + } return } func (s *Store) Keys() (keys []string, err error) { err = s.Db.View(func(txn *badger.Txn) error { - iter := txn.NewIterator(badger.DefaultIteratorOptions) + iterOpts := badger.DefaultIteratorOptions + iterOpts.PrefetchValues = false + iter := txn.NewIterator(iterOpts) defer iter.Close() for iter.Rewind(); iter.Valid(); iter.Next() { - keys = append(keys, string(iter.Item().Key())) + key := iter.Item().Key() + resKey := make([]byte, len(key)) + copy(resKey, key) + keys = append(keys, string(resKey)) + } return nil }) - if err != nil { return keys, ErrKeys } + if err != nil { + return keys, ErrKeys + } return } func (s *Store) KeysPrefix(prefix string, trimPrefix bool) (keys []string, err error) { bprefix := []byte(prefix) err = s.Db.View(func(txn *badger.Txn) error { - iter := txn.NewIterator(badger.DefaultIteratorOptions) + iterOpts := badger.DefaultIteratorOptions + iterOpts.PrefetchValues = false + iter := txn.NewIterator(iterOpts) defer iter.Close() for iter.Seek(bprefix); iter.ValidForPrefix(bprefix); iter.Next() { + key := iter.Item().Key() + resKey := make([]byte, len(key)) + copy(resKey, key) if trimPrefix { - s := strings.TrimPrefix(string(iter.Item().Key()), prefix) - keys = append(keys,s) + s := strings.TrimPrefix(string(resKey), prefix) + keys = append(keys, s) } else { - keys = append(keys, string(iter.Item().Key())) + keys = append(keys, string(resKey)) } } return nil }) - if err != nil { return keys, ErrKeys } + if err != nil { + return keys, ErrKeys + } return } func (s *Store) Delete(key string) (err error) { err = s.Db.Update(func(txn *badger.Txn) error { + return txn.Delete([]byte(key)) }) - if err != nil { return ErrDelete } + if err != nil { + return ErrDelete + } return } +func (s *Store) DeleteMulti(keys []string) (err error) { + txn := s.Db.NewTransaction(true) + + for _, key := range keys { + errDel := txn.Delete([]byte(key)) + if errDel != nil { + if err == badger.ErrTxnTooBig { + txn.Commit(nil) // commit current transaction + txn = s.Db.NewTransaction(true) // replace with new transaction + txn.Delete([]byte(key)) // add Delete which produced error to new transaction + } else { + txn.Discard() + return ErrDelete + } + } + } + txn.Commit(nil) + txn.Discard() + return nil +} func Open(path string) (store *Store, err error) { store = &Store{ Path: path, } if err = store.Open(); err != nil { - return nil,err + return nil, err } return } diff --git a/service/rpc_server.go b/service/rpc_server.go index 8a338ca..1a3dc6c 100644 --- a/service/rpc_server.go +++ b/service/rpc_server.go @@ -52,6 +52,30 @@ type server struct { listenAddrWeb string } +func (s *server) DeleteStoredUSBSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) { + e = &pb.Empty{} + err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_USB_SETTINGS + name.Msg) + return +} + +func (s *server) DeleteStoredWifiSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) { + e = &pb.Empty{} + err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_WIFI_SETTINGS + name.Msg) + return +} + +func (s *server) DeleteStoredEthernetInterfaceSettings(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) { + e = &pb.Empty{} + err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_ETHERNET_INTERFACE_SETTINGS + name.Msg) + return +} + +func (s *server) DeleteStoredTriggerActionSet(ctx context.Context, name *pb.StringMessage) (e *pb.Empty, err error) { + e = &pb.Empty{} + err = s.rootSvc.SubSysDataStore.Delete(cSTORE_PREFIX_TRIGGER_ACTION_SET + name.Msg) + return +} + func (s *server) DBBackup(ctx context.Context, filename *pb.StringMessage) (e *pb.Empty, err error) { e = &pb.Empty{} fname := filename.Msg @@ -65,13 +89,14 @@ func (s *server) DBBackup(ctx context.Context, filename *pb.StringMessage) (e *p } func (s *server) DBRestore(ctx context.Context, filename *pb.StringMessage) (e *pb.Empty, err error) { + fmt.Println("DB restore: ", filename.Msg) e = &pb.Empty{} fname := filename.Msg ext := filepath.Ext(fname) if lext := strings.ToLower(ext); lext != ".db" { fname = fname + ".db" } - err = s.rootSvc.SubSysDataStore.Restore(PATH_DATA_STORE_BACKUP + "/" + fname) + err = s.rootSvc.SubSysDataStore.Restore(PATH_DATA_STORE_BACKUP + "/" + fname, true) return }