From 971ec8de014a55a41656d449fa436218e28d1d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernd-Ren=C3=A9=20Predota?= Date: Tue, 11 May 2021 09:45:04 +0200 Subject: [PATCH] uuid with ulid replaced --- .gitignore | 1 + go.mod | 6 +-- go.sum | 11 ++-- lib/cache/cache.go | 107 ++++++++++++++++++++++++++++++++++++++ lib/common/common.go | 54 ++----------------- lib/common/guid.go | 51 ++++++++++++++++++ main.go | 1 - models/token.go | 5 +- repositories/repsitory.go | 12 +++++ 9 files changed, 188 insertions(+), 60 deletions(-) create mode 100644 lib/cache/cache.go create mode 100644 lib/common/guid.go create mode 100644 repositories/repsitory.go diff --git a/.gitignore b/.gitignore index 5c092e4..192fd66 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ .env .ignore .unison* +.unotes/ /.vscode __pycache__/ _vendor-*/ diff --git a/go.mod b/go.mod index d8c2548..2e9ae8e 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module git.devices.local/mawas/golang-api-skeleton go 1.15 require ( - github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect - github.com/dgraph-io/badger/v2 v2.2007.2 // indirect + github.com/dgraph-io/badger/v2 v2.2007.2 github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-ozzo/ozzo-validation v3.6.0+incompatible github.com/go-ozzo/ozzo-validation/v4 v4.3.0 - github.com/google/uuid v1.2.0 github.com/magiconair/properties v1.8.4 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/oklog/ulid v1.3.1 + github.com/oklog/ulid/v2 v2.0.2 github.com/pelletier/go-toml v1.8.1 // indirect github.com/sirupsen/logrus v1.7.0 github.com/spf13/afero v1.5.1 // indirect diff --git a/go.sum b/go.sum index eb9888f..c620e86 100644 --- a/go.sum +++ b/go.sum @@ -11,11 +11,11 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -103,8 +103,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -246,8 +244,12 @@ github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/oklog/ulid/v2 v2.0.2 h1:r4fFzBm+bv0wNKNh5eXTwU7i85y5x+uwkxCUTNVQqLc= +github.com/oklog/ulid/v2 v2.0.2/go.mod h1:mtBL0Qe/0HAx6/a4Z30qxVIAL1eQDweXq5lxOEiwQ68= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= @@ -292,6 +294,7 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= @@ -476,8 +479,8 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= diff --git a/lib/cache/cache.go b/lib/cache/cache.go new file mode 100644 index 0000000..7889ac7 --- /dev/null +++ b/lib/cache/cache.go @@ -0,0 +1,107 @@ +package cache + +import ( + "fmt" + "time" + + badger "github.com/dgraph-io/badger/v2" + // log "github.com/sirupsen/logrus" +) + +type Reader interface { + Get(key string) (*string, error) +} + +type Writer interface { + Set(key string, value string) error + SetWithTTL(key string, value string, ttl int) error +} + +type Cache interface { + Reader + Writer + Close() +} + +type bCache struct { + db *badger.DB + // logger *log.Entry +} + +func NewCache(db *badger.DB) Cache { + return &bCache{ + db: db, + } +} + +func (cache bCache) close() { + cache.db.Close() +} + +// Get gets a cache entry by key +func (cache bCache) Get(key string) (*string, error) { + var result []byte + if err := cache.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { + if err.Error() != "Key not found" { + return nil + } + } + if err == nil || (item != nil && !item.IsDeletedOrExpired()) { + if err := item.Value(func(val []byte) error { + if err != nil { + return err + } + result = append([]byte{}, val...) + return nil + }); err != nil { + return err + } + } + return nil + }); err != nil { + return nil, err + } + r := string(result) + return &r, nil +} + +// SetWithTTL sets a cache entry +func (cache bCache) Set(key string, value string) error { + if err := cache.db.Update(func(txn *badger.Txn) error { + err := txn.Set([]byte(key), []byte(value)) + return err + }); err != nil { + // cache.logger.WithFields(log.Fields{"module": "cache"}).Errorln(err.Error()) + return err + } + return nil +} + +// SetWithTTL sets a cache entry with defined ttl +func (cache bCache) SetWithTTL(key string, value string, ttl int) error { + if err := cache.db.Update(func(txn *badger.Txn) error { + e := badger.NewEntry([]byte(key), []byte(value)).WithTTL(time.Duration(ttl) * time.Second) + err := txn.SetEntry(e) + return err + }); err != nil { + // cache.logger.WithFields(log.Fields{"module": "cache"}).Errorln(err.Error()) + return err + } + return nil +} + +func (cache bCache) Close() { + cache.close() +} + +func Bootstrap() (Cache, error) { + db, err := badger.Open(badger.DefaultOptions("").WithInMemory(true)) + if err != nil { + return nil, fmt.Errorf("Unable to open badger db: %s", err.Error()) + } + + cache := NewCache(db) + return cache, err +} diff --git a/lib/common/common.go b/lib/common/common.go index faf40dc..ea02a61 100644 --- a/lib/common/common.go +++ b/lib/common/common.go @@ -1,59 +1,13 @@ package common import ( - "database/sql/driver" + "crypto/rand" "time" - "github.com/google/uuid" + "github.com/oklog/ulid" "gorm.io/gorm" ) -type UsernameContextKey string - -// GUID -> new datatype -type GUID uuid.UUID - -// StringToGUID -> parse string to GUID -func StringToGUID(s string) (GUID, error) { - id, err := uuid.Parse(s) - return GUID(id), err -} - -// String -> String Representation of Binary16 -func (guid GUID) String() string { - return uuid.UUID(guid).String() -} - -// GormDataType -> sets type to binary(16) -func (guid GUID) GormDataType() string { - return "binary(16)" -} - -func (guid GUID) MarshalJSON() ([]byte, error) { - s := uuid.UUID(guid) - str := "\"" + s.String() + "\"" - return []byte(str), nil -} - -func (guid *GUID) UnmarshalJSON(by []byte) error { - s, err := uuid.ParseBytes(by) - *guid = GUID(s) - return err -} - -// Scan --> tells GORM how to receive from the database -func (guid *GUID) Scan(value interface{}) error { - bytes, _ := value.([]byte) - parseByte, err := uuid.FromBytes(bytes) - *guid = GUID(parseByte) - return err -} - -// Value -> tells GORM how to save into the database -func (guid GUID) Value() (driver.Value, error) { - return uuid.UUID(guid).MarshalBinary() -} - type BasicFields struct { CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` CreatedByDB GUID `gorm:"column:created_by" json:"-"` @@ -97,7 +51,7 @@ type ModelGUIDPK struct { } func (guidPK *ModelGUIDPK) BeforeCreate(tx *gorm.DB) error { - id, err := uuid.NewRandom() + id, err := ulid.New(ulid.Now(), rand.Reader) guidPK.ID = GUID(id) if userID, ok := tx.Get("userID"); ok { if guid, ok := userID.(GUID); ok { @@ -114,7 +68,7 @@ type ModelHiddenGUIDPK struct { } func (guidPK *ModelHiddenGUIDPK) BeforeCreate(tx *gorm.DB) error { - id, err := uuid.NewRandom() + id, err := ulid.New(ulid.Now(), rand.Reader) guidPK.ID = GUID(id) if userID, ok := tx.Get("userID"); ok { if guid, ok := userID.(GUID); ok { diff --git a/lib/common/guid.go b/lib/common/guid.go new file mode 100644 index 0000000..fb8e70b --- /dev/null +++ b/lib/common/guid.go @@ -0,0 +1,51 @@ +package common + +import ( + "database/sql/driver" + + "github.com/oklog/ulid/v2" +) + +// GUID our new ULID based datatype +type GUID ulid.ULID + +// StringToGUID -> parse string to ULID GUID +func StringToGUID(s string) (GUID, error) { + id, err := ulid.Parse(s) // maybe ParseStrict + return GUID(id), err +} + +// String Representation of Binary16 +func (guid GUID) String() string { + return ulid.ULID(guid).String() +} + +// GormDataType sets type to binary(16) +func (guid GUID) GormDataType() string { + return "binary(16)" +} + +func (guid GUID) MarshalJSON() ([]byte, error) { + str := "\"" + ulid.ULID(guid).String() + "\"" + return []byte(str), nil +} + +func (guid *GUID) UnmarshalJSON(data []byte) error { + id, err := ulid.Parse(string(data)) // maybe ParseStrict + *guid = GUID(id) + return err +} + +// Scan tells GORM how to receive from the database +func (guid *GUID) Scan(value interface{}) error { + converted := ulid.ULID(*guid) + c := &converted + err := c.Scan(value) + *guid = GUID(*c) + return err +} + +// Value tells GORM how to save into the database +func (guid GUID) Value() (driver.Value, error) { + return ulid.ULID(guid).Value() +} diff --git a/main.go b/main.go index a8bb6b7..8698efa 100644 --- a/main.go +++ b/main.go @@ -22,7 +22,6 @@ import ( // https://www.voile.com/voile-straps.html // FIXME how to set unique index for username to make soft deletes possible but disallow duplicates... also how to cache a username which got delete and recerated... user ID would get overwritten, maybe pre/suffix deleted username? -// TODO replace uuids with ulids // TODO use badgerdb for permissions? path:[token1, token2...] // TODO add database loadbalancing feature... so allow more hosts use first as default and other ones as fallback // TODO add userid and username to repo diff --git a/models/token.go b/models/token.go index 449aa83..32ee467 100644 --- a/models/token.go +++ b/models/token.go @@ -1,12 +1,13 @@ package models import ( + "crypto/rand" "database/sql" "time" "git.devices.local/mawas/golang-api-skeleton/lib/cache" "git.devices.local/mawas/golang-api-skeleton/lib/common" - "github.com/google/uuid" + "github.com/oklog/ulid" "gorm.io/gorm" ) @@ -23,7 +24,7 @@ type Token struct { } func (token *Token) BeforeCreate(tx *gorm.DB) error { - id, err := uuid.NewRandom() + id, err := ulid.New(ulid.Now(), rand.Reader) token.Token = common.GUID(id) if userID, ok := tx.Get("userID"); ok { if guid, ok := userID.(common.GUID); ok { diff --git a/repositories/repsitory.go b/repositories/repsitory.go new file mode 100644 index 0000000..a004e53 --- /dev/null +++ b/repositories/repsitory.go @@ -0,0 +1,12 @@ +package repositories + +import ( + "git.devices.local/mawas/golang-api-skeleton/lib/common" + "gorm.io/gorm" +) + +type repo struct { + db *gorm.DB + userID common.GUID + username string +}