diff --git a/.gitignore b/.gitignore index 8c992b8c..a2d494ad 100644 --- a/.gitignore +++ b/.gitignore @@ -202,4 +202,4 @@ pkg/**/*.html web/.yarn/ docs/.yarn/ coverage.txt -/locker/ +/badger/ diff --git a/conf/config-dev.yaml b/conf/config-dev.yaml index f90996ea..723c913f 100644 --- a/conf/config-dev.yaml +++ b/conf/config-dev.yaml @@ -24,23 +24,30 @@ database: sslmode: disable redis: - # redis type available: none, external + # redis type available: none, external. Following all of redis config just use reference here. # none: means never use redis # external: means use the specific redis instance type: external url: redis://:sigma@localhost:6379/0 +badger: + # badger is used to implement lock and cache in a single-node mode. + enabled: true + path: ./badger/ + cache: - # the cache type available is: redis, inmemory, database - type: redis - ttl: 72h - # please attention in multi + # the cache type available is: redis, inmemory, badger + # please attention in multi-node mode, you should use redis + type: badger inmemory: + prefix: sigma-cache size: 10240 redis: - database: - size: 10240 - threshold: 0.2 + prefix: sigma-cache + ttl: 72h + badger: + prefix: sigma-cache + ttl: 72h workqueue: # the workqueue type available: redis, kafka, database, inmemory @@ -53,11 +60,12 @@ workqueue: concurrency: 1024 locker: - # the locker type available: redis, database - type: database - database: - path: ./locker - redis: {} + # the locker type available: redis, badger + type: badger + badger: + prefix: sigma-locker + redis: + prefix: sigma-locker namespace: # push image to registry, if namespace not exist, it will be created automatically diff --git a/conf/config-full.yaml b/conf/config-full.yaml index 601d45f0..6373b799 100644 --- a/conf/config-full.yaml +++ b/conf/config-full.yaml @@ -28,22 +28,30 @@ database: deploy: single redis: - # redis type available: none, external + # redis type available: none, external. Following all of redis config just use reference here. # none: means never use redis # external: means use the specific redis instance type: none url: redis://:sigma@localhost:6379/0 +badger: + # badger is used to implement lock and cache in a single-node mode. + enabled: true + path: /var/lib/sigma/badger/ + cache: - # the cache type available is: redis, inmemory, database - type: redis + # the cache type available is: redis, inmemory, badger + # please attention in multi-node mode, you should use redis + type: badger inmemory: + prefix: sigma-cache size: 10240 - ttl: 72h redis: - database: - size: 10240 - threshold: 0.2 + prefix: sigma-cache + ttl: 72h + badger: + prefix: sigma-cache + ttl: 72h workqueue: # the workqueue type available: redis, kafka, database, inmemory @@ -56,10 +64,12 @@ workqueue: concurrency: 1024 locker: - # the locker type available: redis, database - type: database - database: {} - redis: {} + # the locker type available: redis, badger + type: badger + badger: + prefix: sigma-locker + redis: + prefix: sigma-locker namespace: # push image to registry, if namespace not exist, it will be created automatically diff --git a/docs/docs/configuration.mdx b/docs/docs/configuration.mdx index 4b057cfa..7f502355 100644 --- a/docs/docs/configuration.mdx +++ b/docs/docs/configuration.mdx @@ -28,22 +28,30 @@ database: sslmode: disable redis: - # redis type available: none, external + # redis type available: none, external. Following all of redis config just use reference here. # none: means never use redis # external: means use the specific redis instance type: none url: redis://:sigma@localhost:6379/0 +badger: + # badger is used to implement lock and cache in a single-node mode. + enabled: true + path: /var/lib/sigma/badger/ + cache: - # the cache type available is: redis, inmemory, database - type: redis + # the cache type available is: redis, inmemory, badger + # please attention in multi-node mode, you should use redis + type: badger inmemory: + prefix: sigma-cache size: 10240 redis: + prefix: sigma-cache + ttl: 72h + badger: + prefix: sigma-cache ttl: 72h - database: - size: 10240 - threshold: 0.2 workqueue: # the workqueue type available: redis, kafka, database, inmemory @@ -56,10 +64,12 @@ workqueue: concurrency: 1024 locker: - # the locker type available: redis, database - type: database - database: {} - redis: {} + # the locker type available: redis, badger + type: badger + badger: + prefix: sigma-locker + redis: + prefix: sigma-locker namespace: # push image to registry, if namespace not exist, it will be created automatically diff --git a/pkg/storage/cos/cos.go b/pkg/storage/cos/cos.go index 6a137f4d..17e7fcb3 100644 --- a/pkg/storage/cos/cos.go +++ b/pkg/storage/cos/cos.go @@ -21,7 +21,6 @@ import ( "net/http" "net/url" "path" - "reflect" "strings" gonanoid "github.com/matoous/go-nanoid" @@ -30,11 +29,12 @@ import ( "github.com/go-sigma/sigma/pkg/configs" "github.com/go-sigma/sigma/pkg/consts" "github.com/go-sigma/sigma/pkg/storage" + "github.com/go-sigma/sigma/pkg/types/enums" "github.com/go-sigma/sigma/pkg/utils" ) func init() { - utils.PanicIf(storage.RegisterDriverFactory(path.Base(reflect.TypeOf(factory{}).PkgPath()), &factory{})) + utils.PanicIf(storage.RegisterDriverFactory(enums.StorageTypeCos, &factory{})) } type factory struct{} diff --git a/pkg/storage/filesystem/filesystem.go b/pkg/storage/filesystem/filesystem.go index f7a6b141..237f7226 100644 --- a/pkg/storage/filesystem/filesystem.go +++ b/pkg/storage/filesystem/filesystem.go @@ -21,7 +21,6 @@ import ( "os" "path" "path/filepath" - "reflect" gonanoid "github.com/matoous/go-nanoid" "github.com/rs/zerolog/log" @@ -29,6 +28,7 @@ import ( "github.com/go-sigma/sigma/pkg/configs" "github.com/go-sigma/sigma/pkg/consts" "github.com/go-sigma/sigma/pkg/storage" + "github.com/go-sigma/sigma/pkg/types/enums" "github.com/go-sigma/sigma/pkg/utils" ) @@ -38,7 +38,7 @@ type fs struct { } func init() { - utils.PanicIf(storage.RegisterDriverFactory(path.Base(reflect.TypeOf(factory{}).PkgPath()), &factory{})) + utils.PanicIf(storage.RegisterDriverFactory(enums.StorageTypeFilesystem, &factory{})) } type factory struct{} diff --git a/pkg/storage/oss/oss.go b/pkg/storage/oss/oss.go index 371ae791..1d22a031 100644 --- a/pkg/storage/oss/oss.go +++ b/pkg/storage/oss/oss.go @@ -22,7 +22,6 @@ import ( "net/textproto" "os" "path" - "reflect" "strconv" "strings" @@ -32,11 +31,12 @@ import ( "github.com/go-sigma/sigma/pkg/configs" "github.com/go-sigma/sigma/pkg/consts" "github.com/go-sigma/sigma/pkg/storage" + "github.com/go-sigma/sigma/pkg/types/enums" "github.com/go-sigma/sigma/pkg/utils" ) func init() { - utils.PanicIf(storage.RegisterDriverFactory(path.Base(reflect.TypeOf(factory{}).PkgPath()), &factory{})) + utils.PanicIf(storage.RegisterDriverFactory(enums.StorageTypeOss, &factory{})) } type factory struct{} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 3e4e06d5..96ab4c5d 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -21,7 +21,6 @@ import ( "io" "os" "path" - "reflect" "strings" "github.com/aws/aws-sdk-go/aws" @@ -35,6 +34,7 @@ import ( "github.com/go-sigma/sigma/pkg/configs" "github.com/go-sigma/sigma/pkg/consts" "github.com/go-sigma/sigma/pkg/storage" + "github.com/go-sigma/sigma/pkg/types/enums" "github.com/go-sigma/sigma/pkg/utils" "github.com/go-sigma/sigma/pkg/utils/ptr" ) @@ -47,7 +47,7 @@ type awss3 struct { } func init() { - utils.PanicIf(storage.RegisterDriverFactory(path.Base(reflect.TypeOf(factory{}).PkgPath()), &factory{})) + utils.PanicIf(storage.RegisterDriverFactory(enums.StorageTypeS3, &factory{})) } type factory struct{} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 86317d38..a6d95c60 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -21,9 +21,8 @@ import ( "path" "strings" - "github.com/spf13/viper" - "github.com/go-sigma/sigma/pkg/configs" + "github.com/go-sigma/sigma/pkg/types/enums" ) const ( @@ -80,11 +79,11 @@ type Factory interface { New(config configs.Configuration) (StorageDriver, error) } -var driverFactories = make(map[string]Factory) +var driverFactories = make(map[enums.StorageType]Factory) // RegisterDriverFactory registers a storage factory driver by name. // If RegisterDriverFactory is called twice with the same name or if driver is nil, it panics. -func RegisterDriverFactory(name string, factory Factory) error { +func RegisterDriverFactory(name enums.StorageType, factory Factory) error { if _, ok := driverFactories[name]; ok { return fmt.Errorf("driver %q already registered", name) } @@ -115,10 +114,9 @@ func (s *storageDriverFactory) New() StorageDriver { // Initialize initializes the storage driver func Initialize(config configs.Configuration) error { - typ := viper.GetString("storage.type") - factory, ok := driverFactories[typ] + factory, ok := driverFactories[config.Storage.Type] if !ok { - return fmt.Errorf("driver %q not registered", typ) + return fmt.Errorf("driver %q not registered", config.Storage.Type) } var err error Driver, err = factory.New(config) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index eda23210..ded2a61d 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -18,10 +18,10 @@ import ( "fmt" "testing" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/go-sigma/sigma/pkg/configs" + "github.com/go-sigma/sigma/pkg/types/enums" ) type dummyFactory struct{} @@ -37,35 +37,37 @@ func (dummyFactoryError) New(_ configs.Configuration) (StorageDriver, error) { } func TestRegisterDriverFactory(t *testing.T) { - driverFactories = make(map[string]Factory) + driverFactories = make(map[enums.StorageType]Factory) - err := RegisterDriverFactory("dummy", &dummyFactory{}) + err := RegisterDriverFactory(enums.StorageTypeDummy, &dummyFactory{}) assert.NoError(t, err) - err = RegisterDriverFactory("dummy", &dummyFactory{}) + err = RegisterDriverFactory(enums.StorageTypeDummy, &dummyFactory{}) assert.Error(t, err) } func TestInitialize(t *testing.T) { - driverFactories = make(map[string]Factory) + driverFactories = make(map[enums.StorageType]Factory) - err := RegisterDriverFactory("dummy", &dummyFactory{}) + err := RegisterDriverFactory(enums.StorageTypeDummy, &dummyFactory{}) assert.NoError(t, err) - viper.SetDefault("storage.type", "dummy") - err = Initialize(configs.Configuration{}) + err = Initialize(configs.Configuration{ + Storage: configs.ConfigurationStorage{ + Type: enums.StorageTypeDummy, + }, + }) assert.NoError(t, err) - viper.SetDefault("storage.type", "fake") - err = Initialize(configs.Configuration{}) + err = Initialize(configs.Configuration{ + Storage: configs.ConfigurationStorage{ + Type: "fake", + }, + }) assert.Error(t, err) err = RegisterDriverFactory("dummy-error", &dummyFactoryError{}) assert.NoError(t, err) - - viper.SetDefault("storage.type", "dummy-error") - err = Initialize(configs.Configuration{}) - assert.Error(t, err) } func TestSanitizePath(t *testing.T) { diff --git a/pkg/types/enums/enums.go b/pkg/types/enums/enums.go index e65126ea..4c1077ad 100644 --- a/pkg/types/enums/enums.go +++ b/pkg/types/enums/enums.go @@ -254,6 +254,8 @@ type BuilderSource string // s3, // filesystem, // cos, +// oss, +// dummy, // ) type StorageType string diff --git a/pkg/types/enums/enums_enum.go b/pkg/types/enums/enums_enum.go index ee0ea620..da042e89 100644 --- a/pkg/types/enums/enums_enum.go +++ b/pkg/types/enums/enums_enum.go @@ -2342,6 +2342,10 @@ const ( StorageTypeFilesystem StorageType = "filesystem" // StorageTypeCos is a StorageType of type cos. StorageTypeCos StorageType = "cos" + // StorageTypeOss is a StorageType of type oss. + StorageTypeOss StorageType = "oss" + // StorageTypeDummy is a StorageType of type dummy. + StorageTypeDummy StorageType = "dummy" ) var ErrInvalidStorageType = errors.New("not a valid StorageType") @@ -2362,6 +2366,8 @@ var _StorageTypeValue = map[string]StorageType{ "s3": StorageTypeS3, "filesystem": StorageTypeFilesystem, "cos": StorageTypeCos, + "oss": StorageTypeOss, + "dummy": StorageTypeDummy, } // ParseStorageType attempts to convert a string to a StorageType.