From 6cdabc882e72d2f1ca7b213dbdf3205913abee5c Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Sun, 7 Aug 2022 14:04:22 +0200 Subject: [PATCH 1/5] client,providers: add storj DCS as provider option Signed-off-by: Stefan Benten --- .github/workflows/test.yaml | 2 +- CHANGELOG.md | 1 + Makefile | 2 +- README.md | 26 ++++- client/factory.go | 4 + go.mod | 16 ++- go.sum | 40 ++++++- objtesting/foreach.go | 23 +++- providers/storj/storj.go | 207 ++++++++++++++++++++++++++++++++++++ providers/storj/testing.go | 50 +++++++++ scripts/cfggen/main.go | 16 +-- 11 files changed, 360 insertions(+), 27 deletions(-) create mode 100644 providers/storj/storj.go create mode 100644 providers/storj/testing.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 59a43eba..ce436e82 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,7 +69,7 @@ jobs: - name: Run unit tests env: # THANOS_TEST_OBJSTORE_SKIP: AZURE,COS,ALIYUNOSS,BOS - THANOS_TEST_OBJSTORE_SKIP: GCS,S3,SWIFT,AZURE,COS,ALIYUNOSS,BOS,OCI + THANOS_TEST_OBJSTORE_SKIP: GCS,S3,SWIFT,AZURE,COS,ALIYUNOSS,BOS,OCI,STORJ # Variables for Swift testing. OS_AUTH_URL: http://127.0.0.1:5000/v2.0 OS_PASSWORD: s3cr3t diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a3726e..3656f8dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed ### Added +- [#17](https://github.com/thanos-io/objstore/pull/17) Add Storj DCS Object Storage Bucket support. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. ### Changed diff --git a/Makefile b/Makefile index 72d88f6b..34a9a5fa 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ MDOX_VALIDATE_CONFIG ?= .mdox.validate.yaml .PHONY: test-local test-local: - THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI $(MAKE) test + THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,STORJ $(MAKE) test .PHONY: test test: diff --git a/README.md b/README.md index 9bc2435d..2857f312 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ Current object storage client implementations: | [Baidu BOS](#baidu-bos) | Beta | Production Usage | no | ?? | | [Local Filesystem](#filesystem) | Stable | Testing and Demo only | yes | @bwplotka | | [Oracle Cloud Infrastructure Object Storage](#oracle-cloud-infrastructure-object-storage) | Beta | Production Usage | yes | @aarontams,@gaurav-05,@ericrrath | +| [Storj DCS](#storj-dcs) | Beta | Production Usage | yes | @stefanbenten | **Missing support to some object storage?** Check out [how to add your client section](#how-to-add-a-new-client-to-thanos) @@ -137,7 +138,7 @@ NOTE: Currently Thanos requires strong consistency (write-read) for object store Thanos uses the [minio client](https://github.com/minio/minio-go) library to upload Prometheus data into AWS S3. -> NOTE: S3 client was designed for AWS S3, but it can be configured against other S3-compatible object storages e.g Ceph +> NOTE: S3 client was designed for AWS S3, but it can be configured against other S3-compatible object storages e.g. Ceph. The S# object storage yaml configuration definition: @@ -185,13 +186,13 @@ prefix: "" At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. -However if you set `aws_sdk_auth: true` Thanos will use the default authentication methods of the AWS SDK for go based on [known environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) (`AWS_PROFILE`, `AWS_WEB_IDENTITY_TOKEN_FILE` ... etc) and known AWS config files (~/.aws/config). If you turn this on, then the `bucket` and `endpoint` are the required config keys. +However, if you set `aws_sdk_auth: true` Thanos will use the default authentication methods of the AWS SDK for go based on [known environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) (`AWS_PROFILE`, `AWS_WEB_IDENTITY_TOKEN_FILE` ... etc) and known AWS config files (~/.aws/config). If you turn this on, then the `bucket` and `endpoint` are the required config keys. The field `prefix` can be used to transparently use prefixes in your S3 bucket. This allows you to separate blocks coming from different sources into paths with different prefixes, making it easier to understand what's going on (i.e. you don't have to use Thanos tooling to know from where which blocks came). The AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/s3.html). -Make sure you use a correct signature version. Currently AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. +Make sure you use a correct signature version. Currently, AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. You can configure the timeout settings for the HTTP client by setting the `http_config.idle_conn_timeout` and `http_config.response_header_timeout` keys. As a rule of thumb, if you are seeing errors like `timeout awaiting response headers` in your logs, you may want to increase the value of `http_config.response_header_timeout`. @@ -247,7 +248,7 @@ You will also need to apply the following AWS IAM policy for the user to access ###### Credentials -By default Thanos will try to retrieve credentials from the following sources: +By default, Thanos will try to retrieve credentials from the following sources: 1. From config file if BOTH `access_key` and `secret_key` are present. 2. From the standard AWS environment variable - `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` @@ -329,7 +330,7 @@ Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/usin ###### STS Endpoint -If you want to use IAM credential retrieved from an instance profile, Thanos needs to authenticate through AWS STS. For this purposes you can specify your own STS Endpoint. +If you want to use IAM credential retrieved from an instance profile, Thanos needs to authenticate through AWS STS. For this purpose you can specify your own STS Endpoint. By default Thanos will use endpoint: https://sts.amazonaws.com and AWS region corresponding endpoints. @@ -637,6 +638,21 @@ config: You can also include any of the optional configuration just like the example in `Default Provider`. +### Storj DCS + +In order to be able to configure [Storj DCS](https://storj.io/signup) as Thanos Object Store Backend you will need to +provide an access grant and a bucket name to be used to hold the objects. +You can choose an already existing bucket or a new name and the bucket will be created for you. + +The following snippet shows how to add these information to the configuration yaml. + +```yaml +type: STORJ +config: + access: "" + bucket: "" +``` + #### How to add a new client to Thanos? Following checklist allows adding new Go code client to supported providers: diff --git a/client/factory.go b/client/factory.go index bfe4370f..a416e9e9 100644 --- a/client/factory.go +++ b/client/factory.go @@ -23,6 +23,7 @@ import ( "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" ) @@ -38,6 +39,7 @@ const ( ALIYUNOSS ObjProvider = "ALIYUNOSS" BOS ObjProvider = "BOS" OCI ObjProvider = "OCI" + STORJ ObjProvider = "STORJ" ) type BucketConfig struct { @@ -80,6 +82,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = bos.NewBucket(logger, config, component) case string(OCI): bucket, err = oci.NewBucket(logger, config) + case string(STORJ): + bucket, err = storj.NewBucket(logger, config, component) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/go.mod b/go.mod index 8142f569..f137eb79 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,8 @@ require ( google.golang.org/api v0.80.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 + storj.io/common v0.0.0-20220802175255-aae0c09ec9d4 + storj.io/uplink v1.9.0 ) require ( @@ -56,6 +58,7 @@ require ( github.com/aws/smithy-go v1.11.1 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/calebcase/tmpfile v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -63,6 +66,7 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -74,6 +78,7 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.0 // indirect @@ -91,17 +96,22 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/smartystreets/assertions v1.2.1 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.18 // indirect + github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect + github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/goleak v1.1.12 // indirect - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect - golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220524023933-508584e28198 // indirect google.golang.org/grpc v1.46.2 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.57.0 // indirect + storj.io/drpc v0.0.32 // indirect ) diff --git a/go.sum b/go.sum index 88584d5b..0d04cbaa 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= +github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -190,6 +192,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -268,6 +272,7 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20220520215854-d04f2422c8a1 h1:K4bn56FHdjFCfjSo3wWaD6rJL8r9yvmmncJNMhdkKrw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -301,12 +306,15 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -396,6 +404,8 @@ github.com/smartystreets/assertions v1.2.1/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrxf github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spacemonkeygo/monkit/v3 v3.0.18 h1:in58fMOAD+w70PaNy9GLNAsd34tErGYANZsEPWBMuvs= +github.com/spacemonkeygo/monkit/v3 v3.0.18/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -411,11 +421,21 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.34 h1:xm+Pg+6m486y4eugRI7/E4WasbVmpY1hp9QBSRErgp8= github.com/tencentyun/cos-go-sdk-v5 v0.7.34/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= +github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -439,8 +459,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -491,6 +511,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -522,8 +543,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -681,6 +702,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -689,6 +711,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -700,8 +723,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -900,3 +924,9 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +storj.io/common v0.0.0-20220802175255-aae0c09ec9d4 h1:80gL3xdEt7BgL+I1oTkql3KD/wyTrARSTKphqrVPf6g= +storj.io/common v0.0.0-20220802175255-aae0c09ec9d4/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys= +storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= +storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= +storj.io/uplink v1.9.0 h1:Zg1kX1VqOQIKm0yAukteKpLuT68Be3euyNRML612ERM= +storj.io/uplink v1.9.0/go.mod h1:f6D8306j5mnRHnPDKWCiwtPM6ukyGg77to9LaAY9l6k= diff --git a/objtesting/foreach.go b/objtesting/foreach.go index cc96a2fd..a9a04cca 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -9,8 +9,6 @@ import ( "strings" "testing" - "github.com/thanos-io/objstore/providers/oci" - "github.com/efficientgo/tools/core/pkg/testutil" "github.com/thanos-io/objstore" @@ -20,18 +18,20 @@ import ( "github.com/thanos-io/objstore/providers/cos" "github.com/thanos-io/objstore/providers/filesystem" "github.com/thanos-io/objstore/providers/gcs" + "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" ) // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: -// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI. +// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,STORJ. func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { obstores := strings.Split(e, ",") - for _, objstore := range obstores { - if objstore == string(provider) { + for _, store := range obstores { + if store == string(provider) { t.Logf("%s found in THANOS_TEST_OBJSTORE_SKIP array. Skipping.", provider) return true } @@ -185,4 +185,17 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) testFn(t, bkt) }) } + + // Optional STORJ. + if !IsObjStoreSkipped(t, client.STORJ) { + t.Run("oci", func(t *testing.T) { + bkt, closeFn, err := storj.NewTestBucket(t) + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() + + testFn(t, bkt) + }) + } } diff --git a/providers/storj/storj.go b/providers/storj/storj.go new file mode 100644 index 00000000..f8e4131d --- /dev/null +++ b/providers/storj/storj.go @@ -0,0 +1,207 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storj + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/thanos-io/objstore" + "gopkg.in/yaml.v2" + "storj.io/common/fpath" + "storj.io/uplink" +) + +// Config stores the configuration for the storj bucket. +type Config struct { + AccessGrant string `yaml:"access"` + Bucket string `yaml:"bucket"` +} + +// Bucket implements the store.Bucket interface against storj API. +type Bucket struct { + logger log.Logger + project *uplink.Project + bucket *uplink.Bucket +} + +// NewBucket returns a new Bucket using the provided storj config values. +func NewBucket(logger log.Logger, config []byte, component string) (*Bucket, error) { + var conf Config + + err := yaml.Unmarshal(config, &conf) + if err != nil { + return nil, err + } + + return NewBucketWithConfig(logger, conf, component) +} + +// NewBucketWithConfig returns a new Bucket using the provided storj config struct. +func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { + var instance Bucket + + ctx := fpath.WithTempData(context.TODO(), "", true) + + uplConf := &uplink.Config{ + UserAgent: fmt.Sprintf("thanos-%s", component), + } + + parsedAccess, err := uplink.ParseAccess(config.AccessGrant) + if err != nil { + return nil, err + } + + instance.project, err = uplConf.OpenProject(ctx, parsedAccess) + if err != nil { + return nil, err + } + + instance.bucket, err = instance.project.EnsureBucket(ctx, config.Bucket) + if err != nil { + //Ignoring the error to return the one that occurred first, while still trying to clean up. + _ = instance.project.Close() + return nil, err + } + + instance.logger = logger + + return &instance, nil +} + +// Name returns the name of the bucket. +func (b *Bucket) Name() string { + return b.bucket.Name +} + +// Close closes the Bucket. +func (b *Bucket) Close() error { + return b.project.Close() +} + +// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { + _ = level.Debug(b.logger).Log("Listing %s from Storj Bucket", dir) + + if dir != "" { + dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim + } + + opt := uplink.ListObjectsOptions{ + Recursive: objstore.ApplyIterOptions(options...).Recursive, + Prefix: dir, + } + + iter := b.project.ListObjects(ctx, b.bucket.Name, &opt) + + for iter.Next() { + if err := f(iter.Item().Key); err != nil { + return err + } + } + return iter.Err() +} + +// Get returns a reader for the given object name. +func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + _ = level.Debug(b.logger).Log("Getting attributes %s from Storj Bucket", name) + + options := uplink.DownloadOptions{} + + download, err := b.project.DownloadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, &options) + if err != nil { + return nil, err + } + + return download, nil +} + +// GetRange returns a reader to the range for the given object name. +func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + _ = level.Debug(b.logger).Log("Getting range of %s from Storj Bucket", name) + + options := uplink.DownloadOptions{ + Offset: off, + Length: length, + } + + download, err := b.project.DownloadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, &options) + if err != nil { + return nil, err + } + + return download, nil +} + +// Exists returns whether the object with the given name exists or not. +func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { + _ = level.Debug(b.logger).Log("Ensuring %s exists in Storj Bucket", name) + _, err := b.project.StatObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + if err != nil { + if b.IsObjNotFoundErr(err) { + return false, nil + } + } + return true, err +} + +// IsObjNotFoundErr returns whether the given error matches the object stores non found error. +func (b *Bucket) IsObjNotFoundErr(err error) bool { + return errors.Is(err, uplink.ErrObjectNotFound) +} + +// Attributes returns information about the specified object. +func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { + _ = level.Debug(b.logger).Log("Getting attributes %s from Storj Bucket", name) + + attr := objstore.ObjectAttributes{} + + obj, err := b.project.StatObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + if err != nil { + return attr, err + } + + attr.Size = obj.System.ContentLength + attr.LastModified = obj.System.Created + + return attr, nil +} + +// Upload the contents of the reader as an object into the bucket. +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { + var uploadOptions *uplink.UploadOptions + + _ = level.Debug(b.logger).Log("Uploading %s to Storj Bucket", name) + + writer, err := b.project.UploadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, uploadOptions) + if err != nil { + return err + } + + _, err = io.Copy(writer, r) + if err != nil { + //Ignoring the error to return the one that occurred first, while still trying to clean up. + _ = writer.Abort() + return err + } + + return writer.Commit() +} + +// Delete removes the object with the given name. +func (b *Bucket) Delete(ctx context.Context, name string) error { + var err error + + _ = level.Debug(b.logger).Log("Deleting %s from Storj Bucket", name) + + _, err = b.project.DeleteObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + + return err +} diff --git a/providers/storj/testing.go b/providers/storj/testing.go new file mode 100644 index 00000000..ec824818 --- /dev/null +++ b/providers/storj/testing.go @@ -0,0 +1,50 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storj + +import ( + "context" + "os" + "testing" + + "github.com/go-kit/log" + "github.com/thanos-io/objstore" + "gopkg.in/yaml.v2" + "storj.io/uplink" +) + +// NewTestBucket creates test bkt client that before returning creates temporary bucket. +// In a close function it empties and deletes the bucket. +func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { + + ctx := context.Background() + bktName := objstore.CreateTemporaryTestBucketName(t) + access := os.Getenv("STORJ_ACCESS") + + storjConfig, err := yaml.Marshal(Config{AccessGrant: access, Bucket: bktName}) + if err != nil { + return nil, nil, err + } + + bkt, err := NewBucket(log.NewNopLogger(), storjConfig, "testing") + if err != nil { + t.Errorf("failed to create temporary Storj DCS bucket '%s' for testing", bktName) + return nil, nil, err + } + + t.Logf("created temporary Storj DCS bucket '%s' for testing", bkt.Name()) + return bkt, func() { + objstore.EmptyBucket(t, ctx, bkt) + if _, err := bkt.deleteBucket(ctx, bkt.Name()); err != nil { + t.Logf("failed to delete temporary Storj DCS bucket %s for testing: %s", bkt.Name(), err) + } + t.Logf("deleted temporary Storj DCS bucket '%s' for testing", bkt.Name()) + }, nil +} + +// deleteBucket deletes an empty bucket and otherwise returns an error. +// Intentionally not using DeleteBucketWithObjects to ensure not deleting production data. +func (b *Bucket) deleteBucket(ctx context.Context, name string) (*uplink.Bucket, error) { + return b.project.DeleteBucket(ctx, name) +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 57afbe1f..fa754c50 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -11,23 +11,24 @@ import ( "reflect" "strings" - "github.com/thanos-io/objstore/providers/oci" - "github.com/fatih/structtag" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/pkg/errors" + "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/yaml.v2" + "github.com/thanos-io/objstore/client" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" "github.com/thanos-io/objstore/providers/cos" "github.com/thanos-io/objstore/providers/filesystem" "github.com/thanos-io/objstore/providers/gcs" + "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" - "gopkg.in/alecthomas/kingpin.v2" - "gopkg.in/yaml.v2" ) var ( @@ -44,6 +45,7 @@ var ( client.FILESYSTEM: filesystem.Config{}, client.BOS: bos.Config{}, client.OCI: oci.Config{}, + client.STORJ: storj.Config{}, } ) @@ -70,19 +72,19 @@ func main() { errLogger := level.Error(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))) if _, err := app.Parse(os.Args[1:]); err != nil { - errLogger.Log("err", err) + _ = errLogger.Log("err", err) os.Exit(1) } if c, ok := configs[*structName]; ok { if err := generate(c, os.Stdout); err != nil { - errLogger.Log("err", err) + _ = errLogger.Log("err", err) os.Exit(1) } return } - errLogger.Log("err", errors.Errorf("%v struct not found. Possible values %v", *structName, strings.Join(possibleValues, ","))) + _ = errLogger.Log("err", errors.Errorf("%v struct not found. Possible values %v", *structName, strings.Join(possibleValues, ","))) os.Exit(1) } From 16aa4991c60e733b0f4516018936465e12a5826a Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Sun, 7 Aug 2022 15:21:03 +0200 Subject: [PATCH 2/5] Changelog.md: updating PR link Signed-off-by: Stefan Benten --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3656f8dc..0a696e97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Fixed ### Added -- [#17](https://github.com/thanos-io/objstore/pull/17) Add Storj DCS Object Storage Bucket support. +- [#18](https://github.com/thanos-io/objstore/pull/18) Add Storj DCS Object Storage Bucket support. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. ### Changed From 3f6627d90f79af8b4eee305ff7450e1629245358 Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Tue, 9 Aug 2022 12:53:40 +0200 Subject: [PATCH 3/5] providers/storj/storj.go: fix import lint Signed-off-by: Stefan Benten --- providers/storj/storj.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/storj/storj.go b/providers/storj/storj.go index f8e4131d..4953a4f0 100644 --- a/providers/storj/storj.go +++ b/providers/storj/storj.go @@ -5,13 +5,13 @@ package storj import ( "context" - "errors" "fmt" "io" "strings" "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/pkg/errors" "github.com/thanos-io/objstore" "gopkg.in/yaml.v2" "storj.io/common/fpath" From b1989b6ba830789986ff3a8e22d3e66a663d7df9 Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Thu, 13 Apr 2023 21:22:31 +0200 Subject: [PATCH 4/5] providers/storj/storj.go: remove debug log Signed-off-by: Stefan Benten --- providers/storj/storj.go | 1 - 1 file changed, 1 deletion(-) diff --git a/providers/storj/storj.go b/providers/storj/storj.go index 4953a4f0..274b67a4 100644 --- a/providers/storj/storj.go +++ b/providers/storj/storj.go @@ -88,7 +88,6 @@ func (b *Bucket) Close() error { // Iter calls f for each entry in the given directory (not recursive). The argument to f is the full // object name including the prefix of the inspected directory. func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { - _ = level.Debug(b.logger).Log("Listing %s from Storj Bucket", dir) if dir != "" { dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim From 9fd902a46103dfc87ba5398acfbf9bc2d68e4dbd Mon Sep 17 00:00:00 2001 From: Stefan Benten Date: Fri, 23 Jun 2023 13:22:09 +0200 Subject: [PATCH 5/5] objtesting/foreach.go: fix indentation issues Signed-off-by: Stefan Benten --- objtesting/foreach.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/objtesting/foreach.go b/objtesting/foreach.go index da478541..d897315f 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -190,12 +190,12 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) if !IsObjStoreSkipped(t, client.STORJ) { t.Run("oci", func(t *testing.T) { bkt, closeFn, err := storj.NewTestBucket(t) - testutil.Ok(t, err) + testutil.Ok(t, err) t.Parallel() defer closeFn() - - testFn(t, bkt) + + testFn(t, bkt) }) } @@ -209,7 +209,6 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) defer closeFn() testFn(t, bkt) - testFn(t, objstore.NewPrefixedBucket(bkt, "some_prefix")) }) }