diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8b83c6c2..7eea0972 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,7 +69,7 @@ jobs: - name: Run unit tests env: # THANOS_TEST_OBJSTORE_SKIP: AZURE,COS,ALIYUNOSS,BOS - THANOS_TEST_OBJSTORE_SKIP: GCS,S3,SWIFT,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS + THANOS_TEST_OBJSTORE_SKIP: GCS,S3,SWIFT,AZURE,COS,ALIYUNOSS,BOS,OCI,OBS,STORJ # Variables for Swift testing. OS_AUTH_URL: http://127.0.0.1:5000/v2.0 OS_PASSWORD: s3cr3t diff --git a/CHANGELOG.md b/CHANGELOG.md index 05aafa7b..b65ffb00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. ### Added +- [#18](https://github.com/thanos-io/objstore/pull/18) Add Storj DCS Object Storage Bucket support. - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. - [#25](https://github.com/thanos-io/objstore/pull/25) S3: Support specifying S3 storage class. - [#32](https://github.com/thanos-io/objstore/pull/32) Swift: Support authentication using application credentials. diff --git a/Makefile b/Makefile index 09fd0d3e..f114686a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ MDOX_VALIDATE_CONFIG ?= .mdox.validate.yaml .PHONY: test-local test-local: - THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS $(MAKE) test + THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,OBS,STORJ $(MAKE) test .PHONY: test test: diff --git a/README.md b/README.md index 5cf090e3..37fe58f3 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,8 @@ Current object storage client implementations: | [Baidu BOS](#baidu-bos) | Beta | Production Usage | no | @yahaa | | [Local Filesystem](#filesystem) | Stable | Testing and Demo only | yes | @bwplotka | | [Oracle Cloud Infrastructure Object Storage](#oracle-cloud-infrastructure-object-storage) | Beta | Production Usage | yes | @aarontams,@gaurav-05,@ericrrath | -| [HuaweiCloud OBS](#huaweicloud-obs) | Beta | Production Usage | no | @setoru | +| [Storj DCS](#storj-dcs) | Beta | Production Usage | yes | @stefanbenten | +| [HuaweiCloud OBS](#huaweicloud-obs) | Beta | Production Usage | no | @setoru | **Missing support to some object storage?** Check out [how to add your client section](#how-to-add-a-new-client-to-thanos) @@ -138,7 +139,7 @@ NOTE: Currently Thanos requires strong consistency (write-read) for object store Thanos uses the [minio client](https://github.com/minio/minio-go) library to upload Prometheus data into AWS S3. -> NOTE: S3 client was designed for AWS S3, but it can be configured against other S3-compatible object storages e.g Ceph +> NOTE: S3 client was designed for AWS S3, but it can be configured against other S3-compatible object storages e.g. Ceph. The S# object storage yaml configuration definition: @@ -186,13 +187,13 @@ prefix: "" At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. -However if you set `aws_sdk_auth: true` Thanos will use the default authentication methods of the AWS SDK for go based on [known environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) (`AWS_PROFILE`, `AWS_WEB_IDENTITY_TOKEN_FILE` ... etc) and known AWS config files (~/.aws/config). If you turn this on, then the `bucket` and `endpoint` are the required config keys. +However, if you set `aws_sdk_auth: true` Thanos will use the default authentication methods of the AWS SDK for go based on [known environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) (`AWS_PROFILE`, `AWS_WEB_IDENTITY_TOKEN_FILE` ... etc) and known AWS config files (~/.aws/config). If you turn this on, then the `bucket` and `endpoint` are the required config keys. The field `prefix` can be used to transparently use prefixes in your S3 bucket. This allows you to separate blocks coming from different sources into paths with different prefixes, making it easier to understand what's going on (i.e. you don't have to use Thanos tooling to know from where which blocks came). The AWS region to endpoint mapping can be found in this [link](https://docs.aws.amazon.com/general/latest/gr/s3.html). -Make sure you use a correct signature version. Currently AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. +Make sure you use a correct signature version. Currently, AWS requires signature v4, so it needs `signature_version2: false`. If you don't specify it, you will get an `Access Denied` error. On the other hand, several S3 compatible APIs use `signature_version2: true`. You can configure the timeout settings for the HTTP client by setting the `http_config.idle_conn_timeout` and `http_config.response_header_timeout` keys. As a rule of thumb, if you are seeing errors like `timeout awaiting response headers` in your logs, you may want to increase the value of `http_config.response_header_timeout`. @@ -248,7 +249,7 @@ You will also need to apply the following AWS IAM policy for the user to access ###### Credentials -By default Thanos will try to retrieve credentials from the following sources: +By default, Thanos will try to retrieve credentials from the following sources: 1. From config file if BOTH `access_key` and `secret_key` are present. 2. From the standard AWS environment variable - `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` @@ -330,7 +331,7 @@ Details about AWS policies: https://docs.aws.amazon.com/AmazonS3/latest/dev/usin ###### STS Endpoint -If you want to use IAM credential retrieved from an instance profile, Thanos needs to authenticate through AWS STS. For this purposes you can specify your own STS Endpoint. +If you want to use IAM credential retrieved from an instance profile, Thanos needs to authenticate through AWS STS. For this purpose you can specify your own STS Endpoint. By default Thanos will use endpoint: https://sts.amazonaws.com and AWS region corresponding endpoints. @@ -641,6 +642,21 @@ config: You can also include any of the optional configuration just like the example in `Default Provider`. +### Storj DCS + +In order to be able to configure [Storj DCS](https://storj.io/signup) as Thanos Object Store Backend you will need to +provide an access grant and a bucket name to be used to hold the objects. +You can choose an already existing bucket or a new name and the bucket will be created for you. + +The following snippet shows how to add these information to the configuration yaml. + +```yaml +type: STORJ +config: + access: "" + bucket: "" +``` + ##### HuaweiCloud OBS To use HuaweiCloud OBS as an object store, you should apply for a HuaweiCloud Account to create an object storage bucket at first. diff --git a/client/factory.go b/client/factory.go index 12d61892..3d536c61 100644 --- a/client/factory.go +++ b/client/factory.go @@ -18,6 +18,7 @@ import ( "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" "github.com/go-kit/log" @@ -39,6 +40,7 @@ const ( ALIYUNOSS ObjProvider = "ALIYUNOSS" BOS ObjProvider = "BOS" OCI ObjProvider = "OCI" + STORJ ObjProvider = "STORJ" OBS ObjProvider = "OBS" ) @@ -82,6 +84,8 @@ func NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registe bucket, err = bos.NewBucket(logger, config, component) case string(OCI): bucket, err = oci.NewBucket(logger, config) + case string(STORJ): + bucket, err = storj.NewBucket(logger, config, component) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/go.mod b/go.mod index 8a04716c..6f348d69 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( google.golang.org/api v0.80.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v2 v2.4.0 + storj.io/common v0.0.0-20220802175255-aae0c09ec9d4 + storj.io/uplink v1.9.0 ) require ( @@ -48,12 +50,14 @@ require ( github.com/aws/smithy-go v1.11.1 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/calebcase/tmpfile v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -79,13 +83,18 @@ require ( github.com/rs/xid v1.4.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.18 // indirect github.com/stretchr/objx v0.2.0 // indirect + github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect + github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.23.0 // indirect + golang.org/x/crypto v0.3.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220524023933-508584e28198 // indirect google.golang.org/grpc v1.46.2 // indirect @@ -93,6 +102,7 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v3 v3.0.0 // indirect + storj.io/drpc v0.0.32 // indirect ) require ( @@ -101,5 +111,4 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - golang.org/x/crypto v0.3.0 // indirect ) diff --git a/go.sum b/go.sum index d6b3a275..02b7f4c0 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= +github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -172,6 +174,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -249,6 +253,7 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -281,11 +286,13 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -374,6 +381,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spacemonkeygo/monkit/v3 v3.0.18 h1:in58fMOAD+w70PaNy9GLNAsd34tErGYANZsEPWBMuvs= +github.com/spacemonkeygo/monkit/v3 v3.0.18/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -390,11 +399,21 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= +github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= +github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -463,6 +482,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -651,6 +671,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -659,6 +680,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -669,8 +691,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -870,3 +893,9 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +storj.io/common v0.0.0-20220802175255-aae0c09ec9d4 h1:80gL3xdEt7BgL+I1oTkql3KD/wyTrARSTKphqrVPf6g= +storj.io/common v0.0.0-20220802175255-aae0c09ec9d4/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys= +storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= +storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= +storj.io/uplink v1.9.0 h1:Zg1kX1VqOQIKm0yAukteKpLuT68Be3euyNRML612ERM= +storj.io/uplink v1.9.0/go.mod h1:f6D8306j5mnRHnPDKWCiwtPM6ukyGg77to9LaAY9l6k= diff --git a/objtesting/foreach.go b/objtesting/foreach.go index 87d9ed1b..d897315f 100644 --- a/objtesting/foreach.go +++ b/objtesting/foreach.go @@ -19,18 +19,19 @@ import ( "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" "github.com/efficientgo/core/testutil" ) // IsObjStoreSkipped returns true if given provider ID is found in THANOS_TEST_OBJSTORE_SKIP array delimited by comma e.g: -// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI. +// THANOS_TEST_OBJSTORE_SKIP=GCS,S3,AZURE,SWIFT,COS,ALIYUNOSS,BOS,OCI,STORJ. func IsObjStoreSkipped(t *testing.T, provider client.ObjProvider) bool { if e, ok := os.LookupEnv("THANOS_TEST_OBJSTORE_SKIP"); ok { obstores := strings.Split(e, ",") - for _, objstore := range obstores { - if objstore == string(provider) { + for _, store := range obstores { + if store == string(provider) { t.Logf("%s found in THANOS_TEST_OBJSTORE_SKIP array. Skipping.", provider) return true } @@ -185,6 +186,19 @@ func ForeachStore(t *testing.T, testFn func(t *testing.T, bkt objstore.Bucket)) }) } + // Optional STORJ. + if !IsObjStoreSkipped(t, client.STORJ) { + t.Run("oci", func(t *testing.T) { + bkt, closeFn, err := storj.NewTestBucket(t) + testutil.Ok(t, err) + + t.Parallel() + defer closeFn() + + testFn(t, bkt) + }) + } + // Optional OBS. if !IsObjStoreSkipped(t, client.OBS) { t.Run("obs", func(t *testing.T) { diff --git a/providers/storj/storj.go b/providers/storj/storj.go new file mode 100644 index 00000000..274b67a4 --- /dev/null +++ b/providers/storj/storj.go @@ -0,0 +1,206 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storj + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "github.com/thanos-io/objstore" + "gopkg.in/yaml.v2" + "storj.io/common/fpath" + "storj.io/uplink" +) + +// Config stores the configuration for the storj bucket. +type Config struct { + AccessGrant string `yaml:"access"` + Bucket string `yaml:"bucket"` +} + +// Bucket implements the store.Bucket interface against storj API. +type Bucket struct { + logger log.Logger + project *uplink.Project + bucket *uplink.Bucket +} + +// NewBucket returns a new Bucket using the provided storj config values. +func NewBucket(logger log.Logger, config []byte, component string) (*Bucket, error) { + var conf Config + + err := yaml.Unmarshal(config, &conf) + if err != nil { + return nil, err + } + + return NewBucketWithConfig(logger, conf, component) +} + +// NewBucketWithConfig returns a new Bucket using the provided storj config struct. +func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { + var instance Bucket + + ctx := fpath.WithTempData(context.TODO(), "", true) + + uplConf := &uplink.Config{ + UserAgent: fmt.Sprintf("thanos-%s", component), + } + + parsedAccess, err := uplink.ParseAccess(config.AccessGrant) + if err != nil { + return nil, err + } + + instance.project, err = uplConf.OpenProject(ctx, parsedAccess) + if err != nil { + return nil, err + } + + instance.bucket, err = instance.project.EnsureBucket(ctx, config.Bucket) + if err != nil { + //Ignoring the error to return the one that occurred first, while still trying to clean up. + _ = instance.project.Close() + return nil, err + } + + instance.logger = logger + + return &instance, nil +} + +// Name returns the name of the bucket. +func (b *Bucket) Name() string { + return b.bucket.Name +} + +// Close closes the Bucket. +func (b *Bucket) Close() error { + return b.project.Close() +} + +// Iter calls f for each entry in the given directory (not recursive). The argument to f is the full +// object name including the prefix of the inspected directory. +func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error { + + if dir != "" { + dir = strings.TrimSuffix(dir, objstore.DirDelim) + objstore.DirDelim + } + + opt := uplink.ListObjectsOptions{ + Recursive: objstore.ApplyIterOptions(options...).Recursive, + Prefix: dir, + } + + iter := b.project.ListObjects(ctx, b.bucket.Name, &opt) + + for iter.Next() { + if err := f(iter.Item().Key); err != nil { + return err + } + } + return iter.Err() +} + +// Get returns a reader for the given object name. +func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { + _ = level.Debug(b.logger).Log("Getting attributes %s from Storj Bucket", name) + + options := uplink.DownloadOptions{} + + download, err := b.project.DownloadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, &options) + if err != nil { + return nil, err + } + + return download, nil +} + +// GetRange returns a reader to the range for the given object name. +func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { + _ = level.Debug(b.logger).Log("Getting range of %s from Storj Bucket", name) + + options := uplink.DownloadOptions{ + Offset: off, + Length: length, + } + + download, err := b.project.DownloadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, &options) + if err != nil { + return nil, err + } + + return download, nil +} + +// Exists returns whether the object with the given name exists or not. +func (b *Bucket) Exists(ctx context.Context, name string) (bool, error) { + _ = level.Debug(b.logger).Log("Ensuring %s exists in Storj Bucket", name) + _, err := b.project.StatObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + if err != nil { + if b.IsObjNotFoundErr(err) { + return false, nil + } + } + return true, err +} + +// IsObjNotFoundErr returns whether the given error matches the object stores non found error. +func (b *Bucket) IsObjNotFoundErr(err error) bool { + return errors.Is(err, uplink.ErrObjectNotFound) +} + +// Attributes returns information about the specified object. +func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { + _ = level.Debug(b.logger).Log("Getting attributes %s from Storj Bucket", name) + + attr := objstore.ObjectAttributes{} + + obj, err := b.project.StatObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + if err != nil { + return attr, err + } + + attr.Size = obj.System.ContentLength + attr.LastModified = obj.System.Created + + return attr, nil +} + +// Upload the contents of the reader as an object into the bucket. +func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { + var uploadOptions *uplink.UploadOptions + + _ = level.Debug(b.logger).Log("Uploading %s to Storj Bucket", name) + + writer, err := b.project.UploadObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name, uploadOptions) + if err != nil { + return err + } + + _, err = io.Copy(writer, r) + if err != nil { + //Ignoring the error to return the one that occurred first, while still trying to clean up. + _ = writer.Abort() + return err + } + + return writer.Commit() +} + +// Delete removes the object with the given name. +func (b *Bucket) Delete(ctx context.Context, name string) error { + var err error + + _ = level.Debug(b.logger).Log("Deleting %s from Storj Bucket", name) + + _, err = b.project.DeleteObject(fpath.WithTempData(ctx, "", true), b.bucket.Name, name) + + return err +} diff --git a/providers/storj/testing.go b/providers/storj/testing.go new file mode 100644 index 00000000..ec824818 --- /dev/null +++ b/providers/storj/testing.go @@ -0,0 +1,50 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package storj + +import ( + "context" + "os" + "testing" + + "github.com/go-kit/log" + "github.com/thanos-io/objstore" + "gopkg.in/yaml.v2" + "storj.io/uplink" +) + +// NewTestBucket creates test bkt client that before returning creates temporary bucket. +// In a close function it empties and deletes the bucket. +func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { + + ctx := context.Background() + bktName := objstore.CreateTemporaryTestBucketName(t) + access := os.Getenv("STORJ_ACCESS") + + storjConfig, err := yaml.Marshal(Config{AccessGrant: access, Bucket: bktName}) + if err != nil { + return nil, nil, err + } + + bkt, err := NewBucket(log.NewNopLogger(), storjConfig, "testing") + if err != nil { + t.Errorf("failed to create temporary Storj DCS bucket '%s' for testing", bktName) + return nil, nil, err + } + + t.Logf("created temporary Storj DCS bucket '%s' for testing", bkt.Name()) + return bkt, func() { + objstore.EmptyBucket(t, ctx, bkt) + if _, err := bkt.deleteBucket(ctx, bkt.Name()); err != nil { + t.Logf("failed to delete temporary Storj DCS bucket %s for testing: %s", bkt.Name(), err) + } + t.Logf("deleted temporary Storj DCS bucket '%s' for testing", bkt.Name()) + }, nil +} + +// deleteBucket deletes an empty bucket and otherwise returns an error. +// Intentionally not using DeleteBucketWithObjects to ensure not deleting production data. +func (b *Bucket) deleteBucket(ctx context.Context, name string) (*uplink.Bucket, error) { + return b.project.DeleteBucket(ctx, name) +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 424bf9b0..9baced10 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -11,6 +11,13 @@ import ( "reflect" "strings" + "github.com/fatih/structtag" + "github.com/go-kit/log" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/yaml.v2" + "github.com/thanos-io/objstore/client" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" @@ -21,14 +28,8 @@ import ( "github.com/thanos-io/objstore/providers/oci" "github.com/thanos-io/objstore/providers/oss" "github.com/thanos-io/objstore/providers/s3" + "github.com/thanos-io/objstore/providers/storj" "github.com/thanos-io/objstore/providers/swift" - - "github.com/fatih/structtag" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "gopkg.in/alecthomas/kingpin.v2" - "gopkg.in/yaml.v2" ) var ( @@ -45,6 +46,7 @@ var ( client.FILESYSTEM: filesystem.Config{}, client.BOS: bos.Config{}, client.OCI: oci.Config{}, + client.STORJ: storj.Config{}, client.OBS: obs.DefaultConfig, } ) @@ -72,19 +74,19 @@ func main() { errLogger := level.Error(log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))) if _, err := app.Parse(os.Args[1:]); err != nil { - errLogger.Log("err", err) + _ = errLogger.Log("err", err) os.Exit(1) } if c, ok := configs[*structName]; ok { if err := generate(c, os.Stdout); err != nil { - errLogger.Log("err", err) + _ = errLogger.Log("err", err) os.Exit(1) } return } - errLogger.Log("err", errors.Errorf("%v struct not found. Possible values %v", *structName, strings.Join(possibleValues, ","))) + _ = errLogger.Log("err", errors.Errorf("%v struct not found. Possible values %v", *structName, strings.Join(possibleValues, ","))) os.Exit(1) }