Skip to content

Commit

Permalink
Add support for TLS
Browse files Browse the repository at this point in the history
Signed-off-by: Bipul Adhikari <[email protected]>
  • Loading branch information
bipuladh committed Oct 28, 2024
1 parent 568dcb4 commit b873181
Show file tree
Hide file tree
Showing 6 changed files with 248 additions and 15 deletions.
29 changes: 24 additions & 5 deletions cmd/csi-addons/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,17 @@ limitations under the License.
package main

import (
"crypto/x509"
"flag"
"fmt"
"os"

utils "github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes"
"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"github.com/csi-addons/kubernetes-csi-addons/internal/version"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/credentials"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand Down Expand Up @@ -52,6 +55,7 @@ type command struct {
volumeGroupID string
parameters string
volumeIDs string
enableTLS bool
}

// cmd is the single instance of the command struct, used inside main().
Expand All @@ -75,6 +79,7 @@ func init() {
flag.StringVar(&cmd.volumeGroupID, "volumegroupid", "", "ID of the volume group")
flag.StringVar(&cmd.parameters, "parameters", "", "parameters in key=value format separated by commas(Eg:- k1=v1,k2=v2...)")
flag.StringVar(&cmd.volumeIDs, "volumeids", "", "comma separated list of VolumeIds")
flag.BoolVar(&cmd.enableTLS, "tls", true, "enable TLS(enabled by default)")

// output to show when --help is passed
flag.Usage = func() {
Expand Down Expand Up @@ -163,11 +168,25 @@ type grpcClient struct {

// Connect to the endpoint, or panic in case it fails.
func (g *grpcClient) Connect(endpoint string) {
conn, err := grpc.NewClient(
endpoint,
grpc.WithTransportCredentials(insecure.NewCredentials()))
namespace, err := utils.GetNamespace()
if err != nil {
panic(fmt.Sprintf("failed to connect to %q: %v", endpoint, err))
panic(fmt.Errorf("failed to get namespace %v", err))
}
var opts []grpc.DialOption
opts = append(opts, token.WithServiceAccountToken(getKubernetesClient(), namespace, "csi-addons-sa"))
if cmd.enableTLS {
caFile, caError := token.GetServerCert()
if caError != nil {
panic(fmt.Errorf("failed to get server cert %v", err))
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM([]byte(caFile))
creds := credentials.NewClientTLSFromCert(caCertPool, "")
opts = append(opts, grpc.WithTransportCredentials(creds))
}
conn, err := grpc.NewClient(endpoint, opts...)
if err != nil {
panic(fmt.Errorf("failed to connect to %q: %v", endpoint, err))
}

g.Client = conn
Expand Down
16 changes: 16 additions & 0 deletions deploy/controller/csi-addons-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,19 @@ metadata:
data:
"reclaim-space-timeout": "3m"
"max-concurrent-reconciles": "100"
---
apiVersion: v1
kind: Service
metadata:
name: csi-addons-sidecar
# replace the namespace with the namespace where the operator is deployed.
namespace: csi-addons-system
annotations:
service.alpha.openshift.io/serving-cert-secret-name: csi-addons-sidecar-tls
spec:
ports:
- name: grpc
port: 8443
targetPort: 8443
selector:
app: csi-addons-sidecar
21 changes: 21 additions & 0 deletions internal/kubernetes/namespace.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package kubernetes

import (
"io"
"os"
)

func GetNamespace() (string, error) {
namespaceFile := "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
file, err := os.Open(namespaceFile)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}
150 changes: 150 additions & 0 deletions internal/kubernetes/token/grpc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
/*
Copyright 2024 The Kubernetes-CSI-Addons Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package token

import (
"context"
"fmt"
"io"
"os"

"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/status"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

type tokenResolver struct {
kubeclient *kubernetes.Clientset
namespace string
serviceAccount string

token string
expiration metav1.Time
}

func WithServiceAccountToken(kubeclient *kubernetes.Clientset, namespace, serviceAccount string) grpc.DialOption {
tr := tokenResolver{
kubeclient: kubeclient,
namespace: namespace,
serviceAccount: serviceAccount,
expiration: metav1.Now(),
}

return grpc.WithUnaryInterceptor(tr.addAuthorizationHeader)
}

func (tr *tokenResolver) addAuthorizationHeader(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
token, err := tr.getToken(ctx)
if err != nil {
return err
}

authCtx := metadata.AppendToOutgoingContext(ctx, "Authorization", "Bearer "+token)
return invoker(authCtx, method, req, reply, cc, opts...)
}

func (tr *tokenResolver) getToken(ctx context.Context) (string, error) {
now := metav1.Now()
if tr.expiration.Before(&now) {
// token expired
return tr.refreshToken(ctx)
}

return tr.token, nil
}

func (tr *tokenResolver) refreshToken(ctx context.Context) (string, error) {
treq := &authv1.TokenRequest{
Spec: authv1.TokenRequestSpec{
Audiences: []string{"csi-addons"},
},
}

tres, err := tr.kubeclient.CoreV1().ServiceAccounts(tr.namespace).CreateToken(ctx, tr.serviceAccount, treq, metav1.CreateOptions{})
if err != nil {
return "", err
}

tr.token = tres.Status.Token
tr.expiration = tres.Status.ExpirationTimestamp

return tr.token, nil
}

func AuthorizationInterceptor(kubeclient *kubernetes.Clientset) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if err := authorizeConnection(kubeclient, ctx); err != nil {
return nil, err
}
return handler(ctx, req)
}
}

func authorizeConnection(kubeclient *kubernetes.Clientset, ctx context.Context) error {

md, ok := metadata.FromIncomingContext(ctx)
if !ok {
return status.Errorf(codes.Unauthenticated, "missing metadata")
}

authHeader, ok := md["authorization"]
if !ok || len(authHeader) == 0 {
return status.Errorf(codes.Unauthenticated, "missing authorization token")
}

token := authHeader[0]
isValidated, err := validateBearerToken(token, kubeclient)
if !isValidated || err != nil {
return status.Errorf(codes.Unauthenticated, "invalid token")
}
return nil
}

func validateBearerToken(token string, kubeclient *kubernetes.Clientset) (bool, error) {
tokenReview := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: token,
},
}
result, err := kubeclient.AuthenticationV1().TokenReviews().Create(context.TODO(), tokenReview, metav1.CreateOptions{})
if err != nil {
return false, fmt.Errorf("failed to review token %v", err)
}

if result.Status.Authenticated {
return true, nil
}
return false, nil
}

func GetServerCert() (string, error) {
certFile := "/etc/tls/ca.crt"
file, err := os.Open(certFile)
if err != nil {
return "", err
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
return "", err
}
return string(data), nil
}
30 changes: 23 additions & 7 deletions sidecar/internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@ package server

import (
"errors"
"fmt"
"net"

"github.com/csi-addons/kubernetes-csi-addons/internal/kubernetes/token"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
k8s "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)

Expand All @@ -38,24 +42,27 @@ type SidecarServer struct {
// URL components to listen on the tcp port
scheme string
endpoint string
client *k8s.Clientset

server *grpc.Server
services []SidecarService
server *grpc.Server
services []SidecarService
enableTLS bool
}

// NewSidecarServer create a new SidecarServer on the given IP-address and
// port. If the IP-address is an empty string, the server will listen on all
// available IP-addresses. Only tcp ports are supported.
func NewSidecarServer(ip, port string) *SidecarServer {
func NewSidecarServer(ip string, port int, client *k8s.Clientset, enableTLS bool) *SidecarServer {
ss := &SidecarServer{}

if ss.services == nil {
ss.services = make([]SidecarService, 0)
}

ss.scheme = "tcp"
ss.endpoint = ip + ":" + port

ss.endpoint = ip + ":" + fmt.Sprint(port)
ss.client = client
ss.enableTLS = enableTLS
return ss
}

Expand All @@ -69,8 +76,17 @@ func (ss *SidecarServer) RegisterService(svc SidecarService) {
// Init creates the internal gRPC server, and registers the SidecarServices.
// and starts gRPC server.
func (ss *SidecarServer) Start() {
// create the gRPC server and register services
ss.server = grpc.NewServer()
if ss.enableTLS {
creds, err := credentials.NewServerTLSFromFile("/etc/tls/tls.crt", "/etc/tls/tls.key")
if err != nil {
klog.Fatalf("Could not find TLS file: %v", err)
}
// create the gRPC server and register services
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)), grpc.Creds(creds))
}
if !ss.enableTLS {
ss.server = grpc.NewServer(grpc.UnaryInterceptor(token.AuthorizationInterceptor(ss.client)))
}

for _, svc := range ss.services {
svc.RegisterService(ss.server)
Expand Down
17 changes: 14 additions & 3 deletions sidecar/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package main
import (
"context"
"flag"
"fmt"
"time"

"github.com/csi-addons/kubernetes-csi-addons/internal/sidecar/service"
Expand All @@ -43,19 +44,21 @@ func main() {
csiAddonsAddress = flag.String("csi-addons-address", "/run/csi-addons/socket", "CSI Addons endopoint")
nodeID = flag.String("node-id", "", "NodeID")
stagingPath = flag.String("stagingpath", defaultStagingPath, "stagingpath")
controllerPort = flag.String("controller-port", "",
controllerPort = flag.Int("controller-port", 0,
"The TCP network port where the gRPC server for controller request, will listen (example: `8080`)")
controllerIP = flag.String("controller-ip", "",
"The TCP network ip address where the gRPC server for controller request, will listen (example: `192.168.61.228`)")
podName = flag.String("pod", "", "name of the Pod that contains this sidecar")
podNamespace = flag.String("namespace", "", "namespace of the Pod that contains this sidecar")
podUID = flag.String("pod-uid", "", "UID of the Pod that contains this sidecar")
proxyPort = flag.Int("proxy-port", 0, "The TCP port that is available for outside connections to the gRPC server (example: `9070`)")
showVersion = flag.Bool("version", false, "Print Version details")

leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.")
leaderElectionLeaseDuration = flag.Duration("leader-election-lease-duration", 15*time.Second, "Duration, in seconds, that non-leader candidates will wait to force acquire leadership. Defaults to 15 seconds.")
leaderElectionRenewDeadline = flag.Duration("leader-election-renew-deadline", 10*time.Second, "Duration, in seconds, that the acting leader will retry refreshing leadership before giving up. Defaults to 10 seconds.")
leaderElectionRetryPeriod = flag.Duration("leader-election-retry-period", 5*time.Second, "Duration, in seconds, the LeaderElector clients should wait between tries of actions. Defaults to 5 seconds.")
enableTLS = flag.Bool("tls", true, "Enable TLS(enabled by default)")
)
klog.InitFlags(nil)

Expand All @@ -70,7 +73,15 @@ func main() {
return
}

controllerEndpoint, err := sideutil.BuildEndpointURL(*controllerIP, *controllerPort, *podName, *podNamespace)
// if proxyPort is set, use that in the CSIAddonsNode.Endpoint URL
publicPort := *proxyPort
publicIP := ""
if publicPort == 0 {
publicPort = *controllerPort
publicIP = *controllerIP
}

controllerEndpoint, err := sideutil.BuildEndpointURL(publicIP, fmt.Sprint(publicPort), *podName, *podNamespace)
if err != nil {
klog.Fatalf("Failed to validate controller endpoint: %v", err)
}
Expand Down Expand Up @@ -109,7 +120,7 @@ func main() {
klog.Fatalf("Failed to create csiaddonsnode: %v", err)
}

sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort)
sidecarServer := server.NewSidecarServer(*controllerIP, *controllerPort, kubeClient, *enableTLS)
sidecarServer.RegisterService(service.NewIdentityServer(csiClient.GetGRPCClient()))
sidecarServer.RegisterService(service.NewReclaimSpaceServer(csiClient.GetGRPCClient(), kubeClient, *stagingPath))
sidecarServer.RegisterService(service.NewNetworkFenceServer(csiClient.GetGRPCClient(), kubeClient))
Expand Down

0 comments on commit b873181

Please sign in to comment.