From ad73c1e0ee78ecb56261e98a74c1fd182721a88f Mon Sep 17 00:00:00 2001 From: Zachary Seguin Date: Thu, 13 Aug 2020 19:34:41 -0400 Subject: [PATCH 01/26] feat(pid): Write pid file --- api/common/config.go | 2 ++ internal/flags.go | 7 +++++++ main.go | 12 ++++++++++++ 3 files changed, 21 insertions(+) diff --git a/api/common/config.go b/api/common/config.go index e4f23ad5..ba3fcb7d 100644 --- a/api/common/config.go +++ b/api/common/config.go @@ -26,6 +26,8 @@ import ( ) type FlagStorage struct { + PidFile string + // File system MountOptions map[string]string MountPoint string diff --git a/internal/flags.go b/internal/flags.go index f18633a8..41995e5e 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -97,6 +97,11 @@ func NewApp() (app *cli.App) { Usage: "Print this help text and exit successfully.", }, + cli.StringFlag{ + Name: "pid-file", + Usage: "Write a pid file containing the process pid.", + }, + ///////////////////////// // File system ///////////////////////// @@ -319,6 +324,8 @@ func parseOptions(m map[string]string, s string) { // variables into which the flags will parse. func PopulateFlags(c *cli.Context) (ret *FlagStorage) { flags := &FlagStorage{ + PidFile: c.String("pid-file"), + // File system MountOptions: make(map[string]string), DirMode: os.FileMode(c.Int("dir-mode")), diff --git a/main.go b/main.go index e2e06a1f..ef0d9c8d 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,9 @@ package main import ( + "io/ioutil" + "strconv" + goofys "github.com/kahing/goofys/api" . "github.com/kahing/goofys/api/common" . "github.com/kahing/goofys/internal" @@ -220,6 +223,15 @@ func main() { kill(os.Getppid(), syscall.SIGUSR1) } log.Println("File system has been successfully mounted.") + + if flags.PidFile != "" { + // Write out a Pid file + err = ioutil.WriteFile(flags.PidFile, []byte(strconv.Itoa(os.Getpid())), 0644) + if err != nil { + log.Fatalf("Error writing pid file: %v", err) + } + } + // Let the user unmount with Ctrl-C // (SIGINT). But if cache is on, catfs will // receive the signal and we would detect that exiting From 758eba2703dcd49209c96e5786bc1408811f7098 Mon Sep 17 00:00:00 2001 From: Zachary Seguin Date: Thu, 17 Sep 2020 11:57:54 -0400 Subject: [PATCH 02/26] Update pid-file flag --- internal/flags.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/flags.go b/internal/flags.go index 41995e5e..050de61f 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -97,11 +97,6 @@ func NewApp() (app *cli.App) { Usage: "Print this help text and exit successfully.", }, - cli.StringFlag{ - Name: "pid-file", - Usage: "Write a pid file containing the process pid.", - }, - ///////////////////////// // File system ///////////////////////// @@ -266,6 +261,11 @@ func NewApp() (app *cli.App) { Name: "f", Usage: "Run goofys in foreground.", }, + + cli.StringFlag{ + Name: "pid-file", + Usage: "Write a pid file containing the process pid. Does nothing if -f is used.", + }, }, } From d2fb2662eab467306bc2c20411492586395f8ba5 Mon Sep 17 00:00:00 2001 From: William Hearn Date: Sat, 9 Jan 2021 12:40:04 -0500 Subject: [PATCH 03/26] feat(sparse): Allow for sparse file uploads --- internal/file.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/internal/file.go b/internal/file.go index a2ccbaa7..bac756fa 100644 --- a/internal/file.go +++ b/internal/file.go @@ -240,7 +240,7 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { return fh.lastWriteError } - if offset != fh.nextWriteOffset { + if offset < fh.nextWriteOffset { fh.inode.errFuse("WriteFile: only sequential writes supported", fh.nextWriteOffset, offset) fh.lastWriteError = syscall.ENOTSUP return fh.lastWriteError @@ -262,6 +262,37 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { fh.inode.mu.Unlock() } + // fill the hole with zero + if offset > fh.nextWriteOffset { + toSkip := offset - fh.nextWriteOffset + n := 10240 + data := make([]byte, n) + + for { + if fh.buf == nil { + fh.buf = MBuf{}.Init(fh.poolHandle, fh.partSize(), true) + } + + if toSkip < int64(n) { + n = int(toSkip) + } + nCopied, _ := fh.buf.Write(data[:n]) + toSkip -= int64(nCopied) + fh.nextWriteOffset += int64(nCopied) + + if fh.buf.Full() { + err = fh.uploadCurrentBuf(!fh.cloud.Capabilities().NoParallelMultipart) + if err != nil { + return + } + } + + if toSkip == 0 { + break + } + } + } + for { if fh.buf == nil { fh.buf = MBuf{}.Init(fh.poolHandle, fh.partSize(), true) From dbd06bad71e8cb6f0b2b4b1c0bec69c30deb4c06 Mon Sep 17 00:00:00 2001 From: Jose Matsuda Date: Tue, 9 Apr 2024 09:12:06 -0400 Subject: [PATCH 04/26] test: --- internal/backend_s3.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 5bb4c3ba..0204823a 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -661,13 +661,15 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { from := s.bucket + "/" + param.Source - if !s.gcs && *param.Size > COPY_LIMIT { - reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass) - if err != nil { - return nil, err + /* + if !s.gcs && *param.Size > COPY_LIMIT { + reqId, err := s.copyObjectMultipart(int64(*param.Size), from, param.Destination, "", param.ETag, param.Metadata, param.StorageClass) + if err != nil { + return nil, err + } + return &CopyBlobOutput{reqId}, nil } - return &CopyBlobOutput{reqId}, nil - } + */ params := &s3.CopyObjectInput{ Bucket: &s.bucket, From 69a13ab8887d51ee609d4f5621158a655e672be4 Mon Sep 17 00:00:00 2001 From: Jose Matsuda Date: Tue, 9 Apr 2024 15:23:01 -0400 Subject: [PATCH 05/26] forcibly inc buffer size, will definitely crash if above 625 --- internal/backend_s3.go | 21 ++++++--- internal/file.go | 102 ++++++++++++++++++++++++++++++++++------- main.go | 3 +- 3 files changed, 101 insertions(+), 25 deletions(-) diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 0204823a..897a9f85 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -62,8 +62,10 @@ func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, err flags: flags, config: config, cap: Capabilities{ - Name: "s3", - MaxMultipartSize: 5 * 1024 * 1024 * 1024, + Name: "s3-jose-test", + // MaxMultipartSize doesnt seem to be respected? or at least it goes to multipart right away. + MaxMultipartSize: 5 * 1024 * 1024 * 1024, + NoParallelMultipart: true, }, } @@ -507,7 +509,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri return } -func sizeToParts(size int64) (int, int64) { +func sizeToParts(size int64) (int, int64) { // this shouldnt matter, since this mpu shouldnt get called. const MAX_S3_MPU_SIZE int64 = 5 * 1024 * 1024 * 1024 * 1024 if size > MAX_S3_MPU_SIZE { panic(fmt.Sprintf("object size: %v exceeds maximum S3 MPU size: %v", size, MAX_S3_MPU_SIZE)) @@ -552,13 +554,15 @@ func (s *S3Backend) mpuCopyParts(size int64, from string, to string, mpuId strin sem.V(MAX_CONCURRENCY) } +// This shouldnt get reached, the only reference to `copyObjectMultip` was commented out (other than the test) +// Assuming that this backend_s3.go is the one that is used. func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuId string, srcEtag *string, metadata map[string]*string, storageClass *string) (requestId string, err error) { - nParts, partSize := sizeToParts(size) + nParts, partSize := sizeToParts(size) // should be unreachable, and partsize i dont need anything to do with etags := make([]*string, nParts) if mpuId == "" { - params := &s3.CreateMultipartUploadInput{ + params := &s3.CreateMultipartUploadInput{ // should be unreachable, assuming this is the one that is used Bucket: &s.bucket, Key: &to, StorageClass: storageClass, @@ -581,7 +585,7 @@ func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuI params.ACL = &s.config.ACL } - resp, err := s.CreateMultipartUpload(params) + resp, err := s.CreateMultipartUpload(params) // should be unreachable, assuming this is the one that is used. if err != nil { return "", mapAwsError(err) } @@ -820,7 +824,9 @@ func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { }, nil } +// reached from file.go func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { + // references API then mpu := s3.CreateMultipartUploadInput{ Bucket: &s.bucket, Key: ¶m.Key, @@ -843,9 +849,10 @@ func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*Multipa mpu.ACL = &s.config.ACL } + // again reference API resp, err := s.CreateMultipartUpload(&mpu) if err != nil { - s3Log.Errorf("CreateMultipartUpload %v = %v", param.Key, err) + s3Log.Errorf("Jose: CreateMultipartUpload %v = %v", param.Key, err) // this is the err i get? return nil, mapAwsError(err) } diff --git a/internal/file.go b/internal/file.go index 69cf5b6d..fc0ad565 100644 --- a/internal/file.go +++ b/internal/file.go @@ -83,13 +83,14 @@ func NewFileHandle(inode *Inode, opMetadata fuseops.OpContext) *FileHandle { return fh } -func (fh *FileHandle) initWrite() { +func (fh *FileHandle) initWrite() { // ok how is this reached? fh.writeInit.Do(func() { fh.mpuWG.Add(1) - go fh.initMPU() + go fh.initMPU() // starts multipart? }) } +// how is initMPU reached? func (fh *FileHandle) initMPU() { defer func() { fh.mpuWG.Done() @@ -98,6 +99,7 @@ func (fh *FileHandle) initMPU() { fs := fh.inode.fs fh.mpuName = &fh.key + // how is this reached, could I just do a "PutBlob" instead here (no i dont think so, given other ex in this file)? What would break? resp, err := fh.cloud.MultipartBlobBegin(&MultipartBlobBeginInput{ Key: *fh.mpuName, ContentType: fs.flags.GetMimeType(*fh.mpuName), @@ -146,6 +148,40 @@ func (fh *FileHandle) mpuPartNoSpawn(buf *MBuf, part uint32, total int64, last b return } +// JOSE ATTEMPT +// dont think we should try this at all, since it expects things to be split up +func (fh *FileHandle) uploadSingle(buf *MBuf, part uint32, total int64, last bool) (err error) { + fs := fh.inode.fs + + fs.replicators.Take(1, true) + defer fs.replicators.Return(1) + + if part == 0 || part > 10000 { + return errors.New(fmt.Sprintf("invalid part number: %v", part)) + } + + //hmmm + mpu := MultipartBlobAddInput{ + Commit: fh.mpuId, + PartNumber: part, + Body: buf, + Size: uint64(buf.Len()), + Last: last, + Offset: uint64(total - int64(buf.Len())), + } + + defer func() { + if mpu.Body != nil { + bufferLog.Debugf("Free %T", buf) + buf.Free() + } + }() + + _, err = fh.cloud.MultipartBlobAdd(&mpu) + + return +} + func (fh *FileHandle) mpuPart(buf *MBuf, part uint32, total int64) { defer func() { fh.mpuWG.Done() @@ -169,9 +205,13 @@ func (fh *FileHandle) mpuPart(buf *MBuf, part uint32, total int64) { } func (fh *FileHandle) waitForCreateMPU() (err error) { - if fh.mpuId == nil { + // JOSE: does this need to be modified further??? Like when we upload sequentially does it need to still "wait"? + // no we dont want to even SPLIT it up into parts + if fh.mpuId == nil { // only enters here if mpuId = nil, else returns nothing. + // What will happen if we comment this if out? + // THe question is when does it really start the multipart upload? Like when is it assigned "CreaMultiPartUpload" fh.mu.Unlock() - fh.initWrite() + fh.initWrite() // this calls the MPU to start the createMultipUpload fh.mpuWG.Wait() // wait for initMPU fh.mu.Lock() @@ -183,6 +223,7 @@ func (fh *FileHandle) waitForCreateMPU() (err error) { return } +// So this is actually used regardless in WriteFile under 'MBuf{}.Init(fh.poolHandle, fh.partSize(), true)' func (fh *FileHandle) partSize() uint64 { var size uint64 @@ -200,28 +241,45 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } + // Something I could test here is increasing this and then re uploading and trying things + size = 625 * 1024 * 1024 // ~625 MB return size } func (fh *FileHandle) uploadCurrentBuf(parallel bool) (err error) { - err = fh.waitForCreateMPU() - if err != nil { - return - } + // since removing MPU, remove this call, this would mean that we don't encounter / follow the trail to backend_s3.go + // Where we eventually get to 'MultipartBlobBegin' + + /* + err = fh.waitForCreateMPU() //calls the MPU upload + // this MPU is ALWAYS called, at least once it hits this func and then dies in subsequent calls + if err != nil { + return + } + */ + // the thing is we dont want to break it up, so I feel like we dont want these variable declarations at all. fh.lastPartId++ part := fh.lastPartId buf := fh.buf fh.buf = nil - if parallel { - fh.mpuWG.Add(1) - go fh.mpuPart(buf, part, fh.nextWriteOffset) - } else { - err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) - if fh.lastWriteError == nil { - fh.lastWriteError = err - } + /* + if parallel { // we dont want this if since we dont care about parallel + fh.mpuWG.Add(1) + go fh.mpuPart(buf, part, fh.nextWriteOffset) + } else { + err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) + if fh.lastWriteError == nil { + fh.lastWriteError = err + } + } */ + // We still want to upload though, so we will probably need to modify this mpuPartNoSpawn to at least not call a CreatMultiPUpload + //err = fh.mpuPartNoSpawn(buf, part, fh.nextWriteOffset, false) + err = fh.uploadSingle(buf, part, fh.nextWriteOffset, false) // attempt to change this uploadSingle I created + // i dont think we should even try modifying this, since this splits it up into multi parts + if fh.lastWriteError == nil { + fh.lastWriteError = err } return @@ -267,13 +325,23 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { for { if fh.buf == nil { fh.buf = MBuf{}.Init(fh.poolHandle, fh.partSize(), true) + // I need to modify fh.buf here(?), I feel like this sets buf size to be used later on in fh.buf.Full(). + // but can i just make this infinite? + // note that in the Init, the partSize is related to something that requestsMultiple } nCopied, _ := fh.buf.Write(data) fh.nextWriteOffset += int64(nCopied) + // JOSE: if i comment out this uploadCurrentBuf, which eventually calls multipart, what happens? + // like fh.buf.Full, will it just overflow? Maybe just need to `uploadCurrentBuff` but NOT go into multipart + // / modify that uploadCurrentBuff because we will alwyas need to upload the current thing, so this shouldnt be commented out + // may need to reach a state where the buffer is never full lol... or at least to some reasonable number so if people really need to + // they can use mc or whatever... if fh.buf.Full() { err = fh.uploadCurrentBuf(!fh.cloud.Capabilities().NoParallelMultipart) + // i feel like under no 'modifiable' circumstances would trying to change `uploadCurrBuff` work, because + // it splits the file. thing is what do i do once the buffer is full then? if err != nil { return } @@ -284,7 +352,7 @@ func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { } data = data[nCopied:] - } + } // end for loop to upload fh.inode.Attributes.Size = uint64(fh.nextWriteOffset) fh.inode.Attributes.Mtime = time.Now() diff --git a/main.go b/main.go index 112b8595..4e55f900 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,8 @@ func massageArg0() { var Version = "use `make build' to fill version hash correctly" func main() { - VersionNumber = "0.24.0" + // Changing just to make sure when I push im not using regular goofys still + VersionNumber = "0.42.0" VersionHash = Version massagePath() From 8eaaec3430e6246cb5dbf4210c8a80e586bd8e7f Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Wed, 10 Apr 2024 07:21:39 -0400 Subject: [PATCH 06/26] 10Gb --- internal/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index fc0ad565..3778b2b8 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,8 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Something I could test here is increasing this and then re uploading and trying things - size = 625 * 1024 * 1024 // ~625 MB + // Try 10Gb + size = 10000 * 1024 * 1024 // ~10GB return size } From e45008f359d1c3a1ee0b864319d4e0083cd5e241 Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Wed, 10 Apr 2024 11:27:36 -0400 Subject: [PATCH 07/26] 4.5 --- internal/file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index 3778b2b8..db12504b 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,8 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Try 10Gb - size = 10000 * 1024 * 1024 // ~10GB + // Try 4.5Gb + size = 45000 * 1024 * 1024 // ~2.5Gb return size } From a1fb9da08cf7fdeec2c72d7f83f3f1ed03e71106 Mon Sep 17 00:00:00 2001 From: Jose-Matsuda Date: Thu, 11 Apr 2024 12:36:59 -0400 Subject: [PATCH 08/26] a little bit under 5Gb --- internal/file.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/file.go b/internal/file.go index db12504b..aeb5df41 100644 --- a/internal/file.go +++ b/internal/file.go @@ -241,8 +241,7 @@ func (fh *FileHandle) partSize() uint64 { if maxPartSize != 0 { size = MinUInt64(maxPartSize, size) } - // Try 4.5Gb - size = 45000 * 1024 * 1024 // ~2.5Gb + size = 5000 * 1024 * 1024 return size } From 5a80de4ae5ec28a9aeceb9508dd0d166e1150d18 Mon Sep 17 00:00:00 2001 From: Mathis <84033116+mathis-marcotte@users.noreply.github.com> Date: Wed, 19 Jun 2024 15:56:12 -0400 Subject: [PATCH 09/26] replaced log.fatalf with .og.printf --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 4e55f900..3101d78f 100644 --- a/main.go +++ b/main.go @@ -215,8 +215,10 @@ func main() { if !flags.Foreground { kill(os.Getppid(), syscall.SIGUSR2) } - log.Fatalf("Mounting file system: %v", err) + // log.Fatalf("Mounting file system: %v", err) // fatal also terminates itself + log.Printf("Mounting file system: %v", err) + return } else { if !flags.Foreground { kill(os.Getppid(), syscall.SIGUSR1) From be896e812ac2c78e5de36350c389346b0f07075e Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Tue, 9 Jul 2024 12:50:56 +0000 Subject: [PATCH 10/26] updated to point to statcan fork instead of original --- .travis.yml | 2 +- README-azure.md | 2 +- README.md | 24 ++++++++++++------------ api/api.go | 4 ++-- bench/Dockerfile | 8 ++++---- bench/Dockerfile.azure | 8 ++++---- bench/Dockerfile.gcs | 10 +++++----- bench/azure/README.md | 2 +- bench/cache/README.md | 2 +- example/test_api.go | 6 +++--- go.mod | 2 +- internal/aws_test.go | 4 ++-- internal/backend_adlv1.go | 2 +- internal/backend_adlv2.go | 2 +- internal/backend_azblob.go | 2 +- internal/backend_gcs.go | 2 +- internal/backend_gcs3.go | 2 +- internal/backend_gcs_test.go | 2 +- internal/backend_s3.go | 2 +- internal/buffer_pool.go | 2 +- internal/flags.go | 2 +- internal/goofys.go | 2 +- internal/goofys_test.go | 12 ++++++------ internal/minio_test.go | 2 +- main.go | 6 +++--- 25 files changed, 57 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69202383..eb1e6822 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: script: travis_retry ./test/run-tests.sh GoofysTest go: - 1.16 -go_import_path: github.com/kahing/goofys +go_import_path: github.com/StatCan/goofys matrix: include: - name: "S3Proxy" diff --git a/README-azure.md b/README-azure.md index 01d908f0..c272489e 100644 --- a/README-azure.md +++ b/README-azure.md @@ -52,7 +52,7 @@ $ $GOPATH/bin/goofys adl://servicename.azuredatalakestore.net:prefix diff --git a/README.md b/README.md index 564dbb76..57225736 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Goofys is a high-performance, POSIX-ish [Amazon S3](https://aws.amazon.com/s3/) file system written in Go -[![Build Status](https://travis-ci.org/kahing/goofys.svg?branch=master)](https://travis-ci.org/kahing/goofys) -[![Github All Releases](https://img.shields.io/github/downloads/kahing/goofys/total.svg)](https://github.com/kahing/goofys/releases/) +[![Build Status](https://travis-ci.org/StatCan/goofys.svg?branch=master)](https://travis-ci.org/StatCan/goofys) +[![Github All Releases](https://img.shields.io/github/downloads/StatCan/goofys/total.svg)](https://github.com/StatCan/goofys/releases/) [![Twitter Follow](https://img.shields.io/twitter/follow/s3goofys.svg?style=social&label=Follow)](https://twitter.com/s3goofys) [![Stack Overflow Questions](https://img.shields.io/stackexchange/stackoverflow/t/goofys?label=Stack%20Overflow%20questions)](https://stackoverflow.com/search?q=%5Bgoofys%5D+is%3Aquestion) @@ -14,12 +14,12 @@ for performance first and POSIX second. Particularly things that are difficult to support on S3 or would translate into more than one round-trip would either fail (random writes) or faked (no per-file permission). Goofys does not have an on disk data cache (checkout -[catfs](https://github.com/kahing/catfs)), and consistency model is +[catfs](https://github.com/StatCan/catfs)), and consistency model is close-to-open. # Installation -* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). +* On Linux, install via [pre-built binaries](https://github.com/StatCan/goofys/releases/latest/download/goofys). You may also need to install fuse too if you want to mount it on startup. * On macOS, install via [Homebrew](https://brew.sh/): @@ -33,8 +33,8 @@ $ brew install goofys ```ShellSession $ export GOPATH=$HOME/work -$ go get github.com/kahing/goofys -$ go install github.com/kahing/goofys +$ go get github.com/StatCan/goofys +$ go install github.com/StatCan/goofys ``` # Usage @@ -59,16 +59,16 @@ configured for `root`, and can add this to `/etc/fstab`: goofys#bucket /mnt/mountpoint fuse _netdev,allow_other,--file-mode=0666,--dir-mode=0777 0 0 ``` -See also: [Instruction for Azure Blob Storage, Azure Data Lake Gen1, and Azure Data Lake Gen2](https://github.com/kahing/goofys/blob/master/README-azure.md). +See also: [Instruction for Azure Blob Storage, Azure Data Lake Gen1, and Azure Data Lake Gen2](https://github.com/StatCan/goofys/blob/master/README-azure.md). -Got more questions? Check out [questions other people asked](https://github.com/kahing/goofys/issues?utf8=%E2%9C%93&q=is%3Aissue%20label%3Aquestion%20) +Got more questions? Check out [questions other people asked](https://github.com/StatCan/goofys/issues?utf8=%E2%9C%93&q=is%3Aissue%20label%3Aquestion%20) # Benchmark Using `--stat-cache-ttl 1s --type-cache-ttl 1s` for goofys `-ostat_cache_expire=1` for s3fs to simulate cold runs. Detail for the benchmark can be found in -[bench.sh](https://github.com/kahing/goofys/blob/master/bench/bench.sh). [Raw data](https://github.com/kahing/goofys/blob/master/bench/) +[bench.sh](https://github.com/StatCan/goofys/blob/master/bench/bench.sh). [Raw data](https://github.com/StatCan/goofys/blob/master/bench/) is available as well. The test was run on an EC2 m5.4xlarge in us-west-2a connected to a bucket in us-west-2. Units are seconds. @@ -76,11 +76,11 @@ connected to a bucket in us-west-2. Units are seconds. To run the benchmark, configure EC2's instance role to be able to write to `$TESTBUCKET`, and then do: ```ShellSession -$ sudo docker run -e BUCKET=$TESTBUCKET -e CACHE=false --rm --privileged --net=host -v /tmp/cache:/tmp/cache kahing/goofys-bench +$ sudo docker run -e BUCKET=$TESTBUCKET -e CACHE=false --rm --privileged --net=host -v /tmp/cache:/tmp/cache StatCan/goofys-bench # result will be written to $TESTBUCKET ``` -See also: [cached benchmark result](https://github.com/kahing/goofys/blob/master/bench/cache/README.md) and [result on Azure](https://github.com/kahing/goofys/blob/master/bench/azure/README.md). +See also: [cached benchmark result](https://github.com/StatCan/goofys/blob/master/bench/cache/README.md) and [result on Azure](https://github.com/StatCan/goofys/blob/master/bench/azure/README.md). # License @@ -128,7 +128,7 @@ Additionally, goofys also works with the following non-S3 object stores: * Data is stored on [Amazon S3](https://aws.amazon.com/s3/) * [Amazon SDK for Go](https://github.com/aws/aws-sdk-go) * Other related fuse filesystems - * [catfs](https://github.com/kahing/catfs): caching layer that can be used with goofys + * [catfs](https://github.com/StatCan/catfs): caching layer that can be used with goofys * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse): another popular filesystem for S3 * [gcsfuse](https://github.com/googlecloudplatform/gcsfuse): filesystem for diff --git a/api/api.go b/api/api.go index cf606aef..53a2f9ce 100644 --- a/api/api.go +++ b/api/api.go @@ -1,8 +1,8 @@ package goofys import ( - . "github.com/kahing/goofys/api/common" - "github.com/kahing/goofys/internal" + . "github.com/StatCan/goofys/api/common" + "github.com/StatCan/goofys/internal" "context" "fmt" diff --git a/bench/Dockerfile b/bench/Dockerfile index 8c38be8d..adbf5d3c 100644 --- a/bench/Dockerfile +++ b/bench/Dockerfile @@ -24,14 +24,14 @@ RUN git clone --depth 1 https://github.com/s3fs-fuse/s3fs-fuse.git && \ cd s3fs-fuse && ./autogen.sh && ./configure && make -j8 > /dev/null && make install && \ cd .. && rm -Rf s3fs-fuse -RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # ideally I want to clear out all the go deps too but there's no # way to do that with ADD ENV PATH=$PATH:/root/go/bin -ADD . /root/go/src/github.com/kahing/goofys -WORKDIR /root/go/src/github.com/kahing/goofys +ADD . /root/go/src/github.com/StatCan/goofys +WORKDIR /root/go/src/github.com/StatCan/goofys RUN go get . && make install -ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"] +ENTRYPOINT ["/root/go/src/github.com/StatCan/goofys/bench/run_bench.sh"] diff --git a/bench/Dockerfile.azure b/bench/Dockerfile.azure index 6fdab696..9eeb5369 100644 --- a/bench/Dockerfile.azure +++ b/bench/Dockerfile.azure @@ -25,14 +25,14 @@ RUN git clone --depth 1 https://github.com/Azure/azure-storage-fuse.git && \ cd azure-storage-fuse && bash ./build.sh > /dev/null && make -C build install && \ cd .. && rm -Rf azure-storage-fuse -RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # ideally I want to clear out all the go deps too but there's no # way to do that with ADD ENV PATH=$PATH:/root/go/bin -ADD . /root/go/src/github.com/kahing/goofys -WORKDIR /root/go/src/github.com/kahing/goofys +ADD . /root/go/src/github.com/StatCan/goofys +WORKDIR /root/go/src/github.com/StatCan/goofys RUN go get . && make install -ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"] +ENTRYPOINT ["/root/go/src/github.com/StatCan/goofys/bench/run_bench.sh"] diff --git a/bench/Dockerfile.gcs b/bench/Dockerfile.gcs index 8a51c128..5bb3734b 100644 --- a/bench/Dockerfile.gcs +++ b/bench/Dockerfile.gcs @@ -1,7 +1,7 @@ FROM golang:1.14 AS goofys-builder # install goofys -WORKDIR $GOPATH/src/github.com/kahing/goofys +WORKDIR $GOPATH/src/github.com/StatCan/goofys COPY . . @@ -24,7 +24,7 @@ RUN apt-get update && \ && apt-get clean # install catfs, required to run goofys with cache -RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # goofys graph generation @@ -33,12 +33,12 @@ RUN pip install numpy ENV PATH=$PATH:/root/go/bin # copy go binaries -COPY --from=goofys-builder /go/src/github.com/kahing/goofys/goofys /root/go/bin/goofys +COPY --from=goofys-builder /go/src/github.com/StatCan/goofys/goofys /root/go/bin/goofys COPY --from=goofys-builder /go/bin/gcsfuse /root/go/bin/gcsfuse -WORKDIR /root/go/src/github.com/kahing/goofys +WORKDIR /root/go/src/github.com/StatCan/goofys # copy bench scripts COPY bench bench -ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"] +ENTRYPOINT ["/root/go/src/github.com/StatCan/goofys/bench/run_bench.sh"] diff --git a/bench/azure/README.md b/bench/azure/README.md index 9726e6eb..e0475128 100644 --- a/bench/azure/README.md +++ b/bench/azure/README.md @@ -8,6 +8,6 @@ To run the benchmark, do: ```ShellSession $ export AZURE_STORAGE_ACCOUNT=myaccount $ export AZURE_STORAGE_KEY=STORAGE-ACCESS-KEY -$ sudo docker run -e BUCKET=$TESTBUCKET -e AZURE_STORAGE_ACCOUNT=$AZURE_STORAGE_ACCOUNT -e AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY --rm --privileged --net=host -v /mnt/cache:/tmp/cache kahing/goofys-bench:azure-latest +$ sudo docker run -e BUCKET=$TESTBUCKET -e AZURE_STORAGE_ACCOUNT=$AZURE_STORAGE_ACCOUNT -e AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY --rm --privileged --net=host -v /mnt/cache:/tmp/cache StatCan/goofys-bench:azure-latest # result will be written to $TESTBUCKET ``` diff --git a/bench/cache/README.md b/bench/cache/README.md index a2b2ca46..6c31c6eb 100644 --- a/bench/cache/README.md +++ b/bench/cache/README.md @@ -11,6 +11,6 @@ To run the benchmark, configure EC2's instance role to be able to write to `$TES ```ShellSession $ export AWS_ACCESS_KEY_ID=AKID1234567890 $ export AWS_SECRET_ACCESS_KEY=MY-SECRET-KEY -$ sudo docker run -e BUCKET=$TESTBUCKET -e CACHE=true --rm --privileged --net=host -v /tmp/cache:/tmp/cache kahing/goofys-bench +$ sudo docker run -e BUCKET=$TESTBUCKET -e CACHE=true --rm --privileged --net=host -v /tmp/cache:/tmp/cache StatCan/goofys-bench # result will be written to $TESTBUCKET ``` diff --git a/example/test_api.go b/example/test_api.go index 2bb126dc..cc5088b1 100644 --- a/example/test_api.go +++ b/example/test_api.go @@ -1,11 +1,11 @@ package main import ( - goofys "github.com/kahing/goofys/api" - common "github.com/kahing/goofys/api/common" + goofys "github.com/StatCan/goofys/api" + common "github.com/StatCan/goofys/api/common" - "fmt" "context" + "fmt" ) func main() { diff --git a/go.mod b/go.mod index 313fdd39..d02e64ed 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/kahing/goofys +module github.com/StatCan/goofys go 1.14 diff --git a/internal/aws_test.go b/internal/aws_test.go index fc259afe..c90210f3 100644 --- a/internal/aws_test.go +++ b/internal/aws_test.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" . "gopkg.in/check.v1" "fmt" @@ -39,7 +39,7 @@ func (s *AwsTest) SetUpSuite(t *C) { } func (s *AwsTest) TestRegionDetection(t *C) { - s.s3.bucket = "goofys-eu-west-1.kahing.xyz" + s.s3.bucket = "goofys-eu-west-1.StatCan.xyz" err, isAws := s.s3.detectBucketLocationByHEAD() t.Assert(err, IsNil) diff --git a/internal/backend_adlv1.go b/internal/backend_adlv1.go index 3226a9ea..ce9571ac 100644 --- a/internal/backend_adlv1.go +++ b/internal/backend_adlv1.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "bytes" "context" diff --git a/internal/backend_adlv2.go b/internal/backend_adlv2.go index baa6d3e3..11397dec 100644 --- a/internal/backend_adlv2.go +++ b/internal/backend_adlv2.go @@ -16,7 +16,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "context" "encoding/base64" diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index b83fedbb..802562c4 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "bytes" "context" diff --git a/internal/backend_gcs.go b/internal/backend_gcs.go index fb8efdac..dd428c3d 100644 --- a/internal/backend_gcs.go +++ b/internal/backend_gcs.go @@ -1,7 +1,7 @@ package internal import ( - "github.com/kahing/goofys/api/common" + "github.com/StatCan/goofys/api/common" "bytes" "context" diff --git a/internal/backend_gcs3.go b/internal/backend_gcs3.go index e3b4dc72..60bfc80d 100644 --- a/internal/backend_gcs3.go +++ b/internal/backend_gcs3.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "fmt" "io" diff --git a/internal/backend_gcs_test.go b/internal/backend_gcs_test.go index 6cae756e..582e6a2d 100644 --- a/internal/backend_gcs_test.go +++ b/internal/backend_gcs_test.go @@ -1,7 +1,7 @@ package internal import ( - "github.com/kahing/goofys/api/common" + "github.com/StatCan/goofys/api/common" "bytes" "context" diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 897a9f85..f3aa4890 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "fmt" "net/http" diff --git a/internal/buffer_pool.go b/internal/buffer_pool.go index b43c7db2..88266399 100644 --- a/internal/buffer_pool.go +++ b/internal/buffer_pool.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "io" "runtime" diff --git a/internal/flags.go b/internal/flags.go index d871d8e9..e2fd7b1f 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -16,7 +16,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "fmt" "io" diff --git a/internal/goofys.go b/internal/goofys.go index 4f4bd76f..bf51b4c1 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "context" "fmt" diff --git a/internal/goofys_test.go b/internal/goofys_test.go index 16b86773..51732e67 100644 --- a/internal/goofys_test.go +++ b/internal/goofys_test.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" "bufio" "bytes" @@ -166,11 +166,11 @@ func (t *GoofysTest) deleteBlobsParallelly(cloud StorageBackend, blobs []string) // groupByDecresingDepths takes a slice of path strings and returns the paths as // groups where each group has the same `depth` - depth(a/b/c)=2, depth(a/b/)=1 // The groups are returned in decreasing order of depths. -// - Inp: [] Out: [] -// - Inp: ["a/b1/", "a/b/c1", "a/b2", "a/b/c2"] -// Out: [["a/b/c1", "a/b/c2"], ["a/b1/", "a/b2"]] -// - Inp: ["a/b1/", "z/a/b/c1", "a/b2", "z/a/b/c2"] -// Out: [["z/a/b/c1", "z/a/b/c2"], ["a/b1/", "a/b2"] +// - Inp: [] Out: [] +// - Inp: ["a/b1/", "a/b/c1", "a/b2", "a/b/c2"] +// Out: [["a/b/c1", "a/b/c2"], ["a/b1/", "a/b2"]] +// - Inp: ["a/b1/", "z/a/b/c1", "a/b2", "z/a/b/c2"] +// Out: [["z/a/b/c1", "z/a/b/c2"], ["a/b1/", "a/b2"] func groupByDecresingDepths(items []string) [][]string { depthToGroup := map[int][]string{} for _, item := range items { diff --git a/internal/minio_test.go b/internal/minio_test.go index 0f8a78e3..f2201a61 100644 --- a/internal/minio_test.go +++ b/internal/minio_test.go @@ -15,7 +15,7 @@ package internal import ( - . "github.com/kahing/goofys/api/common" + . "github.com/StatCan/goofys/api/common" . "gopkg.in/check.v1" "context" diff --git a/main.go b/main.go index 3089d7f6..0aff8d57 100644 --- a/main.go +++ b/main.go @@ -19,9 +19,9 @@ import ( "io/ioutil" "strconv" - goofys "github.com/kahing/goofys/api" - . "github.com/kahing/goofys/api/common" - . "github.com/kahing/goofys/internal" + goofys "github.com/StatCan/goofys/api" + . "github.com/StatCan/goofys/api/common" + . "github.com/StatCan/goofys/internal" "fmt" "os" From 9db676b41fd4546c06d7fd1df6afbac44d29bde5 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Tue, 9 Jul 2024 15:54:13 +0000 Subject: [PATCH 11/26] added meta-fuse-csi-plugin code and made dockerfile --- .github/workflows/build.yaml | 0 Dockerfile | 35 ++ Makefile | 11 + go.work | 6 + go.work.sum | 63 ++++ .../cmd/fusermount3-proxy/main.go | 141 ++++++++ meta-fuse-csi-plugin/go.mod | 11 + meta-fuse-csi-plugin/go.sum | 8 + .../pkg/fuse_starter/fuse_starter.go | 105 ++++++ meta-fuse-csi-plugin/pkg/util/fdchannel.go | 84 +++++ meta-fuse-csi-plugin/pkg/util/util.go | 149 +++++++++ meta-fuse-csi-plugin/pkg/util/util_test.go | 306 ++++++++++++++++++ meta-fuse-csi-plugin/pkg/util/volume_lock.go | 61 ++++ 13 files changed, 980 insertions(+) create mode 100644 .github/workflows/build.yaml create mode 100644 Dockerfile create mode 100644 go.work create mode 100644 go.work.sum create mode 100644 meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go create mode 100644 meta-fuse-csi-plugin/go.mod create mode 100644 meta-fuse-csi-plugin/go.sum create mode 100644 meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go create mode 100644 meta-fuse-csi-plugin/pkg/util/fdchannel.go create mode 100644 meta-fuse-csi-plugin/pkg/util/util.go create mode 100644 meta-fuse-csi-plugin/pkg/util/util_test.go create mode 100755 meta-fuse-csi-plugin/pkg/util/volume_lock.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0bd23caf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM golang:1.20.7 as fusermount3-proxy-builder + +WORKDIR /meta-fuse-csi-plugin +ADD . . +# Builds the meta-fuse-csi-plugin app +RUN make fusermount3-proxy BINDIR=/bin +# Builds the goofys app +RUN CGO_ENABLED=0 GOOS=linux go build -o goofys + +FROM ubuntu:22.04 + +RUN apt update && apt upgrade -y +RUN apt install -y ca-certificates wget libfuse2 fuse3 + +# prepare for MinIO +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && chmod +x /usr/bin/mc + +COPY </dev/null || git rev-list -n1 HEAD) +export BUILD_DATE ?= $(shell date --iso-8601=minutes) +BINDIR ?= bin +LDFLAGS ?= -s -w -X main.version=${STAGINGVERSION} -X main.builddate=${BUILD_DATE} -extldflags '-static' +FUSERMOUNT3PROXY_BINARY = fusermount3-proxy + run-test: s3proxy.jar ./test/run-tests.sh @@ -14,3 +20,8 @@ build: install: go install -ldflags "-X main.Version=`git rev-parse HEAD`" + +fusermount3-proxy: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${FUSERMOUNT3PROXY_BINARY} meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go + diff --git a/go.work b/go.work new file mode 100644 index 00000000..57ee83a7 --- /dev/null +++ b/go.work @@ -0,0 +1,6 @@ +go 1.20 + +use ( + . + ./meta-fuse-csi-plugin +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 00000000..04e9e374 --- /dev/null +++ b/go.work.sum @@ -0,0 +1,63 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go new file mode 100644 index 00000000..09d621a4 --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go @@ -0,0 +1,141 @@ +/* +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fusermount3proxy + +import ( + "fmt" + "os" + "strconv" + "syscall" + + starter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + flag "github.com/spf13/pflag" + + "k8s.io/klog/v2" +) + +var ( + optUnmount = flag.BoolP("unmount", "u", false, "unmount (NOT SUPPORTED)") + optAutoUnmount = flag.BoolP("auto-unmount", "U", false, "auto-unmount (NOT SUPPORTED)") + optLazy = flag.BoolP("lazy", "z", false, "lazy umount (NOT SUPPORTED)") + optQuiet = flag.BoolP("quiet", "q", false, "quiet (NOT SUPPORTED)") + optHelp = flag.BoolP("help", "h", false, "print help") + optVersion = flag.BoolP("version", "V", false, "print version") + optOptions = flag.StringP("options", "o", "", "mount options") + // This is set at compile time. + version = "unknown" + builddate = "unknown" +) + +var ignoredOptions = map[string]*bool{ + "unmount": optUnmount, + "auto-unmount": optAutoUnmount, + "lazy": optLazy, + "optQuiet": optQuiet, +} + +const ( + ENV_FUSE_COMMFD = "_FUSE_COMMFD" + ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH = "FUSERMOUNT3PROXY_FDPASSING_SOCKPATH" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + if *optHelp { + flag.PrintDefaults() + os.Exit(0) + } + + if *optVersion { + fmt.Printf("fusermount3-dummy version %v (BuildDate %v)\n", version, builddate) + os.Exit(0) + } + + klog.Infof("Running meta-fuse-csi-plugin fusermount3-dummy version %v (BuildDate %v)", version, builddate) + + if *optUnmount { + klog.Warning("'unmount' is not supported.") + os.Exit(0) + } + + if len(flag.Args()) == 0 { + klog.Error("mountpoint is not specified.") + os.Exit(1) + } + + if *optOptions == "" { + klog.Error("options is not specified.") + os.Exit(1) + } + + // fd-passing socket between fusermount3-dummy and csi-driver is passed as env var + fdPassingSocketPath := os.Getenv(ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH) + if fdPassingSocketPath == "" { + klog.Errorf("environment variable %q is not specified.", ENV_FUSERMOUNT3PROXY_FDPASSING_SOCKPATH) + os.Exit(1) + } + klog.Infof("fd-passing socket path is %q", fdPassingSocketPath) + + mntPoint := flag.Args()[0] + klog.Infof("mountpoint is %q, but ignored.", mntPoint) + + for k, v := range ignoredOptions { + if *v { + klog.Warningf("opiton %q is true, but ignored.", k) + } + } + + // TODO: send options to csi-driver and use them? + klog.Infof("options=%q", *optOptions) + + // get unix domain socket from caller + commFdStr := os.Getenv(ENV_FUSE_COMMFD) + commFd, err := strconv.Atoi(commFdStr) + if err != nil { + klog.Errorf("failed to get commFd _FUSE_COMMFD=%q", commFdStr) + os.Exit(1) + } + klog.Infof("commFd from %q is %d", ENV_FUSE_COMMFD, commFd) + + commConn, err := util.GetNetConnFromRawUnixSocketFd(commFd) + if err != nil { + klog.Errorf("failed to convert commFd to net.Conn: %w", err) + os.Exit(1) + } + klog.Infof("net.Conn is acquired from fd %d", commFd) + + // get fd for /dev/fuse from csi-driver + mc, err := starter.PrepareMountConfig(fdPassingSocketPath) + if err != nil { + klog.Errorf("failed to prepare mount config: socket path %q: %w", fdPassingSocketPath, err) + os.Exit(1) + } + defer syscall.Close(mc.FileDescriptor) + klog.Infof("received fd for /dev/fuse from csi-driver via socket %q", fdPassingSocketPath) + + // now already FUSE-fs mounted and fd is ready. + err = util.SendMsg(commConn, mc.FileDescriptor, []byte{0}) + if err != nil { + klog.Errorf("failed to send fd via commFd: %w", err) + os.Exit(1) + } + klog.Infof("sent fd for /dev/fuse via commFd %d", commFd) + klog.Info("exiting fusermount3-dummy...") +} diff --git a/meta-fuse-csi-plugin/go.mod b/meta-fuse-csi-plugin/go.mod new file mode 100644 index 00000000..37c5a69a --- /dev/null +++ b/meta-fuse-csi-plugin/go.mod @@ -0,0 +1,11 @@ +module github.com/StatCan/goofys/meta-fuse-csi-plugin + +go 1.20 + +require ( + github.com/spf13/pflag v1.0.5 + k8s.io/apimachinery v0.28.1 + k8s.io/klog/v2 v2.130.1 +) + +require github.com/go-logr/logr v1.4.1 // indirect diff --git a/meta-fuse-csi-plugin/go.sum b/meta-fuse-csi-plugin/go.sum new file mode 100644 index 00000000..32b660b7 --- /dev/null +++ b/meta-fuse-csi-plugin/go.sum @@ -0,0 +1,8 @@ +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= diff --git a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go new file mode 100644 index 00000000..7a367f3f --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go @@ -0,0 +1,105 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fusestarter + +import ( + "encoding/json" + "fmt" + "net" + "os" + "os/exec" + "syscall" + + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + "k8s.io/klog/v2" +) + +// FuseStarter will be used in the sidecar container to invoke fuse impl. +type FuseStarter struct { + mounterPath string + mounterArgs []string + Cmd *exec.Cmd +} + +// New returns a FuseStarter for the current system. +// It provides an option to specify the path to fuse binary. +func New(mounterPath string, mounterArgs []string) *FuseStarter { + return &FuseStarter{ + mounterPath: mounterPath, + mounterArgs: mounterArgs, + Cmd: nil, + } +} + +type MountConfig struct { + FileDescriptor int `json:"-"` + VolumeName string `json:"volumeName,omitempty"` +} + +func (m *FuseStarter) Mount(mc *MountConfig) (*exec.Cmd, error) { + klog.Infof("start to invoke fuse impl for volume %q", mc.VolumeName) + + klog.Infof("%s mounting with args %v...", m.mounterPath, m.mounterArgs) + cmd := exec.Cmd{ + Path: m.mounterPath, + Args: append([]string{m.mounterPath}, m.mounterArgs...), + ExtraFiles: []*os.File{os.NewFile(uintptr(mc.FileDescriptor), "/dev/fuse")}, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + + m.Cmd = &cmd + + return &cmd, nil +} + +// Fetch the following information from a given socket path: +// 1. Pod volume name +// 2. The file descriptor +// 3. Mount options passing to mounter (passed by the csi mounter). +func PrepareMountConfig(sp string) (*MountConfig, error) { + mc := MountConfig{} + + klog.Infof("connecting to socket %q", sp) + c, err := net.Dial("unix", sp) + if err != nil { + return nil, fmt.Errorf("failed to connect to the socket %q: %w", sp, err) + } + defer func() { + // as we got all the information from the socket, closing the connection and deleting the socket + c.Close() + if err = syscall.Unlink(sp); err != nil { + // csi driver may already removed the socket. + klog.Warningf("failed to close socket %q: %v", sp, err) + } + }() + + fd, msg, err := util.RecvMsg(c) + if err != nil { + return nil, fmt.Errorf("failed to receive mount options from the socket %q: %w", sp, err) + } + + mc.FileDescriptor = fd + + if err := json.Unmarshal(msg, &mc); err != nil { + return nil, fmt.Errorf("failed to unmarshal the mount config: %w", err) + } + + return &mc, nil +} diff --git a/meta-fuse-csi-plugin/pkg/util/fdchannel.go b/meta-fuse-csi-plugin/pkg/util/fdchannel.go new file mode 100644 index 00000000..721349aa --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/fdchannel.go @@ -0,0 +1,84 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "syscall" + + "k8s.io/klog/v2" +) + +func SendMsg(via net.Conn, fd int, msg []byte) error { + klog.V(4).Info("get the underlying socket") + conn, ok := via.(*net.UnixConn) + if !ok { + return fmt.Errorf("failed to cast via to *net.UnixConn") + } + connf, err := conn.File() + if err != nil { + return err + } + socket := int(connf.Fd()) + defer connf.Close() + + klog.V(4).Infof("calling sendmsg...") + rights := syscall.UnixRights(fd) + + return syscall.Sendmsg(socket, msg, rights, nil, 0) +} + +func RecvMsg(via net.Conn) (int, []byte, error) { + klog.V(4).Info("get the underlying socket") + conn, ok := via.(*net.UnixConn) + if !ok { + return 0, nil, fmt.Errorf("failed to cast via to *net.UnixConn") + } + connf, err := conn.File() + if err != nil { + return 0, nil, err + } + socket := int(connf.Fd()) + defer connf.Close() + + klog.V(4).Info("calling recvmsg...") + buf := make([]byte, syscall.CmsgSpace(4)) + b := make([]byte, 500) + //nolint:dogsled + n, _, _, _, err := syscall.Recvmsg(socket, b, buf, 0) + if err != nil { + return 0, nil, err + } + + klog.V(4).Info("parsing SCM...") + var msgs []syscall.SocketControlMessage + msgs, err = syscall.ParseSocketControlMessage(buf) + if err != nil { + return 0, nil, err + } + + klog.V(4).Info("parsing SCM_RIGHTS...") + fds, err := syscall.ParseUnixRights(&msgs[0]) + if err != nil { + return 0, nil, err + } + + return fds[0], b[:n], err +} diff --git a/meta-fuse-csi-plugin/pkg/util/util.go b/meta-fuse-csi-plugin/pkg/util/util.go new file mode 100644 index 00000000..7c418f78 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/util.go @@ -0,0 +1,149 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "net" + "net/url" + "os" + "regexp" + "strings" + + "k8s.io/klog/v2" +) + +const ( + Mb = 1024 * 1024 +) + +// ConvertLabelsStringToMap converts the labels from string to map +// example: "key1=value1,key2=value2" gets converted into {"key1": "value1", "key2": "value2"} +func ConvertLabelsStringToMap(labels string) (map[string]string, error) { + const labelsDelimiter = "," + const labelsKeyValueDelimiter = "=" + + labelsMap := make(map[string]string) + if labels == "" { + return labelsMap, nil + } + + // Following rules enforced for label keys + // 1. Keys have a minimum length of 1 character and a maximum length of 63 characters, and cannot be empty. + // 2. Keys and values can contain only lowercase letters, numeric characters, underscores, and dashes. + // 3. Keys must start with a lowercase letter. + regexKey := regexp.MustCompile(`^\p{Ll}[\p{Ll}0-9_-]{0,62}$`) + checkLabelKeyFn := func(key string) error { + if !regexKey.MatchString(key) { + return fmt.Errorf("label value %q is invalid (should start with lowercase letter / lowercase letter, digit, _ and - chars are allowed / 1-63 characters", key) + } + + return nil + } + + // Values can be empty, and have a maximum length of 63 characters. + regexValue := regexp.MustCompile(`^[\p{Ll}0-9_-]{0,63}$`) + checkLabelValueFn := func(value string) error { + if !regexValue.MatchString(value) { + return fmt.Errorf("label value %q is invalid (lowercase letter, digit, _ and - chars are allowed / 0-63 characters", value) + } + + return nil + } + + keyValueStrings := strings.Split(labels, labelsDelimiter) + for _, keyValue := range keyValueStrings { + keyValue := strings.Split(keyValue, labelsKeyValueDelimiter) + + if len(keyValue) != 2 { + return nil, fmt.Errorf("labels %q are invalid, correct format: 'key1=value1,key2=value2'", labels) + } + + key := strings.TrimSpace(keyValue[0]) + if err := checkLabelKeyFn(key); err != nil { + return nil, err + } + + value := strings.TrimSpace(keyValue[1]) + if err := checkLabelValueFn(value); err != nil { + return nil, err + } + + labelsMap[key] = value + } + + const maxNumberOfLabels = 64 + if len(labelsMap) > maxNumberOfLabels { + return nil, fmt.Errorf("more than %d labels is not allowed, given: %d", maxNumberOfLabels, len(labelsMap)) + } + + return labelsMap, nil +} + +func ParseEndpoint(endpoint string, cleanupSocket bool) (string, string, error) { + u, err := url.Parse(endpoint) + if err != nil { + klog.Fatal(err.Error()) + } + + var addr string + switch u.Scheme { + case "unix": + addr = u.Path + if cleanupSocket { + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + klog.Fatalf("Failed to remove %s, error: %s", addr, err) + } + } + case "tcp": + addr = u.Host + default: + klog.Fatalf("%v endpoint scheme not supported", u.Scheme) + } + + return u.Scheme, addr, nil +} + +func ParsePodIDVolumeFromTargetpath(targetPath string) (string, string, error) { + r := regexp.MustCompile(`/var/lib/kubelet/pods/(.*)/volumes/kubernetes\.io~csi/(.*)/mount`) + matched := r.FindStringSubmatch(targetPath) + if len(matched) < 3 { + return "", "", fmt.Errorf("targetPath %v does not contain Pod ID or volume information", targetPath) + } + podID := matched[1] + volume := matched[2] + + return podID, volume, nil +} + +func GetEmptyDirPath(podId, emptyDirName string) string { + return fmt.Sprintf("/var/lib/kubelet/pods/%s/volumes/kubernetes.io~empty-dir/%s", podId, emptyDirName) +} + +func GetNetConnFromRawUnixSocketFd(fd int) (net.Conn, error) { + f := os.NewFile(uintptr(fd), "unix_socket") + defer f.Close() + + c, err := net.FileConn(f) + if err != nil { + return nil, err + } + + return c, err +} diff --git a/meta-fuse-csi-plugin/pkg/util/util_test.go b/meta-fuse-csi-plugin/pkg/util/util_test.go new file mode 100644 index 00000000..759cdda6 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/util_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "reflect" + "testing" +) + +func TestConvertLabelsStringToMap(t *testing.T) { + t.Parallel() + t.Run("parsing labels string into map", func(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + labels string + expectedOutput map[string]string + expectedError bool + }{ + // Success test cases + { + name: "should return empty map when labels string is empty", + labels: "", + expectedOutput: map[string]string{}, + expectedError: false, + }, + { + name: "single label string", + labels: "key=value", + expectedOutput: map[string]string{ + "key": "value", + }, + expectedError: false, + }, + { + name: "multiple label string", + labels: "key1=value1,key2=value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + { + name: "multiple labels string with whitespaces gets trimmed", + labels: "key1=value1, key2=value2", + expectedOutput: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectedError: false, + }, + // Failure test cases + { + name: "malformed labels string (no keys and values)", + labels: ",,", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (incorrect format)", + labels: "foo,bar", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (missing key)", + labels: "key1=value1,=bar", + expectedOutput: nil, + expectedError: true, + }, + { + name: "malformed labels string (missing key and value)", + labels: "key1=value1,=bar,=", + expectedOutput: nil, + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + output, err := ConvertLabelsStringToMap(tc.labels) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(output, tc.expectedOutput) { + t.Errorf("Got labels %v, but expected %v", output, tc.expectedOutput) + } + } + }) + + t.Run("checking google requirements", func(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + labels string + expectedError bool + }{ + { + name: "64 labels at most", + labels: `k1=v,k2=v,k3=v,k4=v,k5=v,k6=v,k7=v,k8=v,k9=v,k10=v,k11=v,k12=v,k13=v,k14=v,k15=v,k16=v,k17=v,k18=v,k19=v,k20=v, + k21=v,k22=v,k23=v,k24=v,k25=v,k26=v,k27=v,k28=v,k29=v,k30=v,k31=v,k32=v,k33=v,k34=v,k35=v,k36=v,k37=v,k38=v,k39=v,k40=v, + k41=v,k42=v,k43=v,k44=v,k45=v,k46=v,k47=v,k48=v,k49=v,k50=v,k51=v,k52=v,k53=v,k54=v,k55=v,k56=v,k57=v,k58=v,k59=v,k60=v, + k61=v,k62=v,k63=v,k64=v,k65=v`, + expectedError: true, + }, + { + name: "label key must have atleast 1 char", + labels: "=v", + expectedError: true, + }, + { + name: "label key can only contain lowercase chars, digits, _ and -)", + labels: "k*=v", + expectedError: true, + }, + { + name: "label key can only contain lowercase chars)", + labels: "K=v", + expectedError: true, + }, + { + name: "label key may not have over 63 characters", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234=v", + expectedError: true, + }, + { + name: "label value can only contain lowercase chars, digits, _ and -)", + labels: "k1=###", + expectedError: true, + }, + { + name: "label value can only contain lowercase chars)", + labels: "k1=V", + expectedError: true, + }, + { + name: "label key cannot contain . and /", + labels: "kubernetes.io/created-for/pvc/namespace=v", + expectedError: true, + }, + { + name: "label value cannot contain . and /", + labels: "kubernetes_io_created-for_pvc_namespace=v./", + expectedError: true, + }, + { + name: "label value may not have over 63 chars", + labels: "v=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij1234", + expectedError: true, + }, + { + name: "label key can have up to 63 chars", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123=v", + expectedError: false, + }, + { + name: "label value can have up to 63 chars", + labels: "k=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij123", + expectedError: false, + }, + { + name: "label key can contain - and _", + labels: "abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij-_=v", + expectedError: false, + }, + { + name: "label value can contain - and _", + labels: "k=abcdefghijabcdefghijabcdefghijabcdefghijabcdefghijabcdefghij-_", + expectedError: false, + }, + { + name: "label value can have 0 chars", + labels: "kubernetes_io_created-for_pvc_namespace=", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + _, err := ConvertLabelsStringToMap(tc.labels) + + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + + if !tc.expectedError && err != nil { + t.Errorf("Did not expect error but got: %v", err) + } + } + }) +} + +func TestParseEndpoint(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + endpoint string + expectedScheme string + expectedAddress string + expectedError bool + }{ + { + name: "should parse unix endpoint correctly", + endpoint: "unix:/csi/csi.sock", + expectedScheme: "unix", + expectedAddress: "/csi/csi.sock", + expectedError: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + scheme, address, err := ParseEndpoint(tc.endpoint, false) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(scheme, tc.expectedScheme) { + t.Errorf("Got scheme %v, but expected %v", scheme, tc.expectedScheme) + } + + if !reflect.DeepEqual(address, tc.expectedAddress) { + t.Errorf("Got address %v, but expected %v", address, tc.expectedAddress) + } + } +} + +func TestParsePodIDVolumeFromTargetpath(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + targetPath string + expectedPodID string + expectedVolume string + expectedError bool + }{ + { + name: "should parse Pod ID correctly", + targetPath: "/var/lib/kubelet/pods/d2013878-3d56-45f9-89ec-0826612c89b6/volumes/kubernetes.io~csi/test-volume/mount", + expectedPodID: "d2013878-3d56-45f9-89ec-0826612c89b6", + expectedVolume: "test-volume", + expectedError: false, + }, + { + name: "should return error", + targetPath: "/foo/bar/volumes", + expectedPodID: "", + expectedVolume: "", + expectedError: true, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + podID, volume, err := ParsePodIDVolumeFromTargetpath(tc.targetPath) + if tc.expectedError && err == nil { + t.Errorf("Expected error but got none") + } + if err != nil { + if !tc.expectedError { + t.Errorf("Did not expect error but got: %v", err) + } + + continue + } + + if !reflect.DeepEqual(podID, tc.expectedPodID) { + t.Errorf("Got pod ID %v, but expected %v", podID, tc.expectedPodID) + } + if !reflect.DeepEqual(volume, tc.expectedVolume) { + t.Errorf("Got volume %v, but expected %v", volume, tc.expectedVolume) + } + } +} diff --git a/meta-fuse-csi-plugin/pkg/util/volume_lock.go b/meta-fuse-csi-plugin/pkg/util/volume_lock.go new file mode 100755 index 00000000..c64aa7de --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/util/volume_lock.go @@ -0,0 +1,61 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "sync" + + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + VolumeOperationAlreadyExistsFmt = "An operation with the given volume key %s already exists" +) + +// VolumeLocks implements a map with atomic operations. It stores a set of all volume IDs +// with an ongoing operation. +type VolumeLocks struct { + locks sets.Set[string] + mux sync.Mutex +} + +func NewVolumeLocks() *VolumeLocks { + return &VolumeLocks{ + locks: sets.Set[string]{}, + } +} + +// TryAcquire tries to acquire the lock for operating on volumeID and returns true if successful. +// If another operation is already using volumeID, returns false. +func (vl *VolumeLocks) TryAcquire(volumeID string) bool { + vl.mux.Lock() + defer vl.mux.Unlock() + if vl.locks.Has(volumeID) { + return false + } + vl.locks.Insert(volumeID) + + return true +} + +func (vl *VolumeLocks) Release(volumeID string) { + vl.mux.Lock() + defer vl.mux.Unlock() + vl.locks.Delete(volumeID) +} From 34bca56e935aae89777e55ab90cac41e14a70a9b Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Tue, 9 Jul 2024 18:47:08 +0000 Subject: [PATCH 12/26] updated github workflow --- .github/workflows/build-push.yaml | 69 +++++++++++++++++++++++++++++++ .github/workflows/build.yaml | 0 2 files changed, 69 insertions(+) create mode 100644 .github/workflows/build-push.yaml delete mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml new file mode 100644 index 00000000..1ffd9ba8 --- /dev/null +++ b/.github/workflows/build-push.yaml @@ -0,0 +1,69 @@ +name: build-and-push +on: + push: + branches: + - master + pull_request: + types: + - 'opened' + - 'synchronize' + - 'reopened' + +env: + REGISTRY_NAME: k8scc01covidacr + TRIVY_VERSION: "v0.43.1" + HADOLINT_VERSION: "2.12.0" +jobs: + build-push: + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - uses: actions/checkout@v4 + + # Push image to ACR + # Pushes if this is a push to master or an update to a PR that has auto-deploy label + - name: Test if we should push to ACR + id: should-i-push + if: | + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + contains( github.event.pull_request.labels.*.name, 'auto-deploy') + ) + run: echo "::set-output name=boolean::true" + + # Connect to Azure Container registry (ACR) + - uses: azure/docker-login@v1 + with: + login-server: ${{ env.REGISTRY_NAME }}.azurecr.io + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Run Hadolint + run: | + sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint + sudo chmod +x hadolint + ./hadolint ./Dockerfile --no-fail + + - name: Build image locally + run: | + docker build -f Dockerfile -t localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} . + docker push localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} + docker image prune + + - name: Aqua Security Trivy image scan + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }} + trivy image localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} --exit-code 1 --timeout=20m --security-checks vuln --severity CRITICAL + + # Container build and push to a Azure Container registry (ACR) + - name: Push to ACR if necessary + if: steps.should-i-push.outputs.boolean == 'true' + run: | + docker pull localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} + docker tag localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} ${{ env.REGISTRY_NAME }}/mfcp-proxy-goofys-multi-inc:${{ github.sha }} + docker push ${{ env.REGISTRY_NAME }}/mfcp-proxy-goofys-multi-inc:${{ github.sha }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index e69de29b..00000000 From af12b04dcc6d124ace8d808bea93f7e7b423298c Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Tue, 9 Jul 2024 19:02:13 +0000 Subject: [PATCH 13/26] removes fatal and os.exit --- main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 0aff8d57..d64f88e2 100644 --- a/main.go +++ b/main.go @@ -232,7 +232,8 @@ func main() { // Write out a Pid file err = ioutil.WriteFile(flags.PidFile, []byte(strconv.Itoa(os.Getpid())), 0644) if err != nil { - log.Fatalf("Error writing pid file: %v", err) + //log.Fatalf("Error writing pid file: %v", err) + log.Printf("Error writing pid file: %v", err) } } @@ -256,8 +257,9 @@ func main() { err := app.Run(MassageMountFlags(os.Args)) if err != nil { if flags != nil && !flags.Foreground && child != nil { - log.Fatalln("Unable to mount file system, see syslog for details") + //log.Fatalln("Unable to mount file system, see syslog for details") + log.Println("Unable to mount file system, see syslog for details") } - os.Exit(1) + //os.Exit(1) } } From b912810781a07a799639f95d49de6c18fae1da0a Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Thu, 11 Jul 2024 16:33:16 +0000 Subject: [PATCH 14/26] removed push from push step for debugging --- .github/workflows/build-push.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 1ffd9ba8..f0bb1f1b 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -66,4 +66,3 @@ jobs: run: | docker pull localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} docker tag localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} ${{ env.REGISTRY_NAME }}/mfcp-proxy-goofys-multi-inc:${{ github.sha }} - docker push ${{ env.REGISTRY_NAME }}/mfcp-proxy-goofys-multi-inc:${{ github.sha }} From 4ed1be64127519aae700d0d73427e785927dda16 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Thu, 11 Jul 2024 16:37:33 +0000 Subject: [PATCH 15/26] fixed registry name in build-push --- .github/workflows/build-push.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index f0bb1f1b..f81483c1 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -65,4 +65,5 @@ jobs: if: steps.should-i-push.outputs.boolean == 'true' run: | docker pull localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} - docker tag localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} ${{ env.REGISTRY_NAME }}/mfcp-proxy-goofys-multi-inc:${{ github.sha }} + docker tag localhost:5000/mfcp-proxy-goofys-multi-inc:${{ github.sha }} ${{ env.REGISTRY_NAME }}.azurecr.io/mfcp-proxy-goofys-multi-inc:${{ github.sha }} + docker push ${{ env.REGISTRY_NAME }}.azurecr.io/mfcp-proxy-goofys-multi-inc:${{ github.sha }} From 63d32d9ab8545cebff3f1d8e9f20af755986a69e Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Fri, 12 Jul 2024 14:51:26 +0000 Subject: [PATCH 16/26] going back to wget goofys to test build --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0bd23caf..f8a7b8b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ADD . . # Builds the meta-fuse-csi-plugin app RUN make fusermount3-proxy BINDIR=/bin # Builds the goofys app -RUN CGO_ENABLED=0 GOOS=linux go build -o goofys +#RUN CGO_ENABLED=0 GOOS=linux go build -o goofys FROM ubuntu:22.04 @@ -29,7 +29,8 @@ EOF RUN chmod +x /configure_minio.sh #Get goofys build from first step -COPY --from=fusermount3-proxy-builder /meta-fuse-csi-plugin/goofys . +#COPY --from=fusermount3-proxy-builder /meta-fuse-csi-plugin/goofys . +RUN wget https://github.com/mathis-marcotte/goofys/releases/download/5Gb-no-fatal/goofys -O /goofys && chmod +x /goofys COPY --from=fusermount3-proxy-builder /bin/fusermount3-proxy /bin/fusermount3 RUN ln -sf /bin/fusermount3 /bin/fusermount From a1fbdf753f5d2fb891d0f2ea4fcc5018cdde8274 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Fri, 12 Jul 2024 16:58:36 +0000 Subject: [PATCH 17/26] added missing files to meta-fuse folder --- Dockerfile | 2 +- go.work.sum | 280 +++++++++++++- meta-fuse-csi-plugin/Makefile | 126 +++++++ .../cmd/csi_driver/Dockerfile | 68 ++++ meta-fuse-csi-plugin/cmd/csi_driver/main.go | 71 ++++ .../cmd/fuse_starter/Dockerfile | 33 ++ meta-fuse-csi-plugin/cmd/fuse_starter/main.go | 122 +++++++ .../cmd/fusermount3-proxy/main.go | 6 +- .../deploy/csi-driver-daemonset.yaml | 97 +++++ meta-fuse-csi-plugin/deploy/csi-driver.yaml | 21 ++ meta-fuse-csi-plugin/go.mod | 25 +- meta-fuse-csi-plugin/go.sum | 41 ++- meta-fuse-csi-plugin/pkg/csi_driver/driver.go | 145 ++++++++ .../pkg/csi_driver/identity.go | 57 +++ meta-fuse-csi-plugin/pkg/csi_driver/node.go | 304 ++++++++++++++++ .../pkg/csi_driver/node_unimpl.go | 42 +++ meta-fuse-csi-plugin/pkg/csi_driver/server.go | 102 ++++++ meta-fuse-csi-plugin/pkg/csi_driver/utils.go | 87 +++++ .../pkg/csi_mounter/csi_mounter.go | 341 ++++++++++++++++++ .../pkg/csi_mounter/csi_mounter_test.go | 94 +++++ .../pkg/fuse_starter/fuse_starter.go | 2 +- 21 files changed, 2037 insertions(+), 29 deletions(-) create mode 100755 meta-fuse-csi-plugin/Makefile create mode 100755 meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile create mode 100644 meta-fuse-csi-plugin/cmd/csi_driver/main.go create mode 100755 meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile create mode 100644 meta-fuse-csi-plugin/cmd/fuse_starter/main.go create mode 100644 meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml create mode 100644 meta-fuse-csi-plugin/deploy/csi-driver.yaml create mode 100644 meta-fuse-csi-plugin/pkg/csi_driver/driver.go create mode 100755 meta-fuse-csi-plugin/pkg/csi_driver/identity.go create mode 100644 meta-fuse-csi-plugin/pkg/csi_driver/node.go create mode 100755 meta-fuse-csi-plugin/pkg/csi_driver/node_unimpl.go create mode 100644 meta-fuse-csi-plugin/pkg/csi_driver/server.go create mode 100755 meta-fuse-csi-plugin/pkg/csi_driver/utils.go create mode 100644 meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go create mode 100644 meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter_test.go diff --git a/Dockerfile b/Dockerfile index f8a7b8b9..322ea54c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.20.7 as fusermount3-proxy-builder WORKDIR /meta-fuse-csi-plugin -ADD . . +ADD ./meta-fuse-csi-plugin . # Builds the meta-fuse-csi-plugin app RUN make fusermount3-proxy BINDIR=/bin # Builds the goofys app diff --git a/go.work.sum b/go.work.sum index 04e9e374..6e032491 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,61 +1,315 @@ +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM= github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.41.0/go.mod h1:YjmsSWM1VTcWXFSgyrmLADPMZZohioz9onjgkikk59w= +go.opentelemetry.io/otel v1.15.0/go.mod h1:qfwLEbWhLPk5gyWrne4XnF0lC8wtywbuJbgfAE3zbek= +go.opentelemetry.io/otel/metric v0.38.0/go.mod h1:uAtxN5hl8aXh5irD8afBtSwQU5Zjg64WWSz6KheZxBg= +go.opentelemetry.io/otel/trace v1.15.0/go.mod h1:CUsmE2Ht1CRkvE8OsMESvraoZrrcgD1J2W8GV1ev0Y4= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= -k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= +k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= +k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= diff --git a/meta-fuse-csi-plugin/Makefile b/meta-fuse-csi-plugin/Makefile new file mode 100755 index 00000000..b9417153 --- /dev/null +++ b/meta-fuse-csi-plugin/Makefile @@ -0,0 +1,126 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2022 Google LLC +# Copyright 2023 Preferred Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export STAGINGVERSION ?= $(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git rev-list -n1 HEAD) +export BUILD_DATE ?= $(shell date --iso-8601=minutes) +BINDIR ?= bin +LDFLAGS ?= -s -w -X main.version=${STAGINGVERSION} -X main.builddate=${BUILD_DATE} -extldflags '-static' + +DRIVER_BINARY = meta-fuse-csi-plugin +STARTER_BINARY = fuse-starter +FUSERMOUNT3PROXY_BINARY = fusermount3-proxy + +REGISTRY ?= ghcr.io/pfnet-research/meta-fuse-csi-plugin +DRIVER_IMAGE = ${REGISTRY}/${DRIVER_BINARY} +STARTER_IMAGE = ${REGISTRY}/${STARTER_BINARY} +EXAMPLE_IMAGE = ${REGISTRY}/mfcp-example + +DOCKER_BUILD_ARGS ?= --load --build-arg STAGINGVERSION=${STAGINGVERSION} +ifneq ("$(shell docker buildx build --help | grep 'provenance')", "") +DOCKER_BUILD_ARGS += --provenance=false +endif + +LOAD_TO_KIND ?= false +PUBLISH_IMAGE ?= false + +$(info STAGINGVERSION is ${STAGINGVERSION}) +$(info DRIVER_IMAGE is ${DRIVER_IMAGE}) +$(info STARTER_IMAGE is ${STARTER_IMAGE}) + +.PHONY: all build-image-linux-amd64 + +all: build-driver build-examples + +driver: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${DRIVER_BINARY} cmd/csi_driver/main.go + +fuse-starter: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${STARTER_BINARY} cmd/fuse_starter/main.go + +fusermount3-proxy: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${FUSERMOUNT3PROXY_BINARY} cmd/fusermount3-proxy/main.go + +build-driver: + $(eval IMAGE_NAME := ${DRIVER_IMAGE}:${STAGINGVERSION}) + docker buildx build ${DOCKER_BUILD_ARGS} ${DOCKER_CACHE_ARGS} \ + --file ./cmd/csi_driver/Dockerfile \ + --tag ${IMAGE_NAME} \ + --platform linux/amd64 . + if [ "${PUBLISH_IMAGE}" = "true" ]; then \ + docker push ${IMAGE_NAME}; \ + docker tag ${IMAGE_NAME} ${DRIVER_IMAGE}:latest; \ + docker push ${DRIVER_IMAGE}:latest; \ + fi + if [ "${LOAD_TO_KIND}" = "true" ]; then \ + kind load docker-image ${IMAGE_NAME};\ + fi + +define build-example-template +ifneq ("$(EXAMPLES)", "") +EXAMPLES += build-example-$(1)-$(2) +else +EXAMPLES := build-example-$(1)-$(2) +endif + +.PHONY: build-example-$1-$2 +build-example-$(1)-$(2): + $(eval IMAGE_NAME := ${EXAMPLE_IMAGE}-$1-$2:${STAGINGVERSION}) + docker buildx build ${DOCKER_BUILD_ARGS} ${DOCKER_CACHE_ARGS} \ + --file ./examples/$1/$2/Dockerfile \ + --tag ${IMAGE_NAME} \ + --platform linux/amd64 . + if [ "${PUBLISH_IMAGE}" = "true" ]; then \ + docker push ${IMAGE_NAME}; \ + docker tag ${IMAGE_NAME} ${EXAMPLE_IMAGE}-$1-$2:latest; \ + docker push ${EXAMPLE_IMAGE}-$1-$2:latest; \ + fi + if [ "${LOAD_TO_KIND}" = "true" ]; then \ + kind load docker-image ${IMAGE_NAME};\ + fi +endef + +$(eval $(call build-example-template,proxy,goofys)) + +$(info $(EXAMPLES)) + +.PHONY: build-examples +build-examples: $(EXAMPLES) + +define test-example-template +ifneq ("$(EXAMPLES)", "") +EXAMPLE_TESTS += test-example-$(1)-$(2) +else +EXAMPLE_TESTS := test-example-$(1)-$(2) +endif + +.PHONY: test-example-$1-$2 +test-example-$(1)-$(2): + ./examples/check.sh ./$1/$2 mfcp-example-$1-$2 $3 $4 $5 $6 +endef + +$(eval $(call test-example-template,proxy,goofys,starter,/test.txt,busybox,/data/test.txt)) + +.PHONY: test-examples +test-examples: $(EXAMPLE_TESTS) + +.PHONY: test-e2e +test-e2e: + - kind delete cluster + kind create cluster + ./test_e2e.sh diff --git a/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile b/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile new file mode 100755 index 00000000..eb80bd68 --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile @@ -0,0 +1,68 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2022 Google LLC +# Copyright 2023 Preferred Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build driver go binary +FROM golang:1.20.7 as driver-builder + +ARG STAGINGVERSION + +WORKDIR /meta-fuse-csi-plugin +ADD . . +RUN make driver BINDIR=/bin + +# Start from Kubernetes Debian base. +FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as debian +# Install necessary dependencies +RUN clean-install mount bash + +# go/gke-releasing-policies#base-images +# We use `gcr.io/distroless/base` because it includes glibc. +FROM gcr.io/distroless/base-debian11 as distroless-base + +# The distroless amd64 image has a target triplet of x86_64 +FROM distroless-base AS distroless-amd64 +ENV LIB_DIR_PREFIX x86_64 + +FROM distroless-$TARGETARCH as output-image + +# Copy the mount/umount binaries +COPY --from=debian /bin/mount /bin/mount +COPY --from=debian /bin/umount /bin/umount + +# Copy shared libraries into distroless base. +COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/ + +COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \ + /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0 /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/ + +# Build stage used for validation of the output-image +FROM output-image as validation-image +COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \ + /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 /lib/${LIB_DIR_PREFIX}-linux-gnu/ +COPY --from=debian /bin/bash /bin/bash +COPY --from=debian /bin/grep /bin/grep +COPY --from=debian /usr/bin/ldd /usr/bin/ldd +SHELL ["/bin/bash", "-c"] +RUN if ldd /bin/mount | grep "not found"; then echo "!!! Missing deps for mount command !!!" && exit 1; fi +RUN if ldd /bin/umount | grep "not found"; then echo "!!! Missing deps for umount command !!!" && exit 1; fi + +# Final build stage, create the real Docker image with ENTRYPOINT +FROM output-image + +COPY --from=driver-builder /bin/meta-fuse-csi-plugin /meta-fuse-csi-plugin + +ENTRYPOINT ["/meta-fuse-csi-plugin"] diff --git a/meta-fuse-csi-plugin/cmd/csi_driver/main.go b/meta-fuse-csi-plugin/cmd/csi_driver/main.go new file mode 100644 index 00000000..902df1ca --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/csi_driver/main.go @@ -0,0 +1,71 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + + driver "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_driver" + csimounter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_mounter" + "k8s.io/klog/v2" + "k8s.io/mount-utils" +) + +var ( + endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint") + nodeID = flag.String("nodeid", "", "node id") + + // These are set at compile time. + version = "unknown" + builddate = "unknown" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + var err error + var mounter mount.Interface + if *nodeID == "" { + klog.Fatalf("NodeID cannot be empty for node service") + } + + mounter, err = csimounter.New("") + if err != nil { + klog.Fatalf("Failed to prepare CSI mounter: %v", err) + } + + config := &driver.DriverConfig{ + Name: driver.DefaultName, + Version: version, + NodeID: *nodeID, + Mounter: mounter, + } + + d, err := driver.NewDriver(config) + if err != nil { + klog.Fatalf("Failed to initialize meta-fuse-csi-plugin: %v", err) + } + + klog.Infof("Running meta-fuse-csi-plugin version %v (BuildDate %v)", version, builddate) + d.Run(*endpoint) + + os.Exit(0) +} diff --git a/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile b/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile new file mode 100755 index 00000000..549253f5 --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile @@ -0,0 +1,33 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2022 Google LLC +# Copyright 2023 Preferred Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build fuse-starter go binary +FROM golang:1.20.7 as fuse-starter-builder + +ARG STAGINGVERSION + +WORKDIR /meta-fuse-csi-plugin +ADD . . +RUN make fuse-starter BINDIR=/bin + +# go/gke-releasing-policies#base-images +# We use `gcr.io/distroless/base` because it includes glibc. +FROM gcr.io/distroless/base-debian11 + +# Copy the binaries +COPY --from=fuse-starter-builder /bin/fuse-starter /fuse-starter + +ENTRYPOINT ["/meta-fuse-csi-plugin-fuse-starter"] diff --git a/meta-fuse-csi-plugin/cmd/fuse_starter/main.go b/meta-fuse-csi-plugin/cmd/fuse_starter/main.go new file mode 100644 index 00000000..e75aa421 --- /dev/null +++ b/meta-fuse-csi-plugin/cmd/fuse_starter/main.go @@ -0,0 +1,122 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "os/signal" + "sync" + "syscall" + + starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" + "k8s.io/klog/v2" +) + +var ( + fdPassingSocketPath = flag.String("fd-passing-socket-path", "", "unix domain socket path for FUSE fd passing") + // This is set at compile time. + version = "unknown" + builddate = "unknown" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + klog.Infof("Running meta-fuse-csi-plugin fuse-starter version %v (BuildDate %v)", version, builddate) + klog.Infof("fd-passing-socket-path: %q", *fdPassingSocketPath) + + // parsing command args after "--" + mounterArgsIdx := 0 + for ; mounterArgsIdx < len(os.Args); mounterArgsIdx += 1 { + if os.Args[mounterArgsIdx] == "--" { + mounterArgsIdx += 1 + break + } + } + + if len(os.Args) == mounterArgsIdx { + klog.Error("mounter does not specified") + return + } + + mounterPath := os.Args[mounterArgsIdx] + mounterArgs := os.Args[mounterArgsIdx+1:] + klog.Infof("mounter(%s) args are %v", mounterPath, mounterArgs) + + if *fdPassingSocketPath == "" { + klog.Error("fd-passing-socket-path does not specified") + return + } + + mounter := starter.New(mounterPath, mounterArgs) + var wg sync.WaitGroup + + mc, err := starter.PrepareMountConfig(*fdPassingSocketPath) + if err != nil { + klog.Errorf("failed prepare mount config: socket path %q: %v\n", *fdPassingSocketPath, err) + return + } + + c := make(chan os.Signal, 1) + + wg.Add(1) + go func(mc *starter.MountConfig) { + defer wg.Done() + cmd, err := mounter.Mount(mc) + if err != nil { + klog.Errorf("failed to mount volume %q: %v\n", mc.VolumeName, err) + return + } + + if err = cmd.Start(); err != nil { + klog.Errorf("failed to start mounter with error: %v\n", err) + return + } + + // Since the mounter has taken over the file descriptor, + // closing the file descriptor to avoid other process forking it. + syscall.Close(mc.FileDescriptor) + if err = cmd.Wait(); err != nil { + klog.Errorf("mounter exited with error: %v\n", err) + } else { + klog.Infof("[%v] mounter exited normally.", mc.VolumeName) + } + + // Process may exit early. + c <- syscall.SIGTERM + }(mc) + + signal.Notify(c, syscall.SIGTERM) + klog.Info("waiting for SIGTERM signal...") + + <-c // blocking the process + + klog.Info("received SIGTERM signal, waiting for all the mounter processes exit...") + + // TODO: send SIGKILL to kill hang mounter process + err = mounter.Cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + klog.Warning("failed to send SIGTERM signal to mounter process") + } + wg.Wait() + + klog.Info("exiting fuse-starter...") +} diff --git a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go index 09d621a4..f4149f29 100644 --- a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go +++ b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package fusermount3proxy +package main import ( "fmt" @@ -22,8 +22,8 @@ import ( "strconv" "syscall" - starter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/fuse_starter" - "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" flag "github.com/spf13/pflag" "k8s.io/klog/v2" diff --git a/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml b/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml new file mode 100644 index 00000000..849dc08c --- /dev/null +++ b/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: meta-fuse-csi-plugin + namespace: mfcp-system +spec: + selector: + matchLabels: + k8s-app: meta-fuse-csi-plugin + template: + metadata: + annotations: + seccomp.security.alpha.kubernetes.io/pod: runtime/default + labels: + k8s-app: meta-fuse-csi-plugin + spec: + containers: + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --nodeid=$(KUBE_NODE_NAME) + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/meta-fuse-csi-plugin:latest + imagePullPolicy: IfNotPresent + name: meta-fuse-csi-plugin + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 5m + memory: 10Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/meta-fuse-csi-plugin.csi.storage.pfn.io/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + imagePullPolicy: Always + name: csi-driver-registrar + resources: + limits: + cpu: 50m + memory: 100Mi + requests: + cpu: 10m + memory: 10Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + nodeSelector: + kubernetes.io/os: linux + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/pods/ + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/meta-fuse-csi-plugin.csi.storage.pfn.io/ + type: DirectoryOrCreate + name: socket-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate diff --git a/meta-fuse-csi-plugin/deploy/csi-driver.yaml b/meta-fuse-csi-plugin/deploy/csi-driver.yaml new file mode 100644 index 00000000..e9d90369 --- /dev/null +++ b/meta-fuse-csi-plugin/deploy/csi-driver.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mfcp-system +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + labels: + addonmanager.kubernetes.io/mode: Reconcile + k8s-app: meta-fuse-csi-plugin + name: meta-fuse-csi-plugin.csi.storage.pfn.io +spec: + attachRequired: false + fsGroupPolicy: ReadWriteOnceWithFSType + podInfoOnMount: true + requiresRepublish: true + storageCapacity: false + volumeLifecycleModes: + - Ephemeral + diff --git a/meta-fuse-csi-plugin/go.mod b/meta-fuse-csi-plugin/go.mod index 37c5a69a..f085b29b 100644 --- a/meta-fuse-csi-plugin/go.mod +++ b/meta-fuse-csi-plugin/go.mod @@ -1,11 +1,22 @@ -module github.com/StatCan/goofys/meta-fuse-csi-plugin - +module github.com/pfnet-research/meta-fuse-csi-plugin go 1.20 +require k8s.io/klog/v2 v2.100.1 + require ( - github.com/spf13/pflag v1.0.5 - k8s.io/apimachinery v0.28.1 - k8s.io/klog/v2 v2.130.1 + github.com/container-storage-interface/spec v1.8.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/kubernetes-csi/csi-lib-utils v0.15.0 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/net v0.13.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + k8s.io/apimachinery v0.28.1 // indirect + k8s.io/mount-utils v0.28.1 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect ) - -require github.com/go-logr/logr v1.4.1 // indirect diff --git a/meta-fuse-csi-plugin/go.sum b/meta-fuse-csi-plugin/go.sum index 32b660b7..19a18fed 100644 --- a/meta-fuse-csi-plugin/go.sum +++ b/meta-fuse-csi-plugin/go.sum @@ -1,8 +1,41 @@ -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/container-storage-interface/spec v1.8.0 h1:D0vhF3PLIZwlwZEf2eNbpujGCNwspwTYf2idJRJx4xI= +github.com/container-storage-interface/spec v1.8.0/go.mod h1:ROLik+GhPslwwWRNFF1KasPzroNARibH2rfz1rkg4H0= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/kubernetes-csi/csi-lib-utils v0.15.0 h1:YTMO6WilRUmjGh5/73kF4KjNcXev+V37O4bx8Uoxy5A= +github.com/kubernetes-csi/csi-lib-utils v0.15.0/go.mod h1:fsoR7g1fOfl1z0WDpA1WvWPtt4oVvgzChgSUgR3JWDw= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/mount-utils v0.28.1 h1:oyPtn8ZVxniBfwSlQaBF4fr7QVNYzUuk+gkuxEJgil0= +k8s.io/mount-utils v0.28.1/go.mod h1:AyP8LmZSLgpGdFQr+vzHTerlPiGvXUdP99n98Er47jw= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/driver.go b/meta-fuse-csi-plugin/pkg/csi_driver/driver.go new file mode 100644 index 00000000..096a79d2 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/driver.go @@ -0,0 +1,145 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + "k8s.io/klog/v2" + "k8s.io/mount-utils" +) + +const DefaultName = "meta-fuse-csi-plugin.csi.storage.pfn.io" + +type DriverConfig struct { + Name string // Driver name + Version string // Driver version + NodeID string // Node name + Mounter mount.Interface +} + +type Driver struct { + config *DriverConfig + + // CSI RPC servers + ids csi.IdentityServer + ns csi.NodeServer + + // Plugin capabilities + vcap map[csi.VolumeCapability_AccessMode_Mode]*csi.VolumeCapability_AccessMode + nscap []*csi.NodeServiceCapability +} + +func NewDriver(config *DriverConfig) (*Driver, error) { + if config.Name == "" { + return nil, fmt.Errorf("driver name missing") + } + if config.Version == "" { + return nil, fmt.Errorf("driver version missing") + } + + driver := &Driver{ + config: config, + vcap: map[csi.VolumeCapability_AccessMode_Mode]*csi.VolumeCapability_AccessMode{}, + } + + // TODO: is this ok? + vcam := []csi.VolumeCapability_AccessMode_Mode{ + csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, + } + driver.addVolumeCapabilityAccessModes(vcam) + + driver.ids = newIdentityServer(driver) + nscap := []csi.NodeServiceCapability_RPC_Type{} + driver.ns = newNodeServer(driver, config.Mounter) + driver.addNodeServiceCapabilities(nscap) + + return driver, nil +} + +func (driver *Driver) addVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) { + for _, c := range vc { + klog.Infof("Enabling volume access mode: %v", c.String()) + mode := NewVolumeCapabilityAccessMode(c) + driver.vcap[mode.Mode] = mode + } +} + +func (driver *Driver) validateVolumeCapabilities(caps []*csi.VolumeCapability) error { + if len(caps) == 0 { + return fmt.Errorf("volume capabilities must be provided") + } + + for _, c := range caps { + if err := driver.validateVolumeCapability(c); err != nil { + return err + } + } + + return nil +} + +func (driver *Driver) validateVolumeCapability(c *csi.VolumeCapability) error { + if c == nil { + return fmt.Errorf("volume capability must be provided") + } + + // Validate access mode + accessMode := c.GetAccessMode() + if accessMode == nil { + return fmt.Errorf("volume capability access mode not set") + } + if driver.vcap[accessMode.Mode] == nil { + return fmt.Errorf("driver does not support access mode: %v", accessMode.Mode.String()) + } + + // Validate access type + accessType := c.GetAccessType() + if accessType == nil { + return fmt.Errorf("volume capability access type not set") + } + mountType := c.GetMount() + if mountType == nil { + return fmt.Errorf("driver only supports mount access type volume capability") + } + + return nil +} + +func (driver *Driver) addNodeServiceCapabilities(nl []csi.NodeServiceCapability_RPC_Type) { + nsc := []*csi.NodeServiceCapability{} + for _, n := range nl { + klog.Infof("Enabling node service capability: %v", n.String()) + nsc = append(nsc, NewNodeServiceCapability(n)) + } + driver.nscap = nsc +} + +func (driver *Driver) Run(endpoint string) { + klog.Infof("Running driver: %v", driver.config.Name) + + s := NewNonBlockingGRPCServer() + s.Start(endpoint, driver.ids, nil, driver.ns) + s.Wait() +} diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/identity.go b/meta-fuse-csi-plugin/pkg/csi_driver/identity.go new file mode 100755 index 00000000..dd14a7fb --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/identity.go @@ -0,0 +1,57 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + csi "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" +) + +type identityServer struct { + driver *Driver +} + +func newIdentityServer(driver *Driver) csi.IdentityServer { + return &identityServer{driver: driver} +} + +func (s *identityServer) GetPluginInfo(_ context.Context, _ *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + return &csi.GetPluginInfoResponse{ + Name: s.driver.config.Name, + VendorVersion: s.driver.config.Version, + }, nil +} + +func (s *identityServer) GetPluginCapabilities(_ context.Context, _ *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + }, + }, nil +} + +func (s *identityServer) Probe(_ context.Context, _ *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{}, nil +} diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/node.go b/meta-fuse-csi-plugin/pkg/csi_driver/node.go new file mode 100644 index 00000000..adb7a79c --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/node.go @@ -0,0 +1,304 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "os" + "path/filepath" + "strings" + "time" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + csimounter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_mounter" + "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + mount "k8s.io/mount-utils" +) + +// NodePublishVolume VolumeContext parameters. +const ( + VolumeContextKeyServiceAccountName = "csi.storage.k8s.io/serviceAccount.name" + //nolint:gosec + VolumeContextKeyServiceAccountToken = "csi.storage.k8s.io/serviceAccount.tokens" + VolumeContextKeyPodName = "csi.storage.k8s.io/pod.name" + VolumeContextKeyPodNamespace = "csi.storage.k8s.io/pod.namespace" + VolumeContextKeyEphemeral = "csi.storage.k8s.io/ephemeral" + VolumeContextKeyMountOptions = "mountOptions" + VolumeContextKeyFdPassingEmptyDirName = "fdPassingEmptyDirName" + VolumeContextKeyFdPassingSocketName = "fdPassingSocketName" + + UmountTimeout = time.Second * 5 +) + +// nodeServer handles mounting and unmounting of GCS FUSE volumes on a node. +type nodeServer struct { + driver *Driver + mounter mount.Interface + volumeLocks *util.VolumeLocks +} + +func newNodeServer(driver *Driver, mounter mount.Interface) csi.NodeServer { + return &nodeServer{ + driver: driver, + mounter: mounter, + volumeLocks: util.NewVolumeLocks(), + } +} + +func (s *nodeServer) NodeGetInfo(_ context.Context, _ *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + return &csi.NodeGetInfoResponse{ + NodeId: s.driver.config.NodeID, + }, nil +} + +func (s *nodeServer) NodeGetCapabilities(_ context.Context, _ *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: s.driver.nscap, + }, nil +} + +func (s *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + vc := req.GetVolumeContext() + + fuseMountOptions := []string{} + if req.GetReadonly() { + fuseMountOptions = joinMountOptions(fuseMountOptions, []string{"ro"}) + } else { + fuseMountOptions = joinMountOptions(fuseMountOptions, []string{"rw"}) + } + if capMount := req.GetVolumeCapability().GetMount(); capMount != nil { + fuseMountOptions = joinMountOptions(fuseMountOptions, capMount.GetMountFlags()) + } + if mountOptions, ok := vc[VolumeContextKeyMountOptions]; ok { + fuseMountOptions = joinMountOptions(fuseMountOptions, strings.Split(mountOptions, ",")) + } + + if vc[VolumeContextKeyEphemeral] != "true" { + return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume VolumeContext %q must be provided for ephemeral storage", VolumeContextKeyEphemeral) + } + + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodePublishVolume target path must be provided") + } + + if err := s.driver.validateVolumeCapabilities([]*csi.VolumeCapability{req.GetVolumeCapability()}); err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + // Acquire a lock on the target path instead of volumeID, since we do not want to serialize multiple node publish calls on the same volume. + if acquired := s.volumeLocks.TryAcquire(targetPath); !acquired { + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, targetPath) + } + defer s.volumeLocks.Release(targetPath) + + // Parse targetPath to get volumeName + podId, volumeName, err := util.ParsePodIDVolumeFromTargetpath(targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to parse targetPath %q", targetPath) + } + + fdPassingEmptyDirName, ok := vc[VolumeContextKeyFdPassingEmptyDirName] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume VolumeContext %q must be provided", VolumeContextKeyFdPassingEmptyDirName) + } + + fdPassingSocketName, ok := vc[VolumeContextKeyFdPassingSocketName] + if !ok { + return nil, status.Errorf(codes.InvalidArgument, "NodePublishVolume VolumeContext %q must be provided", VolumeContextKeyFdPassingSocketName) + } + + // Check if the target path is already mounted + mounted, err := s.isDirMounted(targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to check if path %q is already mounted: %v", targetPath, err) + } + + if mounted { + // Already mounted + klog.V(4).Infof("NodePublishVolume succeeded on volume %q to target path %q, mount already exists.", volumeName, targetPath) + + return &csi.NodePublishVolumeResponse{}, nil + } + + emptyDir := util.GetEmptyDirPath(podId, fdPassingEmptyDirName) + if _, err := os.Stat(emptyDir); err != nil { + return nil, status.Errorf(codes.Internal, "directory %q for emptyDir %q does not exist", emptyDir, fdPassingEmptyDirName) + } + + sockPath := filepath.Join(emptyDir, fdPassingSocketName) + if _, err := os.Stat(sockPath); err == nil { + // Unix domain socket already waits for connection + klog.V(4).Infof("NodePublishVolume succeeded on volume %q to target path %q, unix domain socket already exists.", volumeName, targetPath) + + return &csi.NodePublishVolumeResponse{}, nil + } + + klog.V(4).Infof("NodePublishVolume attempting mkdir for path %q", targetPath) + if err := os.MkdirAll(targetPath, 0o750); err != nil { + return nil, status.Errorf(codes.Internal, "mkdir failed for path %q: %v", targetPath, err) + } + + // fuseMountOptions[0] is fdPassingSockPath + fuseMountOptions = append([]string{sockPath}, fuseMountOptions...) + + // Start to mount + if err = s.mounter.Mount(volumeName, targetPath, "fuse", fuseMountOptions); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount volume %q to target path %q: %v", volumeName, targetPath, err) + } + + klog.V(4).Infof("NodePublishVolume succeeded on volume %q to target path %q", volumeName, targetPath) + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (s *nodeServer) NodeUnpublishVolume(_ context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + // Validate arguments + targetPath := req.GetTargetPath() + if len(targetPath) == 0 { + return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume target path must be provided") + } + + // Acquire a lock on the target path instead of volumeID, since we do not want to serialize multiple node unpublish calls on the same volume. + if acquired := s.volumeLocks.TryAcquire(targetPath); !acquired { + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, targetPath) + } + defer s.volumeLocks.Release(targetPath) + + // Check if the target path is already mounted + if mounted, err := s.isDirMounted(targetPath); mounted || err != nil { + if err != nil { + klog.Errorf("failed to check if path %q is already mounted: %v", targetPath, err) + } + // Force unmount the target path + // Try to do force unmount firstly because if the file descriptor was not closed, + // mount.CleanupMountPoint() call will hang. + forceUnmounter, ok := s.mounter.(mount.MounterForceUnmounter) + if ok { + if err = forceUnmounter.UnmountWithForce(targetPath, UmountTimeout); err != nil { + return nil, status.Errorf(codes.Internal, "failed to force unmount target path %q: %v", targetPath, err) + } + } else { + klog.Warningf("failed to cast the mounter to a forceUnmounter, proceed with the default mounter Unmount") + if err = s.mounter.Unmount(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to unmount target path %q: %v", targetPath, err) + } + } + } + + // Remove all files in the path + isMounted, err := s.mounter.IsMountPoint(targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to check %q is mounted: %v", targetPath, err) + } + + // If nothing is mounted and files are written, following mount.CleanupMountPoint will fail. + // To avoid the failure, cleanup childs in mount path's directory. + // CAUTION: This can cause data loss. + // TODO: Not to remove childs. + if !isMounted { + if err = removeChilds(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "failed to remove childs in %q: %v", targetPath, err) + } + } + + // Cleanup the mount point + if err := mount.CleanupMountPoint(targetPath, s.mounter, false /* bind mount */); err != nil { + return nil, status.Errorf(codes.Internal, "failed to cleanup the mount point %q: %v", targetPath, err) + } + + // Checking the fd-passing socket is closed. + // If not closed, close it and wait for the acception goroutine exits. + // NOTE: The acception goroutine owns FUSE fd, and floated FUSE fd causes hang. + // When the acception goroutine exis, FUSE fd is also closed. + csiMounter, ok := s.mounter.(*csimounter.Mounter) + if !ok { + klog.Error("failed to cast the mounter to a csimounter.Mounter.") + } else if !csiMounter.FdPassingSockets.Exist(targetPath) { + klog.V(4).Infof("fd-passing socket for %q is already unregistered.", targetPath) + } else { + klog.V(4).Infof("closing fd-passing socket for %q.", targetPath) + if err := csiMounter.FdPassingSockets.CloseAndUnregister(targetPath, true); err != nil { + klog.Warningf("fd-passing socket for %q is already unregistered.", targetPath) + } else { + csiMounter.FdPassingSockets.WaitForExit(targetPath) + klog.V(4).Infof("fd-passing socket for %q is closed.", targetPath) + } + } + + klog.V(4).Infof("NodeUnpublishVolume succeeded on target path %q", targetPath) + + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +// isDirMounted checks if the path is already a mount point. +func (s *nodeServer) isDirMounted(targetPath string) (bool, error) { + mps, err := s.mounter.List() + if err != nil { + return false, err + } + for _, m := range mps { + if m.Path == targetPath { + return true, nil + } + } + + return false, nil +} + +// joinMountOptions joins mount options eliminating duplicates. +func joinMountOptions(userOptions []string, systemOptions []string) []string { + allMountOptions := sets.NewString() + + for _, mountOption := range userOptions { + if len(mountOption) > 0 { + allMountOptions.Insert(mountOption) + } + } + + for _, mountOption := range systemOptions { + allMountOptions.Insert(mountOption) + } + + return allMountOptions.List() +} + +// removeChilds remove all childs in the directory +func removeChilds(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + names, err := d.Readdirnames(-1) + if err != nil { + return err + } + for _, name := range names { + err = os.RemoveAll(filepath.Join(dir, name)) + if err != nil { + return err + } + } + return nil +} diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/node_unimpl.go b/meta-fuse-csi-plugin/pkg/csi_driver/node_unimpl.go new file mode 100755 index 00000000..4acc5221 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/node_unimpl.go @@ -0,0 +1,42 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + csi "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *nodeServer) NodeStageVolume(_ context.Context, _ *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "NodeStageVolumeResponse unsupported") +} + +func (s *nodeServer) NodeUnstageVolume(_ context.Context, _ *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "NodeUnstageVolumeResponse unsupported") +} + +func (s *nodeServer) NodeGetVolumeStats(_ context.Context, _ *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { + return nil, status.Error(codes.Unimplemented, "NodeGetVolumeStats unsupported") +} + +func (s *nodeServer) NodeExpandVolume(_ context.Context, _ *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "NodeUnStageVolume unsupported") +} diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/server.go b/meta-fuse-csi-plugin/pkg/csi_driver/server.go new file mode 100644 index 00000000..901b9a8d --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/server.go @@ -0,0 +1,102 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "net" + "sync" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +// Defines Non blocking GRPC server interfaces. +type NonBlockingGRPCServer interface { + // Start services at the endpoint + Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) + // Waits for the service to stop + Wait() + // Stops the service gracefully + Stop() + // Stops the service forcefully + ForceStop() +} + +func NewNonBlockingGRPCServer() NonBlockingGRPCServer { + return &nonBlockingGRPCServer{} +} + +// NonBlocking server. +type nonBlockingGRPCServer struct { + wg sync.WaitGroup + server *grpc.Server +} + +func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { + s.wg.Add(1) + + go s.serve(endpoint, ids, ns) +} + +func (s *nonBlockingGRPCServer) Wait() { + s.wg.Wait() +} + +func (s *nonBlockingGRPCServer) Stop() { + s.server.GracefulStop() +} + +func (s *nonBlockingGRPCServer) ForceStop() { + s.server.Stop() +} + +func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, ns csi.NodeServer) { + scheme, addr, err := util.ParseEndpoint(endpoint, true) + if err != nil { + klog.Fatalf("failed to parse endpoint %v", err) + } + + klog.Infof("Start listening with scheme %v, addr %v", scheme, addr) + listener, err := net.Listen(scheme, addr) + if err != nil { + klog.Fatalf("Failed to listen: %v", err) + } + + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(logGRPC), + } + server := grpc.NewServer(opts...) + s.server = server + + if ids != nil { + csi.RegisterIdentityServer(server, ids) + } + if ns != nil { + csi.RegisterNodeServer(server, ns) + } + + klog.Infof("Listening for connections on address: %#v", listener.Addr()) + + err = server.Serve(listener) + if err != nil { + klog.Fatal(err.Error()) + } +} diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/utils.go b/meta-fuse-csi-plugin/pkg/csi_driver/utils.go new file mode 100755 index 00000000..8ef3090a --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_driver/utils.go @@ -0,0 +1,87 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + + csi "github.com/container-storage-interface/spec/lib/go/csi" + pbSanitizer "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" + "golang.org/x/net/context" + "google.golang.org/grpc" + "k8s.io/klog/v2" +) + +const ( + CreateVolumeCSIFullMethod = "/csi.v1.Controller/CreateVolume" + DeleteVolumeCSIFullMethod = "/csi.v1.Controller/DeleteVolume" + NodePublishVolumeCSIFullMethod = "/csi.v1.Node/NodePublishVolume" +) + +func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode { + return &csi.VolumeCapability_AccessMode{Mode: mode} +} + +func NewNodeServiceCapability(c csi.NodeServiceCapability_RPC_Type) *csi.NodeServiceCapability { + return &csi.NodeServiceCapability{ + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: c, + }, + }, + } +} + +func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + var strippedReq string + switch info.FullMethod { + case CreateVolumeCSIFullMethod: + strippedReq = pbSanitizer.StripSecrets(req).String() + case DeleteVolumeCSIFullMethod: + strippedReq = pbSanitizer.StripSecrets(req).String() + case NodePublishVolumeCSIFullMethod: + if nodePublishReq, ok := req.(*csi.NodePublishVolumeRequest); ok { + if token, ok := nodePublishReq.VolumeContext[VolumeContextKeyServiceAccountToken]; ok { + nodePublishReq.VolumeContext[VolumeContextKeyServiceAccountToken] = "***stripped***" + strippedReq = fmt.Sprintf("%+v", nodePublishReq) + nodePublishReq.VolumeContext[VolumeContextKeyServiceAccountToken] = token + } else { + strippedReq = fmt.Sprintf("%+v", req) + } + } else { + klog.Errorf("failed to case req to *csi.NodePublishVolumeRequest") + } + default: + strippedReq = fmt.Sprintf("%+v", req) + } + + klog.V(4).Infof("%s called with request: %v", info.FullMethod, strippedReq) + resp, err := handler(ctx, req) + if err != nil { + klog.Errorf("%s failed with error: %v", info.FullMethod, err) + } else { + if fmt.Sprintf("%v", resp) == "" { + klog.V(4).Infof("%s succeeded.", info.FullMethod) + } else { + klog.V(4).Infof("%s succeeded with response: %s", info.FullMethod, resp) + } + } + + return resp, err +} diff --git a/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go new file mode 100644 index 00000000..68f7cb93 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go @@ -0,0 +1,341 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csimounter + +import ( + "encoding/json" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + + starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + "k8s.io/mount-utils" +) + +const ( + // See the nonroot user discussion: https://github.com/GoogleContainerTools/distroless/issues/443 + NobodyUID = 65534 + NobodyGID = 65534 +) + +// Mounter provides the meta-fuse-csi-plugin implementation of mount.Interface +// for the linux platform. +type Mounter struct { + mount.MounterForceUnmounter + chdirMu sync.Mutex + FdPassingSockets *FdPassingSockets +} + +// New returns a mount.MounterForceUnmounter for the current system. +// It provides options to override the default mounter behavior. +// mounterPath allows using an alternative to `/bin/mount` for mounting. +func New(mounterPath string) (mount.Interface, error) { + m, ok := mount.New(mounterPath).(mount.MounterForceUnmounter) + if !ok { + return nil, fmt.Errorf("failed to cast mounter to MounterForceUnmounter") + } + + return &Mounter{ + m, + sync.Mutex{}, + newFdPassingSockets(), + }, nil +} + +func (m *Mounter) Mount(source string, target string, fstype string, options []string) error { + if len(options) == 1 { + options = append(options, "") + } + + fdPassingSocketPath := options[0] + fdPassingSocketDir, fdPassingSocketName := filepath.Split(fdPassingSocketPath) + klog.V(4).Infof("start to mount (fdPassingSocketDir=%s fdPassingSocketName=%s)", fdPassingSocketDir, fdPassingSocketName) + + options = options[1:] + + csiMountOptions, _ := prepareMountOptions(options[1:]) + + klog.V(4).Info("passing the descriptor") + + err := m.createAndRegisterFdPassingSocket(target, fdPassingSocketDir, fdPassingSocketName) + if err != nil { + return fmt.Errorf("failed to create fd-passing socket: %w", err) + } + + // Prepare sidecar mounter MountConfig + mc := starter.MountConfig{ + VolumeName: source, + } + mcb, err := json.Marshal(mc) + if err != nil { + return fmt.Errorf("failed to marshal sidecar mounter MountConfig %v: %w", mc, err) + } + + // Asynchronously waiting for the sidecar container to connect to the listener + go func(mounter *Mounter, target, fstype string, csiMountOptions []string, msg []byte) { + defer func() { + err = mounter.FdPassingSockets.CloseAndUnregister(target, false) + if err != nil { + klog.Errorf("failed to close and unregister fd-passing socket for %q: %w", target, err) + } + }() + + podID, volumeName, _ := util.ParsePodIDVolumeFromTargetpath(target) + logPrefix := fmt.Sprintf("[Pod %v, VolumeName %v]", podID, volumeName) + + klog.V(4).Infof("%v start to accept connections to the listener.", logPrefix) + a, err := mounter.FdPassingSockets.accept(target) + if err != nil { + klog.Errorf("%v failed to accept connections to the listener: %v", logPrefix, err) + return + } + defer a.Close() + + klog.V(4).Info("opening the device /dev/fuse") + fuseFd, err := syscall.Open("/dev/fuse", syscall.O_RDWR, 0o644) + if err != nil { + klog.Errorf("failed to open the device /dev/fuse: %w", err) + return + } + defer syscall.Close(fuseFd) + csiMountOptions = append(csiMountOptions, fmt.Sprintf("fd=%v", fuseFd)) + + // fuse-impl expects fuse is mounted. + klog.V(4).Info("mounting the fuse filesystem") + err = mounter.MountSensitiveWithoutSystemdWithMountFlags(volumeName, target, fstype, csiMountOptions, nil, []string{"--internal-only"}) + if err != nil { + klog.Errorf("failed to mount the fuse filesystem: %w", err) + return + } + + klog.V(4).Infof("%v start to send file descriptor and mount options", logPrefix) + if err = util.SendMsg(a, fuseFd, msg); err != nil { + klog.Errorf("%v failed to send file descriptor and mount options: %v", logPrefix, err) + return + } + + klog.V(4).Infof("%v exiting the goroutine.", logPrefix) + }(m, target, fstype, csiMountOptions, mcb) + + return nil +} + +func (m *Mounter) createAndRegisterFdPassingSocket(target, sockDir, sockName string) error { + m.chdirMu.Lock() + defer m.chdirMu.Unlock() + + // Need to change the current working directory to the temp volume base path, + // because the socket absolute path is longer than 104 characters, + // which will cause "bind: invalid argument" errors. + exPwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get the current directory to %w", err) + } + if err = os.Chdir(sockDir); err != nil { + return fmt.Errorf("failed to change directory to %q: %w", sockDir, err) + } + + klog.V(4).Infof("creating a listener for the socket at %q", sockDir) + l, err := net.Listen("unix", sockName) + if err != nil { + return fmt.Errorf("failed to create the listener for the socket: %w", err) + } + + unixListner := l.(*net.UnixListener) + + // Change the socket ownership + err = os.Chown(sockDir, NobodyUID, NobodyGID) + if err != nil { + return fmt.Errorf("failed to change ownership on emptyDirBasePath: %w", err) + } + err = os.Chown(sockName, NobodyUID, NobodyGID) + if err != nil { + return fmt.Errorf("failed to change ownership on socket: %w", err) + } + + if err = os.Chdir(exPwd); err != nil { + return fmt.Errorf("failed to change directory to %q: %w", exPwd, err) + } + + sockPath := filepath.Join(sockDir, sockName) + if err = m.FdPassingSockets.register(target, sockPath, unixListner); err != nil { + return fmt.Errorf("failed to register socket at %q: %w", sockPath, err) + } + + return nil +} + +func prepareMountOptions(options []string) ([]string, []string) { + allowedOptions := map[string]bool{ + "exec": true, + "noexec": true, + "atime": true, + "noatime": true, + "sync": true, + "async": true, + "dirsync": true, + } + + csiMountOptions := []string{ + "nodev", + "nosuid", + "allow_other", + "default_permissions", + "rootmode=40000", + fmt.Sprintf("user_id=%d", os.Getuid()), + fmt.Sprintf("group_id=%d", os.Getgid()), + } + + // users may pass options that should be used by Linux mount(8), + // filter out these options and not pass to the sidecar mounter. + validMountOptions := []string{"rw", "ro"} + optionSet := sets.NewString(options...) + for _, o := range validMountOptions { + if optionSet.Has(o) { + csiMountOptions = append(csiMountOptions, o) + optionSet.Delete(o) + } + } + + for _, o := range optionSet.List() { + if strings.HasPrefix(o, "o=") { + v := o[2:] + if allowedOptions[v] { + csiMountOptions = append(csiMountOptions, v) + } else { + klog.Warningf("got invalid mount option %q. Will discard invalid options and continue to mount.", v) + } + optionSet.Delete(o) + } + } + + return csiMountOptions, optionSet.List() +} + +type FdPassingSockets struct { + // key is target path + sockets map[string]*FdPassingSocket + socketsMutex sync.Mutex +} + +type FdPassingSocket struct { + socketPath string + listener *net.UnixListener + exitChan chan bool + closed bool +} + +func newFdPassingSockets() *FdPassingSockets { + return &FdPassingSockets{ + map[string]*FdPassingSocket{}, + sync.Mutex{}, + } +} + +func (fds *FdPassingSockets) register(targetPath string, sockPath string, listener *net.UnixListener) error { + fds.socketsMutex.Lock() + defer fds.socketsMutex.Unlock() + + // if the socket is already registered, return error + if _, ok := fds.sockets[targetPath]; ok { + return fmt.Errorf("fd-passing socket for %q is already registered", sockPath) + } + + fdSock := &FdPassingSocket{ + socketPath: sockPath, + listener: listener, + exitChan: make(chan bool, 5), + closed: false, + } + + fds.sockets[targetPath] = fdSock + + return nil +} + +func (fds *FdPassingSockets) get(targetPath string) *FdPassingSocket { + fds.socketsMutex.Lock() + defer fds.socketsMutex.Unlock() + + return fds.sockets[targetPath] +} + +func (fds *FdPassingSockets) accept(targetPath string) (net.Conn, error) { + sock := fds.get(targetPath) + if sock == nil { + return nil, fmt.Errorf("") + } + + return sock.listener.Accept() +} + +func (fds *FdPassingSockets) Exist(targetPath string) bool { + sock := fds.get(targetPath) + return sock != nil +} + +func (fds *FdPassingSockets) WaitForExit(targetPath string) { + sock := fds.get(targetPath) + if sock == nil { + return + } + + // wait for exit signal + <-sock.exitChan +} + +func (fds *FdPassingSockets) CloseAndUnregister(targetPath string, onlyClose bool) error { + fds.socketsMutex.Lock() + defer fds.socketsMutex.Unlock() + + sock, ok := fds.sockets[targetPath] + if !ok { + // fd-passing socket is already unregistered + return nil + } + + if !sock.closed { + sock.listener.Close() + sock.closed = true + } + + if onlyClose { + return nil + } + + // if unix domain socket exists, remove it. + if _, err := os.Stat(sock.socketPath); err == nil { + syscall.Unlink(sock.socketPath) + } + + // notify that listener has been closed via channel + sock.exitChan <- true + + // remove registered socket + delete(fds.sockets, targetPath) + + return nil +} diff --git a/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter_test.go b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter_test.go new file mode 100644 index 00000000..fad475e0 --- /dev/null +++ b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC +Copyright 2023 Preferred Networks, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csimounter + +import ( + "fmt" + "os" + "reflect" + "testing" +) + +var defaultCsiMountOptions = []string{ + "nodev", + "nosuid", + "allow_other", + "default_permissions", + "rootmode=40000", + fmt.Sprintf("user_id=%d", os.Getuid()), + fmt.Sprintf("group_id=%d", os.Getgid()), +} + +func TestPrepareMountArgs(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + inputMountOptions []string + expecteCsiMountOptions []string + expecteSidecarMountOptions []string + }{ + { + name: "should return valid options correctly with empty input", + inputMountOptions: []string{}, + expecteCsiMountOptions: defaultCsiMountOptions, + expecteSidecarMountOptions: []string{}, + }, + { + name: "should return valid options correctly with CSI mount options only", + inputMountOptions: []string{"ro", "o=noexec", "o=noatime", "o=invalid"}, + expecteCsiMountOptions: append(defaultCsiMountOptions, "ro", "noexec", "noatime"), + expecteSidecarMountOptions: []string{}, + }, + { + name: "should return valid options correctly with sidecar mount options only", + inputMountOptions: []string{"implicit-dirs", "max-conns-per-host=10"}, + expecteCsiMountOptions: defaultCsiMountOptions, + expecteSidecarMountOptions: []string{"implicit-dirs", "max-conns-per-host=10"}, + }, + { + name: "should return valid options correctly with CSI and sidecar mount options", + inputMountOptions: []string{"ro", "implicit-dirs", "max-conns-per-host=10", "o=noexec", "o=noatime", "o=invalid"}, + expecteCsiMountOptions: append(defaultCsiMountOptions, "ro", "noexec", "noatime"), + expecteSidecarMountOptions: []string{"implicit-dirs", "max-conns-per-host=10"}, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %s", tc.name) + + c, s := prepareMountOptions(tc.inputMountOptions) + if !reflect.DeepEqual(countOptionOccurrence(c), countOptionOccurrence(tc.expecteCsiMountOptions)) { + t.Errorf("Got options %v, but expected %v", c, tc.expecteCsiMountOptions) + } + + if !reflect.DeepEqual(countOptionOccurrence(s), countOptionOccurrence(tc.expecteSidecarMountOptions)) { + t.Errorf("Got options %v, but expected %v", s, tc.expecteSidecarMountOptions) + } + } +} + +func countOptionOccurrence(options []string) map[string]int { + dict := make(map[string]int) + for _, o := range options { + dict[o]++ + } + + return dict +} diff --git a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go index 7a367f3f..beb0118d 100644 --- a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go +++ b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go @@ -26,7 +26,7 @@ import ( "os/exec" "syscall" - "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" + "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" "k8s.io/klog/v2" ) From e1011bb6aad083eb3b8f6441877de931c69bd543 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Fri, 12 Jul 2024 17:34:14 +0000 Subject: [PATCH 18/26] updated goofys build step in dockerfile --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 322ea54c..cb8b51c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,16 @@ WORKDIR /meta-fuse-csi-plugin ADD ./meta-fuse-csi-plugin . # Builds the meta-fuse-csi-plugin app RUN make fusermount3-proxy BINDIR=/bin + +FROM golang:1.20.7 as goofys-builder + +WORKDIR /goofys +ADD . . +# Removes unecessary files +# RUN rm -rf meta-fuse-csi-plugin +# RUN rm go.work go.work.sum # Builds the goofys app -#RUN CGO_ENABLED=0 GOOS=linux go build -o goofys +RUN CGO_ENABLED=0 GOOS=linux go build -o goofys FROM ubuntu:22.04 @@ -29,8 +37,7 @@ EOF RUN chmod +x /configure_minio.sh #Get goofys build from first step -#COPY --from=fusermount3-proxy-builder /meta-fuse-csi-plugin/goofys . -RUN wget https://github.com/mathis-marcotte/goofys/releases/download/5Gb-no-fatal/goofys -O /goofys && chmod +x /goofys +COPY --from=goofys-builder /goofys/goofys . COPY --from=fusermount3-proxy-builder /bin/fusermount3-proxy /bin/fusermount3 RUN ln -sf /bin/fusermount3 /bin/fusermount From 6d91bc06b4edb714cbe7d16c8d1c088295fa5fe7 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Fri, 12 Jul 2024 19:19:50 +0000 Subject: [PATCH 19/26] removed unecessary files --- .../cmd/csi_driver/Dockerfile | 68 ---------- meta-fuse-csi-plugin/cmd/csi_driver/main.go | 71 ---------- .../cmd/fuse_starter/Dockerfile | 33 ----- meta-fuse-csi-plugin/cmd/fuse_starter/main.go | 122 ------------------ .../deploy/csi-driver-daemonset.yaml | 97 -------------- meta-fuse-csi-plugin/deploy/csi-driver.yaml | 21 --- 6 files changed, 412 deletions(-) delete mode 100755 meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile delete mode 100644 meta-fuse-csi-plugin/cmd/csi_driver/main.go delete mode 100755 meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile delete mode 100644 meta-fuse-csi-plugin/cmd/fuse_starter/main.go delete mode 100644 meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml delete mode 100644 meta-fuse-csi-plugin/deploy/csi-driver.yaml diff --git a/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile b/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile deleted file mode 100755 index eb80bd68..00000000 --- a/meta-fuse-csi-plugin/cmd/csi_driver/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# Copyright 2023 Preferred Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build driver go binary -FROM golang:1.20.7 as driver-builder - -ARG STAGINGVERSION - -WORKDIR /meta-fuse-csi-plugin -ADD . . -RUN make driver BINDIR=/bin - -# Start from Kubernetes Debian base. -FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as debian -# Install necessary dependencies -RUN clean-install mount bash - -# go/gke-releasing-policies#base-images -# We use `gcr.io/distroless/base` because it includes glibc. -FROM gcr.io/distroless/base-debian11 as distroless-base - -# The distroless amd64 image has a target triplet of x86_64 -FROM distroless-base AS distroless-amd64 -ENV LIB_DIR_PREFIX x86_64 - -FROM distroless-$TARGETARCH as output-image - -# Copy the mount/umount binaries -COPY --from=debian /bin/mount /bin/mount -COPY --from=debian /bin/umount /bin/umount - -# Copy shared libraries into distroless base. -COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/ - -COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \ - /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre2-8.so.0 /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/ - -# Build stage used for validation of the output-image -FROM output-image as validation-image -COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \ - /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 /lib/${LIB_DIR_PREFIX}-linux-gnu/ -COPY --from=debian /bin/bash /bin/bash -COPY --from=debian /bin/grep /bin/grep -COPY --from=debian /usr/bin/ldd /usr/bin/ldd -SHELL ["/bin/bash", "-c"] -RUN if ldd /bin/mount | grep "not found"; then echo "!!! Missing deps for mount command !!!" && exit 1; fi -RUN if ldd /bin/umount | grep "not found"; then echo "!!! Missing deps for umount command !!!" && exit 1; fi - -# Final build stage, create the real Docker image with ENTRYPOINT -FROM output-image - -COPY --from=driver-builder /bin/meta-fuse-csi-plugin /meta-fuse-csi-plugin - -ENTRYPOINT ["/meta-fuse-csi-plugin"] diff --git a/meta-fuse-csi-plugin/cmd/csi_driver/main.go b/meta-fuse-csi-plugin/cmd/csi_driver/main.go deleted file mode 100644 index 902df1ca..00000000 --- a/meta-fuse-csi-plugin/cmd/csi_driver/main.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. -Copyright 2022 Google LLC -Copyright 2023 Preferred Networks, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - - driver "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_driver" - csimounter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_mounter" - "k8s.io/klog/v2" - "k8s.io/mount-utils" -) - -var ( - endpoint = flag.String("endpoint", "unix:/tmp/csi.sock", "CSI endpoint") - nodeID = flag.String("nodeid", "", "node id") - - // These are set at compile time. - version = "unknown" - builddate = "unknown" -) - -func main() { - klog.InitFlags(nil) - flag.Parse() - - var err error - var mounter mount.Interface - if *nodeID == "" { - klog.Fatalf("NodeID cannot be empty for node service") - } - - mounter, err = csimounter.New("") - if err != nil { - klog.Fatalf("Failed to prepare CSI mounter: %v", err) - } - - config := &driver.DriverConfig{ - Name: driver.DefaultName, - Version: version, - NodeID: *nodeID, - Mounter: mounter, - } - - d, err := driver.NewDriver(config) - if err != nil { - klog.Fatalf("Failed to initialize meta-fuse-csi-plugin: %v", err) - } - - klog.Infof("Running meta-fuse-csi-plugin version %v (BuildDate %v)", version, builddate) - d.Run(*endpoint) - - os.Exit(0) -} diff --git a/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile b/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile deleted file mode 100755 index 549253f5..00000000 --- a/meta-fuse-csi-plugin/cmd/fuse_starter/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2018 The Kubernetes Authors. -# Copyright 2022 Google LLC -# Copyright 2023 Preferred Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build fuse-starter go binary -FROM golang:1.20.7 as fuse-starter-builder - -ARG STAGINGVERSION - -WORKDIR /meta-fuse-csi-plugin -ADD . . -RUN make fuse-starter BINDIR=/bin - -# go/gke-releasing-policies#base-images -# We use `gcr.io/distroless/base` because it includes glibc. -FROM gcr.io/distroless/base-debian11 - -# Copy the binaries -COPY --from=fuse-starter-builder /bin/fuse-starter /fuse-starter - -ENTRYPOINT ["/meta-fuse-csi-plugin-fuse-starter"] diff --git a/meta-fuse-csi-plugin/cmd/fuse_starter/main.go b/meta-fuse-csi-plugin/cmd/fuse_starter/main.go deleted file mode 100644 index e75aa421..00000000 --- a/meta-fuse-csi-plugin/cmd/fuse_starter/main.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. -Copyright 2022 Google LLC -Copyright 2023 Preferred Networks, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "os" - "os/signal" - "sync" - "syscall" - - starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" - "k8s.io/klog/v2" -) - -var ( - fdPassingSocketPath = flag.String("fd-passing-socket-path", "", "unix domain socket path for FUSE fd passing") - // This is set at compile time. - version = "unknown" - builddate = "unknown" -) - -func main() { - klog.InitFlags(nil) - flag.Parse() - - klog.Infof("Running meta-fuse-csi-plugin fuse-starter version %v (BuildDate %v)", version, builddate) - klog.Infof("fd-passing-socket-path: %q", *fdPassingSocketPath) - - // parsing command args after "--" - mounterArgsIdx := 0 - for ; mounterArgsIdx < len(os.Args); mounterArgsIdx += 1 { - if os.Args[mounterArgsIdx] == "--" { - mounterArgsIdx += 1 - break - } - } - - if len(os.Args) == mounterArgsIdx { - klog.Error("mounter does not specified") - return - } - - mounterPath := os.Args[mounterArgsIdx] - mounterArgs := os.Args[mounterArgsIdx+1:] - klog.Infof("mounter(%s) args are %v", mounterPath, mounterArgs) - - if *fdPassingSocketPath == "" { - klog.Error("fd-passing-socket-path does not specified") - return - } - - mounter := starter.New(mounterPath, mounterArgs) - var wg sync.WaitGroup - - mc, err := starter.PrepareMountConfig(*fdPassingSocketPath) - if err != nil { - klog.Errorf("failed prepare mount config: socket path %q: %v\n", *fdPassingSocketPath, err) - return - } - - c := make(chan os.Signal, 1) - - wg.Add(1) - go func(mc *starter.MountConfig) { - defer wg.Done() - cmd, err := mounter.Mount(mc) - if err != nil { - klog.Errorf("failed to mount volume %q: %v\n", mc.VolumeName, err) - return - } - - if err = cmd.Start(); err != nil { - klog.Errorf("failed to start mounter with error: %v\n", err) - return - } - - // Since the mounter has taken over the file descriptor, - // closing the file descriptor to avoid other process forking it. - syscall.Close(mc.FileDescriptor) - if err = cmd.Wait(); err != nil { - klog.Errorf("mounter exited with error: %v\n", err) - } else { - klog.Infof("[%v] mounter exited normally.", mc.VolumeName) - } - - // Process may exit early. - c <- syscall.SIGTERM - }(mc) - - signal.Notify(c, syscall.SIGTERM) - klog.Info("waiting for SIGTERM signal...") - - <-c // blocking the process - - klog.Info("received SIGTERM signal, waiting for all the mounter processes exit...") - - // TODO: send SIGKILL to kill hang mounter process - err = mounter.Cmd.Process.Signal(syscall.SIGTERM) - if err != nil { - klog.Warning("failed to send SIGTERM signal to mounter process") - } - wg.Wait() - - klog.Info("exiting fuse-starter...") -} diff --git a/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml b/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml deleted file mode 100644 index 849dc08c..00000000 --- a/meta-fuse-csi-plugin/deploy/csi-driver-daemonset.yaml +++ /dev/null @@ -1,97 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: meta-fuse-csi-plugin - namespace: mfcp-system -spec: - selector: - matchLabels: - k8s-app: meta-fuse-csi-plugin - template: - metadata: - annotations: - seccomp.security.alpha.kubernetes.io/pod: runtime/default - labels: - k8s-app: meta-fuse-csi-plugin - spec: - containers: - - args: - - --v=5 - - --endpoint=unix:/csi/csi.sock - - --nodeid=$(KUBE_NODE_NAME) - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/meta-fuse-csi-plugin:latest - imagePullPolicy: IfNotPresent - name: meta-fuse-csi-plugin - resources: - limits: - cpu: 200m - memory: 200Mi - requests: - cpu: 5m - memory: 10Mi - securityContext: - privileged: true - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /var/lib/kubelet/pods - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /csi - name: socket-dir - - args: - - --v=5 - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - env: - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/meta-fuse-csi-plugin.csi.storage.pfn.io/csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 - imagePullPolicy: Always - name: csi-driver-registrar - resources: - limits: - cpu: 50m - memory: 100Mi - requests: - cpu: 10m - memory: 10Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - all - readOnlyRootFilesystem: true - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /registration - name: registration-dir - nodeSelector: - kubernetes.io/os: linux - securityContext: - seccompProfile: - type: RuntimeDefault - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /var/lib/kubelet/pods/ - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/meta-fuse-csi-plugin.csi.storage.pfn.io/ - type: DirectoryOrCreate - name: socket-dir - updateStrategy: - rollingUpdate: - maxUnavailable: 10% - type: RollingUpdate diff --git a/meta-fuse-csi-plugin/deploy/csi-driver.yaml b/meta-fuse-csi-plugin/deploy/csi-driver.yaml deleted file mode 100644 index e9d90369..00000000 --- a/meta-fuse-csi-plugin/deploy/csi-driver.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: mfcp-system ---- -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - labels: - addonmanager.kubernetes.io/mode: Reconcile - k8s-app: meta-fuse-csi-plugin - name: meta-fuse-csi-plugin.csi.storage.pfn.io -spec: - attachRequired: false - fsGroupPolicy: ReadWriteOnceWithFSType - podInfoOnMount: true - requiresRepublish: true - storageCapacity: false - volumeLifecycleModes: - - Ephemeral - From 31f09038b3b11ff7122ff2d6593f51425d5d1616 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Mon, 15 Jul 2024 12:49:37 +0000 Subject: [PATCH 20/26] replaced build command with makefile command --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cb8b51c8..a76c047d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ADD . . # RUN rm -rf meta-fuse-csi-plugin # RUN rm go.work go.work.sum # Builds the goofys app -RUN CGO_ENABLED=0 GOOS=linux go build -o goofys +RUN make build FROM ubuntu:22.04 From a3ccbd42781c46e33fbda47ea062b2f8f0f1ba67 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Mon, 15 Jul 2024 13:38:50 +0000 Subject: [PATCH 21/26] clean up dockerfile --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a76c047d..9e31cd60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,6 @@ FROM golang:1.20.7 as goofys-builder WORKDIR /goofys ADD . . -# Removes unecessary files -# RUN rm -rf meta-fuse-csi-plugin -# RUN rm go.work go.work.sum # Builds the goofys app RUN make build From 758dfd59478e3aa6be03c70122c227237d21ee54 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Mon, 15 Jul 2024 13:42:56 +0000 Subject: [PATCH 22/26] cleaned up makefile --- Makefile | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Makefile b/Makefile index 0b7fba0a..4d33aee8 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,5 @@ export CGO_ENABLED=0 -export STAGINGVERSION ?= $(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git rev-list -n1 HEAD) -export BUILD_DATE ?= $(shell date --iso-8601=minutes) -BINDIR ?= bin -LDFLAGS ?= -s -w -X main.version=${STAGINGVERSION} -X main.builddate=${BUILD_DATE} -extldflags '-static' -FUSERMOUNT3PROXY_BINARY = fusermount3-proxy - run-test: s3proxy.jar ./test/run-tests.sh @@ -20,8 +14,3 @@ build: install: go install -ldflags "-X main.Version=`git rev-parse HEAD`" - -fusermount3-proxy: - mkdir -p ${BINDIR} - CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -ldflags "${LDFLAGS}" -o ${BINDIR}/${FUSERMOUNT3PROXY_BINARY} meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go - From e62c2e1c93ad177214a2e835f69efcca493d6fd9 Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Mon, 15 Jul 2024 13:53:38 +0000 Subject: [PATCH 23/26] updated repo reference --- meta-fuse-csi-plugin/Makefile | 2 +- meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go | 4 ++-- meta-fuse-csi-plugin/go.mod | 2 +- meta-fuse-csi-plugin/pkg/csi_driver/node.go | 4 ++-- meta-fuse-csi-plugin/pkg/csi_driver/server.go | 2 +- meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go | 4 ++-- meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/meta-fuse-csi-plugin/Makefile b/meta-fuse-csi-plugin/Makefile index b9417153..cacb9553 100755 --- a/meta-fuse-csi-plugin/Makefile +++ b/meta-fuse-csi-plugin/Makefile @@ -23,7 +23,7 @@ DRIVER_BINARY = meta-fuse-csi-plugin STARTER_BINARY = fuse-starter FUSERMOUNT3PROXY_BINARY = fusermount3-proxy -REGISTRY ?= ghcr.io/pfnet-research/meta-fuse-csi-plugin +REGISTRY ?= ghcr.io/StatCan/goofys/meta-fuse-csi-plugin DRIVER_IMAGE = ${REGISTRY}/${DRIVER_BINARY} STARTER_IMAGE = ${REGISTRY}/${STARTER_BINARY} EXAMPLE_IMAGE = ${REGISTRY}/mfcp-example diff --git a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go index f4149f29..75aab351 100644 --- a/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go +++ b/meta-fuse-csi-plugin/cmd/fusermount3-proxy/main.go @@ -22,8 +22,8 @@ import ( "strconv" "syscall" - starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" - "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + starter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" flag "github.com/spf13/pflag" "k8s.io/klog/v2" diff --git a/meta-fuse-csi-plugin/go.mod b/meta-fuse-csi-plugin/go.mod index f085b29b..e84aaf2a 100644 --- a/meta-fuse-csi-plugin/go.mod +++ b/meta-fuse-csi-plugin/go.mod @@ -1,4 +1,4 @@ -module github.com/pfnet-research/meta-fuse-csi-plugin +module github.com/StatCan/goofys/meta-fuse-csi-plugin go 1.20 require k8s.io/klog/v2 v2.100.1 diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/node.go b/meta-fuse-csi-plugin/pkg/csi_driver/node.go index adb7a79c..60fd20bc 100644 --- a/meta-fuse-csi-plugin/pkg/csi_driver/node.go +++ b/meta-fuse-csi-plugin/pkg/csi_driver/node.go @@ -24,9 +24,9 @@ import ( "strings" "time" + csimounter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/csi_mounter" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" csi "github.com/container-storage-interface/spec/lib/go/csi" - csimounter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/csi_mounter" - "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/meta-fuse-csi-plugin/pkg/csi_driver/server.go b/meta-fuse-csi-plugin/pkg/csi_driver/server.go index 901b9a8d..706aacb6 100644 --- a/meta-fuse-csi-plugin/pkg/csi_driver/server.go +++ b/meta-fuse-csi-plugin/pkg/csi_driver/server.go @@ -22,8 +22,8 @@ import ( "net" "sync" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" "google.golang.org/grpc" "k8s.io/klog/v2" ) diff --git a/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go index 68f7cb93..7cf80a3c 100644 --- a/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go +++ b/meta-fuse-csi-plugin/pkg/csi_mounter/csi_mounter.go @@ -28,8 +28,8 @@ import ( "sync" "syscall" - starter "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/fuse_starter" - "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + starter "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/fuse_starter" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" "k8s.io/mount-utils" diff --git a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go index beb0118d..7a367f3f 100644 --- a/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go +++ b/meta-fuse-csi-plugin/pkg/fuse_starter/fuse_starter.go @@ -26,7 +26,7 @@ import ( "os/exec" "syscall" - "github.com/pfnet-research/meta-fuse-csi-plugin/pkg/util" + "github.com/StatCan/goofys/meta-fuse-csi-plugin/pkg/util" "k8s.io/klog/v2" ) From 1109e13dc57c96d5802533e4339c1d262770a74b Mon Sep 17 00:00:00 2001 From: Mathis Marcotte Date: Mon, 15 Jul 2024 17:52:58 +0000 Subject: [PATCH 24/26] fixed curl url --- bench/Dockerfile | 2 +- bench/Dockerfile.azure | 2 +- bench/Dockerfile.gcs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/Dockerfile b/bench/Dockerfile index adbf5d3c..425d72f0 100644 --- a/bench/Dockerfile +++ b/bench/Dockerfile @@ -24,7 +24,7 @@ RUN git clone --depth 1 https://github.com/s3fs-fuse/s3fs-fuse.git && \ cd s3fs-fuse && ./autogen.sh && ./configure && make -j8 > /dev/null && make install && \ cd .. && rm -Rf s3fs-fuse -RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # ideally I want to clear out all the go deps too but there's no diff --git a/bench/Dockerfile.azure b/bench/Dockerfile.azure index 9eeb5369..159ea27f 100644 --- a/bench/Dockerfile.azure +++ b/bench/Dockerfile.azure @@ -25,7 +25,7 @@ RUN git clone --depth 1 https://github.com/Azure/azure-storage-fuse.git && \ cd azure-storage-fuse && bash ./build.sh > /dev/null && make -C build install && \ cd .. && rm -Rf azure-storage-fuse -RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # ideally I want to clear out all the go deps too but there's no diff --git a/bench/Dockerfile.gcs b/bench/Dockerfile.gcs index 5bb3734b..ec4394b4 100644 --- a/bench/Dockerfile.gcs +++ b/bench/Dockerfile.gcs @@ -24,7 +24,7 @@ RUN apt-get update && \ && apt-get clean # install catfs, required to run goofys with cache -RUN curl -L -O https://github.com/StatCan/catfs/releases/download/v0.8.0/catfs && \ +RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \ mv catfs /usr/bin && chmod 0755 /usr/bin/catfs # goofys graph generation From 2dee47d46dcf710d03660bcdbe3b1594074002b9 Mon Sep 17 00:00:00 2001 From: Jose Matsuda Date: Thu, 22 Aug 2024 13:45:43 -0400 Subject: [PATCH 25/26] chore(documentation): point out meta-fuse-csi-plugin --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 57225736..90236d79 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,11 @@ Additionally, goofys also works with the following non-S3 object stores: * Azure Data Lake Gen1 * Azure Data Lake Gen2 +## Integration with [meta-fuse-csi-plugin](https://github.com/pfnet-research/meta-fuse-csi-plugin/tree/main) + +This repository also contains files from the the meta-fuse-csi-plugin repository with more information being found [here](https://github.com/StatCan/aaw/blob/master/docs/dev/features/netapp-mounting/overview.md#integration-with-netapp). The initial commit to add them is [here](https://github.com/StatCan/goofys/commit/9db676b41fd4546c06d7fd1df6afbac44d29bde5) with subsequent commits refining the process. +The purpose of this was to [simplify the process described here](https://github.com/StatCan/aaw/blob/master/docs/dev/features/netapp-mounting/overview.md#deployment-checklist). + # References * Data is stored on [Amazon S3](https://aws.amazon.com/s3/) From a4aa306ca63e4dd0d4bf4c903c270efc75f0ae1e Mon Sep 17 00:00:00 2001 From: Bryan Paget <8212170+bryanpaget@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:06:35 -0700 Subject: [PATCH 26/26] update(security): Ubuntu to alpine, update go pkgs (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update(security): Ubuntu to alpine, update go pkgs The Ubuntu image had some vulnerablities and the easiest way to fix it was to switch to alpine. This results in a smaller image with fewer attack surfaces. Since this is a small image with a single application it makes sense to use a small base. I've also updated some of the go packages, see the diff. * update(security): Ubuntu to alpine, update go pkgs The Ubuntu image had some vulnerablities and the easiest way to fix it was to switch to alpine. This results in a smaller image with fewer attack surfaces. Since this is a small image with a single application it makes sense to use a small base. I've also updated some of the go packages, see the diff. * update(Dockerfile): add bash * update(Dockerfile): add missing packages * update(Dockerfile): modify symlink * add apk upgrade to fix alpine cves Some new CVEs have popped up but adding apk update && apk upgrade to the Dockerfile ensures we are using the latest and most secure packages. goofys (alpine 3.20.2) Total: 2 (UNKNOWN: 0, LOW: 0, MEDIUM: 2, HIGH: 0, CRITICAL: 0) ┌────────────┬───────────────┬──────────┬────────┬───────────────────┬───────────────┬─────────────────────────────────────────────────────────┐ │ Library │ Vulnerability │ Severity │ Status │ Installed Version │ Fixed Version │ Title │ ├────────────┼───────────────┼──────────┼────────┼───────────────────┼───────────────┼─────────────────────────────────────────────────────────┤ │ libcrypto3 │ CVE-2024-6119 │ MEDIUM │ fixed │ 3.3.1-r3 │ 3.3.2-r0 │ Issue summary: Applications performing certificate name │ │ │ │ │ │ │ │ checks (e.g., ... │ │ │ │ │ │ │ │ https://avd.aquasec.com/nvd/cve-2024-6119 │ ├────────────┤ │ │ │ │ │ │ │ libssl3 │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ │ └────────────┴───────────────┴──────────┴────────┴───────────────────┴───────────────┴─────────────────────────────────────────────────────────┘ * update(Dockerfile): pin alpine image version to 3.20.3 I've also reverted the script to use bash instead of sh since we installed bash for compatibility reasons. I've reverted the golang builder containers to use version 1.20.7 since we don't want to introduce new unknown issues. --------- Co-authored-by: Bryan Paget --- Dockerfile | 36 +- go.mod | 16 +- go.sum | 1367 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 1350 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e31cd60..4b87636b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,40 @@ -FROM golang:1.20.7 as fusermount3-proxy-builder +FROM golang:1.20.7-alpine AS fusermount3-proxy-builder + +# Install required build dependencies +RUN apk update && apk upgrade && apk --no-cache add make gcc g++ libc-dev fuse-dev WORKDIR /meta-fuse-csi-plugin ADD ./meta-fuse-csi-plugin . -# Builds the meta-fuse-csi-plugin app +# Build the fusermount3-proxy RUN make fusermount3-proxy BINDIR=/bin -FROM golang:1.20.7 as goofys-builder +FROM golang:1.20.7-alpine AS goofys-builder + +# Install required build dependencies +RUN apk update && apk upgrade && apk --no-cache add git make gcc g++ libc-dev fuse-dev WORKDIR /goofys ADD . . -# Builds the goofys app +# Build the goofys app RUN make build -FROM ubuntu:22.04 +# 3.20.3 is the latest as of this commit (September 09 2024) +FROM alpine:3.20.3 -RUN apt update && apt upgrade -y -RUN apt install -y ca-certificates wget libfuse2 fuse3 +# Install necessary runtime dependencies +RUN apk update && apk upgrade && apk --no-cache add ca-certificates bash wget -# prepare for MinIO -RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && chmod +x /usr/bin/mc +# Download MinIO client (mc) +RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && \ + chmod +x /usr/bin/mc && \ + apk del wget && rm -rf /var/cache/apk/* +# Copy the test file COPY <