From 39c0f09c527a4696d6693341f5142474e7bd06a4 Mon Sep 17 00:00:00 2001 From: ebozduman Date: Sat, 29 Oct 2022 11:08:27 -0700 Subject: [PATCH] Adds missing HeadObject and GetObject support changes (#702) --- Minio.Functional.Tests/FunctionalTest.cs | 74 +++++++++++++++--------- Minio/DataModel/ObjectArgs.cs | 2 + Minio/DataModel/ObjectOperationsArgs.cs | 12 +++- Minio/Helper/OperationsHelper.cs | 6 +- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/Minio.Functional.Tests/FunctionalTest.cs b/Minio.Functional.Tests/FunctionalTest.cs index cfbf3e6ef..4b9c29501 100644 --- a/Minio.Functional.Tests/FunctionalTest.cs +++ b/Minio.Functional.Tests/FunctionalTest.cs @@ -2395,36 +2395,43 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio) #endregion + internal static MemoryStream CreateZipFile(string prefix, int nFiles) { + // CreateZipFile creates a zip file, populates it with many + // small files, each prefixed with and in bytes size plus a single + // 1MB file. It generates and returns a memory stream of the zip file. + // The names of these files are arranged in ".bin" format, + // like "127.bin" is created as a small binary file in 127 bytes size. var outputMemStream = new MemoryStream(); var zipStream = new ZipOutputStream(outputMemStream); zipStream.SetLevel(3); //0-9, 9 being the highest level of compression byte[] bytes = null; - for (var i = 0; i <= nFiles; i++) + Directory.CreateDirectory(prefix); + for (var i = 1; i <= nFiles; i++) { - // Make one large, compressible file. + // Make a single 1Mb file if (i == nFiles) i = 1000000; - var fileName = prefix + "file-" + i + ".bin"; - Directory.CreateDirectory(prefix); + var fileName = prefix + i + ".bin"; var newEntry = new ZipEntry(fileName); newEntry.DateTime = DateTime.Now; zipStream.PutNextEntry(newEntry); bytes = rsg.GenerateStreamFromSeed(i).ToArray(); var inStream = new MemoryStream(bytes); - if (i == 0) StreamUtils.Copy(inStream, zipStream, new byte[128]); - else StreamUtils.Copy(inStream, zipStream, new byte[i * 128]); + StreamUtils.Copy(inStream, zipStream, new byte[i * 128]); inStream.Close(); zipStream.CloseEntry(); } - zipStream.IsStreamOwner = false; // False stops the Close also Closing the underlying stream. - zipStream.Close(); // Must finish the ZipOutputStream before using outputMemStream. + // Setting ownership to False keeps the underlying stream open + zipStream.IsStreamOwner = false; + // Must finish the ZipOutputStream before using outputMemStream + zipStream.Close(); outputMemStream.Position = 0; outputMemStream.Seek(0, SeekOrigin.Begin); @@ -2458,49 +2465,62 @@ internal static async Task GetObjectS3Zip_Test1(MinioClient minio) .WithObjectSize(memStream.Length); await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false); + var extractHeader = new Dictionary(); + extractHeader.Add("x-minio-extract", "true"); + + // GeObject api test + var r = new Random(); + var singleFileName = r.Next(1, nFiles - 1) + ".bin"; + var singleObjectName = objectName + "/" + path + singleFileName; + // File names in the zip file also show the sizes of the files + // For example file "35.bin" has a size of 35Bytes + var expectedFileSize = Path.GetFileNameWithoutExtension(singleFileName); var getObjectArgs = new GetObjectArgs() .WithBucket(bucketName) .WithFile(randomFileName) - .WithObject(objectName); + .WithObject(singleObjectName) + .WithHeaders(extractHeader); var resp = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false); + // Verify the size of the file from the returned info + Assert.AreEqual(expectedFileSize, resp.Size.ToString()); + // HeadObject api test var statArgs = new StatObjectArgs() .WithBucket(bucketName) - .WithObject(objectName); + .WithObject(singleObjectName) + .WithHeaders(extractHeader); var stat = await minio.StatObjectAsync(statArgs).ConfigureAwait(false); + // Verify the size of the file from the returned info + Assert.AreEqual(expectedFileSize, resp.Size.ToString()); - var lOpts = new Dictionary(); - lOpts.Add("x-minio-extract", "true"); - - // Test with different prefix values + // ListObject api test with different prefix values // prefix value="", expected number of files listed=1 var prefix = ""; - ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, 1, true, headers: extractHeader); - // prefix value="/", expected number of files listed=nFiles+1 + // prefix value="/", expected number of files listed=nFiles prefix = objectName + "/"; - ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader); - // prefix value="/test", expected number of files listed=nFiles + 1 + // prefix value="/test", expected number of files listed=nFiles prefix = objectName + "/test"; - ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader); - // prefix value="/test/", expected number of files listed=nFiles+1 + // prefix value="/test/", expected number of files listed=nFiles prefix = objectName + "/test/"; - ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader); - // prefix value="/test", expected number of files listed=nFiles+1 + // prefix value="/test", expected number of files listed=nFiles prefix = objectName + "/test/small"; - ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader); - // prefix value="/test", expected number of files listed=nFiles+1 + // prefix value="/test", expected number of files listed=nFiles prefix = objectName + "/test/small/"; - ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader); // prefix value="/test", expected number of files listed=1 - prefix = objectName + "/test/small/" + "file-1.bin"; - ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts); + ListObjects_Test(minio, bucketName, singleObjectName, 1, true, headers: extractHeader); new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.PASS, DateTime.Now - startTime, args: args).Log(); diff --git a/Minio/DataModel/ObjectArgs.cs b/Minio/DataModel/ObjectArgs.cs index b33022c62..f15a95b77 100644 --- a/Minio/DataModel/ObjectArgs.cs +++ b/Minio/DataModel/ObjectArgs.cs @@ -19,6 +19,8 @@ namespace Minio; public abstract class ObjectArgs : BucketArgs where T : ObjectArgs { + protected const string S3ZipExtractKey = "X-Minio-Extract"; + internal string ObjectName { get; set; } internal byte[] RequestBody { get; set; } diff --git a/Minio/DataModel/ObjectOperationsArgs.cs b/Minio/DataModel/ObjectOperationsArgs.cs index d82c5b2a3..69e452173 100644 --- a/Minio/DataModel/ObjectOperationsArgs.cs +++ b/Minio/DataModel/ObjectOperationsArgs.cs @@ -228,7 +228,11 @@ public StatObjectArgs() internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder) { - if (!string.IsNullOrEmpty(VersionId)) requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}"); + if (!string.IsNullOrEmpty(VersionId)) + requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}"); + if (Headers.ContainsKey(S3ZipExtractKey)) + requestMessageBuilder.AddQueryParameter(S3ZipExtractKey, Headers[S3ZipExtractKey]); + return requestMessageBuilder; } @@ -257,7 +261,7 @@ internal override void Validate() private void Populate() { - Headers = new Dictionary(); + Headers = Headers ?? new Dictionary(); if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers); if (OffsetLengthSet) { @@ -530,7 +534,7 @@ internal override void Validate() private void Populate() { - Headers = new Dictionary(); + Headers = Headers ?? new Dictionary(); if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers); if (OffsetLengthSet) @@ -548,6 +552,8 @@ internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuild { if (!string.IsNullOrEmpty(VersionId)) requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}"); requestMessageBuilder.ResponseWriter = CallBack; + if (Headers.ContainsKey(S3ZipExtractKey)) + requestMessageBuilder.AddQueryParameter(S3ZipExtractKey, Headers[S3ZipExtractKey]); return requestMessageBuilder; } diff --git a/Minio/Helper/OperationsHelper.cs b/Minio/Helper/OperationsHelper.cs index e6b598a4e..0201f7cd5 100644 --- a/Minio/Helper/OperationsHelper.cs +++ b/Minio/Helper/OperationsHelper.cs @@ -37,6 +37,7 @@ private async Task getObjectHelper(GetObjectArgs args, CancellationT { // StatObject is called to both verify the existence of the object and return it with GetObject. // NOTE: This avoids writing the error body to the action stream passed (Do not remove). + var statArgs = new StatObjectArgs() .WithBucket(args.BucketName) .WithObject(args.ObjectName) @@ -45,7 +46,8 @@ private async Task getObjectHelper(GetObjectArgs args, CancellationT .WithNotMatchETag(args.NotMatchETag) .WithModifiedSince(args.ModifiedSince) .WithUnModifiedSince(args.UnModifiedSince) - .WithServerSideEncryption(args.SSE); + .WithServerSideEncryption(args.SSE) + .WithHeaders(args.Headers); if (args.OffsetLengthSet) statArgs.WithOffsetAndLength(args.ObjectOffset, args.ObjectLength); var objStat = await StatObjectAsync(statArgs, cancellationToken).ConfigureAwait(false); args.Validate(); @@ -294,7 +296,7 @@ public class OperationsUtil private static readonly List SupportedHeaders = new() { "cache-control", "content-encoding", "content-type", - "x-amz-acl", "content-disposition" + "x-amz-acl", "content-disposition", "x-minio-extract" }; private static readonly List SSEHeaders = new()