Skip to content

Commit

Permalink
Adds missing HeadObject and GetObject support changes (#702)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebozduman
ebozduman authored Oct 29, 2022
1 parent cc9d016 commit 39c0f09
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 32 deletions.
74 changes: 47 additions & 27 deletions Minio.Functional.Tests/FunctionalTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2395,36 +2395,43 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio)

#endregion


internal static MemoryStream CreateZipFile(string prefix, int nFiles)
{
// CreateZipFile creates a zip file, populates it with <nFiles> many
// small files, each prefixed with <prefix> and in bytes size plus a single
// 1MB file. It generates and returns a memory stream of the zip file.
// The names of these files are arranged in "<file-size>.bin" format,
// like "127.bin" is created as a small binary file in 127 bytes size.
var outputMemStream = new MemoryStream();
var zipStream = new ZipOutputStream(outputMemStream);

zipStream.SetLevel(3); //0-9, 9 being the highest level of compression
byte[] bytes = null;

for (var i = 0; i <= nFiles; i++)
Directory.CreateDirectory(prefix);
for (var i = 1; i <= nFiles; i++)
{
// Make one large, compressible file.
// Make a single 1Mb file
if (i == nFiles) i = 1000000;

var fileName = prefix + "file-" + i + ".bin";
Directory.CreateDirectory(prefix);
var fileName = prefix + i + ".bin";
var newEntry = new ZipEntry(fileName);
newEntry.DateTime = DateTime.Now;
zipStream.PutNextEntry(newEntry);

bytes = rsg.GenerateStreamFromSeed(i).ToArray();
var inStream = new MemoryStream(bytes);
if (i == 0) StreamUtils.Copy(inStream, zipStream, new byte[128]);
else StreamUtils.Copy(inStream, zipStream, new byte[i * 128]);
StreamUtils.Copy(inStream, zipStream, new byte[i * 128]);

inStream.Close();
zipStream.CloseEntry();
}

zipStream.IsStreamOwner = false; // False stops the Close also Closing the underlying stream.
zipStream.Close(); // Must finish the ZipOutputStream before using outputMemStream.
// Setting ownership to False keeps the underlying stream open
zipStream.IsStreamOwner = false;
// Must finish the ZipOutputStream before using outputMemStream
zipStream.Close();

outputMemStream.Position = 0;
outputMemStream.Seek(0, SeekOrigin.Begin);
Expand Down Expand Up @@ -2458,49 +2465,62 @@ internal static async Task GetObjectS3Zip_Test1(MinioClient minio)
.WithObjectSize(memStream.Length);
await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false);

var extractHeader = new Dictionary<string, string>();
extractHeader.Add("x-minio-extract", "true");

// GeObject api test
var r = new Random();
var singleFileName = r.Next(1, nFiles - 1) + ".bin";
var singleObjectName = objectName + "/" + path + singleFileName;
// File names in the zip file also show the sizes of the files
// For example file "35.bin" has a size of 35Bytes
var expectedFileSize = Path.GetFileNameWithoutExtension(singleFileName);
var getObjectArgs = new GetObjectArgs()
.WithBucket(bucketName)
.WithFile(randomFileName)
.WithObject(objectName);
.WithObject(singleObjectName)
.WithHeaders(extractHeader);

var resp = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false);
// Verify the size of the file from the returned info
Assert.AreEqual(expectedFileSize, resp.Size.ToString());

// HeadObject api test
var statArgs = new StatObjectArgs()
.WithBucket(bucketName)
.WithObject(objectName);
.WithObject(singleObjectName)
.WithHeaders(extractHeader);
var stat = await minio.StatObjectAsync(statArgs).ConfigureAwait(false);
// Verify the size of the file from the returned info
Assert.AreEqual(expectedFileSize, resp.Size.ToString());

var lOpts = new Dictionary<string, string>();
lOpts.Add("x-minio-extract", "true");

// Test with different prefix values
// ListObject api test with different prefix values
// prefix value="", expected number of files listed=1
var prefix = "";
ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, 1, true, headers: extractHeader);

// prefix value="/", expected number of files listed=nFiles+1
// prefix value="/", expected number of files listed=nFiles
prefix = objectName + "/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader);

// prefix value="/test", expected number of files listed=nFiles + 1
// prefix value="/test", expected number of files listed=nFiles
prefix = objectName + "/test";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader);

// prefix value="/test/", expected number of files listed=nFiles+1
// prefix value="/test/", expected number of files listed=nFiles
prefix = objectName + "/test/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader);

// prefix value="/test", expected number of files listed=nFiles+1
// prefix value="/test", expected number of files listed=nFiles
prefix = objectName + "/test/small";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader);

// prefix value="/test", expected number of files listed=nFiles+1
// prefix value="/test", expected number of files listed=nFiles
prefix = objectName + "/test/small/";
ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, prefix, nFiles, true, headers: extractHeader);

// prefix value="/test", expected number of files listed=1
prefix = objectName + "/test/small/" + "file-1.bin";
ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);
ListObjects_Test(minio, bucketName, singleObjectName, 1, true, headers: extractHeader);

new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.PASS,
DateTime.Now - startTime, args: args).Log();
Expand Down
2 changes: 2 additions & 0 deletions Minio/DataModel/ObjectArgs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ namespace Minio;
public abstract class ObjectArgs<T> : BucketArgs<T>
where T : ObjectArgs<T>
{
protected const string S3ZipExtractKey = "X-Minio-Extract";

internal string ObjectName { get; set; }
internal byte[] RequestBody { get; set; }

Expand Down
12 changes: 9 additions & 3 deletions Minio/DataModel/ObjectOperationsArgs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,11 @@ public StatObjectArgs()

internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder)
{
if (!string.IsNullOrEmpty(VersionId)) requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}");
if (!string.IsNullOrEmpty(VersionId))
requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}");
if (Headers.ContainsKey(S3ZipExtractKey))
requestMessageBuilder.AddQueryParameter(S3ZipExtractKey, Headers[S3ZipExtractKey]);

return requestMessageBuilder;
}

Expand Down Expand Up @@ -257,7 +261,7 @@ internal override void Validate()

private void Populate()
{
Headers = new Dictionary<string, string>();
Headers = Headers ?? new Dictionary<string, string>();
if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers);
if (OffsetLengthSet)
{
Expand Down Expand Up @@ -530,7 +534,7 @@ internal override void Validate()

private void Populate()
{
Headers = new Dictionary<string, string>();
Headers = Headers ?? new Dictionary<string, string>();
if (SSE != null && SSE.GetType().Equals(EncryptionType.SSE_C)) SSE.Marshal(Headers);

if (OffsetLengthSet)
Expand All @@ -548,6 +552,8 @@ internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuild
{
if (!string.IsNullOrEmpty(VersionId)) requestMessageBuilder.AddQueryParameter("versionId", $"{VersionId}");
requestMessageBuilder.ResponseWriter = CallBack;
if (Headers.ContainsKey(S3ZipExtractKey))
requestMessageBuilder.AddQueryParameter(S3ZipExtractKey, Headers[S3ZipExtractKey]);

return requestMessageBuilder;
}
Expand Down
6 changes: 4 additions & 2 deletions Minio/Helper/OperationsHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ private async Task<ObjectStat> getObjectHelper(GetObjectArgs args, CancellationT
{
// StatObject is called to both verify the existence of the object and return it with GetObject.
// NOTE: This avoids writing the error body to the action stream passed (Do not remove).

var statArgs = new StatObjectArgs()
.WithBucket(args.BucketName)
.WithObject(args.ObjectName)
Expand All @@ -45,7 +46,8 @@ private async Task<ObjectStat> getObjectHelper(GetObjectArgs args, CancellationT
.WithNotMatchETag(args.NotMatchETag)
.WithModifiedSince(args.ModifiedSince)
.WithUnModifiedSince(args.UnModifiedSince)
.WithServerSideEncryption(args.SSE);
.WithServerSideEncryption(args.SSE)
.WithHeaders(args.Headers);
if (args.OffsetLengthSet) statArgs.WithOffsetAndLength(args.ObjectOffset, args.ObjectLength);
var objStat = await StatObjectAsync(statArgs, cancellationToken).ConfigureAwait(false);
args.Validate();
Expand Down Expand Up @@ -294,7 +296,7 @@ public class OperationsUtil
private static readonly List<string> SupportedHeaders = new()
{
"cache-control", "content-encoding", "content-type",
"x-amz-acl", "content-disposition"
"x-amz-acl", "content-disposition", "x-minio-extract"
};

private static readonly List<string> SSEHeaders = new()
Expand Down

0 comments on commit 39c0f09

Please sign in to comment.