From c0fe1e04807c9b3a9141dd49edb5e2bf7a894f6b Mon Sep 17 00:00:00 2001 From: Stan York Date: Tue, 3 Mar 2015 18:17:09 -0500 Subject: [PATCH] V0.90 Connector Enhancement - RAW support This update provides the ability to access the 'raw' attributes exposed by release 2.8 of the Cloud Elements API. Specifically, support for the email address of the last modified by user (Google Drive and Box) --- Cloud Element Test Form/Form1.cs | 5 +- .../CloudElementsConnector.cs | 108 ++++++++++++++---- Cloud Elements Connector/CloudFile.cs | 24 ++++ Cloud Elements Connector/FileOperations.cs | 24 +++- Cloud Elements Connector/Tools.cs | 25 ++++ 5 files changed, 159 insertions(+), 27 deletions(-) diff --git a/Cloud Element Test Form/Form1.cs b/Cloud Element Test Form/Form1.cs index dd5f0ca..e01a14b 100644 --- a/Cloud Element Test Form/Form1.cs +++ b/Cloud Element Test Form/Form1.cs @@ -518,7 +518,7 @@ private async void toolStripButton1_Click_1(object sender, EventArgs e) if (!chkWithTags.Checked || !currentRow.HasTags) { StatusMsg("Getting current Tag(s).... "); - currentRow = await APIConnector.GetDocEntryMetaData(currentRow.EntryType, Cloud_Elements_API.CloudElementsConnector.FileSpecificationType.ID, currentRow.id); + currentRow = await APIConnector.GetDocEntryMetaData(currentRow.EntryType, Cloud_Elements_API.CloudElementsConnector.FileSpecificationType.ID, currentRow.id,false); } StatusMsg("Storing Tag: " + TagToSet); @@ -579,7 +579,8 @@ private async void getMetadataByPathToolStripMenuItem_Click(object sender, Event if (CloudFileInfoByPath == null) StatusMsg("Nothing Returned! (not expecting not found)"); else { - StatusMsg(string.Format("OK: ID is {0}", CloudFileInfoByPath.id)); + StatusMsg(string.Format("OK: ID is {0}, by [{2}], hash {1}", CloudFileInfoByPath.id, Cloud_Elements_API.FileOperations.SHA1(APIConnector, CloudFileInfoByPath), + Cloud_Elements_API.FileOperations.LastWrittenBy(APIConnector, CloudFileInfoByPath))); } } catch (Exception ex) diff --git a/Cloud Elements Connector/CloudElementsConnector.cs b/Cloud Elements Connector/CloudElementsConnector.cs index c6073be..45fc8cf 100644 --- a/Cloud Elements Connector/CloudElementsConnector.cs +++ b/Cloud Elements Connector/CloudElementsConnector.cs @@ -61,6 +61,9 @@ public Cloud_Elements_API.CloudAuthorization APIAuthorization } } + /// + /// Returns max requests per second to the current endpoint + /// public int EndpointMaxRequestsPerSecond { get @@ -70,7 +73,7 @@ public int EndpointMaxRequestsPerSecond { if (EndpointSettings.ContainsKey(Endpoint)) { - EndpointOptions options = new EndpointOptions(); + EndpointOptions options = EndpointSettings[Endpoint]; result = options.MaxRqPerSecond; } } @@ -87,6 +90,39 @@ public int EndpointMaxRequestsPerSecond } } + /// + /// Returns the time when the last 'rate exceeded' result was detected by the connector + /// + public DateTime WhenRateLastExceeded + { + get + { + DateTime result = DateTime.MinValue; + if (EndpointSettings.ContainsKey(Endpoint)) + { + EndpointOptions options = EndpointSettings[Endpoint]; + result = options.LastRateExceeded; + } + return result; + } + } + + /// + /// Returns the current endpoint options in use by this connector + /// + public EndpointOptions EndpointOptions + { + get + { + EndpointOptions result = null; + if (EndpointSettings.ContainsKey(Endpoint)) + { + result = EndpointSettings[Endpoint]; + } + return result; + } + } + #region "Constructors" public CloudElementsConnector(string elementsURL) @@ -156,6 +192,11 @@ private void AssureEnpointControlData(string endpointName) case "box": options.MaxRqPerSecond = 6; options.LogHighwaterThroughput = true; + options.FileHashAlgorithmName = "SHA1"; + options.ModifiedByRawIDPath = "modified_by.login"; + break; + case "googledrive": + options.ModifiedByRawIDPath = "lastModifyingUser.emailAddress"; break; default: options.MaxRqPerSecond = 32; @@ -238,7 +279,7 @@ public async Task GetStorageAvailable() /// Specifies if the identifier is an ID or a PATH /// Specifying an ID that does not exist results in an error response. /// - public async Task GetDocEntryMetaData(DirectoryEntryType entryType, FileSpecificationType fileSpecType, string identifier) + public async Task GetDocEntryMetaData(DirectoryEntryType entryType, FileSpecificationType fileSpecType, string identifier, bool withRaw) { CloudFile Result; HttpResponseMessage response; @@ -248,15 +289,15 @@ public async Task GetDocEntryMetaData(DirectoryEntryType entryType, F switch (fileSpecType) { case FileSpecificationType.ID: - RequestURL = "hubs/documents/{1}/{0}/metadata"; + RequestURL = "hubs/documents/{1}/{0}/metadata?raw={2}"; break; case FileSpecificationType.Path: - RequestURL = "hubs/documents/{1}/metadata?path={0}"; + RequestURL = "hubs/documents/{1}/metadata?path={0}&raw={2}"; break; default: throw new ArgumentException("unsupported File Specification Type - " + fileSpecType.ToString()); } - RequestURL = string.Format(RequestURL, System.Net.WebUtility.UrlEncode(identifier), URLEntryType); + RequestURL = string.Format(RequestURL, System.Net.WebUtility.UrlEncode(identifier), URLEntryType,withRaw); response = await APIExecuteGet(RequestURL); Result = await response.Content.ReadAsAsync(); return Result; @@ -415,7 +456,7 @@ public async Task DeleteFolder(string path, Boolean withTrash) public async Task GetFolderMetaData(FileSpecificationType fileSpecType, string identifier) { CloudFile Result; - Result = await GetDocEntryMetaData(DirectoryEntryType.Folder, fileSpecType, identifier); + Result = await GetDocEntryMetaData(DirectoryEntryType.Folder, fileSpecType, identifier,true); return Result; } @@ -510,23 +551,7 @@ public async Task FileLinks(FileSpecificationType fileSpecType, strin /// public async Task GetFileMetaData(FileSpecificationType fileSpecType, string identifier) { - CloudFile Result; - HttpResponseMessage response; - string RequestURL; - switch (fileSpecType) - { - case FileSpecificationType.ID: - RequestURL = string.Format("hubs/documents/files/{0}/metadata", System.Net.WebUtility.UrlEncode(identifier)); - break; - case FileSpecificationType.Path: - RequestURL = string.Format("hubs/documents/files/metadata?path={0}", System.Net.WebUtility.UrlEncode(identifier)); - break; - default: - throw new ArgumentException("unsupported File Specification Type - " + fileSpecType.ToString()); - } - response = await APIExecuteGet(RequestURL); - Result = await response.Content.ReadAsAsync(); - return Result; + return await GetDocEntryMetaData(DirectoryEntryType.File, fileSpecType, identifier, true); } /// @@ -857,11 +882,12 @@ async Task APIExecuteVerb(HttpVerb verb, string URI, HttpCo if (EndpointSettings.ContainsKey(Endpoint)) { options = EndpointSettings[Endpoint]; + options.LastRateExceeded = DateTime.Now; if ((options.MaxRqPerSecond <= 0) || (options.MaxRqPerSecond > options.HighwaterGeneratedRequestsPerSecond)) options.MaxRqPerSecond = (int)options.HighwaterGeneratedRequestsPerSecond; if ((options.MaxRqPerSecond > 2) && (DateTime.Now.Subtract(options.LastAutoLimit).TotalSeconds > 1)) { options.MaxRqPerSecond--; - options.LastAutoLimit = DateTime.Now; + options.LastAutoLimit = options.LastRateExceeded; OnDiagTrace(string.Format("ce(throughput) [{0}] rate limit exceeded: inferred new target of {1}r/s", Endpoint, options.MaxRqPerSecond)); } } @@ -930,11 +956,45 @@ public DateTime LastAutoLimit get { return _LastAutoLimit; } internal set { _LastAutoLimit = value; } } + public DateTime LastRateExceeded + { + get { return _LastRateExceeded; } + internal set { _LastRateExceeded = value; } + } + + public bool HasFileHashAlgorithm + { + get { return ((_FileHashCyproName != null) && (_FileHashCyproName.Length > 0)); } + + } + + public bool HasModifiedBy + { + get { return ((_ModifiedByRawPath != null) && (_ModifiedByRawPath.Length > 0)); } + + } + + public string FileHashAlgorithmName + { + get { return _FileHashCyproName; } + internal set { _FileHashCyproName = value; } + } + + protected internal string ModifiedByRawIDPath + { + get { return _ModifiedByRawPath; } + set { _ModifiedByRawPath = value; } + } + + public bool LogThrottleDelays; public bool LogHighwaterThroughput; private double _HighwaterGeneratedRequestsPerSecond; private DateTime _LastAutoLimit; + private DateTime _LastRateExceeded; + private string _FileHashCyproName; + private string _ModifiedByRawPath; } diff --git a/Cloud Elements Connector/CloudFile.cs b/Cloud Elements Connector/CloudFile.cs index 45c6730..8ae8270 100644 --- a/Cloud Elements Connector/CloudFile.cs +++ b/Cloud Elements Connector/CloudFile.cs @@ -16,11 +16,16 @@ public class CloudFile public string modifiedDate { get; set; } // optional public string id { get; set; } // optional public Boolean directory { get; set; } // optional + public Newtonsoft.Json.Linq.JObject raw { get; set; } // optional [Newtonsoft.Json.JsonIgnore] public Boolean HasTags { get { return ((tags != null) && (tags.Length > 0)); } } + [Newtonsoft.Json.JsonIgnore] + public Boolean HasRaw { get { return ((raw != null)) ; } } + + [Newtonsoft.Json.JsonIgnore] public Cloud_Elements_API.CloudElementsConnector.DirectoryEntryType EntryType { @@ -65,6 +70,25 @@ public DateTime WhenModified() return (_WhenModified); } + public string RawValue(string valuePath) + { + string[] pathPart = valuePath.Split('.'); + if (pathPart.GetUpperBound(0) < 1) return string.Empty; + Newtonsoft.Json.Linq.JToken valueToken = raw.GetValue(pathPart[0]); + if (valueToken == null) return string.Empty; + for (int i = 1; i < pathPart.GetUpperBound(0) - 1; i++) + { + if (!valueToken.HasValues) return string.Empty; + if (!(valueToken is Newtonsoft.Json.Linq.JObject)) return string.Empty; + valueToken = ((Newtonsoft.Json.Linq.JObject)valueToken).GetValue(pathPart[i]); + if (valueToken == null) return string.Empty; + } + + valueToken = ((Newtonsoft.Json.Linq.JObject)valueToken).GetValue(pathPart[pathPart.GetUpperBound(0)]); + if (valueToken == null) return string.Empty; + return valueToken.ToString(); + } + /// /// Adds a tag to the tag collection with support for Key-Value pair tags (name=value) diff --git a/Cloud Elements Connector/FileOperations.cs b/Cloud Elements Connector/FileOperations.cs index 2322a1e..83c6083 100644 --- a/Cloud Elements Connector/FileOperations.cs +++ b/Cloud Elements Connector/FileOperations.cs @@ -26,7 +26,7 @@ public static async Task Copy(CloudElementsConnector connector, Cloud /// - /// Obtains information about a cloud file by ID or path; returns NULL + /// Obtains information about a cloud file by ID or path; returns NULL for not found /// /// connection to use /// specifies format of supplied file specification @@ -92,7 +92,29 @@ public static async Task Rename(CloudElementsConnector connector, Clo return targetFile; } + /// + /// Returns SHA1, if available + /// + /// + public static string SHA1(CloudElementsConnector connector, CloudFile targetFile) + { + if (!targetFile.HasRaw) return string.Empty; + if (!connector.EndpointOptions.HasFileHashAlgorithm) return string.Empty; + Newtonsoft.Json.Linq.JToken valueToken = targetFile.raw.GetValue("sha1"); + if (valueToken == null) return string.Empty; + return valueToken.ToString(); + } + /// + /// Returns email address of last file writer, if available + /// + /// + public static string LastWrittenBy(CloudElementsConnector connector, CloudFile targetFile) + { + if (!targetFile.HasRaw) return string.Empty; + if (!connector.EndpointOptions.HasModifiedBy) return string.Empty; + return targetFile.RawValue(connector.EndpointOptions.ModifiedByRawIDPath); + } } } diff --git a/Cloud Elements Connector/Tools.cs b/Cloud Elements Connector/Tools.cs index 2a5d910..ae83dec 100644 --- a/Cloud Elements Connector/Tools.cs +++ b/Cloud Elements Connector/Tools.cs @@ -229,5 +229,30 @@ public static string TraceTimeNow() } + /// + /// Returns a digest (hash) of the stream. + /// + /// + /// MD5,SHA1, SHA256, SHA512 (etc) + /// + public static string HashForBuffer(System.IO.Stream inStream, string useCrypto) + { + System.Security.Cryptography.HashAlgorithm viaCrypto = default(System.Security.Cryptography.HashAlgorithm); + try + { + viaCrypto = (System.Security.Cryptography.HashAlgorithm)System.Security.Cryptography.CryptoConfig.CreateFromName(useCrypto.ToUpper()); + } + catch (Exception ex) + { + System.Diagnostics.Trace.TraceError("{0}: <#> {1}({3}) - {2}; Check crypto name", DateTime.Now.ToString(TraceTimeFormat), "HashForBuffer", ex.ToString(), useCrypto); + throw ex; + } + Byte[] hashedBytes = viaCrypto.ComputeHash(inStream); + string hashedText = BitConverter.ToString(hashedBytes).Replace("-", ""); + return hashedText; + } + + + } }