diff --git a/Cloud Element Test Form/Form1.cs b/Cloud Element Test Form/Form1.cs index dd5f0ca..e01a14b 100644 --- a/Cloud Element Test Form/Form1.cs +++ b/Cloud Element Test Form/Form1.cs @@ -518,7 +518,7 @@ private async void toolStripButton1_Click_1(object sender, EventArgs e) if (!chkWithTags.Checked || !currentRow.HasTags) { StatusMsg("Getting current Tag(s).... "); - currentRow = await APIConnector.GetDocEntryMetaData(currentRow.EntryType, Cloud_Elements_API.CloudElementsConnector.FileSpecificationType.ID, currentRow.id); + currentRow = await APIConnector.GetDocEntryMetaData(currentRow.EntryType, Cloud_Elements_API.CloudElementsConnector.FileSpecificationType.ID, currentRow.id,false); } StatusMsg("Storing Tag: " + TagToSet); @@ -579,7 +579,8 @@ private async void getMetadataByPathToolStripMenuItem_Click(object sender, Event if (CloudFileInfoByPath == null) StatusMsg("Nothing Returned! (not expecting not found)"); else { - StatusMsg(string.Format("OK: ID is {0}", CloudFileInfoByPath.id)); + StatusMsg(string.Format("OK: ID is {0}, by [{2}], hash {1}", CloudFileInfoByPath.id, Cloud_Elements_API.FileOperations.SHA1(APIConnector, CloudFileInfoByPath), + Cloud_Elements_API.FileOperations.LastWrittenBy(APIConnector, CloudFileInfoByPath))); } } catch (Exception ex) diff --git a/Cloud Elements Connector/CloudElementsConnector.cs b/Cloud Elements Connector/CloudElementsConnector.cs index c6073be..45fc8cf 100644 --- a/Cloud Elements Connector/CloudElementsConnector.cs +++ b/Cloud Elements Connector/CloudElementsConnector.cs @@ -61,6 +61,9 @@ public Cloud_Elements_API.CloudAuthorization APIAuthorization } } + /// + /// Returns max requests per second to the current endpoint + /// public int EndpointMaxRequestsPerSecond { get @@ -70,7 +73,7 @@ public int EndpointMaxRequestsPerSecond { if (EndpointSettings.ContainsKey(Endpoint)) { - EndpointOptions options = new EndpointOptions(); + EndpointOptions options = EndpointSettings[Endpoint]; result = options.MaxRqPerSecond; } } @@ -87,6 +90,39 @@ public int EndpointMaxRequestsPerSecond } } + /// + /// Returns the time when the last 'rate exceeded' result was detected by the connector + /// + public DateTime WhenRateLastExceeded + { + get + { + DateTime result = DateTime.MinValue; + if (EndpointSettings.ContainsKey(Endpoint)) + { + EndpointOptions options = EndpointSettings[Endpoint]; + result = options.LastRateExceeded; + } + return result; + } + } + + /// + /// Returns the current endpoint options in use by this connector + /// + public EndpointOptions EndpointOptions + { + get + { + EndpointOptions result = null; + if (EndpointSettings.ContainsKey(Endpoint)) + { + result = EndpointSettings[Endpoint]; + } + return result; + } + } + #region "Constructors" public CloudElementsConnector(string elementsURL) @@ -156,6 +192,11 @@ private void AssureEnpointControlData(string endpointName) case "box": options.MaxRqPerSecond = 6; options.LogHighwaterThroughput = true; + options.FileHashAlgorithmName = "SHA1"; + options.ModifiedByRawIDPath = "modified_by.login"; + break; + case "googledrive": + options.ModifiedByRawIDPath = "lastModifyingUser.emailAddress"; break; default: options.MaxRqPerSecond = 32; @@ -238,7 +279,7 @@ public async Task GetStorageAvailable() /// Specifies if the identifier is an ID or a PATH /// Specifying an ID that does not exist results in an error response. /// - public async Task GetDocEntryMetaData(DirectoryEntryType entryType, FileSpecificationType fileSpecType, string identifier) + public async Task GetDocEntryMetaData(DirectoryEntryType entryType, FileSpecificationType fileSpecType, string identifier, bool withRaw) { CloudFile Result; HttpResponseMessage response; @@ -248,15 +289,15 @@ public async Task GetDocEntryMetaData(DirectoryEntryType entryType, F switch (fileSpecType) { case FileSpecificationType.ID: - RequestURL = "hubs/documents/{1}/{0}/metadata"; + RequestURL = "hubs/documents/{1}/{0}/metadata?raw={2}"; break; case FileSpecificationType.Path: - RequestURL = "hubs/documents/{1}/metadata?path={0}"; + RequestURL = "hubs/documents/{1}/metadata?path={0}&raw={2}"; break; default: throw new ArgumentException("unsupported File Specification Type - " + fileSpecType.ToString()); } - RequestURL = string.Format(RequestURL, System.Net.WebUtility.UrlEncode(identifier), URLEntryType); + RequestURL = string.Format(RequestURL, System.Net.WebUtility.UrlEncode(identifier), URLEntryType,withRaw); response = await APIExecuteGet(RequestURL); Result = await response.Content.ReadAsAsync(); return Result; @@ -415,7 +456,7 @@ public async Task DeleteFolder(string path, Boolean withTrash) public async Task GetFolderMetaData(FileSpecificationType fileSpecType, string identifier) { CloudFile Result; - Result = await GetDocEntryMetaData(DirectoryEntryType.Folder, fileSpecType, identifier); + Result = await GetDocEntryMetaData(DirectoryEntryType.Folder, fileSpecType, identifier,true); return Result; } @@ -510,23 +551,7 @@ public async Task FileLinks(FileSpecificationType fileSpecType, strin /// public async Task GetFileMetaData(FileSpecificationType fileSpecType, string identifier) { - CloudFile Result; - HttpResponseMessage response; - string RequestURL; - switch (fileSpecType) - { - case FileSpecificationType.ID: - RequestURL = string.Format("hubs/documents/files/{0}/metadata", System.Net.WebUtility.UrlEncode(identifier)); - break; - case FileSpecificationType.Path: - RequestURL = string.Format("hubs/documents/files/metadata?path={0}", System.Net.WebUtility.UrlEncode(identifier)); - break; - default: - throw new ArgumentException("unsupported File Specification Type - " + fileSpecType.ToString()); - } - response = await APIExecuteGet(RequestURL); - Result = await response.Content.ReadAsAsync(); - return Result; + return await GetDocEntryMetaData(DirectoryEntryType.File, fileSpecType, identifier, true); } /// @@ -857,11 +882,12 @@ async Task APIExecuteVerb(HttpVerb verb, string URI, HttpCo if (EndpointSettings.ContainsKey(Endpoint)) { options = EndpointSettings[Endpoint]; + options.LastRateExceeded = DateTime.Now; if ((options.MaxRqPerSecond <= 0) || (options.MaxRqPerSecond > options.HighwaterGeneratedRequestsPerSecond)) options.MaxRqPerSecond = (int)options.HighwaterGeneratedRequestsPerSecond; if ((options.MaxRqPerSecond > 2) && (DateTime.Now.Subtract(options.LastAutoLimit).TotalSeconds > 1)) { options.MaxRqPerSecond--; - options.LastAutoLimit = DateTime.Now; + options.LastAutoLimit = options.LastRateExceeded; OnDiagTrace(string.Format("ce(throughput) [{0}] rate limit exceeded: inferred new target of {1}r/s", Endpoint, options.MaxRqPerSecond)); } } @@ -930,11 +956,45 @@ public DateTime LastAutoLimit get { return _LastAutoLimit; } internal set { _LastAutoLimit = value; } } + public DateTime LastRateExceeded + { + get { return _LastRateExceeded; } + internal set { _LastRateExceeded = value; } + } + + public bool HasFileHashAlgorithm + { + get { return ((_FileHashCyproName != null) && (_FileHashCyproName.Length > 0)); } + + } + + public bool HasModifiedBy + { + get { return ((_ModifiedByRawPath != null) && (_ModifiedByRawPath.Length > 0)); } + + } + + public string FileHashAlgorithmName + { + get { return _FileHashCyproName; } + internal set { _FileHashCyproName = value; } + } + + protected internal string ModifiedByRawIDPath + { + get { return _ModifiedByRawPath; } + set { _ModifiedByRawPath = value; } + } + + public bool LogThrottleDelays; public bool LogHighwaterThroughput; private double _HighwaterGeneratedRequestsPerSecond; private DateTime _LastAutoLimit; + private DateTime _LastRateExceeded; + private string _FileHashCyproName; + private string _ModifiedByRawPath; } diff --git a/Cloud Elements Connector/CloudFile.cs b/Cloud Elements Connector/CloudFile.cs index 45c6730..8ae8270 100644 --- a/Cloud Elements Connector/CloudFile.cs +++ b/Cloud Elements Connector/CloudFile.cs @@ -16,11 +16,16 @@ public class CloudFile public string modifiedDate { get; set; } // optional public string id { get; set; } // optional public Boolean directory { get; set; } // optional + public Newtonsoft.Json.Linq.JObject raw { get; set; } // optional [Newtonsoft.Json.JsonIgnore] public Boolean HasTags { get { return ((tags != null) && (tags.Length > 0)); } } + [Newtonsoft.Json.JsonIgnore] + public Boolean HasRaw { get { return ((raw != null)) ; } } + + [Newtonsoft.Json.JsonIgnore] public Cloud_Elements_API.CloudElementsConnector.DirectoryEntryType EntryType { @@ -65,6 +70,25 @@ public DateTime WhenModified() return (_WhenModified); } + public string RawValue(string valuePath) + { + string[] pathPart = valuePath.Split('.'); + if (pathPart.GetUpperBound(0) < 1) return string.Empty; + Newtonsoft.Json.Linq.JToken valueToken = raw.GetValue(pathPart[0]); + if (valueToken == null) return string.Empty; + for (int i = 1; i < pathPart.GetUpperBound(0) - 1; i++) + { + if (!valueToken.HasValues) return string.Empty; + if (!(valueToken is Newtonsoft.Json.Linq.JObject)) return string.Empty; + valueToken = ((Newtonsoft.Json.Linq.JObject)valueToken).GetValue(pathPart[i]); + if (valueToken == null) return string.Empty; + } + + valueToken = ((Newtonsoft.Json.Linq.JObject)valueToken).GetValue(pathPart[pathPart.GetUpperBound(0)]); + if (valueToken == null) return string.Empty; + return valueToken.ToString(); + } + /// /// Adds a tag to the tag collection with support for Key-Value pair tags (name=value) diff --git a/Cloud Elements Connector/FileOperations.cs b/Cloud Elements Connector/FileOperations.cs index 2322a1e..83c6083 100644 --- a/Cloud Elements Connector/FileOperations.cs +++ b/Cloud Elements Connector/FileOperations.cs @@ -26,7 +26,7 @@ public static async Task Copy(CloudElementsConnector connector, Cloud /// - /// Obtains information about a cloud file by ID or path; returns NULL + /// Obtains information about a cloud file by ID or path; returns NULL for not found /// /// connection to use /// specifies format of supplied file specification @@ -92,7 +92,29 @@ public static async Task Rename(CloudElementsConnector connector, Clo return targetFile; } + /// + /// Returns SHA1, if available + /// + /// + public static string SHA1(CloudElementsConnector connector, CloudFile targetFile) + { + if (!targetFile.HasRaw) return string.Empty; + if (!connector.EndpointOptions.HasFileHashAlgorithm) return string.Empty; + Newtonsoft.Json.Linq.JToken valueToken = targetFile.raw.GetValue("sha1"); + if (valueToken == null) return string.Empty; + return valueToken.ToString(); + } + /// + /// Returns email address of last file writer, if available + /// + /// + public static string LastWrittenBy(CloudElementsConnector connector, CloudFile targetFile) + { + if (!targetFile.HasRaw) return string.Empty; + if (!connector.EndpointOptions.HasModifiedBy) return string.Empty; + return targetFile.RawValue(connector.EndpointOptions.ModifiedByRawIDPath); + } } } diff --git a/Cloud Elements Connector/Tools.cs b/Cloud Elements Connector/Tools.cs index 2a5d910..ae83dec 100644 --- a/Cloud Elements Connector/Tools.cs +++ b/Cloud Elements Connector/Tools.cs @@ -229,5 +229,30 @@ public static string TraceTimeNow() } + /// + /// Returns a digest (hash) of the stream. + /// + /// + /// MD5,SHA1, SHA256, SHA512 (etc) + /// + public static string HashForBuffer(System.IO.Stream inStream, string useCrypto) + { + System.Security.Cryptography.HashAlgorithm viaCrypto = default(System.Security.Cryptography.HashAlgorithm); + try + { + viaCrypto = (System.Security.Cryptography.HashAlgorithm)System.Security.Cryptography.CryptoConfig.CreateFromName(useCrypto.ToUpper()); + } + catch (Exception ex) + { + System.Diagnostics.Trace.TraceError("{0}: <#> {1}({3}) - {2}; Check crypto name", DateTime.Now.ToString(TraceTimeFormat), "HashForBuffer", ex.ToString(), useCrypto); + throw ex; + } + Byte[] hashedBytes = viaCrypto.ComputeHash(inStream); + string hashedText = BitConverter.ToString(hashedBytes).Replace("-", ""); + return hashedText; + } + + + } }