From 3341fe212e76a1a1a800a111d36c1952e3f8db2d Mon Sep 17 00:00:00 2001 From: Villermen Date: Wed, 3 May 2017 11:38:38 +0200 Subject: [PATCH] Add all previous functionality to TcpFileDownloader --- .../Cache/Downloader/HttpFileDownloader.cs | 4 +- .../Cache/Downloader/TcpFileDownloader.cs | 331 +++++++++--------- 2 files changed, 173 insertions(+), 162 deletions(-) diff --git a/RuneScapeCacheTools/Cache/Downloader/HttpFileDownloader.cs b/RuneScapeCacheTools/Cache/Downloader/HttpFileDownloader.cs index 761d303..a844cae 100644 --- a/RuneScapeCacheTools/Cache/Downloader/HttpFileDownloader.cs +++ b/RuneScapeCacheTools/Cache/Downloader/HttpFileDownloader.cs @@ -28,9 +28,9 @@ public async Task DownloadFileAsync(Index index, int fileId, CacheFi throw new DownloaderException($"HTTP interface responded with status code: {response.StatusCode}."); } - if (response.ContentLength != fileInfo.CompressedSize - 2) + if (response.ContentLength != fileInfo.CompressedSize) { - throw new DownloaderException($"Downloaded file size {response.ContentLength} does not match expected {fileInfo.CompressedSize - 2}."); + throw new DownloaderException($"Downloaded file size {response.ContentLength} does not match expected {fileInfo.CompressedSize}."); } var dataStream = new MemoryStream(); diff --git a/RuneScapeCacheTools/Cache/Downloader/TcpFileDownloader.cs b/RuneScapeCacheTools/Cache/Downloader/TcpFileDownloader.cs index ee32240..2f48f10 100644 --- a/RuneScapeCacheTools/Cache/Downloader/TcpFileDownloader.cs +++ b/RuneScapeCacheTools/Cache/Downloader/TcpFileDownloader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.IO; +using System.Linq; using System.Net; using System.Net.Sockets; using System.Text.RegularExpressions; @@ -47,9 +48,7 @@ public class TcpFileDownloader : IFileDownloader, IDisposable private readonly string _keyPage; - private readonly object _connectLock = new object(); - - private readonly object _responseProcessorLock = new object(); + private readonly object _processorLock = new object(); /// /// The major version is needed to correctly connect to the content server. @@ -61,7 +60,7 @@ public class TcpFileDownloader : IFileDownloader, IDisposable private bool _connected = false; - private ConcurrentDictionary, FileRequest> PendingFileRequests { get; } = + private ConcurrentDictionary, FileRequest> FileRequests { get; } = new ConcurrentDictionary, FileRequest>(); public TcpFileDownloader(string contentHost = "content.runescape.com", int contentPort = 43594, string keyPage = "http://world2.runescape.com") @@ -73,7 +72,14 @@ public TcpFileDownloader(string contentHost = "content.runescape.com", int conte public async Task DownloadFileAsync(Index index, int fileId, CacheFileInfo fileInfo = null) { - throw new System.NotImplementedException(); + // Add the request, or get an existing one + var request = this.FileRequests.GetOrAdd( + new Tuple(index, fileId), + new FileRequest(index, fileId, fileInfo)); + + Task.Run(new Action(this.ProcessRequests)); + + return await request.WaitForCompletionAsync(); } private string GetKey() @@ -104,83 +110,67 @@ private string GetKey() private void Connect() { - if (!this._connected) + if (this._connected) { - lock (this._connectLock) - { - if (this._connected) - { - // No need for this now is there? - return; - } - - TcpFileDownloader.Logger.Debug("Connecting to content server with TCP."); - - var key = this.GetKey(); - - // Retry connecting with an increasing major version until the server no longer reports we're outdated - var connected = false; - while (!connected) - { - this._contentClient = new TcpClient(this._contentHost, this._contentPort); + throw new DownloaderException("Tried to connect while already connected."); + } - var handshakeWriter = new BinaryWriter(this._contentClient.GetStream()); - var handshakeReader = new BinaryReader(this._contentClient.GetStream()); + TcpFileDownloader.Logger.Debug("Connecting to content server with TCP."); - var handshakeLength = (byte) (9 + key.Length + 1); + var key = this.GetKey(); - handshakeWriter.Write(TcpFileDownloader.HandshakeType); - handshakeWriter.Write(handshakeLength); - handshakeWriter.WriteInt32BigEndian(this._contentVersionMajor); - handshakeWriter.WriteInt32BigEndian(TcpFileDownloader.ContentVersionMinor); - handshakeWriter.WriteNullTerminatedString(key); - handshakeWriter.Write((byte)TcpFileDownloader.Language); - handshakeWriter.Flush(); + // Retry connecting with an increasing major version until the server no longer reports we're outdated + var connected = false; + while (!connected) + { + this._contentClient = new TcpClient(this._contentHost, this._contentPort); - var response = (HandshakeResponse)handshakeReader.ReadByte(); + var handshakeWriter = new BinaryWriter(this._contentClient.GetStream()); + var handshakeReader = new BinaryReader(this._contentClient.GetStream()); - switch (response) - { - case HandshakeResponse.Success: - connected = true; - TcpFileDownloader.Logger.Info($"Successfully connected to content server with major version {this._contentVersionMajor}."); - break; - - case HandshakeResponse.Outdated: - this._contentClient.Dispose(); - this._contentClient = null; - TcpFileDownloader.Logger.Info($"Requested connection used outdated version {this._contentVersionMajor}. Retrying with higher major version."); - this._contentVersionMajor++; - break; - - default: - this._contentClient.Dispose(); - this._contentClient = null; - throw new DownloaderException($"Content server responded to handshake with {response}."); - } - } + var handshakeLength = (byte) (9 + key.Length + 1); - // Required loading element sizes. They are unnsed by this tool and I have no idea what they are for. So yeah... - var contentReader = new BinaryReader(this._contentClient.GetStream()); - contentReader.ReadBytes(TcpFileDownloader.LoadingRequirements * 4); + handshakeWriter.Write(TcpFileDownloader.HandshakeType); + handshakeWriter.Write(handshakeLength); + handshakeWriter.WriteInt32BigEndian(this._contentVersionMajor); + handshakeWriter.WriteInt32BigEndian(TcpFileDownloader.ContentVersionMinor); + handshakeWriter.WriteNullTerminatedString(key); + handshakeWriter.Write((byte)TcpFileDownloader.Language); + handshakeWriter.Flush(); - this.SendTcpConnectionInfo(); + var response = (HandshakeResponse)handshakeReader.ReadByte(); - this._connected = true; + switch (response) + { + case HandshakeResponse.Success: + connected = true; + TcpFileDownloader.Logger.Info($"Successfully connected to content server with major version {this._contentVersionMajor}."); + break; + + case HandshakeResponse.Outdated: + this._contentClient.Dispose(); + this._contentClient = null; + TcpFileDownloader.Logger.Info($"Requested connection used outdated version {this._contentVersionMajor}. Retrying with higher major version."); + this._contentVersionMajor++; + break; + + default: + this._contentClient.Dispose(); + this._contentClient = null; + throw new DownloaderException($"Content server responded to handshake with {response}."); } } - } - /// - /// Sends the initial connection status and login packets to the server. - /// - private void SendTcpConnectionInfo() - { + // Required loading element sizes. They are unnsed by this tool and I have no idea what they are for. So yeah... + var contentReader = new BinaryReader(this._contentClient.GetStream()); + contentReader.ReadBytes(TcpFileDownloader.LoadingRequirements * 4); + + // Send the initial connection status and login packets to the server. TcpFileDownloader.Logger.Debug("Sending initial connection status and login packets."); var writer = new BinaryWriter(this._contentClient.GetStream()); - // I don't know + // I don't know what exactly, but this is how it's done writer.Write((byte)6); writer.WriteUInt24BigEndian(4); writer.WriteInt16BigEndian(0); @@ -190,107 +180,108 @@ private void SendTcpConnectionInfo() writer.WriteUInt24BigEndian(0); writer.WriteInt16BigEndian(0); writer.Flush(); + + this._connected = true; } - private void StartFileDownloadTcp(FileRequest fileRequest) + private void ProcessRequests() { - // TODO: This at least needs a different name, and at most a full rework - - // TODO: Wrap in processorlock in its entirety? Account for early-out late-in - Task.Run(() => + lock (this._processorLock) { + // Check if still needed after lock is obtained + if (!this.FileRequests.Any()) + { + return; + } + if (!this._connected) { this.Connect(); } - TcpFileDownloader.Logger.Debug($"Requesting {fileRequest.Index}/{fileRequest.FileId} using TCP."); - - // Send the request - var writer = new BinaryWriter(this._contentClient.GetStream()); + TcpFileDownloader.Logger.Debug("Starting TCP response processor."); - // Send the file request to the content server - writer.Write((byte)(fileRequest.Index == Index.ReferenceTables ? 1 : 0)); - writer.Write((byte)fileRequest.Index); - writer.WriteInt32BigEndian(fileRequest.FileId); - - // This will process all received TCP chunks until the given requested file is complete (so it might also complete other requested files). - // Only one processor may be running at any given moment - lock (this._responseProcessorLock) + while (this.FileRequests.Any()) { - TcpFileDownloader.Logger.Debug("Starting TCP request processor."); + // Request all unrequested files + foreach (var requestPair in this.FileRequests.Where(request => !request.Value.Requested)) + { + this.RequestFile(requestPair.Value); - while (this.PendingFileRequests.ContainsKey(new Tuple(fileRequest.Index, fileRequest.FileId))) + // TODO: Limit to x amount of pending requests + } + + // Read one chunk + if (this._contentClient.Available >= 5) { - // Read one chunk - if (this._contentClient.Available >= 5) - { - var reader = new BinaryReader(this._contentClient.GetStream()); + var reader = new BinaryReader(this._contentClient.GetStream()); - var readByteCount = 0; + var readByteCount = 0; - var index = (Index)reader.ReadByte(); - var fileId = reader.ReadInt32BigEndian() & 0x7fffffff; + // Check which file this chunk is for + var index = (Index) reader.ReadByte(); + var fileId = reader.ReadInt32BigEndian() & 0x7fffffff; - readByteCount += 5; + readByteCount += 5; - var requestKey = new Tuple(index, fileId); + var requestKey = new Tuple(index, fileId); - if (!this.PendingFileRequests.ContainsKey(requestKey)) - { - throw new DownloaderException("Invalid response received (maybe not all data was consumed by the previous operation?"); - } + if (!this.FileRequests.ContainsKey(requestKey)) + { + throw new DownloaderException("Invalid response received (maybe not all data was consumed by the previous operation?"); + } - var request = (FileRequest)this.PendingFileRequests[requestKey]; - var dataWriter = new BinaryWriter(request.DataStream); + var request = this.FileRequests[requestKey]; - // The first part of the file always contains the filesize, which we need to know, but is also part of the file - if (request.FileSize == 0) - { - var compressionType = (CompressionType)reader.ReadByte(); - var length = reader.ReadInt32BigEndian(); + // The first part of the file always contains the filesize, which we need to know, but is also part of the file + if (!request.MetaWritten) + { + var compressionType = (CompressionType) reader.ReadByte(); + var length = reader.ReadInt32BigEndian(); - readByteCount += 5; + readByteCount += 5; - request.FileSize = 5 + (compressionType != CompressionType.None ? 4 : 0) + length; + request.WriteMeta(compressionType, length); + } - dataWriter.Write((byte)compressionType); - dataWriter.WriteInt32BigEndian(length); - } + var remainingBlockLength = TcpFileDownloader.BlockLength - readByteCount; - var remainingBlockLength = TcpFileDownloader.BlockLength - readByteCount; + if (remainingBlockLength > request.RemainingLength) + { + remainingBlockLength = request.RemainingLength; + } - if (remainingBlockLength > request.RemainingLength) - { - remainingBlockLength = request.RemainingLength; - } + request.WriteContent(reader.ReadBytes(remainingBlockLength)); - dataWriter.Write(reader.ReadBytes(remainingBlockLength)); + if (request.Completed) + { + // The request got completed, remove it from the list of pending requests + FileRequest removedRequest; + this.FileRequests.TryRemove(requestKey, out removedRequest); + } + } + } - if (request.RemainingLength == 0) - { - // The request got completed, remove it from the list of pending requests - FileRequest removedRequest; - this.PendingFileRequests.TryRemove(requestKey, out removedRequest); + TcpFileDownloader.Logger.Debug("TCP request processor finished."); + } + } - // TODO: this.AppendVersionToRequestData(removedRequest); + private void RequestFile(FileRequest request) + { + if (request.Requested) + { + throw new DownloaderException("File to be requested is already requested."); + } - removedRequest.Complete(); + TcpFileDownloader.Logger.Debug($"Requesting {request.Index}/{request.FileId} using TCP."); - // Exit the loop if this was the file originally requested - if (removedRequest == fileRequest) - { - break; - } - } - } + var writer = new BinaryWriter(this._contentClient.GetStream()); - // var leftoverBytes = new BinaryReader(TcpContentClient.GetStream()).ReadBytes(TcpContentClient.Available); - } + writer.Write((byte) (request.Index == Index.ReferenceTables ? 1 : 0)); + writer.Write((byte) request.Index); + writer.WriteInt32BigEndian(request.FileId); - TcpFileDownloader.Logger.Debug("TCP request processor finished."); - } - }); + request.Requested = true; } public void Dispose() @@ -309,52 +300,72 @@ private enum HandshakeResponse private class FileRequest { - protected FileRequest(Index index, int fileId, CacheFileInfo cacheFileInfo) + private readonly CacheFileInfo _cacheFileInfo; + private readonly MemoryStream _dataStream = new MemoryStream(); + private readonly TaskCompletionSource _completionSource = new TaskCompletionSource(); + private int _fileSize; + + public Index Index { get; } + public int FileId { get; } + public int RemainingLength => (int)(this._fileSize - this._dataStream.Length); + public bool Completed { get; private set; } + public bool Requested { get; set; } + public bool MetaWritten { get; private set; } + + public FileRequest(Index index, int fileId, CacheFileInfo cacheFileInfo) { this.Index = index; this.FileId = fileId; - this.CacheFileInfo = cacheFileInfo; + this._cacheFileInfo = cacheFileInfo; } - public int FileSize { get; set; } + public void WriteMeta(CompressionType compressionType, int length) + { + if (this._dataStream.Length != 0) + { + throw new DownloaderException("File metadata must be written before anything else."); + } - public int RemainingLength => (int)(this.FileSize - this.DataStream.Length); + var writer = new BinaryWriter(this._dataStream); + writer.Write((byte)compressionType); + writer.WriteInt32BigEndian(length); - public CacheFileInfo CacheFileInfo { get; } - public MemoryStream DataStream { get; } = new MemoryStream(); - public int FileId { get; } - public Index Index { get; } - private TaskCompletionSource CompletionSource { get; } = new TaskCompletionSource(); + this._fileSize = 5 + (compressionType != CompressionType.None ? 4 : 0) + length; - public void Write(byte[] data) + this.MetaWritten = true; + } + + public void WriteContent(byte[] data) { + if (!this.MetaWritten) + { + throw new DownloaderException("File content must be written after metadata"); + } + if (data.Length > this.RemainingLength) { throw new DownloaderException("Tried to write more bytes than were remaining in the file."); } - this.DataStream.Write(data, 0, data.Length); + this._dataStream.Write(data, 0, data.Length); if (this.RemainingLength == 0) { - this.Complete(); - } - } + // Append file version if possible + if (this._cacheFileInfo?.Version != null) + { + var writer = new BinaryWriter(this._dataStream); + writer.WriteUInt16BigEndian((ushort)this._cacheFileInfo.Version); + } - public void Complete() - { - this.CompletionSource.SetResult(this.DataStream.ToArray()); + this.Completed = true; + this._completionSource.SetResult(RuneTek5FileDecoder.DecodeFile(this._dataStream.ToArray(), this._cacheFileInfo)); + } } - public byte[] WaitForCompletion() + public async Task WaitForCompletionAsync() { - // Wait for CompletionSource with a timeout - if (Task.WhenAny(this.CompletionSource.Task, Task.Delay(TimeSpan.FromSeconds(10))).Result == this.CompletionSource.Task) - { - return this.CompletionSource.Task.Result; - } - - throw new TimeoutException("The file request was not fulfilled within 10 seconds."); + return await this._completionSource.Task; } } }