Skip to content

Commit

Permalink
Merge pull request #69 from villermen/nxt-support
Browse files Browse the repository at this point in the history 
nxt-support
  • Loading branch information
@villermen
villermen authored Sep 22, 2020
2 parents 4bc4aa8 + 281ad94 commit 79dbb4e
Show file tree
Hide file tree
Showing 57 changed files with 1,253 additions and 804 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,3 +251,5 @@ paket-files/
# JetBrains Rider
.idea/
*.sln.iml

/dist
28 changes: 14 additions & 14 deletions RuneScapeCacheTools.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RuneScapeCacheToolsTest", "
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Debug|x64 = Debug|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Release|Any CPU.Build.0 = Release|Any CPU
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Release|Any CPU.Build.0 = Release|Any CPU
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Release|Any CPU.ActiveCfg = Release|Any CPU
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Release|Any CPU.Build.0 = Release|Any CPU
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Release|x64.ActiveCfg = Release|x64
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Release|x64.Build.0 = Release|x64
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Debug|x64.ActiveCfg = Debug|x64
{A2C77AE9-5E36-4BDE-B6DE-EA5ADA87DAD2}.Debug|x64.Build.0 = Debug|x64
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Release|x64.ActiveCfg = Release|x64
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Release|x64.Build.0 = Release|x64
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Debug|x64.ActiveCfg = Debug|x64
{305EFE8D-2664-44D9-A62F-2BFCB33D566D}.Debug|x64.Build.0 = Debug|x64
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Release|x64.ActiveCfg = Release|x64
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Release|x64.Build.0 = Release|x64
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Debug|x64.ActiveCfg = Debug|x64
{55E68712-4819-4BC6-B829-A09F7FF04E78}.Debug|x64.Build.0 = Debug|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
11 changes: 6 additions & 5 deletions RuneScapeCacheTools/Cache/DownloaderCache.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Villermen.RuneScapeCacheTools.Cache.Downloader;
using Villermen.RuneScapeCacheTools.Exception;
using Villermen.RuneScapeCacheTools.File;
using Villermen.RuneScapeCacheTools.Model;
using Villermen.RuneScapeCacheTools.Utility;

namespace Villermen.RuneScapeCacheTools.Cache
{
Expand All @@ -24,7 +25,7 @@ public class DownloaderCache : ReferenceTableCache

private readonly HttpFileDownloader _httpFileDownloader;

public DownloaderCache()
public DownloaderCache() : base(new RuneTek5CacheFileDecoder())
{
this._tcpFileDownloader = new TcpFileDownloader();
this._httpFileDownloader = new HttpFileDownloader();
Expand All @@ -47,7 +48,7 @@ public MasterReferenceTableFile GetMasterReferenceTable()
return this._cachedMasterReferenceTable;
}

public override byte[] GetFileData(CacheIndex index, int fileId)
public override byte[] GetFileData(CacheIndex index, int fileId, CacheFileInfo? info)
{
if (DownloaderCache.HttpInterfaceIndexes.Contains(index))
{
Expand All @@ -59,9 +60,9 @@ public override byte[] GetFileData(CacheIndex index, int fileId)
return this._tcpFileDownloader.DownloadFileData(index, fileId);
}

protected override void PutFileData(CacheIndex index, int fileId, byte[] data)
protected override void PutFileData(CacheIndex index, int fileId, byte[] data, CacheFileInfo? info)
{
throw new NotSupportedException("I am a downloader, stop trying to put things in me!");
throw new CacheException("If only what you were trying to do made sense...");
}

public override void Dispose()
Expand Down
6 changes: 3 additions & 3 deletions RuneScapeCacheTools/Cache/JavaClientCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public class JavaClientCache : ReferenceTableCache
/// </summary>
/// <param name="cacheDirectory"></param>
/// <param name="readOnly"></param>
public JavaClientCache(string? cacheDirectory = null, bool readOnly = true)
public JavaClientCache(string? cacheDirectory = null, bool readOnly = true) : base(new RuneTek5CacheFileDecoder())
{
this.CacheDirectory = PathExtensions.FixDirectory(cacheDirectory ?? JavaClientCache.DefaultCacheDirectory);
this.ReadOnly = readOnly;
Expand All @@ -65,7 +65,7 @@ public override void Dispose()
this.CloseStreams();
}

public override byte[] GetFileData(CacheIndex index, int fileId)
public override byte[] GetFileData(CacheIndex index, int fileId, CacheFileInfo? info)
{
// Read the sectors and take their payload data up to the size of the contained file.
return this
Expand All @@ -79,7 +79,7 @@ public override byte[] GetFileData(CacheIndex index, int fileId)
.ToArray();
}

protected override void PutFileData(CacheIndex index, int fileId, byte[] data)
protected override void PutFileData(CacheIndex index, int fileId, byte[] data, CacheFileInfo? info)
{
if (this.ReadOnly)
{
Expand Down
161 changes: 153 additions & 8 deletions RuneScapeCacheTools/Cache/NxtClientCache.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Data.Sqlite;
using Serilog;
using Villermen.RuneScapeCacheTools.Exception;
using Villermen.RuneScapeCacheTools.Model;
using Villermen.RuneScapeCacheTools.Utility;

Expand All @@ -17,10 +21,15 @@ public class NxtClientCache : ReferenceTableCache

public bool ReadOnly { get; }

private object _ioLock = new object();
private readonly object _ioLock = new object();

public NxtClientCache(string? cacheDirectory = null, bool readOnly = true)
private readonly Dictionary<CacheIndex, SqliteConnection> _connections = new Dictionary<CacheIndex, SqliteConnection>();

public NxtClientCache(string? cacheDirectory = null, bool readOnly = true) : base(new RuneTek7CacheFileDecoder())
{
// Configure SQLite provider for NXT operations.
SQLitePCL.raw.SetProvider(new SQLitePCL.SQLite3Provider_sqlite3());

this.CacheDirectory = PathExtensions.FixDirectory(cacheDirectory ?? NxtClientCache.DefaultCacheDirectory);
this.ReadOnly = readOnly;

Expand All @@ -29,26 +38,162 @@ public NxtClientCache(string? cacheDirectory = null, bool readOnly = true)
{
Directory.CreateDirectory(this.CacheDirectory);
}

this.OpenConnections();
}

public override IEnumerable<CacheIndex> GetAvailableIndexes()
{
throw new System.NotImplementedException();
return this._connections.Keys;
}

public override byte[] GetFileData(CacheIndex index, int fileId, CacheFileInfo? info)
{
// Reference tables are stored as file 1 in a separate table in the database of the index.
var dbTable = "cache";
if (index == CacheIndex.ReferenceTables)
{
index = (CacheIndex)fileId;
dbTable = "cache_index";
fileId = 1;
}

lock (this._ioLock)
{
if (!this._connections.Keys.Contains(index))
{
throw new CacheFileNotFoundException($"Index {(int)index} is not available.");
}

var connection = this._connections[index];

using var command = connection.CreateCommand();
command.CommandText = $"SELECT DATA, VERSION, CRC FROM {dbTable} WHERE KEY = @key";
command.Parameters.AddWithValue("key", fileId);
using var resultReader = command.ExecuteReader();
if (!resultReader.HasRows)
{
throw new CacheFileNotFoundException($"File {(int)index}/{fileId} does not exist in the cache.");
}

resultReader.Read();
var data = (byte[])resultReader.GetValue(0);
var version = resultReader.GetInt32(1);
var crc = resultReader.GetInt32(2);

// Version and CRC do not match data but do have to match the passed info.
if (info?.Version != null && version != info.Version)
{
throw new DecodeException($"Retrieved version ({version}) does not match expected ({info.Version}).");
}
if (info?.Crc != null && crc != info.Crc)
{
// They actually mess around with the CRC =S
var message = $"Retrieved CRC ({crc}) does not match expected ({info.Crc}).";
if (crc - 1 == info.Crc)
{
Log.Debug(message + " (allowed)");
}
else
{
throw new DecodeException(message);
}
}

return data;
}
}

protected override void PutFileData(CacheIndex index, int fileId, byte[] data, CacheFileInfo? info)
{
if (this.ReadOnly)
{
throw new CacheException("Can't write data in readonly mode.");
}

var dbTable = "cache";
if (index == CacheIndex.ReferenceTables)
{
index = (CacheIndex)fileId;
dbTable = "cache_index";
fileId = 1;
}

lock (this._ioLock)
{
if (!this._connections.Keys.Contains(index))
{
this.OpenConnection(index, true);
}

var connection = this._connections[index];

using var command = connection.CreateCommand();
command.CommandText = $"INSERT OR REPLACE INTO {dbTable} (KEY, DATA, VERSION, CRC) VALUES (@key, @data, @version, @crc)";
command.Parameters.AddWithValue("key", fileId);
command.Parameters.AddWithValue("data", data);
command.Parameters.AddWithValue("version", info?.Version ?? DateTimeOffset.UtcNow.ToUnixTimeSeconds());
command.Parameters.AddWithValue("crc", info?.Crc ?? 0);
command.ExecuteNonQuery();
}
}

private void OpenConnections()
{
for (var indexId = 0; indexId <= 255; indexId++)
{
this.OpenConnection((CacheIndex)indexId, false);
}
}

public override byte[] GetFileData(CacheIndex index, int fileId)
private void OpenConnection(CacheIndex index, bool force)
{
throw new System.NotImplementedException();
var indexPath = Path.Combine(this.CacheDirectory, $"js5-{(int)index}.jcache");

var createTables = false;
if (!System.IO.File.Exists(indexPath))
{
if (!force)
{
return;
}

createTables = true;
}

var connectionStringBuilder = new SqliteConnectionStringBuilder
{
DataSource = indexPath,
Mode = this.ReadOnly ? SqliteOpenMode.ReadOnly : SqliteOpenMode.ReadWriteCreate,
};
var connection = new SqliteConnection(connectionStringBuilder.ConnectionString);
connection.Open();

if (createTables)
{
using var cacheCommand = connection.CreateCommand();
cacheCommand.CommandText = "CREATE TABLE cache (KEY INTEGER PRIMARY KEY, DATA BLOB, VERSION INTEGER, CRC INTEGER)";
cacheCommand.ExecuteNonQuery();
using var cacheIndexCommand = connection.CreateCommand();
cacheIndexCommand.CommandText = "CREATE TABLE cache_index (KEY INTEGER PRIMARY KEY, DATA BLOB, VERSION INTEGER, CRC INTEGER)";
cacheIndexCommand.ExecuteNonQuery();
}

this._connections[index] = connection;
}

protected override void PutFileData(CacheIndex index, int fileId, byte[] data)
private void CloseConnections()
{
throw new System.NotImplementedException();
foreach (var connection in this._connections.Values)
{
connection.Close();
}
this._connections.Clear();
}

public override void Dispose()
{
throw new System.NotImplementedException();
this.CloseConnections();
}
}
}
36 changes: 27 additions & 9 deletions RuneScapeCacheTools/Cache/ReferenceTableCache.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Villermen.RuneScapeCacheTools.Exception;
using Villermen.RuneScapeCacheTools.File;
using Villermen.RuneScapeCacheTools.Model;
using Villermen.RuneScapeCacheTools.Utility;

namespace Villermen.RuneScapeCacheTools.Cache
{
Expand All @@ -13,6 +14,13 @@ namespace Villermen.RuneScapeCacheTools.Cache
/// </summary>
public abstract class ReferenceTableCache : ICache
{
public readonly ICacheFileDecoder FileDecoder;

protected ReferenceTableCache(ICacheFileDecoder fileDecoder)
{
this.FileDecoder = fileDecoder;
}

/// <summary>
/// Reference tables are kept in memory so they don't have to be obtained again for every file.
/// </summary>
Expand All @@ -37,7 +45,10 @@ public ReferenceTableFile GetReferenceTable(CacheIndex index, bool createIfNotFo
}
catch (CacheFileNotFoundException) when (createIfNotFound)
{
return new ReferenceTableFile();
return new ReferenceTableFile
{
Options = ReferenceTableOptions.Sizes,
};
}
});
}
Expand All @@ -50,8 +61,9 @@ public IEnumerable<int> GetAvailableFileIds(CacheIndex index)
public CacheFile GetFile(CacheIndex index, int fileId)
{
var fileInfo = this.GetFileInfo(index, fileId);
var fileData = this.GetFileData(index, fileId);
return CacheFile.Decode(fileData, fileInfo);
var fileData = this.GetFileData(index, fileId, fileInfo);

return this.FileDecoder.DecodeFile(fileData, fileInfo);
}

public CacheFileInfo GetFileInfo(CacheIndex index, int fileId)
Expand All @@ -70,7 +82,7 @@ public CacheFileInfo GetFileInfo(CacheIndex index, int fileId)
return this.GetReferenceTable(index).GetFileInfo(fileId);
}

public abstract byte[] GetFileData(CacheIndex index, int fileId);
public abstract byte[] GetFileData(CacheIndex index, int fileId, CacheFileInfo? info);

public void PutFile(CacheIndex index, int fileId, CacheFile file)
{
Expand All @@ -79,17 +91,23 @@ public void PutFile(CacheIndex index, int fileId, CacheFile file)
throw new ArgumentException("Manually writing files to the reference table index is not allowed.");
}

this.PutFileData(index, fileId, file.Encode());
var info = file.Info.Clone();

this.PutFileData(index, fileId, this.FileDecoder.EncodeFile(file, info), info);

// Write updated reference table.
var referenceTable = this.GetReferenceTable(index, true);
referenceTable.SetFileInfo(fileId, file.Info);
referenceTable.SetFileInfo(fileId, info);
var referenceTableFile = new CacheFile(referenceTable.Encode());
referenceTableFile.Info.CompressionType = CompressionType.Bzip2;
this.PutFileData(CacheIndex.ReferenceTables, (int)index, referenceTableFile.Encode());
this.PutFileData(
CacheIndex.ReferenceTables,
(int)index,
this.FileDecoder.EncodeFile(referenceTableFile, null),
null
);
}

protected abstract void PutFileData(CacheIndex index, int fileId, byte[] data);
protected abstract void PutFileData(CacheIndex index, int fileId, byte[] data, CacheFileInfo? info);

public void ClearCachedReferenceTables()
{
Expand Down
Loading

0 comments on commit 79dbb4e

Please sign in to comment.