Database Improvements
Brought in all the myriad improvements to the database tools from the Stream Pixels project Added configurable nested entry grouping
This commit is contained in:
parent
526a737a4d
commit
968783bf9c
37 changed files with 865 additions and 580 deletions
|
|
@ -1,5 +1,8 @@
|
|||
using System;
|
||||
|
||||
[AttributeUsage(AttributeTargets.Class)]
|
||||
public class GlobalSettingsAttribute : Attribute { }
|
||||
|
||||
[AttributeUsage(AttributeTargets.Class)]
|
||||
public class ExternalEntryAttribute : Attribute { }
|
||||
|
||||
|
|
@ -8,15 +11,6 @@ public class InspectableAttribute : Attribute { }
|
|||
[AttributeUsage(AttributeTargets.Field)]
|
||||
public abstract class PreferredInspectorAttribute : InspectableAttribute { }
|
||||
|
||||
public class InspectableUnityObjectAttribute : PreferredInspectorAttribute
|
||||
{
|
||||
public Type ObjectType;
|
||||
|
||||
public InspectableUnityObjectAttribute(Type type)
|
||||
{
|
||||
ObjectType = type;
|
||||
}
|
||||
}
|
||||
public class InspectableTextAttribute : PreferredInspectorAttribute { }
|
||||
public class InspectablePrefabAttribute : PreferredInspectorAttribute { }
|
||||
public class InspectableTextureAttribute : PreferredInspectorAttribute { }
|
||||
|
|
@ -25,6 +19,16 @@ public class InspectableTemperatureAttribute : PreferredInspectorAttribute { }
|
|||
public class InspectableAnimationCurveAttribute : PreferredInspectorAttribute { }
|
||||
public class InspectableColorAttribute : PreferredInspectorAttribute { }
|
||||
|
||||
public class InspectableEnumValuesAttribute : PreferredInspectorAttribute
|
||||
{
|
||||
public Type EnumType;
|
||||
|
||||
public InspectableEnumValuesAttribute(Type enumType)
|
||||
{
|
||||
EnumType = enumType;
|
||||
}
|
||||
}
|
||||
|
||||
public class InspectableDatabaseLinkAttribute : PreferredInspectorAttribute
|
||||
{
|
||||
public readonly Type EntryType;
|
||||
|
|
@ -35,22 +39,22 @@ public class InspectableDatabaseLinkAttribute : PreferredInspectorAttribute
|
|||
}
|
||||
}
|
||||
|
||||
public class RangedFloatInspectableAttribute : PreferredInspectorAttribute
|
||||
public class InspectableRangedFloatAttribute : PreferredInspectorAttribute
|
||||
{
|
||||
public readonly float Min, Max;
|
||||
|
||||
public RangedFloatInspectableAttribute(float min, float max)
|
||||
public InspectableRangedFloatAttribute(float min, float max)
|
||||
{
|
||||
Min = min;
|
||||
Max = max;
|
||||
}
|
||||
}
|
||||
|
||||
public class RangedIntInspectableAttribute : PreferredInspectorAttribute
|
||||
public class InspectableRangedIntAttribute : PreferredInspectorAttribute
|
||||
{
|
||||
public readonly int Min, Max;
|
||||
|
||||
public RangedIntInspectableAttribute(int min, int max)
|
||||
public InspectableRangedIntAttribute(int min, int max)
|
||||
{
|
||||
Min = min;
|
||||
Max = max;
|
||||
|
|
@ -107,4 +111,24 @@ public class RethinkTableAttribute : Attribute
|
|||
{
|
||||
TableName = tableName;
|
||||
}
|
||||
}
|
||||
|
||||
public class NameAttribute : Attribute
|
||||
{
|
||||
public string Name;
|
||||
|
||||
public NameAttribute(string name)
|
||||
{
|
||||
Name = name;
|
||||
}
|
||||
}
|
||||
|
||||
public class OrderAttribute : Attribute
|
||||
{
|
||||
public int Order;
|
||||
|
||||
public OrderAttribute(int order)
|
||||
{
|
||||
Order = order;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
public static class CollectionExtensions
|
||||
{
|
||||
// https://stackoverflow.com/a/800469
|
||||
public static string GetHashSHA1(this byte[] data)
|
||||
{
|
||||
using (var sha1 = new System.Security.Cryptography.SHA1CryptoServiceProvider())
|
||||
{
|
||||
return string.Concat(sha1.ComputeHash(data).Select(x => x.ToString("X2")));
|
||||
}
|
||||
}
|
||||
|
||||
// https://stackoverflow.com/a/3188835
|
||||
public static T MaxBy<T, U>(this IEnumerable<T> items, Func<T, U> selector)
|
||||
{
|
||||
if (!items.Any())
|
||||
{
|
||||
throw new InvalidOperationException("Empty input sequence");
|
||||
}
|
||||
|
||||
var comparer = Comparer<U>.Default;
|
||||
T maxItem = items.First();
|
||||
U maxValue = selector(maxItem);
|
||||
|
||||
foreach (T item in items.Skip(1))
|
||||
{
|
||||
// Get the value of the item and compare it to the current max.
|
||||
U value = selector(item);
|
||||
if (comparer.Compare(value, maxValue) > 0)
|
||||
{
|
||||
maxValue = value;
|
||||
maxItem = item;
|
||||
}
|
||||
}
|
||||
|
||||
return maxItem;
|
||||
}
|
||||
|
||||
public static T MinBy<T, U>(this IEnumerable<T> items, Func<T, U> selector)
|
||||
{
|
||||
if (!items.Any())
|
||||
{
|
||||
throw new InvalidOperationException("Empty input sequence");
|
||||
}
|
||||
|
||||
var comparer = Comparer<U>.Default;
|
||||
T maxItem = items.First();
|
||||
U maxValue = selector(maxItem);
|
||||
|
||||
foreach (T item in items.Skip(1))
|
||||
{
|
||||
// Get the value of the item and compare it to the current max.
|
||||
U value = selector(item);
|
||||
if (comparer.Compare(value, maxValue) < 0)
|
||||
{
|
||||
maxValue = value;
|
||||
maxItem = item;
|
||||
}
|
||||
}
|
||||
|
||||
return maxItem;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
fileFormatVersion: 2
|
||||
guid: eed05df00a9a6b84292c81be1662a66e
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
|
|
@ -14,10 +14,10 @@ public class CultCache
|
|||
{
|
||||
public event Action<DatabaseEntry> OnDataInsertLocal;
|
||||
public event Action<DatabaseEntry> OnDataUpdateLocal;
|
||||
public event Action<DatabaseEntry> OnDataDeleteLocal;
|
||||
public event Action<DatabaseEntry> OnDataRemoveLocal;
|
||||
public event Action<DatabaseEntry> OnDataInsertRemote;
|
||||
public event Action<DatabaseEntry> OnDataUpdateRemote;
|
||||
public event Action<DatabaseEntry> OnDataDeleteRemote;
|
||||
public event Action<DatabaseEntry> OnDataRemoveRemote;
|
||||
|
||||
private readonly object addLock = new object();
|
||||
|
||||
|
|
@ -26,6 +26,7 @@ public class CultCache
|
|||
|
||||
private readonly Dictionary<Guid, DatabaseEntry> _entries = new Dictionary<Guid, DatabaseEntry>();
|
||||
|
||||
private readonly Dictionary<Type, DatabaseEntry> _globals = new Dictionary<Type, DatabaseEntry>();
|
||||
private readonly Dictionary<Type, HashSet<DatabaseEntry>> _types = new Dictionary<Type, HashSet<DatabaseEntry>>();
|
||||
|
||||
private readonly Dictionary<Type, (DirectoryInfo directory, List<Guid> entries)> _externalTypes = new Dictionary<Type, (DirectoryInfo directory, List<Guid> entries)>();
|
||||
|
|
@ -48,6 +49,7 @@ public class CultCache
|
|||
|
||||
public CultCache(string filePath)
|
||||
{
|
||||
DatabaseLinkBase.Cache = this;
|
||||
RegisterResolver.Register();
|
||||
_filePath = filePath;
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
|
|
@ -61,6 +63,7 @@ public class CultCache
|
|||
var linkFields = type.GetFields().Where(f => f.FieldType.IsAssignableToGenericType(typeof(DatabaseLink<>))).ToArray();
|
||||
if(linkFields.Length > 0)
|
||||
_linkFields[type] = linkFields;
|
||||
|
||||
if (type.GetCustomAttribute<ExternalEntryAttribute>() != null)
|
||||
{
|
||||
var typeDirectory = dataDirectory.CreateSubdirectory(type.Name);
|
||||
|
|
@ -79,6 +82,12 @@ public class CultCache
|
|||
}
|
||||
}
|
||||
else _types[type] = new HashSet<DatabaseEntry>();
|
||||
|
||||
if (type.GetCustomAttribute<GlobalSettingsAttribute>() != null)
|
||||
{
|
||||
_globals[type] = null;
|
||||
Add(Activator.CreateInstance(type) as DatabaseEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -90,19 +99,9 @@ public class CultCache
|
|||
{
|
||||
var exists = _entries.ContainsKey(entry.ID);
|
||||
|
||||
// If an external entry is added, store it separately
|
||||
var type = entry.GetType();
|
||||
if (_linkFields.ContainsKey(type))
|
||||
{
|
||||
foreach (var linkField in _linkFields[type])
|
||||
{
|
||||
var link = (DatabaseLinkBase) linkField.GetValue(entry);
|
||||
if (link != null)
|
||||
{
|
||||
link.Cache = this;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If an external entry is added, store it separately
|
||||
if (_externalTypes.ContainsKey(type))
|
||||
{
|
||||
if (_externalEntries.ContainsKey(entry.ID))
|
||||
|
|
@ -119,6 +118,16 @@ public class CultCache
|
|||
}
|
||||
else
|
||||
{
|
||||
if (_globals.ContainsKey(type))
|
||||
{
|
||||
if(_globals[type]!=null)
|
||||
{
|
||||
Remove(_globals[type], remote);
|
||||
exists = true;
|
||||
}
|
||||
_globals[type] = entry;
|
||||
}
|
||||
|
||||
_entries[entry.ID] = entry;
|
||||
_types[type].Add(entry);
|
||||
foreach (var parentType in type.GetParentTypes())
|
||||
|
|
@ -146,6 +155,9 @@ public class CultCache
|
|||
}
|
||||
}
|
||||
|
||||
public bool IsExternal(DatabaseEntry entry) => _externalTypes.ContainsKey(entry.GetType());
|
||||
public bool IsGlobal(DatabaseEntry entry) => _globals.ContainsKey(entry.GetType());
|
||||
|
||||
public void AddAll(IEnumerable<DatabaseEntry> entries, bool remote = false)
|
||||
{
|
||||
foreach (var entry in entries)
|
||||
|
|
@ -178,6 +190,17 @@ public class CultCache
|
|||
return Get(guid) as T;
|
||||
}
|
||||
|
||||
public DatabaseEntry GetGlobal(Type type)
|
||||
{
|
||||
if (_globals.ContainsKey(type)) return _globals[type];
|
||||
return null;
|
||||
}
|
||||
|
||||
public T GetGlobal<T>() where T : DatabaseEntry
|
||||
{
|
||||
return GetGlobal(typeof(T)) as T;
|
||||
}
|
||||
|
||||
public IEnumerable<DatabaseEntry> GetAll(Type type)
|
||||
{
|
||||
if (_externalTypes.ContainsKey(type))
|
||||
|
|
@ -197,14 +220,19 @@ public class CultCache
|
|||
return !_types.ContainsKey(type) ? Enumerable.Empty<T>() : _types[type].Cast<T>();
|
||||
}
|
||||
|
||||
public void Delete(DatabaseEntry entry, bool remote = false)
|
||||
public void Remove(DatabaseEntry entry, bool remote = false)
|
||||
{
|
||||
_entries.Remove(entry.ID);
|
||||
foreach (var parentType in entry.GetType().GetParentTypes())
|
||||
{
|
||||
if(_types.ContainsKey(parentType))
|
||||
_types[parentType].Remove(entry);
|
||||
}
|
||||
|
||||
if (remote)
|
||||
OnDataDeleteRemote?.Invoke(entry);
|
||||
OnDataRemoveRemote?.Invoke(entry);
|
||||
else
|
||||
OnDataDeleteLocal?.Invoke(entry);
|
||||
OnDataRemoveLocal?.Invoke(entry);
|
||||
}
|
||||
|
||||
public void Load()
|
||||
|
|
|
|||
|
|
@ -55,6 +55,17 @@ public abstract class DatabaseEntry
|
|||
{
|
||||
[JsonProperty("id"), Key(0)]
|
||||
public Guid ID = Guid.NewGuid();
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return ID.GetHashCode();
|
||||
}
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
if (obj is DatabaseEntry entry) return entry.ID == ID;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
[MessagePackObject, JsonObject(MemberSerialization.OptIn)]
|
||||
|
|
@ -71,5 +82,10 @@ public class DatabaseLinkBase
|
|||
public Guid LinkID;
|
||||
|
||||
[IgnoreMember]
|
||||
public CultCache Cache;
|
||||
public static CultCache Cache;
|
||||
}
|
||||
|
||||
public interface ITintInspector
|
||||
{
|
||||
float3 TintColor { get; }
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
public static class MathExtensions
|
||||
{
|
||||
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
fileFormatVersion: 2
|
||||
guid: 5effe288f2074414930e040d4531709f
|
||||
timeCreated: 1617057296
|
||||
|
|
@ -2,9 +2,11 @@ using System;
|
|||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
public static class CultCacheExtensions
|
||||
public static class ReflectionExtensions
|
||||
{
|
||||
private static Dictionary<Type,Type[]> InterfaceClasses = new Dictionary<Type, Type[]>();
|
||||
public static Type[] GetAllInterfaceClasses(this Type type)
|
||||
|
|
@ -85,9 +87,34 @@ public static class CultCacheExtensions
|
|||
|
||||
return IsAssignableToGenericType(baseType, genericType);
|
||||
}
|
||||
|
||||
public static async void WrapAwait(this Task task)
|
||||
|
||||
public static string SplitCamelCase( this string str )
|
||||
{
|
||||
await task;
|
||||
return Regex.Replace(
|
||||
Regex.Replace(
|
||||
str,
|
||||
@"(\P{Ll})(\P{Ll}\p{Ll})",
|
||||
"$1 $2"
|
||||
),
|
||||
@"(\p{Ll})(\P{Ll})",
|
||||
"$1 $2"
|
||||
);
|
||||
}
|
||||
|
||||
public static string FormatTypeName(this string typeName)
|
||||
{
|
||||
return (typeName.EndsWith("Data")
|
||||
? typeName.Substring(0, typeName.Length - 4)
|
||||
: typeName).SplitCamelCase();
|
||||
}
|
||||
|
||||
public static string GetFullName(this Type t)
|
||||
{
|
||||
if (!t.IsGenericType)
|
||||
return t.Name.FormatTypeName();
|
||||
|
||||
var name = t.Name.FormatTypeName();
|
||||
return
|
||||
$"{name.Substring(0, name.LastIndexOf("`"))}{t.GetGenericArguments().Aggregate("<", (aggregate, type) => $"{aggregate}{(aggregate == "<" ? "" : ",")}{GetFullName(type)}")}>";
|
||||
}
|
||||
}
|
||||
147
Assets/Scripts/ServerShared/CultCache/RethinkConnection.cs
Normal file
147
Assets/Scripts/ServerShared/CultCache/RethinkConnection.cs
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
using RethinkDb.Driver;
|
||||
using RethinkDb.Driver.Ast;
|
||||
using RethinkDb.Driver.Net;
|
||||
|
||||
public static class RethinkConnection
|
||||
{
|
||||
public static RethinkDB R = RethinkDB.R;
|
||||
|
||||
public static RethinkQueryStatus RethinkConnect(CultCache cache, string connectionString, string dbName, bool syncLocalChanges = true)
|
||||
{
|
||||
// Add Unity.Mathematics serialization support to RethinkDB Driver
|
||||
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
|
||||
{
|
||||
Converters = new List<JsonConverter>
|
||||
{
|
||||
new MathJsonConverter(),
|
||||
Converter.DateTimeConverter,
|
||||
Converter.BinaryConverter,
|
||||
Converter.GroupingConverter,
|
||||
Converter.PocoExprConverter
|
||||
}
|
||||
};
|
||||
|
||||
var syncTables = typeof(DatabaseEntry).GetAllChildClasses()
|
||||
.Select(t => t.GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Default").Distinct();
|
||||
|
||||
var connectionStringDomainLength = connectionString.IndexOf(':');
|
||||
if (connectionStringDomainLength < 1)
|
||||
throw new ArgumentException("Illegal Connection String: must include port!");
|
||||
|
||||
var portString = connectionString.Substring(connectionStringDomainLength + 1);
|
||||
if (!int.TryParse(portString, out var port))
|
||||
throw new ArgumentException($"Illegal connection string! \"{portString}\" is not a valid port number!");
|
||||
|
||||
var connection = R.Connection()
|
||||
.Hostname(connectionString.Substring(0,connectionStringDomainLength))
|
||||
.Port(port).Timeout(60).Connect();
|
||||
|
||||
var tables = R.Db(dbName).TableList().RunAtom<string[]>(connection);
|
||||
|
||||
foreach (var st in syncTables.Where(st => !tables.Contains(st)))
|
||||
R.Db(dbName).TableCreate(st).RunNoReply(connection);
|
||||
|
||||
if (syncLocalChanges)
|
||||
{
|
||||
// When entries are changed locally, push the changes to RethinkDB
|
||||
cache.OnDataUpdateLocal += async entry =>
|
||||
{
|
||||
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
|
||||
var result = await R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.Get(entry.ID)
|
||||
.Replace(entry)
|
||||
.RunAsync(connection);
|
||||
};
|
||||
|
||||
cache.OnDataInsertLocal += async entry =>
|
||||
{
|
||||
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
|
||||
var result = await R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.Insert(entry)
|
||||
.RunAsync(connection);
|
||||
};
|
||||
|
||||
cache.OnDataRemoveLocal += async entry =>
|
||||
{
|
||||
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
|
||||
var result = await R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.Get(entry.ID)
|
||||
.Delete()
|
||||
.RunAsync(connection);
|
||||
};
|
||||
}
|
||||
|
||||
var status = new RethinkQueryStatus();
|
||||
|
||||
foreach (var table in syncTables)
|
||||
{
|
||||
// Get entries from RethinkDB
|
||||
Task.Run(async () =>
|
||||
{
|
||||
status.TotalEntries += R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.Count().RunAtom<int>(connection);
|
||||
|
||||
var result = await R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.RunCursorAsync<DatabaseEntry>(connection);
|
||||
while (await result.MoveNextAsync())
|
||||
{
|
||||
var entry = result.Current;
|
||||
cache.Add(entry, true);
|
||||
status.RetrievedEntries++;
|
||||
}
|
||||
}).WrapAwait();
|
||||
|
||||
// Subscribe to changes from RethinkDB
|
||||
Task.Run(async () =>
|
||||
{
|
||||
var result = await R
|
||||
.Db(dbName)
|
||||
.Table(table)
|
||||
.Changes()
|
||||
.RunChangesAsync<DatabaseEntry>(connection);
|
||||
while (await result.MoveNextAsync())
|
||||
{
|
||||
var change = result.Current;
|
||||
cache.Add(change.NewValue, true);
|
||||
}
|
||||
}).WrapAwait();
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
public static class AsyncAwaitExtensions
|
||||
{
|
||||
public static async void WrapAwait(this Task task)
|
||||
{
|
||||
await task;
|
||||
}
|
||||
}
|
||||
|
||||
public class RethinkQueryStatus
|
||||
{
|
||||
public int RetrievedEntries;
|
||||
public int TotalEntries;
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
fileFormatVersion: 2
|
||||
guid: a8fce60db4f3e614c8d19c3da4cede0e
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using MessagePack;
|
||||
using MessagePack.ReactivePropertyExtension;
|
||||
using MessagePack.Resolvers;
|
||||
|
||||
public static class RegisterResolver
|
||||
|
|
@ -16,8 +15,7 @@ public static class RegisterResolver
|
|||
MathResolver.Instance,
|
||||
TypeResolver.Instance,
|
||||
NativeGuidResolver.Instance,
|
||||
StandardResolver.Instance,
|
||||
ReactivePropertyResolver.Instance
|
||||
StandardResolver.Instance
|
||||
);
|
||||
var options = MessagePackSerializerOptions.Standard.WithResolver(resolver);
|
||||
MessagePackSerializer.DefaultOptions = options;
|
||||
|
|
|
|||
Loading…
Reference in a new issue