Database Improvements

Brought in all the myriad improvements to the database tools from the Stream Pixels project
Added configurable nested entry grouping
This commit is contained in:
Reinier Vens 2021-04-07 01:27:35 +01:00
commit 968783bf9c
37 changed files with 865 additions and 580 deletions

View file

@ -1,5 +1,8 @@
using System;
[AttributeUsage(AttributeTargets.Class)]
public class GlobalSettingsAttribute : Attribute { }
[AttributeUsage(AttributeTargets.Class)]
public class ExternalEntryAttribute : Attribute { }
@ -8,15 +11,6 @@ public class InspectableAttribute : Attribute { }
[AttributeUsage(AttributeTargets.Field)]
public abstract class PreferredInspectorAttribute : InspectableAttribute { }
public class InspectableUnityObjectAttribute : PreferredInspectorAttribute
{
public Type ObjectType;
public InspectableUnityObjectAttribute(Type type)
{
ObjectType = type;
}
}
public class InspectableTextAttribute : PreferredInspectorAttribute { }
public class InspectablePrefabAttribute : PreferredInspectorAttribute { }
public class InspectableTextureAttribute : PreferredInspectorAttribute { }
@ -25,6 +19,16 @@ public class InspectableTemperatureAttribute : PreferredInspectorAttribute { }
public class InspectableAnimationCurveAttribute : PreferredInspectorAttribute { }
public class InspectableColorAttribute : PreferredInspectorAttribute { }
public class InspectableEnumValuesAttribute : PreferredInspectorAttribute
{
public Type EnumType;
public InspectableEnumValuesAttribute(Type enumType)
{
EnumType = enumType;
}
}
public class InspectableDatabaseLinkAttribute : PreferredInspectorAttribute
{
public readonly Type EntryType;
@ -35,22 +39,22 @@ public class InspectableDatabaseLinkAttribute : PreferredInspectorAttribute
}
}
public class RangedFloatInspectableAttribute : PreferredInspectorAttribute
public class InspectableRangedFloatAttribute : PreferredInspectorAttribute
{
public readonly float Min, Max;
public RangedFloatInspectableAttribute(float min, float max)
public InspectableRangedFloatAttribute(float min, float max)
{
Min = min;
Max = max;
}
}
public class RangedIntInspectableAttribute : PreferredInspectorAttribute
public class InspectableRangedIntAttribute : PreferredInspectorAttribute
{
public readonly int Min, Max;
public RangedIntInspectableAttribute(int min, int max)
public InspectableRangedIntAttribute(int min, int max)
{
Min = min;
Max = max;
@ -107,4 +111,24 @@ public class RethinkTableAttribute : Attribute
{
TableName = tableName;
}
}
public class NameAttribute : Attribute
{
public string Name;
public NameAttribute(string name)
{
Name = name;
}
}
public class OrderAttribute : Attribute
{
public int Order;
public OrderAttribute(int order)
{
Order = order;
}
}

View file

@ -0,0 +1,67 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography;
public static class CollectionExtensions
{
// https://stackoverflow.com/a/800469
public static string GetHashSHA1(this byte[] data)
{
using (var sha1 = new System.Security.Cryptography.SHA1CryptoServiceProvider())
{
return string.Concat(sha1.ComputeHash(data).Select(x => x.ToString("X2")));
}
}
// https://stackoverflow.com/a/3188835
public static T MaxBy<T, U>(this IEnumerable<T> items, Func<T, U> selector)
{
if (!items.Any())
{
throw new InvalidOperationException("Empty input sequence");
}
var comparer = Comparer<U>.Default;
T maxItem = items.First();
U maxValue = selector(maxItem);
foreach (T item in items.Skip(1))
{
// Get the value of the item and compare it to the current max.
U value = selector(item);
if (comparer.Compare(value, maxValue) > 0)
{
maxValue = value;
maxItem = item;
}
}
return maxItem;
}
public static T MinBy<T, U>(this IEnumerable<T> items, Func<T, U> selector)
{
if (!items.Any())
{
throw new InvalidOperationException("Empty input sequence");
}
var comparer = Comparer<U>.Default;
T maxItem = items.First();
U maxValue = selector(maxItem);
foreach (T item in items.Skip(1))
{
// Get the value of the item and compare it to the current max.
U value = selector(item);
if (comparer.Compare(value, maxValue) < 0)
{
maxValue = value;
maxItem = item;
}
}
return maxItem;
}
}

View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: eed05df00a9a6b84292c81be1662a66e
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View file

@ -14,10 +14,10 @@ public class CultCache
{
public event Action<DatabaseEntry> OnDataInsertLocal;
public event Action<DatabaseEntry> OnDataUpdateLocal;
public event Action<DatabaseEntry> OnDataDeleteLocal;
public event Action<DatabaseEntry> OnDataRemoveLocal;
public event Action<DatabaseEntry> OnDataInsertRemote;
public event Action<DatabaseEntry> OnDataUpdateRemote;
public event Action<DatabaseEntry> OnDataDeleteRemote;
public event Action<DatabaseEntry> OnDataRemoveRemote;
private readonly object addLock = new object();
@ -26,6 +26,7 @@ public class CultCache
private readonly Dictionary<Guid, DatabaseEntry> _entries = new Dictionary<Guid, DatabaseEntry>();
private readonly Dictionary<Type, DatabaseEntry> _globals = new Dictionary<Type, DatabaseEntry>();
private readonly Dictionary<Type, HashSet<DatabaseEntry>> _types = new Dictionary<Type, HashSet<DatabaseEntry>>();
private readonly Dictionary<Type, (DirectoryInfo directory, List<Guid> entries)> _externalTypes = new Dictionary<Type, (DirectoryInfo directory, List<Guid> entries)>();
@ -48,6 +49,7 @@ public class CultCache
public CultCache(string filePath)
{
DatabaseLinkBase.Cache = this;
RegisterResolver.Register();
_filePath = filePath;
var fileInfo = new FileInfo(filePath);
@ -61,6 +63,7 @@ public class CultCache
var linkFields = type.GetFields().Where(f => f.FieldType.IsAssignableToGenericType(typeof(DatabaseLink<>))).ToArray();
if(linkFields.Length > 0)
_linkFields[type] = linkFields;
if (type.GetCustomAttribute<ExternalEntryAttribute>() != null)
{
var typeDirectory = dataDirectory.CreateSubdirectory(type.Name);
@ -79,6 +82,12 @@ public class CultCache
}
}
else _types[type] = new HashSet<DatabaseEntry>();
if (type.GetCustomAttribute<GlobalSettingsAttribute>() != null)
{
_globals[type] = null;
Add(Activator.CreateInstance(type) as DatabaseEntry);
}
}
}
@ -90,19 +99,9 @@ public class CultCache
{
var exists = _entries.ContainsKey(entry.ID);
// If an external entry is added, store it separately
var type = entry.GetType();
if (_linkFields.ContainsKey(type))
{
foreach (var linkField in _linkFields[type])
{
var link = (DatabaseLinkBase) linkField.GetValue(entry);
if (link != null)
{
link.Cache = this;
}
}
}
// If an external entry is added, store it separately
if (_externalTypes.ContainsKey(type))
{
if (_externalEntries.ContainsKey(entry.ID))
@ -119,6 +118,16 @@ public class CultCache
}
else
{
if (_globals.ContainsKey(type))
{
if(_globals[type]!=null)
{
Remove(_globals[type], remote);
exists = true;
}
_globals[type] = entry;
}
_entries[entry.ID] = entry;
_types[type].Add(entry);
foreach (var parentType in type.GetParentTypes())
@ -146,6 +155,9 @@ public class CultCache
}
}
public bool IsExternal(DatabaseEntry entry) => _externalTypes.ContainsKey(entry.GetType());
public bool IsGlobal(DatabaseEntry entry) => _globals.ContainsKey(entry.GetType());
public void AddAll(IEnumerable<DatabaseEntry> entries, bool remote = false)
{
foreach (var entry in entries)
@ -178,6 +190,17 @@ public class CultCache
return Get(guid) as T;
}
public DatabaseEntry GetGlobal(Type type)
{
if (_globals.ContainsKey(type)) return _globals[type];
return null;
}
public T GetGlobal<T>() where T : DatabaseEntry
{
return GetGlobal(typeof(T)) as T;
}
public IEnumerable<DatabaseEntry> GetAll(Type type)
{
if (_externalTypes.ContainsKey(type))
@ -197,14 +220,19 @@ public class CultCache
return !_types.ContainsKey(type) ? Enumerable.Empty<T>() : _types[type].Cast<T>();
}
public void Delete(DatabaseEntry entry, bool remote = false)
public void Remove(DatabaseEntry entry, bool remote = false)
{
_entries.Remove(entry.ID);
foreach (var parentType in entry.GetType().GetParentTypes())
{
if(_types.ContainsKey(parentType))
_types[parentType].Remove(entry);
}
if (remote)
OnDataDeleteRemote?.Invoke(entry);
OnDataRemoveRemote?.Invoke(entry);
else
OnDataDeleteLocal?.Invoke(entry);
OnDataRemoveLocal?.Invoke(entry);
}
public void Load()

View file

@ -55,6 +55,17 @@ public abstract class DatabaseEntry
{
[JsonProperty("id"), Key(0)]
public Guid ID = Guid.NewGuid();
public override int GetHashCode()
{
return ID.GetHashCode();
}
public override bool Equals(object obj)
{
if (obj is DatabaseEntry entry) return entry.ID == ID;
return false;
}
}
[MessagePackObject, JsonObject(MemberSerialization.OptIn)]
@ -71,5 +82,10 @@ public class DatabaseLinkBase
public Guid LinkID;
[IgnoreMember]
public CultCache Cache;
public static CultCache Cache;
}
public interface ITintInspector
{
float3 TintColor { get; }
}

View file

@ -1,4 +0,0 @@
public static class MathExtensions
{
}

View file

@ -1,3 +0,0 @@
fileFormatVersion: 2
guid: 5effe288f2074414930e040d4531709f
timeCreated: 1617057296

View file

@ -2,9 +2,11 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
public static class CultCacheExtensions
public static class ReflectionExtensions
{
private static Dictionary<Type,Type[]> InterfaceClasses = new Dictionary<Type, Type[]>();
public static Type[] GetAllInterfaceClasses(this Type type)
@ -85,9 +87,34 @@ public static class CultCacheExtensions
return IsAssignableToGenericType(baseType, genericType);
}
public static async void WrapAwait(this Task task)
public static string SplitCamelCase( this string str )
{
await task;
return Regex.Replace(
Regex.Replace(
str,
@"(\P{Ll})(\P{Ll}\p{Ll})",
"$1 $2"
),
@"(\p{Ll})(\P{Ll})",
"$1 $2"
);
}
public static string FormatTypeName(this string typeName)
{
return (typeName.EndsWith("Data")
? typeName.Substring(0, typeName.Length - 4)
: typeName).SplitCamelCase();
}
public static string GetFullName(this Type t)
{
if (!t.IsGenericType)
return t.Name.FormatTypeName();
var name = t.Name.FormatTypeName();
return
$"{name.Substring(0, name.LastIndexOf("`"))}{t.GetGenericArguments().Aggregate("<", (aggregate, type) => $"{aggregate}{(aggregate == "<" ? "" : ",")}{GetFullName(type)}")}>";
}
}

View file

@ -0,0 +1,147 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using Newtonsoft.Json;
using RethinkDb.Driver;
using RethinkDb.Driver.Ast;
using RethinkDb.Driver.Net;
public static class RethinkConnection
{
public static RethinkDB R = RethinkDB.R;
public static RethinkQueryStatus RethinkConnect(CultCache cache, string connectionString, string dbName, bool syncLocalChanges = true)
{
// Add Unity.Mathematics serialization support to RethinkDB Driver
JsonConvert.DefaultSettings = () => new JsonSerializerSettings
{
Converters = new List<JsonConverter>
{
new MathJsonConverter(),
Converter.DateTimeConverter,
Converter.BinaryConverter,
Converter.GroupingConverter,
Converter.PocoExprConverter
}
};
var syncTables = typeof(DatabaseEntry).GetAllChildClasses()
.Select(t => t.GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Default").Distinct();
var connectionStringDomainLength = connectionString.IndexOf(':');
if (connectionStringDomainLength < 1)
throw new ArgumentException("Illegal Connection String: must include port!");
var portString = connectionString.Substring(connectionStringDomainLength + 1);
if (!int.TryParse(portString, out var port))
throw new ArgumentException($"Illegal connection string! \"{portString}\" is not a valid port number!");
var connection = R.Connection()
.Hostname(connectionString.Substring(0,connectionStringDomainLength))
.Port(port).Timeout(60).Connect();
var tables = R.Db(dbName).TableList().RunAtom<string[]>(connection);
foreach (var st in syncTables.Where(st => !tables.Contains(st)))
R.Db(dbName).TableCreate(st).RunNoReply(connection);
if (syncLocalChanges)
{
// When entries are changed locally, push the changes to RethinkDB
cache.OnDataUpdateLocal += async entry =>
{
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
var result = await R
.Db(dbName)
.Table(table)
.Get(entry.ID)
.Replace(entry)
.RunAsync(connection);
};
cache.OnDataInsertLocal += async entry =>
{
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
var result = await R
.Db(dbName)
.Table(table)
.Insert(entry)
.RunAsync(connection);
};
cache.OnDataRemoveLocal += async entry =>
{
var table = entry.GetType().GetCustomAttribute<RethinkTableAttribute>()?.TableName ?? "Other";
var result = await R
.Db(dbName)
.Table(table)
.Get(entry.ID)
.Delete()
.RunAsync(connection);
};
}
var status = new RethinkQueryStatus();
foreach (var table in syncTables)
{
// Get entries from RethinkDB
Task.Run(async () =>
{
status.TotalEntries += R
.Db(dbName)
.Table(table)
.Count().RunAtom<int>(connection);
var result = await R
.Db(dbName)
.Table(table)
.RunCursorAsync<DatabaseEntry>(connection);
while (await result.MoveNextAsync())
{
var entry = result.Current;
cache.Add(entry, true);
status.RetrievedEntries++;
}
}).WrapAwait();
// Subscribe to changes from RethinkDB
Task.Run(async () =>
{
var result = await R
.Db(dbName)
.Table(table)
.Changes()
.RunChangesAsync<DatabaseEntry>(connection);
while (await result.MoveNextAsync())
{
var change = result.Current;
cache.Add(change.NewValue, true);
}
}).WrapAwait();
}
return status;
}
}
public static class AsyncAwaitExtensions
{
public static async void WrapAwait(this Task task)
{
await task;
}
}
public class RethinkQueryStatus
{
public int RetrievedEntries;
public int TotalEntries;
}

View file

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: a8fce60db4f3e614c8d19c3da4cede0e
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View file

@ -1,7 +1,6 @@
using System.Collections;
using System.Collections.Generic;
using MessagePack;
using MessagePack.ReactivePropertyExtension;
using MessagePack.Resolvers;
public static class RegisterResolver
@ -16,8 +15,7 @@ public static class RegisterResolver
MathResolver.Instance,
TypeResolver.Instance,
NativeGuidResolver.Instance,
StandardResolver.Instance,
ReactivePropertyResolver.Instance
StandardResolver.Instance
);
var options = MessagePackSerializerOptions.Standard.WithResolver(resolver);
MessagePackSerializer.DefaultOptions = options;