AquaSynth/src/AquaSynth.Core/PackedNeuralNetwork.cs
2026-05-19 18:59:47 +01:00

639 lines
22 KiB
C#

namespace AquaSynth.Dsl;
public sealed record PackedNeuralTrainingExample(
IReadOnlyList<float> Input,
IReadOnlyList<float> Target);
public sealed record PackedNeuralTrainingOptions(
int Epochs = 200,
float LearningRate = 0.01f,
int Seed = 1337,
PackedNeuralOptimizer Optimizer = PackedNeuralOptimizer.Adam,
int BatchSize = 32,
float Beta1 = 0.9f,
float Beta2 = 0.999f,
float Epsilon = 0.00000001f,
bool Shuffle = true);
public enum PackedNeuralOptimizer
{
Sgd,
Adam
}
public sealed record PackedNeuralTrainingStep(int Epoch, float Loss);
public sealed record PackedNeuralTrainingResult(
PackedNeuralNetwork Network,
IReadOnlyList<PackedNeuralTrainingStep> Steps);
public sealed record PackedNeuralBackpropagation(
float Loss,
float[] Output,
float[] InputGradient);
public sealed class PackedNeuralNetwork
{
private readonly Layer[] layers;
private PackedNeuralNetwork(int inputSize, int outputSize, Layer[] layers)
{
InputSize = inputSize;
OutputSize = outputSize;
this.layers = layers;
HiddenLayerSizes = layers.Take(layers.Length - 1).Select(layer => layer.OutputSize).ToArray();
}
public int InputSize { get; }
public int OutputSize { get; }
public IReadOnlyList<int> HiddenLayerSizes { get; }
public static PackedNeuralNetwork Create(
int inputSize,
int outputSize,
IReadOnlyList<int>? hiddenLayerSizes = null,
int seed = 1337)
{
if (inputSize <= 0) throw new ArgumentOutOfRangeException(nameof(inputSize), "input size must be positive");
if (outputSize <= 0) throw new ArgumentOutOfRangeException(nameof(outputSize), "output size must be positive");
hiddenLayerSizes ??= [64, 64, 32];
var random = new Random(seed);
var sizes = new List<int> { inputSize };
sizes.AddRange(hiddenLayerSizes);
sizes.Add(outputSize);
var layers = new Layer[sizes.Count - 1];
for (var index = 0; index < layers.Length; index++)
{
var activation = index == layers.Length - 1
? Activation.Sigmoid
: Activation.Tanh;
layers[index] = Layer.Create(sizes[index], sizes[index + 1], activation, random);
}
return new PackedNeuralNetwork(inputSize, outputSize, layers);
}
public float[] Predict(IReadOnlyList<float> input)
{
if (input.Count != InputSize)
{
throw new ArgumentException($"input must have {InputSize} values", nameof(input));
}
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
Copy(input, scratch.Activations[0]);
Forward(scratch);
return scratch.Output.ToArray();
}
public PackedNeuralTrainingResult Train(
IReadOnlyList<PackedNeuralTrainingExample> examples,
PackedNeuralTrainingOptions? options = null)
{
if (examples.Count == 0)
{
throw new ArgumentException("at least one training example is required", nameof(examples));
}
options ??= new PackedNeuralTrainingOptions();
Validate(options);
var steps = new List<PackedNeuralTrainingStep>();
var order = Enumerable.Range(0, examples.Count).ToArray();
var random = new Random(options.Seed);
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
var gradients = LayerBuffers.Create(layers);
var adamState = options.Optimizer == PackedNeuralOptimizer.Adam
? OptimizerState.Create(layers)
: null;
var updateStep = 0;
for (var epoch = 0; epoch < options.Epochs; epoch++)
{
if (options.Shuffle)
{
Shuffle(order, random);
}
var loss = 0f;
for (var batchStart = 0; batchStart < order.Length; batchStart += options.BatchSize)
{
var batchEnd = Math.Min(batchStart + options.BatchSize, order.Length);
gradients.Clear();
for (var orderIndex = batchStart; orderIndex < batchEnd; orderIndex++)
{
var example = examples[order[orderIndex]];
CopyChecked(example.Input, scratch.Activations[0], "input");
CopyChecked(example.Target, scratch.Target, "target");
Forward(scratch);
loss += MeanSquaredError(scratch.Output, scratch.Target);
AccumulateGradients(scratch, gradients);
}
updateStep++;
var scale = 1f / (batchEnd - batchStart);
if (options.Optimizer == PackedNeuralOptimizer.Adam)
{
adamState!.ApplyAdam(layers, gradients, scale, options.LearningRate, options.Beta1, options.Beta2, options.Epsilon, updateStep);
}
else
{
ApplySgd(gradients, scale, options.LearningRate);
}
}
steps.Add(new PackedNeuralTrainingStep(epoch + 1, loss / examples.Count));
}
return new PackedNeuralTrainingResult(this, steps);
}
public PackedNeuralBackpropagation TrainSingle(
IReadOnlyList<float> input,
IReadOnlyList<float> target,
float learningRate)
{
if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive");
if (target.Count != OutputSize) throw new ArgumentException($"target must have {OutputSize} values", nameof(target));
var outputGradient = new float[OutputSize];
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
CopyChecked(input, scratch.Activations[0], "input");
Copy(target, scratch.Target);
Forward(scratch);
var loss = MeanSquaredError(scratch.Output, scratch.Target);
for (var index = 0; index < outputGradient.Length; index++)
{
outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]) / Math.Max(1, OutputSize);
}
return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss);
}
public PackedNeuralBackpropagation TrainSingleFromOutputGradient(
IReadOnlyList<float> input,
IReadOnlyList<float> outputGradient,
float learningRate,
float loss = 0)
{
if (learningRate <= 0) throw new ArgumentOutOfRangeException(nameof(learningRate), "learning rate must be positive");
if (outputGradient.Count != OutputSize) throw new ArgumentException($"output gradient must have {OutputSize} values", nameof(outputGradient));
var scratch = TrainingScratch.Create(layers, InputSize, OutputSize);
CopyChecked(input, scratch.Activations[0], "input");
Forward(scratch);
return TrainSingleFromOutputGradient(scratch, outputGradient, learningRate, loss);
}
private static void Validate(PackedNeuralTrainingOptions options)
{
if (options.Epochs <= 0) throw new ArgumentOutOfRangeException(nameof(options), "epoch count must be positive");
if (options.LearningRate <= 0) throw new ArgumentOutOfRangeException(nameof(options), "learning rate must be positive");
if (options.BatchSize <= 0) throw new ArgumentOutOfRangeException(nameof(options), "batch size must be positive");
}
private void Forward(TrainingScratch scratch)
{
for (var index = 0; index < layers.Length; index++)
{
layers[index].Forward(scratch.Activations[index], scratch.Activations[index + 1]);
}
}
private PackedNeuralBackpropagation TrainSingleFromOutputGradient(
TrainingScratch scratch,
IReadOnlyList<float> outputGradient,
float learningRate,
float loss)
{
var gradients = LayerBuffers.Create(layers);
AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients);
var inputGradient = InputGradient(scratch);
ApplySgd(gradients, 1f, learningRate);
return new PackedNeuralBackpropagation(loss, scratch.Output.ToArray(), inputGradient);
}
private void AccumulateGradients(TrainingScratch scratch, LayerBuffers gradients)
{
var outputGradient = new float[OutputSize];
for (var index = 0; index < outputGradient.Length; index++)
{
outputGradient[index] = 2f * (scratch.Output[index] - scratch.Target[index]);
}
AccumulateGradientsFromOutputGradient(scratch, outputGradient, gradients);
}
private void AccumulateGradientsFromOutputGradient(TrainingScratch scratch, IReadOnlyList<float> outputGradient, LayerBuffers gradients)
{
var lastLayerIndex = layers.Length - 1;
var outputDelta = scratch.Deltas[lastLayerIndex];
var output = scratch.Activations[^1];
for (var index = 0; index < outputDelta.Length; index++)
{
var actual = output[index];
outputDelta[index] = outputGradient[index] * layers[lastLayerIndex].Derivative(actual);
}
for (var layerIndex = lastLayerIndex - 1; layerIndex >= 0; layerIndex--)
{
layers[layerIndex].BackpropagateDelta(layers[layerIndex + 1], scratch.Deltas[layerIndex + 1], scratch.Activations[layerIndex + 1], scratch.Deltas[layerIndex]);
}
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
gradients.Accumulate(layerIndex, scratch.Activations[layerIndex], scratch.Deltas[layerIndex]);
}
}
private float[] InputGradient(TrainingScratch scratch)
{
var inputGradient = new float[InputSize];
layers[0].BackpropagateInputGradient(scratch.Deltas[0], inputGradient);
return inputGradient;
}
private void ApplySgd(LayerBuffers gradients, float scale, float learningRate)
{
for (var layerIndex = 0; layerIndex < layers.Length; layerIndex++)
{
layers[layerIndex].ApplySgd(gradients.WeightBuffers[layerIndex], gradients.BiasBuffers[layerIndex], scale, learningRate);
}
}
private static void CopyChecked(IReadOnlyList<float> source, float[] target, string name)
{
if (source.Count != target.Length)
{
throw new ArgumentException($"{name} must have {target.Length} values");
}
Copy(source, target);
}
private static void Copy(IReadOnlyList<float> source, float[] target)
{
for (var index = 0; index < target.Length; index++)
{
target[index] = source[index];
}
}
private static void Shuffle(int[] values, Random random)
{
for (var index = values.Length - 1; index > 0; index--)
{
var swapIndex = random.Next(index + 1);
(values[index], values[swapIndex]) = (values[swapIndex], values[index]);
}
}
private static float MeanSquaredError(IReadOnlyList<float> actual, IReadOnlyList<float> target)
{
var sum = 0f;
for (var index = 0; index < actual.Count; index++)
{
var difference = actual[index] - target[index];
sum += difference * difference;
}
return sum / actual.Count;
}
private sealed class TrainingScratch
{
private TrainingScratch(float[][] activations, float[][] deltas, float[] target)
{
Activations = activations;
Deltas = deltas;
Target = target;
}
public float[][] Activations { get; }
public float[][] Deltas { get; }
public float[] Target { get; }
public float[] Output => Activations[^1];
public static TrainingScratch Create(IReadOnlyList<Layer> layers, int inputSize, int outputSize)
{
var activations = new float[layers.Count + 1][];
var deltas = new float[layers.Count][];
activations[0] = new float[inputSize];
for (var index = 0; index < layers.Count; index++)
{
activations[index + 1] = new float[layers[index].OutputSize];
deltas[index] = new float[layers[index].OutputSize];
}
return new TrainingScratch(activations, deltas, new float[outputSize]);
}
}
private sealed class LayerBuffers
{
private LayerBuffers(float[][] weightBuffers, float[][] biasBuffers)
{
WeightBuffers = weightBuffers;
BiasBuffers = biasBuffers;
}
public float[][] WeightBuffers { get; }
public float[][] BiasBuffers { get; }
public static LayerBuffers Create(IReadOnlyList<Layer> layers)
{
var weights = new float[layers.Count][];
var biases = new float[layers.Count][];
for (var index = 0; index < layers.Count; index++)
{
weights[index] = new float[layers[index].Weights.Length];
biases[index] = new float[layers[index].Biases.Length];
}
return new LayerBuffers(weights, biases);
}
public void Clear()
{
for (var index = 0; index < WeightBuffers.Length; index++)
{
Array.Clear(WeightBuffers[index]);
Array.Clear(BiasBuffers[index]);
}
}
public void Accumulate(int layerIndex, float[] input, float[] delta)
{
var weightGradients = WeightBuffers[layerIndex];
var biasGradients = BiasBuffers[layerIndex];
var inputSize = input.Length;
unsafe
{
fixed (float* inputBase = input)
fixed (float* deltaBase = delta)
fixed (float* weightBase = weightGradients)
fixed (float* biasBase = biasGradients)
{
for (var output = 0; output < delta.Length; output++)
{
var deltaValue = deltaBase[output];
var weightRow = weightBase + output * inputSize;
for (var column = 0; column < inputSize; column++)
{
weightRow[column] += deltaValue * inputBase[column];
}
biasBase[output] += deltaValue;
}
}
}
}
}
private sealed class Layer
{
private readonly Activation activation;
private Layer(int inputSize, int outputSize, float[] weights, float[] biases, Activation activation)
{
InputSize = inputSize;
OutputSize = outputSize;
Weights = weights;
Biases = biases;
this.activation = activation;
}
public int InputSize { get; }
public int OutputSize { get; }
public float[] Weights { get; }
public float[] Biases { get; }
public static Layer Create(int inputSize, int outputSize, Activation activation, Random random)
{
var scale = MathF.Sqrt(2f / Math.Max(1, inputSize));
var weights = new float[inputSize * outputSize];
for (var index = 0; index < weights.Length; index++)
{
weights[index] = ((float)random.NextDouble() * 2f - 1f) * scale;
}
return new Layer(inputSize, outputSize, weights, new float[outputSize], activation);
}
public void Forward(float[] input, float[] output)
{
unsafe
{
fixed (float* inputBase = input)
fixed (float* outputBase = output)
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
{
for (var row = 0; row < OutputSize; row++)
{
var sum = biasBase[row];
var weightRow = weightBase + row * InputSize;
for (var column = 0; column < InputSize; column++)
{
sum += weightRow[column] * inputBase[column];
}
outputBase[row] = Activate(sum);
}
}
}
}
public void BackpropagateDelta(Layer nextLayer, float[] nextDelta, float[] activationValues, float[] delta)
{
unsafe
{
fixed (float* nextDeltaBase = nextDelta)
fixed (float* activationBase = activationValues)
fixed (float* deltaBase = delta)
fixed (float* nextWeightBase = nextLayer.Weights)
{
for (var output = 0; output < OutputSize; output++)
{
var sum = 0f;
for (var nextOutput = 0; nextOutput < nextLayer.OutputSize; nextOutput++)
{
sum += nextWeightBase[nextOutput * nextLayer.InputSize + output] * nextDeltaBase[nextOutput];
}
deltaBase[output] = sum * Derivative(activationBase[output]);
}
}
}
}
public void BackpropagateInputGradient(float[] nextDelta, float[] inputGradient)
{
unsafe
{
fixed (float* nextDeltaBase = nextDelta)
fixed (float* inputGradientBase = inputGradient)
fixed (float* weightBase = Weights)
{
for (var input = 0; input < InputSize; input++)
{
var sum = 0f;
for (var output = 0; output < OutputSize; output++)
{
sum += weightBase[output * InputSize + input] * nextDeltaBase[output];
}
inputGradientBase[input] = sum;
}
}
}
}
public float Derivative(float activated) =>
activation switch
{
Activation.Sigmoid => activated * (1f - activated),
Activation.Tanh => 1f - activated * activated,
_ => 1
};
public void ApplySgd(float[] weightGradients, float[] biasGradients, float scale, float learningRate)
{
unsafe
{
fixed (float* weightBase = Weights)
fixed (float* biasBase = Biases)
fixed (float* weightGradientBase = weightGradients)
fixed (float* biasGradientBase = biasGradients)
{
for (var index = 0; index < Weights.Length; index++)
{
weightBase[index] -= learningRate * weightGradientBase[index] * scale;
}
for (var index = 0; index < Biases.Length; index++)
{
biasBase[index] -= learningRate * biasGradientBase[index] * scale;
}
}
}
}
private float Activate(float value) =>
activation switch
{
Activation.Sigmoid => 1f / (1f + MathF.Exp(-value)),
Activation.Tanh => MathF.Tanh(value),
_ => value
};
}
private enum Activation
{
Tanh,
Sigmoid
}
private sealed class OptimizerState
{
private readonly LayerBuffers firstMoment;
private readonly LayerBuffers secondMoment;
private OptimizerState(LayerBuffers firstMoment, LayerBuffers secondMoment)
{
this.firstMoment = firstMoment;
this.secondMoment = secondMoment;
}
public static OptimizerState Create(IReadOnlyList<Layer> layers) =>
new(LayerBuffers.Create(layers), LayerBuffers.Create(layers));
public void ApplyAdam(
IReadOnlyList<Layer> layers,
LayerBuffers gradients,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
int step)
{
var beta1Correction = 1f - MathF.Pow(beta1, step);
var beta2Correction = 1f - MathF.Pow(beta2, step);
for (var layerIndex = 0; layerIndex < layers.Count; layerIndex++)
{
ApplyAdam(
layers[layerIndex].Weights,
gradients.WeightBuffers[layerIndex],
firstMoment.WeightBuffers[layerIndex],
secondMoment.WeightBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
ApplyAdam(
layers[layerIndex].Biases,
gradients.BiasBuffers[layerIndex],
firstMoment.BiasBuffers[layerIndex],
secondMoment.BiasBuffers[layerIndex],
scale,
learningRate,
beta1,
beta2,
epsilon,
beta1Correction,
beta2Correction);
}
}
private static void ApplyAdam(
float[] values,
float[] gradients,
float[] firstMoment,
float[] secondMoment,
float scale,
float learningRate,
float beta1,
float beta2,
float epsilon,
float beta1Correction,
float beta2Correction)
{
unsafe
{
fixed (float* valueBase = values)
fixed (float* gradientBase = gradients)
fixed (float* firstBase = firstMoment)
fixed (float* secondBase = secondMoment)
{
for (var index = 0; index < values.Length; index++)
{
var gradient = gradientBase[index] * scale;
firstBase[index] = beta1 * firstBase[index] + (1f - beta1) * gradient;
secondBase[index] = beta2 * secondBase[index] + (1f - beta2) * gradient * gradient;
var mHat = firstBase[index] / beta1Correction;
var vHat = secondBase[index] / beta2Correction;
valueBase[index] -= learningRate * mHat / (MathF.Sqrt(vHat) + epsilon);
}
}
}
}
}
}