3098 lines
121 KiB
C#
3098 lines
121 KiB
C#
using System.Globalization;
|
|
using System.Diagnostics;
|
|
using System.Net.Http.Json;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
using AquaSynth.Dsl;
|
|
using AquaSynth.Faust;
|
|
using NAudio.Wave;
|
|
|
|
var command = args.FirstOrDefault();
|
|
if (string.IsNullOrWhiteSpace(command) || command is "-h" or "--help")
|
|
{
|
|
PrintHelp();
|
|
return 0;
|
|
}
|
|
|
|
var options = ParseOptions(args.Skip(1).ToArray());
|
|
try
|
|
{
|
|
switch (command)
|
|
{
|
|
case "seed":
|
|
await SeedAsync(options);
|
|
return 0;
|
|
case "score":
|
|
await ScoreAsync(options);
|
|
return 0;
|
|
case "song-prepare":
|
|
await SongPrepareAsync(options);
|
|
return 0;
|
|
case "song-score":
|
|
await SongScoreAsync(options);
|
|
return 0;
|
|
case "dump":
|
|
await DumpAsync(options);
|
|
return 0;
|
|
case "distill":
|
|
await DistillAsync(options);
|
|
return 0;
|
|
case "search":
|
|
await SearchAsync(options);
|
|
return 0;
|
|
case "index":
|
|
await IndexAsync(options);
|
|
return 0;
|
|
case "show":
|
|
await ShowAsync(options);
|
|
return 0;
|
|
default:
|
|
Console.Error.WriteLine($"Unknown command `{command}`.");
|
|
PrintHelp();
|
|
return 2;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine(ex);
|
|
return 1;
|
|
}
|
|
|
|
static async Task SeedAsync(Dictionary<string, string> options)
|
|
{
|
|
var artifactRoot = Required(options, "artifact-root");
|
|
var batchId = Value(options, "batch-id", "five-seed-trials");
|
|
var store = Value(options, "store", Path.Combine(artifactRoot, "ipa-trial-results.cc"));
|
|
var result = await IpaTrialOrchestrator.RunAsync(
|
|
artifactRoot,
|
|
options: new IpaTrialOrchestrationOptions(BatchId: batchId));
|
|
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, result.TrialResults);
|
|
Console.WriteLine(result.ArtifactDirectory);
|
|
Console.WriteLine(store);
|
|
}
|
|
|
|
static async Task ScoreAsync(Dictionary<string, string> options)
|
|
{
|
|
var patchRoot = Required(options, "patch-root");
|
|
var artifactRoot = Required(options, "artifact-root");
|
|
var batchId = Value(options, "batch-id", $"agent-candidates-{DateTimeOffset.UtcNow:yyyyMMddTHHmmssfff}");
|
|
var store = Value(options, "store", Path.Combine(artifactRoot, "ipa-trial-results.cc"));
|
|
var hypothesizer = Value(options, "hypothesizer", "external-codex-hypothesis-worker");
|
|
var candidates = CandidateScripts(patchRoot, hypothesizer);
|
|
var result = await IpaTrialOrchestrator.RunCandidateScriptsAsync(
|
|
artifactRoot,
|
|
candidates,
|
|
options: new IpaTrialOrchestrationOptions(BatchId: batchId, HypothesizerId: hypothesizer));
|
|
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, result.TrialResults);
|
|
Console.WriteLine(result.ArtifactDirectory);
|
|
Console.WriteLine(store);
|
|
}
|
|
|
|
static async Task SongPrepareAsync(Dictionary<string, string> options)
|
|
{
|
|
var source = Required(options, "source");
|
|
var artifactRoot = Required(options, "artifact-root");
|
|
var durationSeconds = FloatValue(options, "duration-seconds", 10f);
|
|
var sampleRate = IntValue(options, "sample-rate", 44100);
|
|
var seed = IntValue(options, "seed", RandomNumberGenerator.GetInt32(1, int.MaxValue));
|
|
var challengeId = Value(options, "challenge-id", $"song-snippet-{DateTimeOffset.UtcNow:yyyyMMddTHHmmssfff}");
|
|
var output = Value(options, "output", Path.Combine(artifactRoot, "challenge.json"));
|
|
|
|
if (!File.Exists(source))
|
|
{
|
|
throw new FileNotFoundException("Song source file was not found.", source);
|
|
}
|
|
|
|
Directory.CreateDirectory(artifactRoot);
|
|
var decoded = DecodeMono(source);
|
|
var resampled = Resample(decoded.Samples, decoded.SampleRate, sampleRate);
|
|
var clipLength = Math.Min(resampled.Length, Math.Max(1, (int)MathF.Round(durationSeconds * sampleRate)));
|
|
var maxStart = Math.Max(0, resampled.Length - clipLength);
|
|
var random = new Random(seed);
|
|
var startSample = maxStart == 0 ? 0 : random.Next(0, maxStart + 1);
|
|
var clip = new float[clipLength];
|
|
Array.Copy(resampled, startSample, clip, 0, clipLength);
|
|
NormalizePeak(clip, .9f);
|
|
|
|
var referenceWav = Path.Combine(artifactRoot, "reference.wav");
|
|
WriteWav(referenceWav, clip, sampleRate);
|
|
var analyzer = new AudioAnalyzer(new AudioAnalysisConfig(SampleRate: sampleRate));
|
|
var analysis = analyzer.Analyze(clip);
|
|
var tempo = EstimateTempo(clip, sampleRate);
|
|
var register = EstimateRegister(clip, sampleRate, analysis.Features.SpectralCentroidHz);
|
|
var spectrogramPath = Path.Combine(artifactRoot, "logmel-spectrogram.csv");
|
|
var bandStatsPath = Path.Combine(artifactRoot, "logmel-band-stats.csv");
|
|
var envelopePath = Path.Combine(artifactRoot, "rms-envelope.csv");
|
|
var envelopeAutocorrPath = Path.Combine(artifactRoot, "rms-envelope-autocorr.csv");
|
|
var analysisReportPath = Path.Combine(artifactRoot, "analysis.md");
|
|
await File.WriteAllTextAsync(spectrogramPath, SpectrogramCsv(analysis.LogMelSpectrogram), Encoding.UTF8);
|
|
await File.WriteAllTextAsync(bandStatsPath, SpectrogramBandStatsCsv(analysis.LogMelSpectrogram), Encoding.UTF8);
|
|
await File.WriteAllTextAsync(envelopePath, EnvelopeCsv(analysis.RmsEnvelope, sampleRate, analyzer.Config.HopSize), Encoding.UTF8);
|
|
await File.WriteAllTextAsync(envelopeAutocorrPath, AutocorrelationCsv(analysis.RmsEnvelope, sampleRate / (float)Math.Max(1, analyzer.Config.HopSize)), Encoding.UTF8);
|
|
var features = new SongChallengeFeatures(
|
|
analysis.Features.DurationSeconds,
|
|
analysis.Features.Peak,
|
|
analysis.Features.Rms,
|
|
analysis.Features.ZeroCrossingRate,
|
|
analysis.Features.SpectralCentroidHz,
|
|
analysis.Features.SpectralRolloffHz,
|
|
ActiveDuty(clip, sampleRate),
|
|
SpectralFlux(clip, sampleRate),
|
|
tempo.Bpm,
|
|
tempo.BeatSeconds,
|
|
tempo.Confidence,
|
|
register.DominantHz,
|
|
register.LowHz,
|
|
register.HighHz,
|
|
register.RootNote,
|
|
register.SuggestedScale,
|
|
string.Join(",", register.ScaleFrequencies.Select(value => value.ToString("0.###", CultureInfo.InvariantCulture))));
|
|
var artifacts = new SongChallengeAnalysisArtifacts(
|
|
spectrogramPath,
|
|
bandStatsPath,
|
|
envelopePath,
|
|
envelopeAutocorrPath,
|
|
analysisReportPath);
|
|
var challenge = new SongChallenge(
|
|
challengeId,
|
|
source,
|
|
Path.GetFileName(source),
|
|
seed,
|
|
startSample / (float)sampleRate,
|
|
clipLength / (float)sampleRate,
|
|
sampleRate,
|
|
referenceWav,
|
|
features,
|
|
artifacts);
|
|
await File.WriteAllTextAsync(analysisReportPath, SongAnalysisReport(challenge, analysis), Encoding.UTF8);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
await File.WriteAllTextAsync(output, JsonSerializer.Serialize(challenge, JsonOptions()), Encoding.UTF8);
|
|
await File.WriteAllTextAsync(Path.Combine(artifactRoot, "challenge.md"), SongChallengeReport(challenge), Encoding.UTF8);
|
|
Console.WriteLine(output);
|
|
}
|
|
|
|
static async Task SongScoreAsync(Dictionary<string, string> options)
|
|
{
|
|
var patchRoot = Required(options, "patch-root");
|
|
var challengePath = Required(options, "challenge");
|
|
var artifactRoot = Required(options, "artifact-root");
|
|
var batchId = Value(options, "batch-id", $"song-round-{DateTimeOffset.UtcNow:yyyyMMddTHHmmssfff}");
|
|
var store = Value(options, "store", Path.Combine(artifactRoot, "song-trial-results.cc"));
|
|
var hypothesizer = Value(options, "hypothesizer", "external-codex-song-worker");
|
|
var challenge = JsonSerializer.Deserialize<SongChallenge>(
|
|
await File.ReadAllTextAsync(challengePath, Encoding.UTF8),
|
|
JsonOptions()) ?? throw new InvalidDataException($"Could not read song challenge `{challengePath}`.");
|
|
var referenceSamples = ReadMonoPcm16Wav(challenge.ReferenceWavPath);
|
|
var candidates = SongCandidateScripts(patchRoot, hypothesizer);
|
|
if (candidates.Count == 0)
|
|
{
|
|
throw new InvalidDataException($"No .aqua song candidates found under `{patchRoot}`.");
|
|
}
|
|
|
|
var batchDirectory = Path.Combine(artifactRoot, batchId);
|
|
var candidateRoot = Path.Combine(batchDirectory, "song-candidates");
|
|
var timelineRoot = Path.Combine(batchDirectory, "song-timelines");
|
|
Directory.CreateDirectory(candidateRoot);
|
|
Directory.CreateDirectory(timelineRoot);
|
|
var analyzer = new AudioAnalyzer(new AudioAnalysisConfig(SampleRate: challenge.SampleRate));
|
|
var challengeEvidence = SongChallengeEvidenceDocuments(challenge).ToArray();
|
|
await IpaTrialResultCultCacheStore.UpsertSongChallengeEvidenceAsync(store, challengeEvidence);
|
|
var results = new List<IpaTrialResult>();
|
|
|
|
foreach (var candidate in candidates)
|
|
{
|
|
var startedAt = DateTimeOffset.UtcNow;
|
|
var startedStamp = Stopwatch.GetTimestamp();
|
|
var safeCandidateId = SafeName(candidate.CandidateId);
|
|
var candidateDir = Path.Combine(candidateRoot, safeCandidateId);
|
|
var audioDir = Path.Combine(candidateDir, "audio");
|
|
Directory.CreateDirectory(audioDir);
|
|
var scriptCopy = Path.Combine(candidateDir, "candidate.aqua");
|
|
File.Copy(candidate.ScriptPath, scriptCopy, overwrite: true);
|
|
var script = await File.ReadAllTextAsync(scriptCopy, Encoding.UTF8);
|
|
var timelinePath = Path.Combine(timelineRoot, $"{safeCandidateId}.csv");
|
|
var dspPath = Path.Combine(candidateDir, "candidate.dsp");
|
|
|
|
var candidateWav = Path.Combine(audioDir, "candidate.wav");
|
|
var metrics = new List<SpeechScoreMetric>();
|
|
var artifacts = new List<SpeechRenderArtifact>
|
|
{
|
|
Artifact("song-challenge", challengePath),
|
|
Artifact("reference-wav", challenge.ReferenceWavPath),
|
|
Artifact("candidate-script", scriptCopy),
|
|
Artifact("primitive-timeline", timelinePath)
|
|
};
|
|
metrics.AddRange(SongTargetMetrics(challenge, challengeEvidence));
|
|
artifacts.AddRange(SongChallengeArtifacts(challenge));
|
|
artifacts.AddRange(challengeEvidence.Select(SongChallengeEvidenceArtifact));
|
|
var timelineFacts = Array.Empty<PrimitiveTimelineFact>();
|
|
string verdict;
|
|
string evaluation;
|
|
try
|
|
{
|
|
var patch = PatchScript.Parse(script);
|
|
var timelineNetwork = patch.VocalNetworks.FirstOrDefault();
|
|
var timeline = timelineNetwork is null
|
|
? Array.Empty<ProbeTimelineSample>()
|
|
: ProbeTimelineReport.Build(patch, timelineNetwork.Name, 12);
|
|
await File.WriteAllTextAsync(timelinePath, ProbeTimelineReport.ToCsv(timeline), Encoding.UTF8);
|
|
timelineFacts = PrimitiveTimelineFactExtractor.Extract(timeline).ToArray();
|
|
var source = FaustEmitter.EmitScript(script, new FaustExportOptions(safeCandidateId)).Source;
|
|
await File.WriteAllTextAsync(dspPath, source, Encoding.UTF8);
|
|
artifacts.Add(Artifact("candidate-dsp", dspPath));
|
|
|
|
var render = await FaustCompiler.RenderAsync(
|
|
source,
|
|
new FaustRenderOptions(challenge.SampleRate, challenge.DurationSeconds));
|
|
if (render is null || render.Samples.Length == 0)
|
|
{
|
|
verdict = "render-failed";
|
|
evaluation = render is null
|
|
? "Faust was not available to render this song candidate."
|
|
: $"Faust render produced no samples. stderr: {render.Stderr}";
|
|
metrics.Add(new SpeechScoreMetric("render_failed", 1, 1));
|
|
}
|
|
else
|
|
{
|
|
var matched = MatchLength(render.Samples, referenceSamples.Length);
|
|
NormalizePeak(matched, .9f);
|
|
WriteWav(candidateWav, matched, challenge.SampleRate);
|
|
artifacts.Add(Artifact("candidate-wav", candidateWav));
|
|
var comparison = analyzer.Compare(referenceSamples, matched);
|
|
metrics.AddRange(SongComparisonMetrics(comparison));
|
|
verdict = SongVerdict(comparison);
|
|
evaluation = SongEvaluationSentence(challenge, comparison, verdict);
|
|
var comparisonPath = Path.Combine(audioDir, "comparison.txt");
|
|
await File.WriteAllTextAsync(comparisonPath, SongComparisonReport(challenge, candidate, comparison, verdict), Encoding.UTF8);
|
|
artifacts.Add(Artifact("comparison-report", comparisonPath));
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is PatchScriptException or InvalidOperationException or InvalidDataException)
|
|
{
|
|
await File.WriteAllTextAsync(timelinePath, ProbeTimelineReport.ToCsv([]), Encoding.UTF8);
|
|
verdict = "render-failed";
|
|
evaluation = $"Candidate could not be parsed, lowered, or prepared for render: {ex.GetType().Name}: {ex.Message}";
|
|
metrics.Add(new SpeechScoreMetric("render_failed", 1, 1));
|
|
metrics.Add(new SpeechScoreMetric("candidate_invalid", 1, 1));
|
|
}
|
|
|
|
var latency = Stopwatch.GetElapsedTime(startedStamp).TotalMilliseconds;
|
|
results.Add(new IpaTrialResult(
|
|
$"{batchId}:song-snippet:{safeCandidateId}",
|
|
batchId,
|
|
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
|
"song-snippet",
|
|
["song", "snippet", "alien-gibberish"],
|
|
challenge.ChallengeId,
|
|
safeCandidateId,
|
|
candidate.HypothesizerId,
|
|
candidate.Hypothesis,
|
|
scriptCopy,
|
|
challenge.ReferenceWavPath,
|
|
File.Exists(candidateWav) ? candidateWav : "",
|
|
timelinePath,
|
|
metrics.ToArray(),
|
|
artifacts.ToArray(),
|
|
"song-snippet-audio-evaluator",
|
|
evaluation,
|
|
verdict,
|
|
[
|
|
"This is a local-only song clip challenge; reference audio is not redistributed by the repo.",
|
|
$"The target is a randomly selected {challenge.DurationSeconds:0.###}-second scene-audio snippet, not IPA articulation truth.",
|
|
"Challenge spectrogram, derivative, envelope, and autocorrelation artifacts are stored as typed CultMesh/CultCache evidence documents in the .cc database.",
|
|
"Full-patch FM/AM/noise/scene modeling is allowed; primitive timeline facts remain diagnostic when present."
|
|
],
|
|
[
|
|
new SpeechTimingReceipt(
|
|
"song-snippet-render-score",
|
|
startedAt.ToString("O", CultureInfo.InvariantCulture),
|
|
latency,
|
|
0,
|
|
SongConfidence(metrics),
|
|
"Local worker rendered an AquaSynth candidate against a frozen song snippet and wrote CultCache trial result data.")
|
|
],
|
|
timelineFacts));
|
|
}
|
|
|
|
await IpaTrialResultCultCacheStore.UpsertResultsAsync(store, results);
|
|
var summaryPath = Path.Combine(batchDirectory, "summary.csv");
|
|
await File.WriteAllTextAsync(summaryPath, SongSummaryCsv(results), Encoding.UTF8);
|
|
var reportPath = Path.Combine(batchDirectory, "evaluator-report.md");
|
|
await File.WriteAllTextAsync(reportPath, SongEvaluatorReport(challenge, results), Encoding.UTF8);
|
|
Console.WriteLine(batchDirectory);
|
|
Console.WriteLine(store);
|
|
}
|
|
|
|
static async Task DumpAsync(Dictionary<string, string> options)
|
|
{
|
|
var store = Required(options, "store");
|
|
var output = Required(options, "output");
|
|
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
await File.WriteAllTextAsync(output, StoreReport(store, results), Encoding.UTF8);
|
|
Console.WriteLine(output);
|
|
}
|
|
|
|
static async Task DistillAsync(Dictionary<string, string> options)
|
|
{
|
|
var store = Required(options, "store");
|
|
var outputStore = Required(options, "output-store");
|
|
var output = Required(options, "output");
|
|
var maxResults = int.TryParse(Value(options, "max-results", "40"), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedMax)
|
|
? Math.Clamp(parsedMax, 1, 500)
|
|
: 40;
|
|
var minCosine = FloatValue(options, "min-cosine", .35f);
|
|
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
|
var evidence = await IpaTrialResultCultCacheStore.ReadSongChallengeEvidenceAsync(store);
|
|
var selected = DistillResults(results, maxResults, minCosine).ToArray();
|
|
var selectedReferenceIds = selected.Select(result => result.ReferenceId).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
var selectedEvidence = evidence
|
|
.Where(document => selectedReferenceIds.Contains(document.ChallengeId))
|
|
.ToArray();
|
|
var distillation = SongTrialDistillation(store, results, selected, minCosine);
|
|
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputStore))!);
|
|
if (File.Exists(outputStore))
|
|
{
|
|
File.Delete(outputStore);
|
|
}
|
|
|
|
await IpaTrialResultCultCacheStore.UpsertResultsAsync(outputStore, selected);
|
|
await IpaTrialResultCultCacheStore.UpsertSongChallengeEvidenceAsync(outputStore, selectedEvidence);
|
|
await IpaTrialResultCultCacheStore.UpsertSongTrialDistillationsAsync(outputStore, [distillation]);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
await File.WriteAllTextAsync(output, DistillationReport(store, outputStore, results, selected, evidence, selectedEvidence, distillation, minCosine), Encoding.UTF8);
|
|
Console.WriteLine(outputStore);
|
|
Console.WriteLine(output);
|
|
}
|
|
|
|
static async Task SearchAsync(Dictionary<string, string> options)
|
|
{
|
|
var store = Required(options, "store");
|
|
var query = Required(options, "query");
|
|
var output = Required(options, "output");
|
|
var limit = int.TryParse(Value(options, "limit", "12"), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)
|
|
? Math.Clamp(parsed, 1, 100)
|
|
: 12;
|
|
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
var searchOptions = SearchOptionsFrom(options);
|
|
var ranked = await SearchRankAsync(results, query, limit, store, searchOptions);
|
|
await File.WriteAllTextAsync(output, SearchReport(store, query, results, ranked, searchOptions), Encoding.UTF8);
|
|
Console.WriteLine(output);
|
|
}
|
|
|
|
static async Task IndexAsync(Dictionary<string, string> options)
|
|
{
|
|
var store = Required(options, "store");
|
|
var output = Value(options, "output", "");
|
|
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
|
var searchOptions = SearchOptionsFrom(options);
|
|
var index = await EnsureVectorIndexAsync(results, store, searchOptions, force: BoolValue(options, "force"));
|
|
var report = new StringBuilder();
|
|
report.AppendLine("# IPA Trial Vector Index");
|
|
report.AppendLine();
|
|
report.AppendLine($"store: `{store}`");
|
|
report.AppendLine($"collection: `{searchOptions.Collection}`");
|
|
report.AppendLine($"qdrant: `{searchOptions.QdrantUrl}`");
|
|
report.AppendLine($"ollama: `{searchOptions.OllamaUrl}`");
|
|
report.AppendLine($"embedder: `{searchOptions.EmbedModel}`");
|
|
report.AppendLine($"chunks: {index.ChunkCount}");
|
|
report.AppendLine($"vector_dimensions: {index.VectorSize}");
|
|
report.AppendLine($"store_key: `{StoreKey(store)}`");
|
|
if (!string.IsNullOrWhiteSpace(output))
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
await File.WriteAllTextAsync(output, report.ToString(), Encoding.UTF8);
|
|
Console.WriteLine(output);
|
|
}
|
|
else
|
|
{
|
|
Console.Write(report.ToString());
|
|
}
|
|
}
|
|
|
|
static async Task<IReadOnlyList<SearchHit>> SearchRankAsync(
|
|
IReadOnlyList<IpaTrialResult> results,
|
|
string query,
|
|
int limit,
|
|
string store,
|
|
SearchOptions options)
|
|
{
|
|
if (!options.UseVector)
|
|
{
|
|
return Rank(results, query, limit, store, "lexical-disabled");
|
|
}
|
|
|
|
try
|
|
{
|
|
if (!options.SkipIndex)
|
|
{
|
|
await EnsureVectorIndexAsync(results, store, options, force: false);
|
|
}
|
|
|
|
var vectorHits = await QueryVectorIndexAsync(query, limit * 3, store, options);
|
|
if (vectorHits.Count > 0)
|
|
{
|
|
return MergeVectorAndLexicalHits(results, query, limit, store, vectorHits);
|
|
}
|
|
}
|
|
catch (Exception ex) when (!options.RequireVector)
|
|
{
|
|
return Rank(results, query, limit, store, $"vector-fallback:{ex.GetType().Name}:{ex.Message}");
|
|
}
|
|
|
|
return Rank(results, query, limit, store, "vector-empty-fallback");
|
|
}
|
|
|
|
static async Task<VectorIndexResult> EnsureVectorIndexAsync(
|
|
IReadOnlyList<IpaTrialResult> results,
|
|
string store,
|
|
SearchOptions options,
|
|
bool force)
|
|
{
|
|
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds) };
|
|
var chunks = BuildEvidenceChunks(results, store).ToArray();
|
|
if (chunks.Length == 0)
|
|
{
|
|
return new VectorIndexResult(0, 0);
|
|
}
|
|
|
|
if (force)
|
|
{
|
|
await DeleteCollectionIfExistsAsync(http, options);
|
|
}
|
|
|
|
var sampleVector = await EmbedOneAsync(http, options, chunks[0].Text);
|
|
await EnsureQdrantCollectionAsync(http, options, sampleVector.Length);
|
|
|
|
const int batchSize = 32;
|
|
for (var index = 0; index < chunks.Length; index += batchSize)
|
|
{
|
|
var batch = chunks.Skip(index).Take(batchSize).ToArray();
|
|
var vectors = await EmbedManyAsync(http, options, batch.Select(chunk => chunk.Text).ToArray());
|
|
if (vectors.Length != batch.Length)
|
|
{
|
|
throw new InvalidDataException($"Ollama returned {vectors.Length} vectors for {batch.Length} evidence chunks.");
|
|
}
|
|
|
|
var points = batch.Select((chunk, offset) => new QdrantPoint(
|
|
StableUuid(chunk.Id),
|
|
vectors[offset],
|
|
QdrantPayloadFrom(chunk, options.EmbedModel))).ToArray();
|
|
var upsert = new QdrantUpsertRequest(points);
|
|
var response = await http.PutAsJsonAsync(
|
|
$"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}/points?wait=true",
|
|
upsert,
|
|
JsonOptions());
|
|
response.EnsureSuccessStatusCode();
|
|
}
|
|
|
|
await EnsurePayloadIndexesAsync(http, options);
|
|
return new VectorIndexResult(chunks.Length, sampleVector.Length);
|
|
}
|
|
|
|
static async Task<IReadOnlyList<VectorSearchHit>> QueryVectorIndexAsync(
|
|
string query,
|
|
int limit,
|
|
string store,
|
|
SearchOptions options)
|
|
{
|
|
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(options.TimeoutSeconds) };
|
|
var queryVector = await EmbedOneAsync(http, options, query);
|
|
var request = new QdrantSearchRequest(
|
|
queryVector,
|
|
Math.Clamp(limit, 1, 100),
|
|
true,
|
|
false,
|
|
QdrantFilter.Store(StoreKey(store), options.EmbedModel));
|
|
var response = await http.PostAsJsonAsync(
|
|
$"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}/points/search",
|
|
request,
|
|
JsonOptions());
|
|
response.EnsureSuccessStatusCode();
|
|
var result = await response.Content.ReadFromJsonAsync<QdrantSearchResponse>(JsonOptions())
|
|
?? new QdrantSearchResponse([]);
|
|
return result.Result
|
|
.Select(VectorSearchHitFrom)
|
|
.Where(hit => !string.IsNullOrWhiteSpace(hit.TrialId))
|
|
.ToArray();
|
|
}
|
|
|
|
static IReadOnlyList<SearchHit> MergeVectorAndLexicalHits(
|
|
IReadOnlyList<IpaTrialResult> results,
|
|
string query,
|
|
int limit,
|
|
string store,
|
|
IReadOnlyList<VectorSearchHit> vectorHits)
|
|
{
|
|
var lexical = Rank(results, query, Math.Max(limit, results.Count), store, "hybrid-lexical");
|
|
var lexicalByTrial = lexical.ToDictionary(hit => hit.Result.TrialId, StringComparer.OrdinalIgnoreCase);
|
|
var vectorByTrial = vectorHits
|
|
.GroupBy(hit => hit.TrialId, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(group => group.Key, group => group.OrderByDescending(hit => hit.Score).First(), StringComparer.OrdinalIgnoreCase);
|
|
var resultsByTrial = results.ToDictionary(result => result.TrialId, StringComparer.OrdinalIgnoreCase);
|
|
return vectorByTrial.Values
|
|
.Where(hit => resultsByTrial.ContainsKey(hit.TrialId))
|
|
.Select(hit =>
|
|
{
|
|
var result = resultsByTrial[hit.TrialId];
|
|
var lexicalHit = lexicalByTrial.GetValueOrDefault(result.TrialId);
|
|
var matchedByField = lexicalHit?.MatchedByField ?? new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
|
|
var matchedTerms = lexicalHit?.MatchedTerms ?? [];
|
|
var tags = EvidenceTags(result, SearchDocument.From(result, DocumentFields(result, store)));
|
|
var lexicalScore = lexicalHit?.Score ?? 0;
|
|
var queryTerms = ExpandTerms(Tokenize(query)).ToArray();
|
|
var score = hit.Score * 10f
|
|
+ MathF.Min(lexicalScore, 8f)
|
|
+ MetricBias(result, queryTerms)
|
|
+ ClassAffinityBias(result, queryTerms);
|
|
return new SearchHit(
|
|
result,
|
|
score,
|
|
matchedTerms,
|
|
matchedByField,
|
|
tags,
|
|
"qdrant-ollama",
|
|
hit.Score,
|
|
hit.ChunkId,
|
|
hit.Text);
|
|
})
|
|
.OrderByDescending(hit => hit.Score)
|
|
.ThenByDescending(hit => ScoreSort(hit.Result))
|
|
.Take(limit)
|
|
.ToArray();
|
|
}
|
|
|
|
static async Task DeleteCollectionIfExistsAsync(HttpClient http, SearchOptions options)
|
|
{
|
|
var response = await http.GetAsync($"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}");
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var delete = await http.DeleteAsync($"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}");
|
|
delete.EnsureSuccessStatusCode();
|
|
}
|
|
|
|
static async Task EnsureQdrantCollectionAsync(HttpClient http, SearchOptions options, int vectorSize)
|
|
{
|
|
var url = $"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}";
|
|
var existing = await http.GetAsync(url);
|
|
if (existing.IsSuccessStatusCode)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var create = new QdrantCreateCollectionRequest(
|
|
new QdrantVectorConfig(vectorSize, "Cosine", true),
|
|
true,
|
|
new Dictionary<string, object>
|
|
{
|
|
["managedBy"] = "aquasynth",
|
|
["corpusKind"] = "ipa_trial_result",
|
|
["embedderId"] = options.EmbedModel,
|
|
["schemaVersion"] = 1
|
|
});
|
|
var response = await http.PutAsJsonAsync(url, create, JsonOptions());
|
|
response.EnsureSuccessStatusCode();
|
|
}
|
|
|
|
static async Task EnsurePayloadIndexesAsync(HttpClient http, SearchOptions options)
|
|
{
|
|
foreach (var field in new[] { "storeKey", "trialId", "candidateId", "targetSetId", "chunkKind", "embedderId" })
|
|
{
|
|
var response = await http.PutAsJsonAsync(
|
|
$"{options.QdrantUrl.TrimEnd('/')}/collections/{Uri.EscapeDataString(options.Collection)}/index?wait=true",
|
|
new QdrantPayloadIndexRequest(field, "keyword"),
|
|
JsonOptions());
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
// Existing indexes may be reported as an error by older Qdrant builds.
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
static async Task<float[]> EmbedOneAsync(HttpClient http, SearchOptions options, string text) =>
|
|
(await EmbedManyAsync(http, options, [text])).FirstOrDefault()
|
|
?? throw new InvalidDataException("Ollama returned no embedding.");
|
|
|
|
static async Task<float[][]> EmbedManyAsync(HttpClient http, SearchOptions options, string[] texts)
|
|
{
|
|
var response = await http.PostAsJsonAsync(
|
|
$"{options.OllamaUrl.TrimEnd('/')}/api/embed",
|
|
new OllamaEmbedRequest(options.EmbedModel, texts),
|
|
JsonOptions());
|
|
response.EnsureSuccessStatusCode();
|
|
var body = await response.Content.ReadFromJsonAsync<OllamaEmbedResponse>(JsonOptions())
|
|
?? throw new InvalidDataException("Ollama returned an empty embedding response.");
|
|
if (body.Embeddings is { Length: > 0 })
|
|
{
|
|
return body.Embeddings;
|
|
}
|
|
|
|
if (body.Embedding is { Length: > 0 })
|
|
{
|
|
return [body.Embedding];
|
|
}
|
|
|
|
throw new InvalidDataException("Ollama returned no embeddings.");
|
|
}
|
|
|
|
static async Task ShowAsync(Dictionary<string, string> options)
|
|
{
|
|
var store = Required(options, "store");
|
|
var trialId = Required(options, "trial-id");
|
|
var output = Required(options, "output");
|
|
var results = await IpaTrialResultCultCacheStore.ReadResultsAsync(store);
|
|
var result = results.FirstOrDefault(item =>
|
|
item.TrialId.Equals(trialId, StringComparison.OrdinalIgnoreCase) ||
|
|
item.CandidateId.Equals(trialId, StringComparison.OrdinalIgnoreCase))
|
|
?? throw new InvalidDataException($"No trial or candidate found for `{trialId}`.");
|
|
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!);
|
|
await File.WriteAllTextAsync(output, DetailReport(store, result), Encoding.UTF8);
|
|
Console.WriteLine(output);
|
|
}
|
|
|
|
static IReadOnlyList<IpaTrialScriptCandidate> CandidateScripts(string patchRoot, string hypothesizer)
|
|
{
|
|
if (!Directory.Exists(patchRoot))
|
|
{
|
|
throw new DirectoryNotFoundException(patchRoot);
|
|
}
|
|
|
|
var targetIds = IpaTrialOrchestrator.DefaultFiveSeedTrialSets
|
|
.SelectMany(set => set.Targets)
|
|
.Select(target => target.Target.Id)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.OrderByDescending(id => id.Length)
|
|
.ToArray();
|
|
return Directory.EnumerateFiles(patchRoot, "*.aqua", SearchOption.AllDirectories)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.Select(path =>
|
|
{
|
|
var stem = Path.GetFileNameWithoutExtension(path);
|
|
var target = TargetIdFromStem(stem, targetIds);
|
|
return new IpaTrialScriptCandidate(
|
|
target,
|
|
stem,
|
|
path,
|
|
hypothesizer,
|
|
$"External Codex-authored IPA patch candidate `{stem}` for target `{target}`.");
|
|
})
|
|
.ToArray();
|
|
}
|
|
|
|
static IReadOnlyList<IpaTrialScriptCandidate> SongCandidateScripts(string patchRoot, string hypothesizer)
|
|
{
|
|
if (!Directory.Exists(patchRoot))
|
|
{
|
|
throw new DirectoryNotFoundException(patchRoot);
|
|
}
|
|
|
|
return Directory.EnumerateFiles(patchRoot, "*.aqua", SearchOption.AllDirectories)
|
|
.Order(StringComparer.OrdinalIgnoreCase)
|
|
.Select(path =>
|
|
{
|
|
var stem = Path.GetFileNameWithoutExtension(path);
|
|
return new IpaTrialScriptCandidate(
|
|
"song-snippet",
|
|
stem,
|
|
path,
|
|
hypothesizer,
|
|
$"External Codex-authored song-snippet patch candidate `{stem}`.");
|
|
})
|
|
.ToArray();
|
|
}
|
|
|
|
static string TargetIdFromStem(string stem, IReadOnlyList<string> targetIds)
|
|
{
|
|
var explicitSplit = stem.Split(["__"], StringSplitOptions.None);
|
|
if (explicitSplit.Length > 1 && targetIds.Contains(explicitSplit[0], StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
return explicitSplit[0];
|
|
}
|
|
|
|
var matched = targetIds.FirstOrDefault(id =>
|
|
stem.Equals(id, StringComparison.OrdinalIgnoreCase) ||
|
|
stem.StartsWith($"{id}-", StringComparison.OrdinalIgnoreCase) ||
|
|
stem.StartsWith($"{id}_", StringComparison.OrdinalIgnoreCase));
|
|
if (!string.IsNullOrWhiteSpace(matched))
|
|
{
|
|
return matched;
|
|
}
|
|
|
|
throw new InvalidDataException(
|
|
$"Could not infer IPA target id from `{stem}`. Name files as `<targetId>__candidate-name.aqua`; known target ids: {string.Join(", ", targetIds)}.");
|
|
}
|
|
|
|
static IReadOnlyList<SearchHit> Rank(
|
|
IReadOnlyList<IpaTrialResult> results,
|
|
string query,
|
|
int limit,
|
|
string store,
|
|
string retrievalMode = "lexical")
|
|
{
|
|
var queryTerms = ExpandTerms(Tokenize(query)).ToArray();
|
|
var documents = results
|
|
.Select(result => SearchDocument.From(result, DocumentFields(result, store)))
|
|
.ToArray();
|
|
var documentFrequency = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
|
foreach (var document in documents)
|
|
{
|
|
foreach (var term in document.AllTerms.Distinct(StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
documentFrequency[term] = documentFrequency.TryGetValue(term, out var existingCount) ? existingCount + 1 : 1;
|
|
}
|
|
}
|
|
|
|
var count = Math.Max(1, documents.Length);
|
|
return results
|
|
.Zip(documents)
|
|
.Select(pair =>
|
|
{
|
|
var (result, document) = pair;
|
|
var matchedByField = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase);
|
|
var lexical = 0f;
|
|
foreach (var (field, weightedTerms) in document.Fields)
|
|
{
|
|
var termCounts = weightedTerms.Terms.GroupBy(term => term, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(group => group.Key, group => group.Count(), StringComparer.OrdinalIgnoreCase);
|
|
var matched = queryTerms.Where(term => termCounts.ContainsKey(term))
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.ToArray();
|
|
if (matched.Length == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
matchedByField[field] = matched;
|
|
foreach (var term in matched)
|
|
{
|
|
var df = documentFrequency.TryGetValue(term, out var value) ? value : 1;
|
|
var idf = MathF.Log(1 + ((count - df + 0.5f) / (df + 0.5f)));
|
|
lexical += weightedTerms.Weight * (1f + MathF.Log(1 + termCounts[term])) * Math.Max(0.1f, idf);
|
|
}
|
|
}
|
|
|
|
var matchedTerms = matchedByField.Values.SelectMany(term => term)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.ToArray();
|
|
var metricBias = MetricBias(result, queryTerms);
|
|
var evidenceBias = EvidenceBias(result, queryTerms, document);
|
|
var classBias = ClassAffinityBias(result, queryTerms);
|
|
var score = lexical + metricBias + evidenceBias + classBias;
|
|
return new SearchHit(result, score, matchedTerms, matchedByField, EvidenceTags(result, document), retrievalMode);
|
|
})
|
|
.Where(item => item.Score > 0)
|
|
.OrderByDescending(item => item.Score)
|
|
.ThenByDescending(item => ScoreSort(item.Result))
|
|
.ThenBy(item => item.Result.TrialId, StringComparer.Ordinal)
|
|
.Take(limit)
|
|
.ToArray();
|
|
}
|
|
|
|
static IEnumerable<string> ExpandTerms(IEnumerable<string> terms)
|
|
{
|
|
var set = new HashSet<string>(terms, StringComparer.OrdinalIgnoreCase);
|
|
var aliases = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
["vowel"] = ["a", "i", "u", "e", "o", "formant", "tongue", "lip", "radiation"],
|
|
["nasal"] = ["m", "n", "ng", "ŋ", "velum", "branch", "velopharynx", "admittance"],
|
|
["sonorant"] = ["nasal", "approximant", "l", "r", "voiced"],
|
|
["fricative"] = ["s", "z", "f", "v", "th", "θ", "turbulence", "noise", "constriction"],
|
|
["stop"] = ["p", "b", "t", "d", "k", "plosive", "closure", "release", "reservoir"],
|
|
["plosive"] = ["stop", "closure", "release", "reservoir", "stored", "pressure"],
|
|
["weak"] = ["render-failed", "weak", "low", "failed", "silent"],
|
|
["promising"] = ["pressure", "promising", "high", "improved"],
|
|
["articulation"] = ["gesture", "primitive", "timeline", "opening", "flow"],
|
|
["dressing"] = ["fm", "am", "modulator", "envelope", "noise", "patch"],
|
|
["owner"] = ["source", "lowering", "tract", "radiation", "gesture", "orchestration"]
|
|
};
|
|
|
|
var queue = set.ToArray();
|
|
foreach (var term in queue)
|
|
{
|
|
if (!aliases.TryGetValue(term, out var expanded))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
foreach (var alias in expanded)
|
|
{
|
|
set.Add(alias);
|
|
}
|
|
}
|
|
|
|
return set;
|
|
}
|
|
|
|
static float MetricBias(IpaTrialResult result, IReadOnlyCollection<string> terms)
|
|
{
|
|
var bias = 0f;
|
|
var cosine = result.Metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? 0;
|
|
var articulation = result.Metrics.FirstOrDefault(metric => metric.Name == "articulation_score")?.Value ?? 0;
|
|
var gesture = result.Metrics.FirstOrDefault(metric => metric.Name == "gesture_score")?.Value ?? 0;
|
|
if (terms.Contains("weak", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
bias += Math.Clamp(1 - cosine, 0, 1);
|
|
bias += Math.Clamp(1 - articulation, 0, 1);
|
|
}
|
|
|
|
if (terms.Contains("promising", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
bias += Math.Clamp(cosine, 0, 1);
|
|
bias += Math.Clamp(articulation, 0, 1);
|
|
}
|
|
|
|
if (terms.Contains("gesture", StringComparer.OrdinalIgnoreCase) ||
|
|
terms.Contains("articulation", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
bias += Math.Clamp(gesture, 0, 1);
|
|
}
|
|
|
|
return bias;
|
|
}
|
|
|
|
static float EvidenceBias(IpaTrialResult result, IReadOnlyCollection<string> terms, SearchDocument document)
|
|
{
|
|
var bias = 0f;
|
|
if (terms.Contains("timeline", StringComparer.OrdinalIgnoreCase) &&
|
|
document.Fields.TryGetValue("artifacts", out var artifacts) &&
|
|
artifacts.Terms.Contains("timeline", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
bias += 1.25f;
|
|
}
|
|
|
|
if (terms.Contains("contrast", StringComparer.OrdinalIgnoreCase) ||
|
|
terms.Contains("pair", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
bias += result.Phonemes.Length > 1 ? 0.25f : 0.5f;
|
|
}
|
|
|
|
if (terms.Contains("source", StringComparer.OrdinalIgnoreCase) &&
|
|
result.Hypothesis.Contains("source", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
bias += 0.75f;
|
|
}
|
|
|
|
return bias;
|
|
}
|
|
|
|
static float ClassAffinityBias(IpaTrialResult result, IReadOnlyCollection<string> terms)
|
|
{
|
|
var intents = QueryClasses(terms).ToArray();
|
|
if (intents.Length == 0)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
if (intents.Contains("dressing", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
return MentionsDressing(result) ? 2.0f : -0.75f;
|
|
}
|
|
|
|
var classes = ResultClasses(result).ToArray();
|
|
var primaryIntents = intents
|
|
.Where(intent => !intent.Equals("mixed", StringComparison.OrdinalIgnoreCase))
|
|
.ToArray();
|
|
if (primaryIntents.Length > 0)
|
|
{
|
|
if (primaryIntents.Intersect(classes, StringComparer.OrdinalIgnoreCase).Any())
|
|
{
|
|
return classes.Contains("mixed", StringComparer.OrdinalIgnoreCase) ? 2.0f : 2.5f;
|
|
}
|
|
|
|
return classes.Contains("mixed", StringComparer.OrdinalIgnoreCase) ? -1.0f : -1.75f;
|
|
}
|
|
|
|
return intents.Intersect(classes, StringComparer.OrdinalIgnoreCase).Any() ? 1.25f : -0.75f;
|
|
}
|
|
|
|
static IEnumerable<string> QueryClasses(IEnumerable<string> terms)
|
|
{
|
|
var set = new HashSet<string>(terms, StringComparer.OrdinalIgnoreCase);
|
|
if (set.Overlaps(["stop", "plosive", "closure", "release", "reservoir"]))
|
|
{
|
|
yield return "stop";
|
|
}
|
|
if (set.Overlaps(["fricative", "sibilant", "constriction", "turbulence", "labiodental", "dental"]))
|
|
{
|
|
yield return "fricative";
|
|
}
|
|
if (set.Overlaps(["nasal", "velum", "veloparynx", "velopharynx", "branch", "admittance"]))
|
|
{
|
|
yield return "nasal";
|
|
}
|
|
if (set.Overlaps(["vowel", "formant", "tongue", "body", "rounded", "unrounded"]))
|
|
{
|
|
yield return "vowel";
|
|
}
|
|
if (set.Overlaps(["mixed", "transfer", "generalization"]))
|
|
{
|
|
yield return "mixed";
|
|
}
|
|
if (set.Overlaps(["dressing", "fm", "am", "modulator", "envelope", "helper"]))
|
|
{
|
|
yield return "dressing";
|
|
}
|
|
}
|
|
|
|
static IEnumerable<string> ResultClasses(IpaTrialResult result)
|
|
{
|
|
var candidateClasses = CandidateSpecificClasses(result.CandidateId).ToArray();
|
|
var target = result.TargetSetId.ToLowerInvariant();
|
|
if (candidateClasses.Length > 0)
|
|
{
|
|
foreach (var candidateClass in candidateClasses)
|
|
{
|
|
yield return candidateClass;
|
|
}
|
|
|
|
if (target.Contains("mixed", StringComparison.Ordinal))
|
|
{
|
|
yield return "mixed";
|
|
}
|
|
|
|
yield break;
|
|
}
|
|
|
|
if (target.Contains("mixed", StringComparison.Ordinal))
|
|
{
|
|
yield return "mixed";
|
|
yield break;
|
|
}
|
|
if (target.Contains("stop", StringComparison.Ordinal))
|
|
{
|
|
yield return "stop";
|
|
yield break;
|
|
}
|
|
if (target.Contains("fricative", StringComparison.Ordinal))
|
|
{
|
|
yield return "fricative";
|
|
yield break;
|
|
}
|
|
if (target.Contains("nasal", StringComparison.Ordinal))
|
|
{
|
|
yield return "nasal";
|
|
yield break;
|
|
}
|
|
if (target.Contains("vowel", StringComparison.Ordinal))
|
|
{
|
|
yield return "vowel";
|
|
yield break;
|
|
}
|
|
|
|
foreach (var phoneme in result.Phonemes)
|
|
{
|
|
var normalized = phoneme.ToLowerInvariant();
|
|
if (normalized is "p" or "b" or "t" or "d" or "k")
|
|
{
|
|
yield return "stop";
|
|
}
|
|
if (normalized is "s" or "z" or "f" or "v" or "th" or "θ")
|
|
{
|
|
yield return "fricative";
|
|
}
|
|
if (normalized is "m" or "n" or "ng" or "ŋ")
|
|
{
|
|
yield return "nasal";
|
|
}
|
|
if (normalized is "a" or "i" or "u" or "e" or "o")
|
|
{
|
|
yield return "vowel";
|
|
}
|
|
}
|
|
}
|
|
|
|
static IEnumerable<string> CandidateSpecificClasses(string candidateId)
|
|
{
|
|
var normalized = candidateId.ToLowerInvariant();
|
|
var prefix = normalized;
|
|
if (prefix.StartsWith("mix-", StringComparison.Ordinal))
|
|
{
|
|
prefix = prefix[4..];
|
|
}
|
|
|
|
prefix = prefix.Split(['-', '_'], StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? prefix;
|
|
if (prefix is "p" or "b" or "t" or "d" or "k")
|
|
{
|
|
yield return "stop";
|
|
}
|
|
if (prefix is "s" or "z" or "f" or "v" or "th" or "θ")
|
|
{
|
|
yield return "fricative";
|
|
}
|
|
if (prefix is "m" or "n" or "ng" or "ŋ")
|
|
{
|
|
yield return "nasal";
|
|
}
|
|
if (prefix is "a" or "i" or "u" or "e" or "o")
|
|
{
|
|
yield return "vowel";
|
|
}
|
|
if (prefix is "l" or "r")
|
|
{
|
|
yield return "approximant";
|
|
}
|
|
}
|
|
|
|
static bool MentionsDressing(IpaTrialResult result)
|
|
{
|
|
var text = string.Join(' ', [
|
|
result.Hypothesis,
|
|
result.EvaluationSummary,
|
|
string.Join(' ', result.KnownLies)
|
|
]);
|
|
return Tokenize(text).Any(term => term is "dressing" or "fm" or "am" or "modulator" or "envelope" or "helper");
|
|
}
|
|
|
|
static Dictionary<string, WeightedTerms> DocumentFields(IpaTrialResult result, string store)
|
|
{
|
|
var fields = new Dictionary<string, WeightedTerms>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
["identity"] = new(4.0f, Tokenize(string.Join(' ', [
|
|
result.TrialId,
|
|
result.BatchId,
|
|
result.TargetSetId,
|
|
result.ReferenceId,
|
|
result.CandidateId,
|
|
result.HypothesizerId,
|
|
result.Verdict
|
|
]))),
|
|
["phonetics"] = new(4.0f, Tokenize(string.Join(' ', result.Phonemes))),
|
|
["hypothesis"] = new(2.5f, Tokenize(result.Hypothesis)),
|
|
["evaluation"] = new(2.5f, Tokenize(result.EvaluationSummary)),
|
|
["known_lies"] = new(2.0f, Tokenize(string.Join(' ', result.KnownLies))),
|
|
["metrics"] = new(2.0f, Tokenize(MetricText(result))),
|
|
["timeline_facts"] = new(2.5f, Tokenize(TimelineFactText(result))),
|
|
["artifacts"] = new(1.75f, Tokenize(ArtifactText(result, store))),
|
|
["timing"] = new(1.25f, Tokenize(TimingText(result)))
|
|
};
|
|
return fields;
|
|
}
|
|
|
|
static string MetricText(IpaTrialResult result)
|
|
{
|
|
var builder = new StringBuilder();
|
|
foreach (var metric in result.Metrics)
|
|
{
|
|
builder.Append(metric.Name).Append(' ');
|
|
builder.Append(metric.Value.ToString("0.######", CultureInfo.InvariantCulture)).Append(' ');
|
|
if (metric.Name.Contains("cosine", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append(metric.Value < 0.2f ? "weak low failed " : "promising high ");
|
|
}
|
|
if (metric.Name.Contains("articulation", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append(metric.Value < 0.3f ? "weak articulation failed " : "promising articulation ");
|
|
}
|
|
if (metric.Name.Contains("gesture", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append(metric.Value < 0.6f ? "weak gesture " : "promising gesture ");
|
|
}
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string TimelineFactText(IpaTrialResult result)
|
|
{
|
|
var builder = new StringBuilder();
|
|
foreach (var fact in result.TimelineFacts ?? [])
|
|
{
|
|
builder.Append(fact.Name).Append(' ');
|
|
builder.Append(fact.Primitive).Append(' ');
|
|
builder.Append(fact.Signal).Append(' ');
|
|
builder.Append(fact.Value.ToString("0.######", CultureInfo.InvariantCulture)).Append(' ');
|
|
builder.Append(fact.Unit).Append(' ');
|
|
builder.Append("blocks ").Append(fact.BlockStart).Append(' ').Append(fact.BlockEnd).Append(' ');
|
|
builder.Append(fact.Summary).Append(' ');
|
|
if (fact.Name.Contains("release", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("closure release burst plosive stop ");
|
|
}
|
|
if (fact.Name.Contains("branch", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("nasal velum admittance branch ");
|
|
}
|
|
if (fact.Name.Contains("radiation", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("radiation lip output flow ");
|
|
}
|
|
if (fact.Name.Contains("passivity", StringComparison.OrdinalIgnoreCase) ||
|
|
fact.Name.Contains("energy", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("passivity energy check ");
|
|
}
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string TimingText(IpaTrialResult result) =>
|
|
string.Join(' ', result.TimingReceipts.Select(receipt =>
|
|
$"{receipt.StageId} {receipt.Notes} latency {receipt.LatencyMilliseconds:0.###} budget {receipt.BudgetMilliseconds:0.###} confidence {receipt.Confidence:0.###}"));
|
|
|
|
static string ArtifactText(IpaTrialResult result, string store)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.Append(result.CandidatePatchUri).Append(' ');
|
|
builder.Append(result.ReferenceArtifactUri).Append(' ');
|
|
builder.Append(result.CandidateArtifactUri).Append(' ');
|
|
builder.Append(result.PrimitiveTimelineUri).Append(' ');
|
|
foreach (var artifact in result.Artifacts)
|
|
{
|
|
builder.Append(artifact.Kind).Append(' ').Append(artifact.Uri).Append(' ').Append(artifact.ContentHash).Append(' ');
|
|
builder.Append(ReadArtifactSnippet(artifact.Uri, store)).Append(' ');
|
|
}
|
|
|
|
builder.Append(ReadArtifactSnippet(result.CandidatePatchUri, store)).Append(' ');
|
|
builder.Append(ReadArtifactSnippet(result.PrimitiveTimelineUri, store)).Append(' ');
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string ReadArtifactSnippet(string uri, string store)
|
|
{
|
|
var path = ResolveArtifactPath(uri, store);
|
|
if (path is null || !File.Exists(path))
|
|
{
|
|
return "";
|
|
}
|
|
|
|
var extension = Path.GetExtension(path);
|
|
if (!new[] { ".aqua", ".md", ".txt", ".csv", ".jsonl", ".yaml", ".yml", ".dsp" }
|
|
.Contains(extension, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
return "";
|
|
}
|
|
|
|
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
|
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
|
|
var buffer = new char[32768];
|
|
var read = reader.Read(buffer, 0, buffer.Length);
|
|
return new string(buffer, 0, read);
|
|
}
|
|
|
|
static string? ResolveArtifactPath(string uri, string store)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(uri))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if (File.Exists(uri))
|
|
{
|
|
return uri;
|
|
}
|
|
|
|
if (Uri.TryCreate(uri, UriKind.Absolute, out var parsed) && parsed.IsFile)
|
|
{
|
|
return parsed.LocalPath;
|
|
}
|
|
|
|
if (uri.StartsWith("file://", StringComparison.OrdinalIgnoreCase) &&
|
|
Uri.TryCreate(uri, UriKind.Absolute, out var fileUri))
|
|
{
|
|
return fileUri.LocalPath;
|
|
}
|
|
|
|
var storeDirectory = Path.GetDirectoryName(Path.GetFullPath(store));
|
|
if (!string.IsNullOrWhiteSpace(storeDirectory))
|
|
{
|
|
var relative = Path.GetFullPath(Path.Combine(storeDirectory, uri));
|
|
if (File.Exists(relative))
|
|
{
|
|
return relative;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
static IEnumerable<EvidenceChunk> BuildEvidenceChunks(IReadOnlyList<IpaTrialResult> results, string store)
|
|
{
|
|
var songEvidenceByChallenge = IpaTrialResultCultCacheStore.ReadSongChallengeEvidenceAsync(store)
|
|
.GetAwaiter()
|
|
.GetResult()
|
|
.GroupBy(document => document.ChallengeId, StringComparer.OrdinalIgnoreCase)
|
|
.ToDictionary(group => group.Key, group => group.ToArray(), StringComparer.OrdinalIgnoreCase);
|
|
var distillations = IpaTrialResultCultCacheStore.ReadSongTrialDistillationsAsync(store)
|
|
.GetAwaiter()
|
|
.GetResult();
|
|
|
|
foreach (var document in distillations)
|
|
{
|
|
var linkedTrialIds = document.KeptTrialIds.Length == 0
|
|
? [document.DistillationId]
|
|
: document.KeptTrialIds;
|
|
foreach (var linkedTrialId in linkedTrialIds)
|
|
{
|
|
yield return new EvidenceChunk(
|
|
$"song-trial-distillation:{StoreKey(store)}:{document.DistillationId}:{linkedTrialId}",
|
|
linkedTrialId,
|
|
"distillation",
|
|
"song-trial-distillation",
|
|
"distillation",
|
|
$"""
|
|
song trial distillation
|
|
id {document.DistillationId}
|
|
linked_trial {linkedTrialId}
|
|
source_store {document.SourceStore}
|
|
input_trials {document.InputTrialCount}
|
|
kept_trials {document.KeptTrialCount}
|
|
summary {document.Summary}
|
|
reusable_scene_roles
|
|
{string.Join(Environment.NewLine, document.ReusableSceneRoles)}
|
|
transfer_rules
|
|
{string.Join(Environment.NewLine, document.TransferRules)}
|
|
failure_patterns
|
|
{string.Join(Environment.NewLine, document.FailurePatterns)}
|
|
aggregate_metrics
|
|
{string.Join(Environment.NewLine, document.AggregateMetrics.Select(metric => $"{metric.Name} {metric.Value.ToString("0.######", CultureInfo.InvariantCulture)} weight {metric.Weight.ToString("0.######", CultureInfo.InvariantCulture)}"))}
|
|
""",
|
|
store);
|
|
}
|
|
}
|
|
|
|
foreach (var result in results)
|
|
{
|
|
var text = new StringBuilder();
|
|
text.AppendLine($"trial {result.TrialId}");
|
|
text.AppendLine($"candidate {result.CandidateId}");
|
|
text.AppendLine($"target {result.TargetSetId}");
|
|
text.AppendLine($"phonemes {string.Join(' ', result.Phonemes)}");
|
|
text.AppendLine($"reference {result.ReferenceId}");
|
|
text.AppendLine($"verdict {result.Verdict}");
|
|
text.AppendLine("metrics");
|
|
foreach (var metric in result.Metrics.OrderBy(metric => metric.Name, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
text.AppendLine($"{metric.Name} {metric.Value.ToString("0.######", CultureInfo.InvariantCulture)} weight {metric.Weight.ToString("0.######", CultureInfo.InvariantCulture)}");
|
|
}
|
|
|
|
text.AppendLine("timeline_facts");
|
|
foreach (var fact in (result.TimelineFacts ?? []).OrderBy(fact => fact.Name, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
text.AppendLine($"{fact.Name} {fact.Primitive} {fact.Signal} {fact.Value.ToString("0.######", CultureInfo.InvariantCulture)} {fact.Unit} blocks {fact.BlockStart}-{fact.BlockEnd} {fact.Summary}");
|
|
}
|
|
|
|
text.AppendLine("hypothesis");
|
|
text.AppendLine(result.Hypothesis);
|
|
text.AppendLine("evaluation");
|
|
text.AppendLine(result.EvaluationSummary);
|
|
text.AppendLine("known_lies");
|
|
text.AppendLine(string.Join(Environment.NewLine, result.KnownLies));
|
|
text.AppendLine("artifacts");
|
|
text.AppendLine(ArtifactText(result, store));
|
|
yield return new EvidenceChunk(
|
|
$"ipa-trial:{StoreKey(store)}:{result.TrialId}:summary",
|
|
result.TrialId,
|
|
result.CandidateId,
|
|
result.TargetSetId,
|
|
"summary",
|
|
text.ToString(),
|
|
store);
|
|
|
|
if (songEvidenceByChallenge.TryGetValue(result.ReferenceId, out var evidenceDocuments))
|
|
{
|
|
foreach (var document in evidenceDocuments)
|
|
{
|
|
yield return new EvidenceChunk(
|
|
$"song-challenge-evidence:{StoreKey(store)}:{result.TrialId}:{document.Kind}",
|
|
result.TrialId,
|
|
result.CandidateId,
|
|
result.TargetSetId,
|
|
$"song-challenge-{document.Kind}",
|
|
$"""
|
|
song challenge evidence
|
|
challenge {document.ChallengeId}
|
|
kind {document.Kind}
|
|
content_type {document.ContentType}
|
|
content_hash {document.ContentHash}
|
|
source {document.SourcePath}
|
|
{EvidenceIndexText(document)}
|
|
""",
|
|
store);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static string EvidenceIndexText(SongChallengeEvidenceDocument document)
|
|
{
|
|
const int maxIndexedChars = 32768;
|
|
if (document.Content.Length <= maxIndexedChars)
|
|
{
|
|
return document.Content;
|
|
}
|
|
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine($"indexed_excerpt_only true");
|
|
builder.AppendLine($"full_content_hash {document.ContentHash}");
|
|
builder.AppendLine($"full_content_chars {document.Content.Length.ToString(CultureInfo.InvariantCulture)}");
|
|
if (document.Kind.Contains("spectrogram", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
builder.AppendLine("spectrogram evidence is stored in full in CultCache; vector index carries a bounded CSV excerpt.");
|
|
}
|
|
|
|
builder.Append(document.Content.AsSpan(0, maxIndexedChars));
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string[] EvidenceTags(IpaTrialResult result, SearchDocument document)
|
|
{
|
|
var tags = new List<string>();
|
|
var cosine = result.Metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value;
|
|
var articulation = result.Metrics.FirstOrDefault(metric => metric.Name == "articulation_score")?.Value;
|
|
var gesture = result.Metrics.FirstOrDefault(metric => metric.Name == "gesture_score")?.Value;
|
|
if (cosine is < 0.2f || articulation is < 0.25f)
|
|
{
|
|
tags.Add("weak");
|
|
}
|
|
if (cosine is > 0.5f || articulation is > 0.4f)
|
|
{
|
|
tags.Add("promising");
|
|
}
|
|
if (gesture is > 0.7f && articulation is < 0.3f)
|
|
{
|
|
tags.Add("gesture-audio-gap");
|
|
}
|
|
if (document.AllTerms.Contains("timeline", StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
tags.Add("timeline");
|
|
}
|
|
if ((result.TimelineFacts ?? []).Any(fact => fact.Name.Contains("release", StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
tags.Add("release-facts");
|
|
}
|
|
if ((result.TimelineFacts ?? []).Any(fact => fact.Name.Contains("passivity", StringComparison.OrdinalIgnoreCase)))
|
|
{
|
|
tags.Add("passivity-facts");
|
|
}
|
|
if (MentionsDressing(result))
|
|
{
|
|
tags.Add("dressing");
|
|
}
|
|
|
|
return tags.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
|
|
}
|
|
|
|
static string[] Tokenize(string text) =>
|
|
text.ToLowerInvariant()
|
|
.Split([' ', '\t', '\r', '\n', ',', '.', ':', ';', '/', '\\', '`', '"', '\'', '[', ']', '(', ')', '{', '}', '<', '>', '|', '_', '-'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
|
.Where(term => term.Length > 1 || "aeioubdklmnprstvzθŋ".Contains(term, StringComparison.Ordinal))
|
|
.ToArray();
|
|
|
|
static string SearchReport(
|
|
string store,
|
|
string query,
|
|
IReadOnlyList<IpaTrialResult> allResults,
|
|
IReadOnlyList<SearchHit> ranked,
|
|
SearchOptions options)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("# IPA Trial Semantic Search");
|
|
builder.AppendLine();
|
|
builder.AppendLine($"store: `{store}`");
|
|
builder.AppendLine($"query: `{query}`");
|
|
builder.AppendLine($"retrieval: `{(ranked.Any(hit => hit.RetrievalMode == "qdrant-ollama") ? "qdrant-ollama" : "lexical-fallback")}`");
|
|
builder.AppendLine($"collection: `{options.Collection}`");
|
|
builder.AppendLine($"embedder: `{options.EmbedModel}`");
|
|
builder.AppendLine($"class_focus: `{string.Join(", ", QueryClasses(ExpandTerms(Tokenize(query))).Distinct(StringComparer.OrdinalIgnoreCase))}`");
|
|
builder.AppendLine($"matches: {ranked.Count}");
|
|
builder.AppendLine();
|
|
foreach (var hit in ranked)
|
|
{
|
|
var result = hit.Result;
|
|
var contrast = ContrastCandidate(result, allResults);
|
|
builder.Append("- ");
|
|
builder.Append(result.TrialId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.CandidateId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.Verdict);
|
|
builder.Append(" / score=");
|
|
builder.Append(hit.Score.ToString("0.###", CultureInfo.InvariantCulture));
|
|
if (hit.VectorScore is not null)
|
|
{
|
|
builder.Append(" / vector=");
|
|
builder.Append(hit.VectorScore.Value.ToString("0.###", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
builder.Append(" / matched=");
|
|
builder.AppendLine(string.Join('|', hit.MatchedTerms.Take(12)));
|
|
builder.Append(" tags: ");
|
|
builder.AppendLine(string.Join(", ", hit.Tags));
|
|
builder.Append(" class_match: ");
|
|
builder.AppendLine(string.Join(", ", ResultClasses(result).Distinct(StringComparer.OrdinalIgnoreCase)));
|
|
builder.Append(" metrics: gesture=");
|
|
builder.Append(Metric(result, "gesture_score"));
|
|
builder.Append(", logMelCosine=");
|
|
builder.Append(Metric(result, "log_mel_cosine"));
|
|
builder.Append(", articulation=");
|
|
builder.Append(Metric(result, "articulation_score"));
|
|
builder.Append(", rmsRatio=");
|
|
builder.AppendLine(Metric(result, "rms_ratio"));
|
|
builder.Append(" timeline: releasePeak=");
|
|
builder.Append(TimelineFact(result, "contact_release_peak"));
|
|
builder.Append(", releaseBlock=");
|
|
builder.Append(TimelineFact(result, "contact_release_peak_block"));
|
|
builder.Append(", branchAdmittance=");
|
|
builder.Append(TimelineFact(result, "branch_admittance_peak"));
|
|
builder.Append(", radiationOutput=");
|
|
builder.Append(TimelineFact(result, "radiation_output_peak"));
|
|
builder.Append(", passivityMax=");
|
|
builder.AppendLine(TimelineFact(result, "path_passivity_max"));
|
|
builder.Append(" hypothesis: ");
|
|
builder.AppendLine(result.Hypothesis);
|
|
builder.Append(" evaluation: ");
|
|
builder.AppendLine(result.EvaluationSummary);
|
|
if (contrast is not null)
|
|
{
|
|
builder.Append(" contrast: ");
|
|
builder.Append(contrast.TrialId);
|
|
builder.Append(" / ");
|
|
builder.Append(contrast.CandidateId);
|
|
builder.Append(" / logMelCosine=");
|
|
builder.Append(Metric(contrast, "log_mel_cosine"));
|
|
builder.Append(" / articulation=");
|
|
builder.AppendLine(Metric(contrast, "articulation_score"));
|
|
}
|
|
if (!string.IsNullOrWhiteSpace(hit.VectorChunkId))
|
|
{
|
|
builder.Append(" vector_chunk: ");
|
|
builder.AppendLine(hit.VectorChunkId);
|
|
}
|
|
if (!string.IsNullOrWhiteSpace(hit.VectorText))
|
|
{
|
|
builder.Append(" vector_excerpt: ");
|
|
builder.AppendLine(OneLine(hit.VectorText, 280));
|
|
}
|
|
builder.Append(" patch: ");
|
|
builder.AppendLine(result.CandidatePatchUri);
|
|
builder.Append(" show: dotnet run --project tools/IpaTrialWorker/IpaTrialWorker.csproj -- show --store \"");
|
|
builder.Append(store);
|
|
builder.Append("\" --trial-id \"");
|
|
builder.Append(result.TrialId);
|
|
builder.AppendLine("\" --output <detail.md>");
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string DetailReport(string store, IpaTrialResult result)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine($"# IPA Trial Detail: {result.TrialId}");
|
|
builder.AppendLine();
|
|
builder.AppendLine($"store: `{store}`");
|
|
builder.AppendLine($"batch: `{result.BatchId}`");
|
|
builder.AppendLine($"target_set: `{result.TargetSetId}`");
|
|
builder.AppendLine($"phonemes: `{string.Join(", ", result.Phonemes)}`");
|
|
builder.AppendLine($"reference: `{result.ReferenceId}`");
|
|
builder.AppendLine($"candidate: `{result.CandidateId}`");
|
|
builder.AppendLine($"verdict: `{result.Verdict}`");
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Hypothesis");
|
|
builder.AppendLine(result.Hypothesis);
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Evaluation");
|
|
builder.AppendLine(result.EvaluationSummary);
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Metrics");
|
|
foreach (var metric in result.Metrics.OrderBy(metric => metric.Name, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(metric.Name);
|
|
builder.Append(": ");
|
|
builder.Append(metric.Value.ToString("0.######", CultureInfo.InvariantCulture));
|
|
builder.Append(" weight=");
|
|
builder.AppendLine(metric.Weight.ToString("0.######", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Primitive Timeline Facts");
|
|
foreach (var fact in (result.TimelineFacts ?? []).OrderBy(fact => fact.Name, StringComparer.OrdinalIgnoreCase))
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(fact.Name);
|
|
builder.Append(": ");
|
|
builder.Append(fact.Value.ToString("0.######", CultureInfo.InvariantCulture));
|
|
builder.Append(' ');
|
|
builder.Append(fact.Unit);
|
|
builder.Append(" / ");
|
|
builder.Append(fact.Primitive);
|
|
builder.Append(' ');
|
|
builder.Append(fact.Signal);
|
|
builder.Append(" / blocks ");
|
|
builder.Append(fact.BlockStart);
|
|
builder.Append('-');
|
|
builder.Append(fact.BlockEnd);
|
|
if (!string.IsNullOrWhiteSpace(fact.Summary))
|
|
{
|
|
builder.Append(" / ");
|
|
builder.Append(fact.Summary);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Artifacts");
|
|
foreach (var artifact in result.Artifacts)
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(artifact.Kind);
|
|
builder.Append(": ");
|
|
builder.Append(artifact.Uri);
|
|
if (!string.IsNullOrWhiteSpace(artifact.ContentHash))
|
|
{
|
|
builder.Append(" ");
|
|
builder.Append(artifact.ContentHash);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Known Lies");
|
|
foreach (var lie in result.KnownLies)
|
|
{
|
|
builder.Append("- ");
|
|
builder.AppendLine(lie);
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string StoreReport(string store, IReadOnlyList<IpaTrialResult> results)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine($"# IPA Trial Results Store");
|
|
builder.AppendLine();
|
|
builder.AppendLine($"store: `{store}`");
|
|
builder.AppendLine($"records: {results.Count}");
|
|
builder.AppendLine();
|
|
foreach (var group in results.GroupBy(result => result.TargetSetId).OrderBy(group => group.Key, StringComparer.Ordinal))
|
|
{
|
|
builder.AppendLine($"## {group.Key}");
|
|
foreach (var result in group.OrderByDescending(ScoreSort).ThenBy(result => result.TrialId, StringComparer.Ordinal))
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(result.TrialId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.CandidateId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.Verdict);
|
|
builder.Append(": ");
|
|
builder.Append("gesture=");
|
|
builder.Append(Metric(result, "gesture_score"));
|
|
builder.Append(", logMelCosine=");
|
|
builder.Append(Metric(result, "log_mel_cosine"));
|
|
builder.Append(", articulation=");
|
|
builder.Append(Metric(result, "articulation_score"));
|
|
builder.Append(", rmsRatio=");
|
|
builder.Append(Metric(result, "rms_ratio"));
|
|
builder.Append(", releasePeak=");
|
|
builder.Append(TimelineFact(result, "contact_release_peak"));
|
|
builder.Append(", radiationOutput=");
|
|
builder.Append(TimelineFact(result, "radiation_output_peak"));
|
|
builder.Append(", passivityMax=");
|
|
builder.Append(TimelineFact(result, "path_passivity_max"));
|
|
builder.AppendLine();
|
|
builder.Append(" patch: ");
|
|
builder.AppendLine(result.CandidatePatchUri);
|
|
builder.Append(" timeline: ");
|
|
builder.AppendLine(result.PrimitiveTimelineUri);
|
|
builder.Append(" evaluation: ");
|
|
builder.AppendLine(result.EvaluationSummary);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static IReadOnlyList<IpaTrialResult> DistillResults(
|
|
IReadOnlyList<IpaTrialResult> results,
|
|
int maxResults,
|
|
float minCosine)
|
|
{
|
|
var keep = new Dictionary<string, IpaTrialResult>(StringComparer.OrdinalIgnoreCase);
|
|
foreach (var result in results
|
|
.Where(result => result.Verdict.Equals("promising", StringComparison.OrdinalIgnoreCase) ||
|
|
result.Verdict.Equals("pressure", StringComparison.OrdinalIgnoreCase) ||
|
|
MetricValue(result, "log_mel_cosine") >= minCosine)
|
|
.OrderByDescending(ScoreSort))
|
|
{
|
|
keep.TryAdd(result.TrialId, result);
|
|
}
|
|
|
|
foreach (var result in results
|
|
.GroupBy(result => result.ReferenceId, StringComparer.OrdinalIgnoreCase)
|
|
.Select(group => group.OrderByDescending(ScoreSort).First()))
|
|
{
|
|
keep.TryAdd(result.TrialId, result);
|
|
}
|
|
|
|
foreach (var result in results
|
|
.GroupBy(result => result.CandidateId, StringComparer.OrdinalIgnoreCase)
|
|
.Select(group => group.OrderByDescending(ScoreSort).First()))
|
|
{
|
|
keep.TryAdd(result.TrialId, result);
|
|
}
|
|
|
|
return keep.Values
|
|
.OrderByDescending(ScoreSort)
|
|
.ThenByDescending(result => MetricValue(result, "audio_score"))
|
|
.ThenBy(result => result.TrialId, StringComparer.Ordinal)
|
|
.Take(maxResults)
|
|
.ToArray();
|
|
}
|
|
|
|
static string DistillationReport(
|
|
string sourceStore,
|
|
string outputStore,
|
|
IReadOnlyList<IpaTrialResult> allResults,
|
|
IReadOnlyList<IpaTrialResult> selected,
|
|
IReadOnlyList<SongChallengeEvidenceDocument> allEvidence,
|
|
IReadOnlyList<SongChallengeEvidenceDocument> selectedEvidence,
|
|
SongTrialDistillationDocument distillation,
|
|
float minCosine)
|
|
{
|
|
var selectedIds = selected.Select(result => result.TrialId).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
|
var dropped = allResults.Where(result => !selectedIds.Contains(result.TrialId)).ToArray();
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("# Song Trial Distillation");
|
|
builder.AppendLine();
|
|
builder.AppendLine($"source_store: `{sourceStore}`");
|
|
builder.AppendLine($"output_store: `{outputStore}`");
|
|
builder.AppendLine($"min_cosine: `{minCosine.ToString("0.######", CultureInfo.InvariantCulture)}`");
|
|
builder.AppendLine($"input_trials: `{allResults.Count}`");
|
|
builder.AppendLine($"kept_trials: `{selected.Count}`");
|
|
builder.AppendLine($"dropped_trials: `{dropped.Length}`");
|
|
builder.AppendLine($"input_evidence_docs: `{allEvidence.Count}`");
|
|
builder.AppendLine($"kept_evidence_docs: `{selectedEvidence.Count}`");
|
|
builder.AppendLine($"distillation_document: `{distillation.DistillationId}`");
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Distilled Signal");
|
|
builder.AppendLine(distillation.Summary);
|
|
builder.AppendLine();
|
|
builder.AppendLine("### Reusable Scene Roles");
|
|
foreach (var role in distillation.ReusableSceneRoles)
|
|
{
|
|
builder.Append("- ");
|
|
builder.AppendLine(role);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("### Transfer Rules");
|
|
foreach (var rule in distillation.TransferRules)
|
|
{
|
|
builder.Append("- ");
|
|
builder.AppendLine(rule);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("### Failure Patterns");
|
|
foreach (var pattern in distillation.FailurePatterns)
|
|
{
|
|
builder.Append("- ");
|
|
builder.AppendLine(pattern);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Kept Trials");
|
|
foreach (var result in selected)
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(result.TrialId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.CandidateId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.Verdict);
|
|
builder.Append(" / cosine=");
|
|
builder.Append(Metric(result, "log_mel_cosine"));
|
|
builder.Append(" / score=");
|
|
builder.Append(Metric(result, "audio_score"));
|
|
builder.Append(" / articulation=");
|
|
builder.Append(Metric(result, "articulation_score"));
|
|
builder.Append(" / reference=");
|
|
builder.AppendLine(result.ReferenceId);
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Candidate Aggregates");
|
|
foreach (var group in selected.GroupBy(result => result.CandidateId, StringComparer.OrdinalIgnoreCase)
|
|
.OrderByDescending(group => group.Average(result => MetricValue(result, "log_mel_cosine"))))
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(group.Key);
|
|
builder.Append(": trials=");
|
|
builder.Append(group.Count());
|
|
builder.Append(", mean_cosine=");
|
|
builder.Append(group.Average(result => MetricValue(result, "log_mel_cosine")).ToString("0.######", CultureInfo.InvariantCulture));
|
|
builder.Append(", min_cosine=");
|
|
builder.Append(group.Min(result => MetricValue(result, "log_mel_cosine")).ToString("0.######", CultureInfo.InvariantCulture));
|
|
builder.Append(", max_cosine=");
|
|
builder.Append(group.Max(result => MetricValue(result, "log_mel_cosine")).ToString("0.######", CultureInfo.InvariantCulture));
|
|
builder.AppendLine();
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Dropped Noise");
|
|
foreach (var result in dropped.OrderBy(ScoreSort).Take(30))
|
|
{
|
|
builder.Append("- ");
|
|
builder.Append(result.TrialId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.CandidateId);
|
|
builder.Append(" / ");
|
|
builder.Append(result.Verdict);
|
|
builder.Append(" / cosine=");
|
|
builder.Append(Metric(result, "log_mel_cosine"));
|
|
builder.AppendLine();
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static SongTrialDistillationDocument SongTrialDistillation(
|
|
string sourceStore,
|
|
IReadOnlyList<IpaTrialResult> allResults,
|
|
IReadOnlyList<IpaTrialResult> selected,
|
|
float minCosine)
|
|
{
|
|
var droppedIds = allResults
|
|
.Where(result => !selected.Any(kept => kept.TrialId.Equals(result.TrialId, StringComparison.OrdinalIgnoreCase)))
|
|
.Select(result => result.TrialId)
|
|
.ToArray();
|
|
var top = selected.OrderByDescending(ScoreSort).Take(5).ToArray();
|
|
var bestCosine = selected.Select(result => MetricValue(result, "log_mel_cosine")).DefaultIfEmpty(0).Max();
|
|
var meanCosine = selected.Select(result => MetricValue(result, "log_mel_cosine")).DefaultIfEmpty(0).Average();
|
|
var meanScore = selected.Select(result => MetricValue(result, "audio_score")).DefaultIfEmpty(0).Average();
|
|
var text = string.Join(' ', selected.Select(result =>
|
|
$"{result.CandidateId} {result.Hypothesis} {result.EvaluationSummary} {string.Join(' ', result.KnownLies)}"));
|
|
var roles = SceneRolesFrom(text).ToArray();
|
|
var failures = FailurePatternsFrom(allResults, selected).ToArray();
|
|
var transferRules = TransferRulesFrom(selected, roles).ToArray();
|
|
var summary = selected.Count == 0
|
|
? $"No high-signal trials met min cosine {minCosine:0.###}; keep only best-per-reference scaffolding."
|
|
: $"Kept {selected.Count} of {allResults.Count} trials. Best cosine {bestCosine:0.######}; kept mean cosine {meanCosine:0.######}; best candidates: {string.Join(", ", top.Select(result => result.CandidateId).Distinct(StringComparer.OrdinalIgnoreCase).Take(5))}.";
|
|
return new SongTrialDistillationDocument(
|
|
$"song-distill-{DateTimeOffset.UtcNow:yyyyMMddTHHmmssfff}-{StableUuid(sourceStore)[..8]}",
|
|
sourceStore,
|
|
DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture),
|
|
allResults.Count,
|
|
selected.Count,
|
|
summary,
|
|
roles,
|
|
transferRules,
|
|
failures,
|
|
selected.Select(result => result.TrialId).ToArray(),
|
|
droppedIds,
|
|
[
|
|
new SpeechScoreMetric("distilled_best_log_mel_cosine", bestCosine, 0),
|
|
new SpeechScoreMetric("distilled_mean_log_mel_cosine", meanCosine, 0),
|
|
new SpeechScoreMetric("distilled_mean_audio_score", meanScore, 0),
|
|
new SpeechScoreMetric("distilled_kept_ratio", allResults.Count == 0 ? 0 : selected.Count / (float)allResults.Count, 0)
|
|
]);
|
|
}
|
|
|
|
static IEnumerable<string> SceneRolesFrom(string text)
|
|
{
|
|
var lower = text.ToLowerInvariant();
|
|
if (lower.Contains("formant", StringComparison.Ordinal) || lower.Contains("vowel", StringComparison.Ordinal))
|
|
{
|
|
yield return "bright formant/vowel lead: owns pitched synthetic body and vocal-ish spectral peaks.";
|
|
}
|
|
if (lower.Contains("dust", StringComparison.Ordinal) || lower.Contains("hat", StringComparison.Ordinal))
|
|
{
|
|
yield return "clocked dust/hat texture: owns short gated high-band transients through `texture role=dust`, not static white hiss.";
|
|
}
|
|
if (lower.Contains("codec", StringComparison.Ordinal) || lower.Contains("bit", StringComparison.Ordinal))
|
|
{
|
|
yield return "codec/bit bed: owns narrow mid-high grit through shaped `texture role=codec` with slow motion.";
|
|
}
|
|
if (lower.Contains("bass", StringComparison.Ordinal) || lower.Contains("sub", StringComparison.Ordinal))
|
|
{
|
|
yield return "rubber/sub bass: owns low rhythmic support and should stay register-bounded.";
|
|
}
|
|
if (lower.Contains("air", StringComparison.Ordinal) || lower.Contains("room", StringComparison.Ordinal) || lower.Contains("bed", StringComparison.Ordinal))
|
|
{
|
|
yield return "air/room bed: owns low-level recording color through shaped moving texture, not broad full-duration noise.";
|
|
}
|
|
}
|
|
|
|
static IEnumerable<string> TransferRulesFrom(IReadOnlyList<IpaTrialResult> selected, IReadOnlyList<string> roles)
|
|
{
|
|
if (roles.Count == 0)
|
|
{
|
|
yield return "Prefer target analysis artifacts over memorized song names; choose roles from tempo, register, band deltas, and envelope autocorr.";
|
|
}
|
|
else
|
|
{
|
|
yield return "Start zero-shot production by mapping target analysis to reusable scene roles, then instantiate one owner per role.";
|
|
}
|
|
|
|
if (selected.Any(result => MetricValue(result, "rms_ratio") is > 1.35f or < .7f))
|
|
{
|
|
yield return "Normalize loudness after role selection; high cosine can still hide bad RMS balance.";
|
|
}
|
|
|
|
yield return "Use `texture` for background/recording noise and reserve raw `wave=noise` for short gated transients with narrow filters.";
|
|
yield return "Treat repeated pressure verdicts as reusable direction, not acceptance; keep the patch family but mutate timing/register/noise shaping.";
|
|
}
|
|
|
|
static IEnumerable<string> FailurePatternsFrom(IReadOnlyList<IpaTrialResult> allResults, IReadOnlyList<IpaTrialResult> selected)
|
|
{
|
|
var weakCount = allResults.Count(result => result.Verdict.Equals("weak", StringComparison.OrdinalIgnoreCase));
|
|
if (weakCount > 0)
|
|
{
|
|
yield return $"{weakCount} weak trials are dropped from retrieval pressure unless they were best for their reference.";
|
|
}
|
|
|
|
if (allResults.Any(result => MetricValue(result, "log_mel_cosine") < 0))
|
|
{
|
|
yield return "Negative cosine candidates are curriculum noise unless preserved as contrast; they should not steer zero-shot production.";
|
|
}
|
|
|
|
if (selected.Any(result => result.KnownLies.Any(lie => lie.Contains("noise", StringComparison.OrdinalIgnoreCase))))
|
|
{
|
|
yield return "Scene/noise helper voices are allowed, but only shaped role owners should transfer.";
|
|
}
|
|
}
|
|
|
|
static float ScoreSort(IpaTrialResult result) =>
|
|
result.Metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? float.NegativeInfinity;
|
|
|
|
static string Metric(IpaTrialResult result, string name) =>
|
|
(result.Metrics.FirstOrDefault(metric => metric.Name == name)?.Value ?? 0)
|
|
.ToString("0.######", CultureInfo.InvariantCulture);
|
|
|
|
static string TimelineFact(IpaTrialResult result, string name) =>
|
|
(result.TimelineFacts?
|
|
.Where(fact => fact.Name.Equals(name, StringComparison.OrdinalIgnoreCase))
|
|
.Select(fact => fact.Value)
|
|
.DefaultIfEmpty(0)
|
|
.Max() ?? 0)
|
|
.ToString("0.######", CultureInfo.InvariantCulture);
|
|
|
|
static SpeechRenderArtifact Artifact(string kind, string path) =>
|
|
new(kind, path, File.Exists(path) ? Sha256(path) : "");
|
|
|
|
static IEnumerable<SpeechRenderArtifact> SongChallengeArtifacts(SongChallenge challenge)
|
|
{
|
|
if (challenge.Artifacts is null)
|
|
{
|
|
yield break;
|
|
}
|
|
|
|
yield return Artifact("target-analysis-report", challenge.Artifacts.AnalysisReportMarkdown);
|
|
yield return Artifact("target-logmel-spectrogram", challenge.Artifacts.LogMelSpectrogramCsv);
|
|
yield return Artifact("target-logmel-band-stats", challenge.Artifacts.LogMelBandStatsCsv);
|
|
yield return Artifact("target-rms-envelope", challenge.Artifacts.RmsEnvelopeCsv);
|
|
yield return Artifact("target-rms-envelope-autocorr", challenge.Artifacts.RmsEnvelopeAutocorrCsv);
|
|
}
|
|
|
|
static SpeechRenderArtifact SongChallengeEvidenceArtifact(SongChallengeEvidenceDocument document) =>
|
|
new(
|
|
$"cultmesh-song-challenge-{document.Kind}",
|
|
$"cultmesh://aquasynth/song-challenge-evidence/{Uri.EscapeDataString(document.EvidenceId)}",
|
|
document.ContentHash);
|
|
|
|
static IReadOnlyList<SongChallengeEvidenceDocument> SongChallengeEvidenceDocuments(SongChallenge challenge)
|
|
{
|
|
var documents = new List<SongChallengeEvidenceDocument>();
|
|
if (challenge.Artifacts is null)
|
|
{
|
|
return documents;
|
|
}
|
|
|
|
AddSongChallengeEvidence(documents, challenge, "analysis-report", "text/markdown", challenge.Artifacts.AnalysisReportMarkdown);
|
|
AddSongChallengeEvidence(documents, challenge, "logmel-spectrogram", "text/csv", challenge.Artifacts.LogMelSpectrogramCsv);
|
|
AddSongChallengeEvidence(documents, challenge, "logmel-band-stats", "text/csv", challenge.Artifacts.LogMelBandStatsCsv);
|
|
AddSongChallengeEvidence(documents, challenge, "rms-envelope", "text/csv", challenge.Artifacts.RmsEnvelopeCsv);
|
|
AddSongChallengeEvidence(documents, challenge, "rms-envelope-autocorr", "text/csv", challenge.Artifacts.RmsEnvelopeAutocorrCsv);
|
|
return documents;
|
|
}
|
|
|
|
static void AddSongChallengeEvidence(
|
|
List<SongChallengeEvidenceDocument> documents,
|
|
SongChallenge challenge,
|
|
string kind,
|
|
string contentType,
|
|
string path)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
|
|
{
|
|
return;
|
|
}
|
|
|
|
documents.Add(new SongChallengeEvidenceDocument(
|
|
$"{challenge.ChallengeId}:{kind}",
|
|
challenge.ChallengeId,
|
|
kind,
|
|
contentType,
|
|
File.ReadAllText(path, Encoding.UTF8),
|
|
Sha256(path),
|
|
challenge.SourcePath,
|
|
DateTimeOffset.UtcNow.ToString("O", CultureInfo.InvariantCulture)));
|
|
}
|
|
|
|
static AutocorrelationPoint ReadAutocorrPeak(string? path)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(path) || !File.Exists(path))
|
|
{
|
|
return new AutocorrelationPoint(0, 0, 0);
|
|
}
|
|
|
|
return File.ReadLines(path, Encoding.UTF8)
|
|
.Skip(1)
|
|
.Select(line => line.Split(','))
|
|
.Where(parts => parts.Length >= 3)
|
|
.Select(parts => new AutocorrelationPoint(
|
|
int.TryParse(parts[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out var lagFrames) ? lagFrames : 0,
|
|
float.TryParse(parts[1], NumberStyles.Float, CultureInfo.InvariantCulture, out var lagSeconds) ? lagSeconds : 0,
|
|
float.TryParse(parts[2], NumberStyles.Float, CultureInfo.InvariantCulture, out var correlation) ? correlation : 0))
|
|
.OrderByDescending(point => point.Correlation)
|
|
.FirstOrDefault() ?? new AutocorrelationPoint(0, 0, 0);
|
|
}
|
|
|
|
static string Sha256(string path)
|
|
{
|
|
using var stream = File.OpenRead(path);
|
|
var hash = SHA256.HashData(stream);
|
|
return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant();
|
|
}
|
|
|
|
static string SafeName(string value) =>
|
|
new(value.Select(character => char.IsLetterOrDigit(character) ? character : '_').ToArray());
|
|
|
|
static string Escape(string value) =>
|
|
value.Contains(',') || value.Contains('"') || value.Contains('\n') || value.Contains('\r')
|
|
? $"\"{value.Replace("\"", "\"\"", StringComparison.Ordinal)}\""
|
|
: value;
|
|
|
|
static IpaTrialResult? ContrastCandidate(IpaTrialResult result, IReadOnlyList<IpaTrialResult> allResults)
|
|
{
|
|
var sameTarget = allResults
|
|
.Where(item => !ReferenceEquals(item, result) && item.TargetSetId.Equals(result.TargetSetId, StringComparison.OrdinalIgnoreCase))
|
|
.OrderByDescending(item => Math.Abs(ScoreSort(item) - ScoreSort(result)))
|
|
.ThenByDescending(item => item.Metrics.FirstOrDefault(metric => metric.Name == "articulation_score")?.Value ?? 0)
|
|
.FirstOrDefault();
|
|
if (sameTarget is not null)
|
|
{
|
|
return sameTarget;
|
|
}
|
|
|
|
return allResults
|
|
.Where(item => !ReferenceEquals(item, result) && item.Phonemes.Intersect(result.Phonemes, StringComparer.OrdinalIgnoreCase).Any())
|
|
.OrderByDescending(item => Math.Abs(ScoreSort(item) - ScoreSort(result)))
|
|
.FirstOrDefault();
|
|
}
|
|
|
|
static string OneLine(string text, int maxLength)
|
|
{
|
|
var normalized = text.Replace('\r', ' ').Replace('\n', ' ').Trim();
|
|
return normalized.Length <= maxLength ? normalized : normalized[..maxLength] + "...";
|
|
}
|
|
|
|
static string StoreKey(string store)
|
|
{
|
|
var fullPath = Path.GetFullPath(store).ToLowerInvariant();
|
|
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(fullPath));
|
|
return Convert.ToHexString(hash)[..16].ToLowerInvariant();
|
|
}
|
|
|
|
static string StableUuid(string id)
|
|
{
|
|
var digest = SHA256.HashData(Encoding.UTF8.GetBytes(id));
|
|
digest[6] = (byte)((digest[6] & 0x0f) | 0x50);
|
|
digest[8] = (byte)((digest[8] & 0x3f) | 0x80);
|
|
var hex = Convert.ToHexString(digest[..16]).ToLowerInvariant();
|
|
return $"{hex[..8]}-{hex[8..12]}-{hex[12..16]}-{hex[16..20]}-{hex[20..32]}";
|
|
}
|
|
|
|
static DecodedAudio DecodeMono(string path)
|
|
{
|
|
using var reader = new AudioFileReader(path);
|
|
var interleaved = new float[4096 * reader.WaveFormat.Channels];
|
|
var mono = new List<float>();
|
|
int read;
|
|
while ((read = reader.Read(interleaved, 0, interleaved.Length)) > 0)
|
|
{
|
|
for (var index = 0; index < read; index += reader.WaveFormat.Channels)
|
|
{
|
|
var sum = 0f;
|
|
for (var channel = 0; channel < reader.WaveFormat.Channels && index + channel < read; channel++)
|
|
{
|
|
sum += interleaved[index + channel];
|
|
}
|
|
|
|
mono.Add(sum / reader.WaveFormat.Channels);
|
|
}
|
|
}
|
|
|
|
return new DecodedAudio(reader.WaveFormat.SampleRate, mono.ToArray());
|
|
}
|
|
|
|
static float[] Resample(IReadOnlyList<float> samples, int sourceRate, int targetRate)
|
|
{
|
|
if (sourceRate == targetRate)
|
|
{
|
|
return samples.ToArray();
|
|
}
|
|
|
|
var result = new float[Math.Max(1, (int)Math.Round(samples.Count * (double)targetRate / sourceRate))];
|
|
for (var index = 0; index < result.Length; index++)
|
|
{
|
|
var position = index * (sourceRate / (double)targetRate);
|
|
var left = Math.Clamp((int)Math.Floor(position), 0, samples.Count - 1);
|
|
var right = Math.Clamp(left + 1, 0, samples.Count - 1);
|
|
var t = (float)(position - left);
|
|
result[index] = samples[left] * (1 - t) + samples[right] * t;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
static float[] MatchLength(IReadOnlyList<float> samples, int length)
|
|
{
|
|
var result = new float[length];
|
|
for (var index = 0; index < result.Length && index < samples.Count; index++)
|
|
{
|
|
result[index] = samples[index];
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
static void NormalizePeak(IList<float> samples, float peak)
|
|
{
|
|
var current = samples.Select(MathF.Abs).DefaultIfEmpty(0).Max();
|
|
if (current <= 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var gain = peak / current;
|
|
for (var index = 0; index < samples.Count; index++)
|
|
{
|
|
samples[index] *= gain;
|
|
}
|
|
}
|
|
|
|
static float[] ReadMonoPcm16Wav(string path)
|
|
{
|
|
using var stream = File.OpenRead(path);
|
|
using var reader = new BinaryReader(stream, Encoding.ASCII);
|
|
if (new string(reader.ReadChars(4)) != "RIFF")
|
|
{
|
|
throw new InvalidDataException($"`{path}` is not a RIFF WAV file.");
|
|
}
|
|
|
|
reader.ReadInt32();
|
|
if (new string(reader.ReadChars(4)) != "WAVE")
|
|
{
|
|
throw new InvalidDataException($"`{path}` is not a WAVE file.");
|
|
}
|
|
|
|
short channels = 1;
|
|
short bitsPerSample = 16;
|
|
byte[]? data = null;
|
|
while (stream.Position < stream.Length)
|
|
{
|
|
var chunkId = new string(reader.ReadChars(4));
|
|
var chunkSize = reader.ReadInt32();
|
|
if (chunkId == "fmt ")
|
|
{
|
|
var format = reader.ReadInt16();
|
|
channels = reader.ReadInt16();
|
|
reader.ReadInt32();
|
|
reader.ReadInt32();
|
|
reader.ReadInt16();
|
|
bitsPerSample = reader.ReadInt16();
|
|
if (chunkSize > 16)
|
|
{
|
|
reader.ReadBytes(chunkSize - 16);
|
|
}
|
|
|
|
if (format != 1 || bitsPerSample != 16)
|
|
{
|
|
throw new InvalidDataException($"`{path}` must be PCM16.");
|
|
}
|
|
}
|
|
else if (chunkId == "data")
|
|
{
|
|
data = reader.ReadBytes(chunkSize);
|
|
}
|
|
else
|
|
{
|
|
reader.ReadBytes(chunkSize);
|
|
}
|
|
}
|
|
|
|
if (data is null)
|
|
{
|
|
throw new InvalidDataException($"`{path}` has no data chunk.");
|
|
}
|
|
|
|
var samples = new List<float>(data.Length / Math.Max(1, channels * sizeof(short)));
|
|
for (var offset = 0; offset + sizeof(short) * channels <= data.Length; offset += sizeof(short) * channels)
|
|
{
|
|
var sum = 0f;
|
|
for (var channel = 0; channel < channels; channel++)
|
|
{
|
|
sum += BitConverter.ToInt16(data, offset + channel * sizeof(short)) / (float)short.MaxValue;
|
|
}
|
|
|
|
samples.Add(sum / channels);
|
|
}
|
|
|
|
return samples.ToArray();
|
|
}
|
|
|
|
static void WriteWav(string path, IReadOnlyList<float> samples, int sampleRate)
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
|
using var stream = File.Create(path);
|
|
using var writer = new BinaryWriter(stream, Encoding.ASCII);
|
|
var dataSize = samples.Count * sizeof(short);
|
|
writer.Write("RIFF"u8);
|
|
writer.Write(36 + dataSize);
|
|
writer.Write("WAVE"u8);
|
|
writer.Write("fmt "u8);
|
|
writer.Write(16);
|
|
writer.Write((short)1);
|
|
writer.Write((short)1);
|
|
writer.Write(sampleRate);
|
|
writer.Write(sampleRate * sizeof(short));
|
|
writer.Write((short)sizeof(short));
|
|
writer.Write((short)16);
|
|
writer.Write("data"u8);
|
|
writer.Write(dataSize);
|
|
foreach (var sample in samples)
|
|
{
|
|
writer.Write((short)Math.Clamp(MathF.Round(sample * short.MaxValue), short.MinValue, short.MaxValue));
|
|
}
|
|
}
|
|
|
|
static float ActiveDuty(IReadOnlyList<float> samples, int sampleRate)
|
|
{
|
|
var frameSize = Math.Max(64, sampleRate / 100);
|
|
var peak = samples.Select(MathF.Abs).DefaultIfEmpty(0).Max();
|
|
var threshold = Math.Max(.01f, peak * .04f);
|
|
var active = 0;
|
|
var frames = 0;
|
|
for (var start = 0; start < samples.Count; start += frameSize)
|
|
{
|
|
var end = Math.Min(samples.Count, start + frameSize);
|
|
var rms = MathF.Sqrt(samples.Skip(start).Take(end - start).Select(sample => sample * sample).DefaultIfEmpty(0).Average());
|
|
if (rms >= threshold)
|
|
{
|
|
active++;
|
|
}
|
|
|
|
frames++;
|
|
}
|
|
|
|
return active / (float)Math.Max(1, frames);
|
|
}
|
|
|
|
static float SpectralFlux(IReadOnlyList<float> samples, int sampleRate)
|
|
{
|
|
var frameSize = Math.Max(64, sampleRate / 100);
|
|
var previous = 0f;
|
|
var flux = 0f;
|
|
var frames = 0;
|
|
for (var start = 0; start < samples.Count; start += frameSize)
|
|
{
|
|
var end = Math.Min(samples.Count, start + frameSize);
|
|
var rms = MathF.Sqrt(samples.Skip(start).Take(end - start).Select(sample => sample * sample).DefaultIfEmpty(0).Average());
|
|
if (frames > 0)
|
|
{
|
|
flux += MathF.Abs(rms - previous);
|
|
}
|
|
|
|
previous = rms;
|
|
frames++;
|
|
}
|
|
|
|
var peak = samples.Select(MathF.Abs).DefaultIfEmpty(0).Max();
|
|
return frames <= 1 || peak <= 0 ? 0 : Math.Clamp(flux / (frames * peak), 0, 1);
|
|
}
|
|
|
|
static TempoEstimate EstimateTempo(IReadOnlyList<float> samples, int sampleRate)
|
|
{
|
|
var hopSize = Math.Max(64, sampleRate / 100);
|
|
var frameRate = sampleRate / (float)hopSize;
|
|
var envelope = RmsEnvelope(samples, hopSize)
|
|
.Select(value => MathF.Log(1e-7f + value))
|
|
.ToArray();
|
|
if (envelope.Length < 4)
|
|
{
|
|
return new TempoEstimate(0, 0, 0);
|
|
}
|
|
|
|
var onset = new float[envelope.Length];
|
|
for (var index = 1; index < envelope.Length; index++)
|
|
{
|
|
onset[index] = MathF.Max(0, envelope[index] - envelope[index - 1]);
|
|
}
|
|
|
|
var mean = onset.Average();
|
|
for (var index = 0; index < onset.Length; index++)
|
|
{
|
|
onset[index] = MathF.Max(0, onset[index] - mean);
|
|
}
|
|
|
|
var minBpm = 60f;
|
|
var maxBpm = 200f;
|
|
var minLag = Math.Max(1, (int)MathF.Floor(frameRate * 60f / maxBpm));
|
|
var maxLag = Math.Min(onset.Length - 1, (int)MathF.Ceiling(frameRate * 60f / minBpm));
|
|
var bestLag = 0;
|
|
var best = 0f;
|
|
var energy = onset.Sum(value => value * value);
|
|
if (energy <= 1e-9f)
|
|
{
|
|
return new TempoEstimate(0, 0, 0);
|
|
}
|
|
|
|
for (var lag = minLag; lag <= maxLag; lag++)
|
|
{
|
|
var sum = 0f;
|
|
for (var index = 0; index + lag < onset.Length; index++)
|
|
{
|
|
sum += onset[index] * onset[index + lag];
|
|
}
|
|
|
|
var normalized = sum / energy;
|
|
if (normalized > best)
|
|
{
|
|
best = normalized;
|
|
bestLag = lag;
|
|
}
|
|
}
|
|
|
|
if (bestLag <= 0)
|
|
{
|
|
return new TempoEstimate(0, 0, 0);
|
|
}
|
|
|
|
var beatSeconds = bestLag / frameRate;
|
|
var bpm = 60f / Math.Max(1e-6f, beatSeconds);
|
|
return new TempoEstimate(bpm, beatSeconds, Math.Clamp(best, 0, 1));
|
|
}
|
|
|
|
static SongRegister EstimateRegister(IReadOnlyList<float> samples, int sampleRate, float spectralCentroidHz)
|
|
{
|
|
var dominantHz = DominantFrequency(samples, sampleRate);
|
|
if (dominantHz <= 0)
|
|
{
|
|
var fallback = Math.Clamp(spectralCentroidHz * .25f, 55f, 880f);
|
|
dominantHz = float.IsFinite(fallback) ? fallback : 220f;
|
|
}
|
|
|
|
var midi = (int)MathF.Round(69f + 12f * MathF.Log2(dominantHz / 440f));
|
|
var rootMidi = midi;
|
|
while (rootMidi > 72)
|
|
{
|
|
rootMidi -= 12;
|
|
}
|
|
|
|
while (rootMidi < 36)
|
|
{
|
|
rootMidi += 12;
|
|
}
|
|
|
|
var rootHz = MidiToFrequency(rootMidi);
|
|
var lowHz = rootHz;
|
|
while (lowHz > 220f)
|
|
{
|
|
lowHz *= .5f;
|
|
}
|
|
|
|
while (lowHz < 55f)
|
|
{
|
|
lowHz *= 2f;
|
|
}
|
|
|
|
var highHz = Math.Min(4000f, lowHz * 8f);
|
|
var scaleName = spectralCentroidHz > 1800f ? "minor-pentatonic-plus-tritone" : "minor-pentatonic";
|
|
var intervals = scaleName == "minor-pentatonic-plus-tritone"
|
|
? new[] { 0, 3, 5, 6, 7, 10, 12, 15, 17, 18, 19, 22, 24 }
|
|
: new[] { 0, 3, 5, 7, 10, 12, 15, 17, 19, 22, 24 };
|
|
var scaleFrequencies = intervals
|
|
.Select(interval => MidiToFrequency(rootMidi + interval))
|
|
.Where(value => value >= lowHz * .9f && value <= highHz * 1.1f)
|
|
.ToArray();
|
|
|
|
return new SongRegister(
|
|
dominantHz,
|
|
lowHz,
|
|
highHz,
|
|
NoteName(rootMidi),
|
|
scaleName,
|
|
scaleFrequencies);
|
|
}
|
|
|
|
static float DominantFrequency(IReadOnlyList<float> samples, int sampleRate)
|
|
{
|
|
if (samples.Count < 128)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
var windowSize = Math.Min(samples.Count, Math.Max(2048, sampleRate / 5));
|
|
var hopSize = Math.Max(256, windowSize / 4);
|
|
var bestStart = 0;
|
|
var bestEnergy = 0f;
|
|
for (var start = 0; start + windowSize <= samples.Count; start += hopSize)
|
|
{
|
|
var energy = 0f;
|
|
for (var index = start; index < start + windowSize; index++)
|
|
{
|
|
energy += samples[index] * samples[index];
|
|
}
|
|
|
|
if (energy > bestEnergy)
|
|
{
|
|
bestEnergy = energy;
|
|
bestStart = start;
|
|
}
|
|
}
|
|
|
|
if (bestEnergy <= 1e-9f)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
var minLag = Math.Max(1, sampleRate / 2000);
|
|
var maxLag = Math.Min(windowSize - 1, sampleRate / 40);
|
|
var bestLag = 0;
|
|
var bestScore = 0f;
|
|
for (var lag = minLag; lag <= maxLag; lag++)
|
|
{
|
|
var sum = 0f;
|
|
var left = 0f;
|
|
var right = 0f;
|
|
for (var offset = 0; offset + lag < windowSize; offset++)
|
|
{
|
|
var a = samples[bestStart + offset];
|
|
var b = samples[bestStart + offset + lag];
|
|
sum += a * b;
|
|
left += a * a;
|
|
right += b * b;
|
|
}
|
|
|
|
var normalized = sum / MathF.Sqrt(Math.Max(1e-12f, left * right));
|
|
if (normalized > bestScore)
|
|
{
|
|
bestScore = normalized;
|
|
bestLag = lag;
|
|
}
|
|
}
|
|
|
|
return bestLag <= 0 || bestScore < .08f ? 0 : sampleRate / (float)bestLag;
|
|
}
|
|
|
|
static float MidiToFrequency(int midi) =>
|
|
440f * MathF.Pow(2f, (midi - 69) / 12f);
|
|
|
|
static string NoteName(int midi)
|
|
{
|
|
ReadOnlySpan<string> names = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"];
|
|
var octave = midi / 12 - 1;
|
|
var pitchClass = ((midi % 12) + 12) % 12;
|
|
return $"{names[pitchClass]}{octave}";
|
|
}
|
|
|
|
static float[] RmsEnvelope(IReadOnlyList<float> samples, int frameSize)
|
|
{
|
|
var envelope = new List<float>();
|
|
for (var start = 0; start < samples.Count; start += frameSize)
|
|
{
|
|
var end = Math.Min(samples.Count, start + frameSize);
|
|
var sum = 0f;
|
|
for (var index = start; index < end; index++)
|
|
{
|
|
sum += samples[index] * samples[index];
|
|
}
|
|
|
|
envelope.Add(MathF.Sqrt(sum / Math.Max(1, end - start)));
|
|
}
|
|
|
|
return envelope.ToArray();
|
|
}
|
|
|
|
static string SpectrogramCsv(Spectrogram spectrogram)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("frame,band,value");
|
|
for (var frame = 0; frame < spectrogram.Frames; frame++)
|
|
{
|
|
for (var band = 0; band < spectrogram.Bands; band++)
|
|
{
|
|
builder.Append(frame).Append(',');
|
|
builder.Append(band).Append(',');
|
|
builder.AppendLine(spectrogram.At(frame, band).ToString("0.########", CultureInfo.InvariantCulture));
|
|
}
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string SpectrogramBandStatsCsv(Spectrogram spectrogram)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("band,mean,min,max,stddev,delta_mean,abs_delta_mean,delta2_mean,abs_delta2_mean");
|
|
foreach (var band in SpectrogramBandStats(spectrogram))
|
|
{
|
|
builder.Append(band.Band).Append(',');
|
|
builder.Append(band.Mean.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.Min.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.Max.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.StdDev.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.DeltaMean.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.AbsDeltaMean.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.Append(band.Delta2Mean.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.AppendLine(band.AbsDelta2Mean.ToString("0.########", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static IReadOnlyList<SpectrogramBandSummary> SpectrogramBandStats(Spectrogram spectrogram)
|
|
{
|
|
var summaries = new List<SpectrogramBandSummary>();
|
|
for (var band = 0; band < spectrogram.Bands; band++)
|
|
{
|
|
var values = new float[Math.Max(0, spectrogram.Frames)];
|
|
for (var frame = 0; frame < values.Length; frame++)
|
|
{
|
|
values[frame] = spectrogram.At(frame, band);
|
|
}
|
|
|
|
var deltas = new float[Math.Max(0, values.Length - 1)];
|
|
for (var index = 1; index < values.Length; index++)
|
|
{
|
|
deltas[index - 1] = values[index] - values[index - 1];
|
|
}
|
|
|
|
var delta2 = new float[Math.Max(0, deltas.Length - 1)];
|
|
for (var index = 1; index < deltas.Length; index++)
|
|
{
|
|
delta2[index - 1] = deltas[index] - deltas[index - 1];
|
|
}
|
|
|
|
summaries.Add(new SpectrogramBandSummary(
|
|
band,
|
|
Mean(values),
|
|
values.Length == 0 ? 0 : values.Min(),
|
|
values.Length == 0 ? 0 : values.Max(),
|
|
StdDev(values),
|
|
Mean(deltas),
|
|
MeanAbs(deltas),
|
|
Mean(delta2),
|
|
MeanAbs(delta2)));
|
|
}
|
|
|
|
return summaries;
|
|
}
|
|
|
|
static string EnvelopeCsv(IReadOnlyList<float> envelope, float sampleRate, int hopSize)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("frame,time_seconds,rms");
|
|
var frameRate = sampleRate / Math.Max(1, hopSize);
|
|
for (var frame = 0; frame < envelope.Count; frame++)
|
|
{
|
|
builder.Append(frame).Append(',');
|
|
builder.Append((frame / Math.Max(1e-6f, frameRate)).ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.AppendLine(envelope[frame].ToString("0.########", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string AutocorrelationCsv(IReadOnlyList<float> values, float frameRate)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("lag_frames,lag_seconds,correlation");
|
|
foreach (var point in Autocorrelation(values, frameRate, Math.Min(Math.Max(1, values.Count - 1), 512)))
|
|
{
|
|
builder.Append(point.LagFrames).Append(',');
|
|
builder.Append(point.LagSeconds.ToString("0.########", CultureInfo.InvariantCulture)).Append(',');
|
|
builder.AppendLine(point.Correlation.ToString("0.########", CultureInfo.InvariantCulture));
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static IReadOnlyList<AutocorrelationPoint> Autocorrelation(IReadOnlyList<float> values, float frameRate, int maxLags)
|
|
{
|
|
if (values.Count < 2)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var mean = Mean(values);
|
|
var centered = values.Select(value => value - mean).ToArray();
|
|
var energy = centered.Sum(value => value * value);
|
|
if (energy <= 1e-12f)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
var limit = Math.Min(maxLags, values.Count - 1);
|
|
var points = new List<AutocorrelationPoint>(limit);
|
|
for (var lag = 1; lag <= limit; lag++)
|
|
{
|
|
var sum = 0f;
|
|
for (var index = 0; index + lag < centered.Length; index++)
|
|
{
|
|
sum += centered[index] * centered[index + lag];
|
|
}
|
|
|
|
points.Add(new AutocorrelationPoint(
|
|
lag,
|
|
lag / Math.Max(1e-6f, frameRate),
|
|
sum / energy));
|
|
}
|
|
|
|
return points;
|
|
}
|
|
|
|
static float Mean(IReadOnlyList<float> values) =>
|
|
values.Count == 0 ? 0 : values.Sum() / values.Count;
|
|
|
|
static float MeanAbs(IReadOnlyList<float> values) =>
|
|
values.Count == 0 ? 0 : values.Sum(value => MathF.Abs(value)) / values.Count;
|
|
|
|
static float StdDev(IReadOnlyList<float> values)
|
|
{
|
|
if (values.Count == 0)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
var mean = Mean(values);
|
|
var variance = values.Sum(value => (value - mean) * (value - mean)) / values.Count;
|
|
return MathF.Sqrt(Math.Max(0, variance));
|
|
}
|
|
|
|
static IReadOnlyList<SpeechScoreMetric> SongTargetMetrics(
|
|
SongChallenge challenge,
|
|
IReadOnlyList<SongChallengeEvidenceDocument> evidence)
|
|
{
|
|
var metrics = new List<SpeechScoreMetric>
|
|
{
|
|
new("target_duration_seconds", challenge.DurationSeconds, 0),
|
|
new("target_rms", challenge.Features.Rms, 0),
|
|
new("target_active_duty", challenge.Features.ActiveDuty, 0),
|
|
new("target_spectral_centroid_hz", challenge.Features.SpectralCentroidHz, 0),
|
|
new("target_spectral_flux", challenge.Features.SpectralFlux, 0),
|
|
new("target_tempo_bpm", challenge.Features.TempoBpm, 0),
|
|
new("target_tempo_confidence", challenge.Features.TempoConfidence, 0),
|
|
new("target_dominant_hz", challenge.Features.DominantHz, 0),
|
|
new("target_register_low_hz", challenge.Features.RegisterLowHz, 0),
|
|
new("target_register_high_hz", challenge.Features.RegisterHighHz, 0),
|
|
new("target_evidence_document_count", evidence.Count, 0)
|
|
};
|
|
|
|
var autocorrPeak = ReadAutocorrPeak(challenge.Artifacts?.RmsEnvelopeAutocorrCsv);
|
|
metrics.Add(new SpeechScoreMetric("target_envelope_autocorr_peak", autocorrPeak.Correlation, 0));
|
|
metrics.Add(new SpeechScoreMetric("target_envelope_autocorr_peak_lag_seconds", autocorrPeak.LagSeconds, 0));
|
|
return metrics;
|
|
}
|
|
|
|
static IReadOnlyList<SpeechScoreMetric> SongComparisonMetrics(AudioComparison comparison) =>
|
|
[
|
|
new("log_mel_cosine", comparison.LogMelCosineSimilarity, .25f),
|
|
new("log_mel_distance", comparison.LogMelDistance, .15f),
|
|
new("audio_score", comparison.Score, .15f),
|
|
new("envelope_distance", comparison.EnvelopeDistance, .10f),
|
|
new("rms_ratio", comparison.RmsRatio, .10f),
|
|
new("centroid_ratio", comparison.CentroidRatio, .10f),
|
|
new("zero_crossing_ratio", comparison.ZeroCrossingRatio, .05f),
|
|
new("articulation_score", comparison.Articulation.ArticulationScore, .05f),
|
|
new("speech_band_ratio", comparison.Articulation.SpeechBandRatio, .05f)
|
|
];
|
|
|
|
static string SongVerdict(AudioComparison comparison)
|
|
{
|
|
if (comparison.LogMelCosineSimilarity >= .70f && comparison.Score >= .45f)
|
|
{
|
|
return "promising";
|
|
}
|
|
|
|
if (comparison.LogMelCosineSimilarity >= .35f || comparison.Score >= .30f)
|
|
{
|
|
return "pressure";
|
|
}
|
|
|
|
return "weak";
|
|
}
|
|
|
|
static float SongConfidence(IReadOnlyList<SpeechScoreMetric> metrics)
|
|
{
|
|
var cosine = metrics.FirstOrDefault(metric => metric.Name == "log_mel_cosine")?.Value ?? 0;
|
|
var score = metrics.FirstOrDefault(metric => metric.Name == "audio_score")?.Value ?? 0;
|
|
return Math.Clamp((cosine + score) * .5f, 0, 1);
|
|
}
|
|
|
|
static string SongEvaluationSentence(SongChallenge challenge, AudioComparison comparison, string verdict) =>
|
|
$"Song snippet `{challenge.ChallengeId}` is `{verdict}`: logMelCosine={comparison.LogMelCosineSimilarity:0.0000}, logMelDistance={comparison.LogMelDistance:0.0000}, score={comparison.Score:0.0000}, rmsRatio={comparison.RmsRatio:0.0000}, centroidRatio={comparison.CentroidRatio:0.0000}.";
|
|
|
|
static string SongAnalysisReport(SongChallenge challenge, AudioAnalysis analysis)
|
|
{
|
|
var bands = SpectrogramBandStats(analysis.LogMelSpectrogram);
|
|
var brightest = bands.OrderByDescending(band => band.Mean).Take(5).Select(band => band.Band).ToArray();
|
|
var mostMoving = bands.OrderByDescending(band => band.AbsDeltaMean).Take(5).Select(band => band.Band).ToArray();
|
|
var mostAccelerating = bands.OrderByDescending(band => band.AbsDelta2Mean).Take(5).Select(band => band.Band).ToArray();
|
|
var autocorr = Autocorrelation(analysis.RmsEnvelope, analysis.RmsEnvelope.Length / Math.Max(1f, challenge.DurationSeconds), maxLags: 64)
|
|
.OrderByDescending(point => point.Correlation)
|
|
.Take(5)
|
|
.ToArray();
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine($"# Song Challenge Analysis: {challenge.ChallengeId}");
|
|
builder.AppendLine();
|
|
builder.AppendLine($"spectrogram_frames: `{analysis.LogMelSpectrogram.Frames}`");
|
|
builder.AppendLine($"spectrogram_bands: `{analysis.LogMelSpectrogram.Bands}`");
|
|
builder.AppendLine($"rms_envelope_frames: `{analysis.RmsEnvelope.Length}`");
|
|
builder.AppendLine($"brightest_bands: `{string.Join(",", brightest)}`");
|
|
builder.AppendLine($"highest_delta_bands: `{string.Join(",", mostMoving)}`");
|
|
builder.AppendLine($"highest_delta2_bands: `{string.Join(",", mostAccelerating)}`");
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Strong Envelope Autocorrelation Lags");
|
|
foreach (var point in autocorr)
|
|
{
|
|
builder.AppendLine(CultureInfo.InvariantCulture, $"- lag_frames `{point.LagFrames}`, lag_seconds `{point.LagSeconds:0.######}`, correlation `{point.Correlation:0.######}`");
|
|
}
|
|
|
|
builder.AppendLine();
|
|
builder.AppendLine("## Artifact Paths");
|
|
builder.AppendLine($"- log_mel_spectrogram_csv: `{challenge.Artifacts?.LogMelSpectrogramCsv}`");
|
|
builder.AppendLine($"- log_mel_band_stats_csv: `{challenge.Artifacts?.LogMelBandStatsCsv}`");
|
|
builder.AppendLine($"- rms_envelope_csv: `{challenge.Artifacts?.RmsEnvelopeCsv}`");
|
|
builder.AppendLine($"- rms_envelope_autocorr_csv: `{challenge.Artifacts?.RmsEnvelopeAutocorrCsv}`");
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string SongChallengeReport(SongChallenge challenge) =>
|
|
$"""
|
|
# Song Snippet Challenge: {challenge.ChallengeId}
|
|
|
|
source: `{challenge.SourcePath}`
|
|
file: `{challenge.SourceFileName}`
|
|
seed: `{challenge.Seed}`
|
|
start_seconds: `{challenge.StartSeconds.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
duration_seconds: `{challenge.DurationSeconds.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
sample_rate: `{challenge.SampleRate}`
|
|
reference_wav: `{challenge.ReferenceWavPath}`
|
|
|
|
## Reference Features
|
|
peak: `{challenge.Features.Peak.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
rms: `{challenge.Features.Rms.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
active_duty: `{challenge.Features.ActiveDuty.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
zero_crossing_rate: `{challenge.Features.ZeroCrossingRate.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
spectral_centroid_hz: `{challenge.Features.SpectralCentroidHz.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
spectral_rolloff_hz: `{challenge.Features.SpectralRolloffHz.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
spectral_flux: `{challenge.Features.SpectralFlux.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
tempo_bpm: `{challenge.Features.TempoBpm.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
beat_seconds: `{challenge.Features.BeatSeconds.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
tempo_confidence: `{challenge.Features.TempoConfidence.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
dominant_hz: `{challenge.Features.DominantHz.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
register_low_hz: `{challenge.Features.RegisterLowHz.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
register_high_hz: `{challenge.Features.RegisterHighHz.ToString("0.######", CultureInfo.InvariantCulture)}`
|
|
root_note: `{challenge.Features.RootNote}`
|
|
suggested_scale: `{challenge.Features.SuggestedScale}`
|
|
scale_frequencies_hz: `{challenge.Features.ScaleFrequenciesHz}`
|
|
|
|
## Analysis Artifacts
|
|
analysis_report: `{challenge.Artifacts?.AnalysisReportMarkdown ?? ""}`
|
|
log_mel_spectrogram_csv: `{challenge.Artifacts?.LogMelSpectrogramCsv ?? ""}`
|
|
log_mel_band_stats_csv: `{challenge.Artifacts?.LogMelBandStatsCsv ?? ""}`
|
|
rms_envelope_csv: `{challenge.Artifacts?.RmsEnvelopeCsv ?? ""}`
|
|
rms_envelope_autocorr_csv: `{challenge.Artifacts?.RmsEnvelopeAutocorrCsv ?? ""}`
|
|
""";
|
|
|
|
static string SongComparisonReport(SongChallenge challenge, IpaTrialScriptCandidate candidate, AudioComparison comparison, string verdict) =>
|
|
$"""
|
|
candidate={candidate.CandidateId}
|
|
challenge={challenge.ChallengeId}
|
|
verdict={verdict}
|
|
logMelCosine={comparison.LogMelCosineSimilarity:0.######}
|
|
logMelDistance={comparison.LogMelDistance:0.######}
|
|
score={comparison.Score:0.######}
|
|
envelopeDistance={comparison.EnvelopeDistance:0.######}
|
|
rmsRatio={comparison.RmsRatio:0.######}
|
|
centroidRatio={comparison.CentroidRatio:0.######}
|
|
zeroCrossingRatio={comparison.ZeroCrossingRatio:0.######}
|
|
articulation={comparison.Articulation.ArticulationScore:0.######}
|
|
speechBandRatio={comparison.Articulation.SpeechBandRatio:0.######}
|
|
""";
|
|
|
|
static string SongSummaryCsv(IReadOnlyList<IpaTrialResult> results)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine("trial_id,candidate_id,reference_id,verdict,log_mel_cosine,log_mel_distance,audio_score,envelope_distance,rms_ratio,centroid_ratio,zero_crossing_ratio,articulation_score");
|
|
foreach (var result in results)
|
|
{
|
|
builder.Append(Escape(result.TrialId)).Append(',');
|
|
builder.Append(Escape(result.CandidateId)).Append(',');
|
|
builder.Append(Escape(result.ReferenceId)).Append(',');
|
|
builder.Append(Escape(result.Verdict)).Append(',');
|
|
builder.Append(Metric(result, "log_mel_cosine")).Append(',');
|
|
builder.Append(Metric(result, "log_mel_distance")).Append(',');
|
|
builder.Append(Metric(result, "audio_score")).Append(',');
|
|
builder.Append(Metric(result, "envelope_distance")).Append(',');
|
|
builder.Append(Metric(result, "rms_ratio")).Append(',');
|
|
builder.Append(Metric(result, "centroid_ratio")).Append(',');
|
|
builder.Append(Metric(result, "zero_crossing_ratio")).Append(',');
|
|
builder.AppendLine(Metric(result, "articulation_score"));
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static string SongEvaluatorReport(SongChallenge challenge, IReadOnlyList<IpaTrialResult> results)
|
|
{
|
|
var builder = new StringBuilder();
|
|
builder.AppendLine($"# Song Snippet Evaluator Report: {challenge.ChallengeId}");
|
|
builder.AppendLine();
|
|
builder.AppendLine(SongChallengeReport(challenge));
|
|
builder.AppendLine();
|
|
builder.AppendLine("| candidate | verdict | logMelCosine | score | envelopeDistance | rmsRatio | centroidRatio |");
|
|
builder.AppendLine("| --- | --- | ---: | ---: | ---: | ---: | ---: |");
|
|
foreach (var result in results.OrderByDescending(result => MetricValue(result, "log_mel_cosine")))
|
|
{
|
|
builder.AppendLine(CultureInfo.InvariantCulture, $"| {result.CandidateId} | {result.Verdict} | {MetricValue(result, "log_mel_cosine"):0.######} | {MetricValue(result, "audio_score"):0.######} | {MetricValue(result, "envelope_distance"):0.######} | {MetricValue(result, "rms_ratio"):0.######} | {MetricValue(result, "centroid_ratio"):0.######} |");
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static float MetricValue(IpaTrialResult result, string name) =>
|
|
result.Metrics.FirstOrDefault(metric => metric.Name.Equals(name, StringComparison.OrdinalIgnoreCase))?.Value ?? 0;
|
|
|
|
static JsonSerializerOptions JsonOptions() => new(JsonSerializerDefaults.Web)
|
|
{
|
|
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
|
WriteIndented = true
|
|
};
|
|
|
|
static SearchOptions SearchOptionsFrom(Dictionary<string, string> options) => new(
|
|
Value(options, "qdrant-url", "http://127.0.0.1:6333"),
|
|
Value(options, "ollama-url", "http://127.0.0.1:11434"),
|
|
Value(options, "embed-model", "qwen3-embedding:0.6b"),
|
|
Value(options, "collection", "aquasynth_ipa_trial_results"),
|
|
int.TryParse(Value(options, "timeout-seconds", "120"), NumberStyles.Integer, CultureInfo.InvariantCulture, out var timeout)
|
|
? Math.Clamp(timeout, 5, 600)
|
|
: 120,
|
|
!BoolValue(options, "no-vector"),
|
|
BoolValue(options, "require-vector"),
|
|
BoolValue(options, "skip-index"));
|
|
|
|
static QdrantPayload QdrantPayloadFrom(EvidenceChunk chunk, string embedderId) => new(
|
|
chunk.Id,
|
|
chunk.TrialId,
|
|
"ipa_trial_result",
|
|
"ipa_trial_result",
|
|
StoreKey(chunk.Store),
|
|
chunk.TrialId,
|
|
chunk.CandidateId,
|
|
chunk.TargetSetId,
|
|
chunk.ChunkKind,
|
|
embedderId,
|
|
chunk.Text);
|
|
|
|
static VectorSearchHit VectorSearchHitFrom(QdrantScoredPoint point)
|
|
{
|
|
var payload = point.Payload ?? new Dictionary<string, JsonElement>();
|
|
return new VectorSearchHit(
|
|
ReadPayloadString(payload, "chunkId"),
|
|
ReadPayloadString(payload, "trialId"),
|
|
ReadPayloadString(payload, "candidateId"),
|
|
ReadPayloadString(payload, "targetSetId"),
|
|
ReadPayloadString(payload, "text"),
|
|
point.Score);
|
|
}
|
|
|
|
static Dictionary<string, string> ParseOptions(IReadOnlyList<string> args)
|
|
{
|
|
var options = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
|
for (var i = 0; i < args.Count; i++)
|
|
{
|
|
var arg = args[i];
|
|
if (!arg.StartsWith("--", StringComparison.Ordinal))
|
|
{
|
|
throw new ArgumentException($"Expected option, got `{arg}`.");
|
|
}
|
|
|
|
var key = arg[2..];
|
|
if (i + 1 >= args.Count || args[i + 1].StartsWith("--", StringComparison.Ordinal))
|
|
{
|
|
options[key] = "true";
|
|
}
|
|
else
|
|
{
|
|
options[key] = args[++i];
|
|
}
|
|
}
|
|
|
|
return options;
|
|
}
|
|
|
|
static string Required(Dictionary<string, string> options, string key) =>
|
|
options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)
|
|
? value
|
|
: throw new ArgumentException($"Missing required --{key} option.");
|
|
|
|
static string Value(Dictionary<string, string> options, string key, string fallback) =>
|
|
options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) ? value : fallback;
|
|
|
|
static bool BoolValue(Dictionary<string, string> options, string key) =>
|
|
options.TryGetValue(key, out var value) &&
|
|
(value.Equals("true", StringComparison.OrdinalIgnoreCase) ||
|
|
value.Equals("1", StringComparison.OrdinalIgnoreCase) ||
|
|
value.Equals("yes", StringComparison.OrdinalIgnoreCase));
|
|
|
|
static int IntValue(Dictionary<string, string> options, string key, int fallback) =>
|
|
int.TryParse(Value(options, key, fallback.ToString(CultureInfo.InvariantCulture)), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)
|
|
? parsed
|
|
: fallback;
|
|
|
|
static float FloatValue(Dictionary<string, string> options, string key, float fallback) =>
|
|
float.TryParse(Value(options, key, fallback.ToString(CultureInfo.InvariantCulture)), NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)
|
|
? parsed
|
|
: fallback;
|
|
|
|
static void PrintHelp()
|
|
{
|
|
Console.WriteLine("""
|
|
IPA trial worker
|
|
|
|
Commands:
|
|
seed --artifact-root <dir> [--batch-id five-seed-trials] [--store <trial-results.cc>]
|
|
score --patch-root <dir> --artifact-root <dir> [--batch-id round-001] [--store <trial-results.cc>] [--hypothesizer id]
|
|
song-prepare --source <audio-file> --artifact-root <dir> [--duration-seconds 10] [--seed n] [--output challenge.json]
|
|
song-score --patch-root <dir> --challenge <challenge.json> --artifact-root <dir> [--batch-id round-001] [--store <trial-results.cc>] [--hypothesizer id]
|
|
dump --store <trial-results.cc> --output <report.md>
|
|
distill --store <trial-results.cc> --output-store <distilled.cc> --output <report.md> [--min-cosine .35] [--max-results 40]
|
|
index --store <trial-results.cc> [--output <report.md>] [--force true]
|
|
search --store <trial-results.cc> --query <text> --output <report.md> [--limit 12] [--no-vector true] [--require-vector true] [--skip-index true]
|
|
show --store <trial-results.cc> --trial-id <id-or-candidate> --output <detail.md>
|
|
|
|
Vector search defaults:
|
|
--qdrant-url http://127.0.0.1:6333
|
|
--ollama-url http://127.0.0.1:11434
|
|
--embed-model qwen3-embedding:0.6b
|
|
--collection aquasynth_ipa_trial_results
|
|
|
|
Agent-authored patch files must be named <targetId>__candidate-name.aqua.
|
|
Known seed target ids: a, i, u, e, o, m, n, ng, l, r, s, z, f, v, th, p, b, t, d, k, mix-a, mix-m, mix-s, mix-p, mix-u.
|
|
""");
|
|
}
|
|
|
|
static string ReadPayloadString(Dictionary<string, JsonElement> payload, string key)
|
|
{
|
|
if (!payload.TryGetValue(key, out var value))
|
|
{
|
|
return "";
|
|
}
|
|
|
|
return value.ValueKind == JsonValueKind.String ? value.GetString() ?? "" : value.ToString();
|
|
}
|
|
|
|
sealed record DecodedAudio(int SampleRate, float[] Samples);
|
|
|
|
sealed record TempoEstimate(float Bpm, float BeatSeconds, float Confidence);
|
|
|
|
sealed record SongChallenge(
|
|
string ChallengeId,
|
|
string SourcePath,
|
|
string SourceFileName,
|
|
int Seed,
|
|
float StartSeconds,
|
|
float DurationSeconds,
|
|
int SampleRate,
|
|
string ReferenceWavPath,
|
|
SongChallengeFeatures Features,
|
|
SongChallengeAnalysisArtifacts? Artifacts = null);
|
|
|
|
sealed record SongChallengeAnalysisArtifacts(
|
|
string LogMelSpectrogramCsv,
|
|
string LogMelBandStatsCsv,
|
|
string RmsEnvelopeCsv,
|
|
string RmsEnvelopeAutocorrCsv,
|
|
string AnalysisReportMarkdown);
|
|
|
|
sealed record SongChallengeFeatures(
|
|
float DurationSeconds,
|
|
float Peak,
|
|
float Rms,
|
|
float ZeroCrossingRate,
|
|
float SpectralCentroidHz,
|
|
float SpectralRolloffHz,
|
|
float ActiveDuty,
|
|
float SpectralFlux,
|
|
float TempoBpm,
|
|
float BeatSeconds,
|
|
float TempoConfidence,
|
|
float DominantHz,
|
|
float RegisterLowHz,
|
|
float RegisterHighHz,
|
|
string RootNote,
|
|
string SuggestedScale,
|
|
string ScaleFrequenciesHz);
|
|
|
|
sealed record SongRegister(
|
|
float DominantHz,
|
|
float LowHz,
|
|
float HighHz,
|
|
string RootNote,
|
|
string SuggestedScale,
|
|
float[] ScaleFrequencies);
|
|
|
|
sealed record SpectrogramBandSummary(
|
|
int Band,
|
|
float Mean,
|
|
float Min,
|
|
float Max,
|
|
float StdDev,
|
|
float DeltaMean,
|
|
float AbsDeltaMean,
|
|
float Delta2Mean,
|
|
float AbsDelta2Mean);
|
|
|
|
sealed record AutocorrelationPoint(
|
|
int LagFrames,
|
|
float LagSeconds,
|
|
float Correlation);
|
|
|
|
sealed record SearchOptions(
|
|
string QdrantUrl,
|
|
string OllamaUrl,
|
|
string EmbedModel,
|
|
string Collection,
|
|
int TimeoutSeconds,
|
|
bool UseVector,
|
|
bool RequireVector,
|
|
bool SkipIndex);
|
|
|
|
sealed record WeightedTerms(float Weight, string[] Terms);
|
|
|
|
sealed record SearchDocument(IpaTrialResult Result, Dictionary<string, WeightedTerms> Fields)
|
|
{
|
|
public string[] AllTerms { get; } = Fields.Values
|
|
.SelectMany(field => field.Terms)
|
|
.Distinct(StringComparer.OrdinalIgnoreCase)
|
|
.ToArray();
|
|
|
|
public static SearchDocument From(IpaTrialResult result, Dictionary<string, WeightedTerms> fields) => new(result, fields);
|
|
}
|
|
|
|
sealed record SearchHit(
|
|
IpaTrialResult Result,
|
|
float Score,
|
|
string[] MatchedTerms,
|
|
IReadOnlyDictionary<string, string[]> MatchedByField,
|
|
string[] Tags,
|
|
string RetrievalMode = "lexical",
|
|
float? VectorScore = null,
|
|
string? VectorChunkId = null,
|
|
string? VectorText = null);
|
|
|
|
sealed record EvidenceChunk(
|
|
string Id,
|
|
string TrialId,
|
|
string CandidateId,
|
|
string TargetSetId,
|
|
string ChunkKind,
|
|
string Text,
|
|
string Store);
|
|
|
|
sealed record VectorIndexResult(int ChunkCount, int VectorSize);
|
|
|
|
sealed record VectorSearchHit(
|
|
string ChunkId,
|
|
string TrialId,
|
|
string CandidateId,
|
|
string TargetSetId,
|
|
string Text,
|
|
float Score);
|
|
|
|
sealed record OllamaEmbedRequest(string Model, string[] Input);
|
|
|
|
sealed record OllamaEmbedResponse(
|
|
[property: JsonPropertyName("embedding")] float[]? Embedding,
|
|
[property: JsonPropertyName("embeddings")] float[][]? Embeddings);
|
|
|
|
sealed record QdrantVectorConfig(
|
|
[property: JsonPropertyName("size")] int Size,
|
|
[property: JsonPropertyName("distance")] string Distance,
|
|
[property: JsonPropertyName("on_disk")] bool OnDisk);
|
|
|
|
sealed record QdrantCreateCollectionRequest(
|
|
[property: JsonPropertyName("vectors")] QdrantVectorConfig Vectors,
|
|
[property: JsonPropertyName("on_disk_payload")] bool OnDiskPayload,
|
|
[property: JsonPropertyName("metadata")] Dictionary<string, object> Metadata);
|
|
|
|
sealed record QdrantPayloadIndexRequest(
|
|
[property: JsonPropertyName("field_name")] string FieldName,
|
|
[property: JsonPropertyName("field_schema")] string FieldSchema);
|
|
|
|
sealed record QdrantPoint(
|
|
[property: JsonPropertyName("id")] string Id,
|
|
[property: JsonPropertyName("vector")] float[] Vector,
|
|
[property: JsonPropertyName("payload")] QdrantPayload Payload);
|
|
|
|
sealed record QdrantUpsertRequest([property: JsonPropertyName("points")] QdrantPoint[] Points);
|
|
|
|
sealed record QdrantPayload(
|
|
[property: JsonPropertyName("chunkId")] string ChunkId,
|
|
[property: JsonPropertyName("sourceId")] string SourceId,
|
|
[property: JsonPropertyName("sourceKind")] string SourceKind,
|
|
[property: JsonPropertyName("corpusKind")] string CorpusKind,
|
|
[property: JsonPropertyName("storeKey")] string StoreKey,
|
|
[property: JsonPropertyName("trialId")] string TrialId,
|
|
[property: JsonPropertyName("candidateId")] string CandidateId,
|
|
[property: JsonPropertyName("targetSetId")] string TargetSetId,
|
|
[property: JsonPropertyName("chunkKind")] string ChunkKind,
|
|
[property: JsonPropertyName("embedderId")] string EmbedderId,
|
|
[property: JsonPropertyName("text")] string Text);
|
|
|
|
sealed record QdrantMatch([property: JsonPropertyName("value")] string Value);
|
|
|
|
sealed record QdrantCondition(
|
|
[property: JsonPropertyName("key")] string Key,
|
|
[property: JsonPropertyName("match")] QdrantMatch Match);
|
|
|
|
sealed record QdrantFilter([property: JsonPropertyName("must")] QdrantCondition[] Must)
|
|
{
|
|
public static QdrantFilter Store(string storeKey, string embedderId) => new([
|
|
new QdrantCondition("storeKey", new QdrantMatch(storeKey)),
|
|
new QdrantCondition("embedderId", new QdrantMatch(embedderId))
|
|
]);
|
|
}
|
|
|
|
sealed record QdrantSearchRequest(
|
|
[property: JsonPropertyName("vector")] float[] Vector,
|
|
[property: JsonPropertyName("limit")] int Limit,
|
|
[property: JsonPropertyName("with_payload")] bool WithPayload,
|
|
[property: JsonPropertyName("with_vector")] bool WithVector,
|
|
[property: JsonPropertyName("filter")] QdrantFilter Filter);
|
|
|
|
sealed record QdrantSearchResponse([property: JsonPropertyName("result")] QdrantScoredPoint[] Result);
|
|
|
|
sealed record QdrantScoredPoint(
|
|
[property: JsonPropertyName("score")] float Score,
|
|
[property: JsonPropertyName("payload")] Dictionary<string, JsonElement>? Payload);
|