GameCult/Mimir
Watch
9
0
Fork
You've already forked Mimir
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Leap point cloud field socket

Browse source
This commit is contained in:
Meta Vens 2026-05-30 22:54:59 +01:00
commit 4fadcfff14
9 changed files with 494 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

94
src/Mimir.BufferSmoke/Program.cs
View file

@ -155,6 +155,11 @@ if (args.Any(arg => string.Equals(arg, "--leap-packed-stereo-depth-smoke", Strin
return RunLeapPackedStereoDepthSmoke();
}
if (args.Any(arg => string.Equals(arg, "--leap-point-cloud-root-smoke", StringComparison.OrdinalIgnoreCase)))
{
return RunLeapPointCloudRootSmoke();
}
if (args.Any(arg => string.Equals(arg, "--ks-camera-driver-smoke", StringComparison.OrdinalIgnoreCase)))
{
return RunKsCameraDriverSmoke(args);
@ -2536,6 +2541,95 @@ static int RunLeapPackedStereoDepthSmoke()
: 1;
}
static int RunLeapPointCloudRootSmoke()
{
const string sourceId = "leap-stereo-ir";
var frameDescriptor = new MimirVideoFrameDescriptor(
Width: 640,
Height: 240,
PixelFormat: MimirVideoPixelFormat.LeapStereoIr,
StrideBytes: 1280,
DeviceTimestampNs: 1_000_000_000L,
NativeHandle: 0xCAFE,
NativeHandleKind: "fensalir-owned-d3d12-texture2d",
ResourceKey: MimirFensalirTextureLeaseClient.ResourceKeyForSource(sourceId),
ProducerFenceHandle: 0xD00D,
ProducerFenceValue: 7);
var sample = new MimirStreamSample(
sourceId,
MimirStreamKind.Video,
MimirStreamOrigin.LocalDevice,
TimestampNs: frameDescriptor.DeviceTimestampNs,
ArrivalNs: frameDescriptor.DeviceTimestampNs,
Sequence: 7,
PayloadHandle: frameDescriptor.NativeHandle,
ByteLength: 0,
VideoFrame: frameDescriptor);
var buffer = new MimirRollingStreamBuffer(
new MimirStreamDescriptor(sourceId, MimirStreamKind.Video, MimirStreamOrigin.LocalDevice),
TimeSpan.FromSeconds(5));
buffer.Append(sample);
var lowerer = new MimirFensalirFieldLowering();
var cameraFrame = lowerer.BuildCameraObservationFrame([buffer]);
var windows = new List<MimirRollingStreamWindow>();
MimirFensalirBridgeMapper.MapWindows([buffer], windows);
var stereoFrame = lowerer.BuildLeapPackedStereoDepthCandidateFrame(windows, cameraFrame.Resources);
var pointCloudFrame = lowerer.BuildLeapPackedStereoPointCloudCandidateFrame(windows);
var mergedResources = stereoFrame.Resources
.Concat(pointCloudFrame.Resources)
.Where(static resource => resource.HasIdentity)
.GroupBy(static resource => resource.ResourceKey, StringComparer.Ordinal)
.Select(static group => group.First())
.ToArray();
var merged = new AquariumFieldEvidenceFrame
{
Domains = [.. stereoFrame.Domains, .. pointCloudFrame.Domains],
Claims = [.. stereoFrame.Claims, .. pointCloudFrame.Claims],
Candidates = [.. stereoFrame.Candidates, .. pointCloudFrame.Candidates],
Resources = mergedResources,
StereoDepthLowerings = stereoFrame.StereoDepthLowerings,
AccumulationWindowSeconds = cameraFrame.AccumulationWindowSeconds,
PresentationDelaySeconds = cameraFrame.PresentationDelaySeconds
};
var validation = AquariumFieldEvidenceValidator.Validate(merged);
var plan = AquariumFieldLoweringPlanner.Plan(merged);
var profile = MimirPointCloudConfigurations.LeapDisparityPointCloudRoot;
var pointResource = pointCloudFrame.Resources.FirstOrDefault(resource =>
string.Equals(resource.ResourceKey, "mimir:resource:point-cloud:leap-stereo-ir:leap-points", StringComparison.Ordinal));
var pointClaim = pointCloudFrame.Claims.FirstOrDefault();
var meshPacket = plan.Packets.FirstOrDefault(packet => packet.Backend == AquariumFieldBackendKind.Mesh);
var stereoPacket = plan.Packets.FirstOrDefault(packet => packet.Backend == AquariumFieldBackendKind.SurfacePage);
Console.WriteLine(
$"leap-point-cloud-root profile={profile.Id} provenance={profile.Provenance} liveDependency={profile.LiveDependency} stride={profile.DefaultSampleStride} baseline={profile.BaselineMeters:0.###} focal={profile.FocalLengthPixels:0.###}");
Console.WriteLine(
$"leap-point-cloud-root-field resources={merged.Resources.Count} pointResources={pointCloudFrame.Resources.Count} claims={pointCloudFrame.Claims.Count} planned={plan.Packets.Count} deferred={plan.DeferredRequests.Count} errors={validation.HasErrors}");
Console.WriteLine(
$"leap-point-cloud-root-output resource={pointResource.ResourceKey} kind={pointResource.Kind} topology={pointResource.Mesh.Topology} layout={pointResource.Mesh.Layout} vertices={pointResource.Mesh.Vertices.Count} indices={pointResource.Mesh.Indices.Count} source={pointResource.SourceUri} backend={meshPacket.Backend}");
return !validation.HasErrors &&
!profile.LiveDependency &&
profile.ProjectionKind == MimirPointCloudProjectionKind.StereoDisparityProjection &&
profile.NegativeChecks.Any(check => check.Contains("no synthetic point cloud", StringComparison.OrdinalIgnoreCase)) &&
pointCloudFrame.Resources.Count == 1 &&
pointCloudFrame.Claims.Count == 1 &&
pointClaim.Encoding == AquariumFieldEncoding.Mesh &&
pointClaim.PayloadHandle == pointResource.ResourceKey &&
pointClaim.ProducerKey == profile.Id &&
pointResource.Kind == AquariumFieldResourceKind.Mesh &&
pointResource.Mesh.Topology == AquariumFieldMeshTopology.PointList &&
pointResource.Mesh.Layout == AquariumFieldMeshLayout.PositionNormalUvColor &&
pointResource.Mesh.Vertices.Count == 320 * 120 &&
pointResource.Mesh.Indices.Count == pointResource.Mesh.Vertices.Count &&
pointResource.SourceUri == "derived-from:mimir:resource:stereo-depth:leap-stereo-ir:disparity-r16f" &&
stereoPacket.Backend == AquariumFieldBackendKind.SurfacePage &&
meshPacket.Backend == AquariumFieldBackendKind.Mesh &&
plan.DeferredRequests.Count == 0
? 0
: 1;
}
static int RunPerfectMachineProfileSmoke()
{
var profiles = MimirPerfectMachineProfiles.All;

21
src/Mimir.Runtime/MimirRuntime.cs
View file

@ -322,7 +322,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
var frame = fieldLowering.BuildFieldEvidenceFrame(windows, observations, constraints, intents);
frame = AddSpectrumFieldEvidence(frame);
return AddLeapStereoDepthFieldEvidence(frame, windows);
return AddLeapGeometryFieldEvidence(frame, windows);
}
private void CommitProducerLeases(IReadOnlyList<MimirRollingStreamWindow> windows)
@ -369,7 +369,7 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
Purpose: MimirSurfaceIntentPurpose.Production);
}
private AquariumFieldEvidenceFrame AddLeapStereoDepthFieldEvidence(
private AquariumFieldEvidenceFrame AddLeapGeometryFieldEvidence(
AquariumFieldEvidenceFrame frame,
IReadOnlyList<MimirRollingStreamWindow> windows)
{
@ -379,16 +379,17 @@ public sealed class MimirRuntime : IAquariumRuntime, IAquariumRuntimeServicesRec
return frame;
}
var pointCloudFrame = fieldLowering.BuildLeapPackedStereoPointCloudCandidateFrame(windows);
return new AquariumFieldEvidenceFrame
{
Domains = [.. frame.Domains, .. stereoFrame.Domains],
Claims = [.. frame.Claims, .. stereoFrame.Claims],
Candidates = [.. frame.Candidates, .. stereoFrame.Candidates],
BackendPackets = [.. frame.BackendPackets, .. stereoFrame.BackendPackets],
Resources = MergeResources(frame.Resources, stereoFrame.Resources),
ResourceUploads = [.. frame.ResourceUploads, .. stereoFrame.ResourceUploads],
TubeSplineLowerings = [.. frame.TubeSplineLowerings, .. stereoFrame.TubeSplineLowerings],
StereoDepthLowerings = [.. frame.StereoDepthLowerings, .. stereoFrame.StereoDepthLowerings],
Domains = [.. frame.Domains, .. stereoFrame.Domains, .. pointCloudFrame.Domains],
Claims = [.. frame.Claims, .. stereoFrame.Claims, .. pointCloudFrame.Claims],
Candidates = [.. frame.Candidates, .. stereoFrame.Candidates, .. pointCloudFrame.Candidates],
BackendPackets = [.. frame.BackendPackets, .. stereoFrame.BackendPackets, .. pointCloudFrame.BackendPackets],
Resources = MergeResources(MergeResources(frame.Resources, stereoFrame.Resources), pointCloudFrame.Resources),
ResourceUploads = [.. frame.ResourceUploads, .. stereoFrame.ResourceUploads, .. pointCloudFrame.ResourceUploads],
TubeSplineLowerings = [.. frame.TubeSplineLowerings, .. stereoFrame.TubeSplineLowerings, .. pointCloudFrame.TubeSplineLowerings],
StereoDepthLowerings = [.. frame.StereoDepthLowerings, .. stereoFrame.StereoDepthLowerings, .. pointCloudFrame.StereoDepthLowerings],
AccumulationWindowSeconds = frame.AccumulationWindowSeconds,
PresentationDelaySeconds = frame.PresentationDelaySeconds,
};

267
src/Mimir.Runtime/Synchronization/MimirFensalirFieldLowering.cs
View file

@ -210,33 +210,59 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
var profile = MimirStereoDepthConfigurations.D3D12SgmLibSgmProvenance;
var sourceSegment = SanitizeResourceSegment(leapWindow.StreamId);
var candidate = new MimirStereoDepthFieldCandidate(
CandidateKey: $"live-{sourceSegment}-{leapWindow.SequenceId}",
CalibrationId: "leapuvc-packed-stereo-ir-calibration-pending",
CameraPairId: leapWindow.StreamId,
ProducerKey: profile.Id,
ProfileId: profile.Id,
LeftObservationKey: $"{leapWindow.WindowId}:left-packed-r",
RightObservationKey: $"{leapWindow.WindowId}:right-packed-g",
LeftResourceKey: packedResourceKey,
RightResourceKey: packedResourceKey,
DisparityResourceKey: $"mimir:resource:stereo-depth:{sourceSegment}:disparity-r16f",
ConfidenceResourceKey: $"mimir:resource:stereo-depth:{sourceSegment}:confidence-r8",
Width: Math.Max(1, leapWindow.SampleDescriptor.Width),
Height: Math.Max(1, leapWindow.SampleDescriptor.Height),
MinDisparity: profile.MinDisparity,
DisparityLevels: profile.DisparityLevels,
AggregationPathCount: profile.AggregationPathCount,
CensusRadius: profile.CensusRadius,
SmoothnessPenaltySmall: profile.SmoothnessPenaltySmall,
SmoothnessPenaltyLarge: profile.SmoothnessPenaltyLarge,
MinDepthMeters: 0.20,
MaxDepthMeters: 4.0,
Confidence: 0.50,
ObservedTimeNs: leapWindow.EdgeNs);
var candidate = BuildLeapPackedStereoDepthCandidate(leapWindow, packedResourceKey, profile, sourceSegment);
return BuildStereoDepthCandidateFrame([candidate], [packedInput]);
}
public AquariumFieldEvidenceFrame BuildLeapPackedStereoPointCloudCandidateFrame(
IEnumerable<MimirRollingStreamWindow> windows)
{
var leapWindow = LatestLiveLeapPackedStereoWindow(windows);
if (string.IsNullOrWhiteSpace(leapWindow.WindowId))
{
return AquariumFieldEvidenceFrame.Empty;
}
var packedResourceKey = MimirFensalirBridgeMapper.ResourceKeyForPayload(leapWindow.Payload);
if (string.IsNullOrWhiteSpace(packedResourceKey))
{
return AquariumFieldEvidenceFrame.Empty;
}
var stereoProfile = MimirStereoDepthConfigurations.D3D12SgmLibSgmProvenance;
var sourceSegment = SanitizeResourceSegment(leapWindow.StreamId);
var depthCandidate = BuildLeapPackedStereoDepthCandidate(leapWindow, packedResourceKey, stereoProfile, sourceSegment);
var pointCloudProfile = MimirPointCloudConfigurations.LeapDisparityPointCloudRoot;
var stride = Math.Max(1, pointCloudProfile.DefaultSampleStride);
var width = Math.Max(1, depthCandidate.Width);
var height = Math.Max(1, depthCandidate.Height);
var sampledWidth = Math.Max(1, (width + stride - 1) / stride);
var sampledHeight = Math.Max(1, (height + stride - 1) / stride);
var candidate = new MimirPointCloudFieldCandidate(
CandidateKey: $"live-{sourceSegment}-point-cloud-{leapWindow.SequenceId}",
CalibrationId: depthCandidate.CalibrationId,
CameraRigId: depthCandidate.CameraPairId,
ProducerKey: pointCloudProfile.Id,
ProfileId: pointCloudProfile.Id,
SourceDisparityResourceKey: depthCandidate.DisparityResourceKey,
SourceConfidenceResourceKey: depthCandidate.ConfidenceResourceKey,
PointCloudResourceKey: $"mimir:resource:point-cloud:{sourceSegment}:leap-points",
Width: width,
Height: height,
SampleStride: stride,
MaxPointCount: sampledWidth * sampledHeight,
BaselineMeters: pointCloudProfile.BaselineMeters,
FocalLengthPixels: pointCloudProfile.FocalLengthPixels,
PrincipalPointX: Math.Clamp(pointCloudProfile.PrincipalPointX, 0.0, width),
PrincipalPointY: Math.Clamp(pointCloudProfile.PrincipalPointY, 0.0, height),
MinDepthMeters: depthCandidate.MinDepthMeters,
MaxDepthMeters: depthCandidate.MaxDepthMeters,
Confidence: Math.Min(depthCandidate.Confidence, 0.45),
ObservedTimeNs: depthCandidate.ObservedTimeNs);
return BuildPointCloudCandidateFrame([candidate]);
}
public AquariumFieldEvidenceFrame BuildStereoDepthCandidateFrame(
IEnumerable<MimirStereoDepthFieldCandidate> depthCandidates,
IEnumerable<AquariumFieldResourceDeclaration>? inputResources = null)
@ -285,6 +311,87 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
};
}
public AquariumFieldEvidenceFrame BuildPointCloudCandidateFrame(
IEnumerable<MimirPointCloudFieldCandidate> pointCloudCandidates)
{
var domains = new List<AquariumFieldDomain>();
var claims = new List<AquariumFieldClaim>();
var candidates = new List<AquariumFieldCandidate>();
var resources = new List<AquariumFieldResourceDeclaration>();
var seenDomains = new HashSet<string>(StringComparer.Ordinal);
var seenResources = new HashSet<string>(StringComparer.Ordinal);
foreach (var pointCloudCandidate in pointCloudCandidates)
{
if (string.IsNullOrWhiteSpace(pointCloudCandidate.CandidateKey) ||
string.IsNullOrWhiteSpace(pointCloudCandidate.PointCloudResourceKey) ||
string.IsNullOrWhiteSpace(pointCloudCandidate.SourceDisparityResourceKey))
{
continue;
}
var domain = DomainForPointCloudCandidate(pointCloudCandidate);
AddDomain(domains, seenDomains, domain);
AddResource(resources, seenResources, PointCloudResourceForCandidate(pointCloudCandidate));
var claim = ClaimForPointCloudCandidate(pointCloudCandidate, domain.DomainKey);
claims.Add(claim);
candidates.Add(CandidateForClaim(claim, AquariumFieldGuide.Valid(claim.Confidence)));
}
return new AquariumFieldEvidenceFrame
{
Domains = domains,
Claims = claims,
Candidates = candidates,
Resources = resources,
AccumulationWindowSeconds = (float)options.AccumulationWindowSeconds,
PresentationDelaySeconds = (float)options.PresentationDelaySeconds
};
}
private static MimirRollingStreamWindow LatestLiveLeapPackedStereoWindow(
IEnumerable<MimirRollingStreamWindow> windows) =>
windows
.Where(static window =>
window.Status == MimirBridgeWindowStatus.Live &&
window.SourceKind == MimirStreamKind.Video &&
window.SampleDescriptor.PixelFormat == MimirVideoPixelFormat.LeapStereoIr &&
window.Payload.HasResource &&
!string.IsNullOrWhiteSpace(window.Payload.ResourceKey))
.OrderByDescending(static window => window.EdgeNs)
.FirstOrDefault();
private static MimirStereoDepthFieldCandidate BuildLeapPackedStereoDepthCandidate(
MimirRollingStreamWindow leapWindow,
string packedResourceKey,
MimirStereoDepthKernelProfile profile,
string sourceSegment) =>
new(
CandidateKey: $"live-{sourceSegment}-{leapWindow.SequenceId}",
CalibrationId: "leapuvc-packed-stereo-ir-calibration-pending",
CameraPairId: leapWindow.StreamId,
ProducerKey: profile.Id,
ProfileId: profile.Id,
LeftObservationKey: $"{leapWindow.WindowId}:left-packed-r",
RightObservationKey: $"{leapWindow.WindowId}:right-packed-g",
LeftResourceKey: packedResourceKey,
RightResourceKey: packedResourceKey,
DisparityResourceKey: $"mimir:resource:stereo-depth:{sourceSegment}:disparity-r16f",
ConfidenceResourceKey: $"mimir:resource:stereo-depth:{sourceSegment}:confidence-r8",
Width: Math.Max(1, leapWindow.SampleDescriptor.Width),
Height: Math.Max(1, leapWindow.SampleDescriptor.Height),
MinDisparity: profile.MinDisparity,
DisparityLevels: profile.DisparityLevels,
AggregationPathCount: profile.AggregationPathCount,
CensusRadius: profile.CensusRadius,
SmoothnessPenaltySmall: profile.SmoothnessPenaltySmall,
SmoothnessPenaltyLarge: profile.SmoothnessPenaltyLarge,
MinDepthMeters: 0.20,
MaxDepthMeters: 4.0,
Confidence: 0.50,
ObservedTimeNs: leapWindow.EdgeNs);
private static void AddDomain(
ICollection<AquariumFieldDomain> domains,
ISet<string> seenDomains,
@ -477,6 +584,25 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
"Fensalir D3D12 compute");
}
private static AquariumFieldDomain DomainForPointCloudCandidate(MimirPointCloudFieldCandidate candidate)
{
var maxDepth = Math.Max(candidate.MinDepthMeters, candidate.MaxDepthMeters);
var halfWidthMeters = (float)Math.Max(0.01, maxDepth * candidate.Width / Math.Max(1.0, candidate.FocalLengthPixels) * 0.5);
var halfHeightMeters = (float)Math.Max(0.01, maxDepth * candidate.Height / Math.Max(1.0, candidate.FocalLengthPixels) * 0.5);
var minDepth = (float)Math.Max(0.0, candidate.MinDepthMeters);
var maxDepthFloat = (float)Math.Max(minDepth + 0.001, maxDepth);
return new AquariumFieldDomain(
DomainKeyForPointCloudCandidate(candidate),
DomainKeyForStereoDepthCandidate(candidate.CalibrationId, candidate.CameraRigId, candidate.SourceDisparityResourceKey),
AquariumFieldDomainKind.Object3D,
Matrix4x4.Identity,
Matrix4x4.Identity,
new Vector3(-halfWidthMeters, -halfHeightMeters, minDepth),
new Vector3(halfWidthMeters, halfHeightMeters, maxDepthFloat),
Vector3.Zero,
"Fensalir D3D12 compute");
}
private static AquariumFieldDomain DomainForSurfaceIntent(MimirSurfaceIntent intent) =>
new(
DomainKeyForSurfaceIntent(intent.IntentKey),
@ -611,6 +737,32 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
Confidence: confidence);
}
private AquariumFieldClaim ClaimForPointCloudCandidate(
MimirPointCloudFieldCandidate pointCloudCandidate,
string domainKey)
{
var confidence = Clamp01((float)pointCloudCandidate.Confidence);
var represented = Math.Max(1, pointCloudCandidate.MaxPointCount);
var proposal = new AquariumFieldProposalPolicy(
AquariumFieldProposalKind.DeterministicStructural,
SourcePdf: 1.0f,
TargetContribution: confidence,
RepresentedCandidateCount: represented,
Seed: StableSeed(pointCloudCandidate.CandidateKey));
return new AquariumFieldClaim(
ClaimKey: $"point-cloud:{pointCloudCandidate.CalibrationId}:{pointCloudCandidate.CameraRigId}:{pointCloudCandidate.CandidateKey}",
DomainKey: domainKey,
ProducerKey: pointCloudCandidate.ProducerKey,
Layer: AquariumFieldLayer.Form,
Encoding: AquariumFieldEncoding.Mesh,
Support: SupportForPointCloudCandidate(pointCloudCandidate),
Proposal: proposal,
PayloadHandle: pointCloudCandidate.PointCloudResourceKey,
ObservedTimeNs: pointCloudCandidate.ObservedTimeNs,
Confidence: confidence);
}
private AquariumFieldClaim ClaimForSurfaceIntent(
MimirSurfaceIntent intent,
string domainKey,
@ -767,6 +919,22 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
TemporalUncertainty: 0.0f);
}
private static AquariumFieldSupport SupportForPointCloudCandidate(MimirPointCloudFieldCandidate pointCloudCandidate)
{
var maxDepth = Math.Max(pointCloudCandidate.MinDepthMeters, pointCloudCandidate.MaxDepthMeters);
var halfWidthMeters = (float)Math.Max(0.01, maxDepth * pointCloudCandidate.Width / Math.Max(1.0, pointCloudCandidate.FocalLengthPixels) * 0.5);
var halfHeightMeters = (float)Math.Max(0.01, maxDepth * pointCloudCandidate.Height / Math.Max(1.0, pointCloudCandidate.FocalLengthPixels) * 0.5);
var depthRange = Math.Max(0.001f, (float)(pointCloudCandidate.MaxDepthMeters - pointCloudCandidate.MinDepthMeters));
return new AquariumFieldSupport(
Center: new Vector3(0.0f, 0.0f, (float)(pointCloudCandidate.MinDepthMeters + pointCloudCandidate.MaxDepthMeters) * 0.5f),
Radius: new Vector3(halfWidthMeters, halfHeightMeters, depthRange * 0.5f),
LocalFrame: Matrix4x4.Identity,
ConservativeRadius: MathF.Max(MathF.Max(halfWidthMeters, halfHeightMeters), depthRange * 0.5f),
ProjectedError: 1.0f / Math.Max(1, pointCloudCandidate.SampleStride),
Curvature: 0.0f,
TemporalUncertainty: 0.0f);
}
private static AquariumFieldResourceDeclaration DisparityResourceForStereoDepthCandidate(
MimirStereoDepthFieldCandidate depthCandidate) =>
new(
@ -810,6 +978,49 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
NativeHandleKind: "fensalir-stereo-depth-confidence");
}
private static AquariumFieldResourceDeclaration PointCloudResourceForCandidate(
MimirPointCloudFieldCandidate pointCloudCandidate)
{
var pointCount = Math.Max(1, pointCloudCandidate.MaxPointCount);
var mesh = new AquariumFieldMeshResource(
Vertices: new AquariumFieldMeshBuffer(
BufferKey: $"{pointCloudCandidate.PointCloudResourceKey}:vertices",
Count: pointCount,
StrideBytes: AquariumFieldMeshResource.PositionNormalUvColorStrideBytes,
NativeHandle: IntPtr.Zero,
NativeHandleKind: "fensalir-generated-vertex-buffer"),
Indices: new AquariumFieldMeshBuffer(
BufferKey: $"{pointCloudCandidate.PointCloudResourceKey}:indices",
Count: pointCount,
StrideBytes: sizeof(uint),
NativeHandle: IntPtr.Zero,
NativeHandleKind: "fensalir-generated-index-buffer"),
Topology: AquariumFieldMeshTopology.PointList,
IndexFormat: AquariumFieldMeshIndexFormat.UInt32,
Layout: AquariumFieldMeshLayout.PositionNormalUvColor,
BoundsMin: new Vector3(-1.0f, -1.0f, (float)Math.Max(0.0, pointCloudCandidate.MinDepthMeters)),
BoundsMax: new Vector3(1.0f, 1.0f, (float)Math.Max(pointCloudCandidate.MinDepthMeters + 0.001, pointCloudCandidate.MaxDepthMeters)),
SubmeshCount: 1);
return new AquariumFieldResourceDeclaration(
ResourceKey: pointCloudCandidate.PointCloudResourceKey,
Kind: AquariumFieldResourceKind.Mesh,
Residency: AquariumFieldResourceResidency.GpuResident,
Access: AquariumFieldShaderAccess.ShaderResource,
Format: "FieldMesh",
Width: pointCount,
Height: 1,
DepthOrCount: pointCount,
StrideBytes: AquariumFieldMeshResource.PositionNormalUvColorStrideBytes,
ValidFromNs: pointCloudCandidate.ObservedTimeNs,
ValidUntilNs: pointCloudCandidate.ObservedTimeNs,
Version: checked((ulong)Math.Max(0L, pointCloudCandidate.ObservedTimeNs)),
NativeHandle: IntPtr.Zero,
NativeHandleKind: "fensalir-leap-point-cloud",
SourceUri: $"derived-from:{pointCloudCandidate.SourceDisparityResourceKey}",
Mesh: mesh);
}
private static AquariumFieldStereoDepthLowering StereoDepthLoweringForCandidate(
MimirStereoDepthFieldCandidate depthCandidate,
string claimKey) =>
@ -1004,7 +1215,13 @@ public sealed class MimirFensalirFieldLowering(MimirFensalirLoweringOptions? opt
$"mimir:visual-marker:{candidate.CalibrationId}:{candidate.MarkerId}:{candidate.CandidateKey}";
private static string DomainKeyForStereoDepthCandidate(MimirStereoDepthFieldCandidate candidate) =>
$"mimir:stereo-depth:{candidate.CalibrationId}:{candidate.CameraPairId}:{candidate.CandidateKey}";
DomainKeyForStereoDepthCandidate(candidate.CalibrationId, candidate.CameraPairId, candidate.CandidateKey);
private static string DomainKeyForStereoDepthCandidate(string calibrationId, string cameraPairId, string candidateKey) =>
$"mimir:stereo-depth:{calibrationId}:{cameraPairId}:{candidateKey}";
private static string DomainKeyForPointCloudCandidate(MimirPointCloudFieldCandidate candidate) =>
$"mimir:point-cloud:{candidate.CalibrationId}:{candidate.CameraRigId}:{candidate.CandidateKey}";
private static string DomainKeyForSurfaceIntent(string intentKey) => $"mimir:intent:{intentKey}";

82
src/Mimir.Runtime/Synchronization/MimirStereoDepthConfiguration.cs
View file

@ -14,6 +14,13 @@ public enum MimirStereoDepthOutputKind
ConfidenceTexture
}
public enum MimirPointCloudProjectionKind
{
StereoDisparityProjection,
VisualFeatureTriangulation,
ExternalDepthSensor
}
public sealed record MimirStereoDepthKernelProfile(
string Id,
string Description,
@ -59,6 +66,45 @@ public sealed record MimirStereoDepthFieldCandidate(
double Confidence,
long ObservedTimeNs);
public sealed record MimirPointCloudProjectionProfile(
string Id,
string Description,
MimirPointCloudProjectionKind ProjectionKind,
string Owner,
string Provenance,
string License,
bool LiveDependency,
string[] RequiredInputResources,
string[] OutputResources,
int DefaultSampleStride,
double BaselineMeters,
double FocalLengthPixels,
double PrincipalPointX,
double PrincipalPointY,
string[] NegativeChecks);
public sealed record MimirPointCloudFieldCandidate(
string CandidateKey,
string CalibrationId,
string CameraRigId,
string ProducerKey,
string ProfileId,
string SourceDisparityResourceKey,
string SourceConfidenceResourceKey,
string PointCloudResourceKey,
int Width,
int Height,
int SampleStride,
int MaxPointCount,
double BaselineMeters,
double FocalLengthPixels,
double PrincipalPointX,
double PrincipalPointY,
double MinDepthMeters,
double MaxDepthMeters,
double Confidence,
long ObservedTimeNs);
public static class MimirStereoDepthConfigurations
{
public static MimirStereoDepthKernelProfile D3D12SgmLibSgmProvenance { get; } = new(
@ -134,3 +180,39 @@ public static class MimirStereoDepthConfigurations
DepthAnythingSmallReference
];
}
public static class MimirPointCloudConfigurations
{
public static MimirPointCloudProjectionProfile LeapDisparityPointCloudRoot { get; } = new(
"leap-disparity-point-cloud-root",
"D3D12 point-cloud projection rooted in the Leap packed-stereo disparity SurfacePage.",
MimirPointCloudProjectionKind.StereoDisparityProjection,
"Fensalir D3D12 compute",
"Derived from calibrated stereo pinhole projection: z = f * baseline / disparity; disparity source follows the libSGM-provenance D3D12 lane.",
"Repo-native implementation; upstream stereo kernel provenance remains Apache-2.0 libSGM.",
LiveDependency: false,
RequiredInputResources:
[
"R16Float disparity SurfacePage",
"R8_UNorm confidence Texture2D",
"stereo intrinsics/extrinsics calibration"
],
OutputResources: ["FieldMesh PointList PositionNormalUvColor"],
DefaultSampleStride: 2,
BaselineMeters: 0.04,
FocalLengthPixels: 250.0,
PrincipalPointX: 320.0,
PrincipalPointY: 120.0,
NegativeChecks:
[
"no synthetic point cloud without a live disparity resource",
"no RGB monocular depth accepted as metric geometry owner",
"no calibration optimizer update before residual ownership exists",
"no CPU point array in the live D3D12 path"
]);
public static IReadOnlyList<MimirPointCloudProjectionProfile> BuiltIn { get; } =
[
LeapDisparityPointCloudRoot
];
}