Prove embedded surface slots across Eve runtimes

This commit is contained in:
Meta Vens 2026-07-08 07:41:08 +01:00
commit dabb3e692f
8 changed files with 255 additions and 85 deletions

View file

@ -130,6 +130,46 @@ static NSData *EVEMediaObservation(NSString *observationId,
return data;
}
static NSArray<NSURL *> *EVEConfiguredURLs(NSString *key) {
NSUserDefaults *defaults = NSUserDefaults.standardUserDefaults;
id configured = [defaults objectForKey:key];
NSMutableArray<NSString *> *values = [NSMutableArray array];
if ([configured isKindOfClass:NSArray.class]) {
for (id item in (NSArray *)configured) {
if ([item isKindOfClass:NSString.class]) {
[values addObject:item];
}
}
} else if ([configured isKindOfClass:NSString.class]) {
NSCharacterSet *separators = [NSCharacterSet characterSetWithCharactersInString:@", \n\t"];
for (NSString *part in [(NSString *)configured componentsSeparatedByCharactersInSet:separators]) {
if (part.length > 0) {
[values addObject:part];
}
}
}
NSString *envValue = NSProcessInfo.processInfo.environment[key];
if ([envValue isKindOfClass:NSString.class] && envValue.length > 0) {
NSCharacterSet *separators = [NSCharacterSet characterSetWithCharactersInString:@", \n\t"];
for (NSString *part in [envValue componentsSeparatedByCharactersInSet:separators]) {
if (part.length > 0) {
[values addObject:part];
}
}
}
NSMutableArray<NSURL *> *urls = [NSMutableArray array];
for (NSString *value in values) {
NSURL *url = [NSURL URLWithString:value];
if (url) {
[urls addObject:url];
}
}
return urls;
}
@interface EVEViewController () <EVEFrameStreamClientDelegate, EVEDashboardClientDelegate, AVCaptureVideoDataOutputSampleBufferDelegate, UIGestureRecognizerDelegate>
@property(nonatomic, strong) EVEGLView *glView;
@ -240,39 +280,23 @@ static NSData *EVEMediaObservation(NSString *observationId,
self.streamScale = 2.0;
self.streamStatus = @"stream idle";
self.streamCodec = @"jpeg";
self.dialogueLine = @"awaiting Mimir";
self.dialogueLine = @"awaiting configured stream";
self.dashboardStatus = @"dashboard idle";
self.dashboardProviderId = @"";
self.activeSurfaceFontScale = 1.0;
self.nodeViews = [NSMutableDictionary dictionary];
self.dashboardNodes = [NSMutableDictionary dictionary];
self.avatarCache = [NSMutableDictionary dictionary];
NSArray<NSURL *> *streamURLs = @[
[NSURL URLWithString:@"ws://127.0.0.1:8792/stream"],
[NSURL URLWithString:@"ws://192.168.1.66:8792/stream"],
];
NSArray<NSURL *> *streamURLs = EVEConfiguredURLs(@"EVE_STREAM_URLS");
self.streamClient = [[EVEFrameStreamClient alloc] initWithURLs:streamURLs delegate:self];
[self.streamClient connect];
NSArray<NSURL *> *dashboardURLs = @[
[NSURL URLWithString:@"ws://127.0.0.1:8797/eve/deck"],
[NSURL URLWithString:@"ws://192.168.1.66:8797/eve/deck"],
[NSURL URLWithString:@"ws://127.0.0.1:8795/eve/deck"],
[NSURL URLWithString:@"ws://192.168.1.66:8795/eve/deck"],
[NSURL URLWithString:@"ws://127.0.0.1:8795/eve/dashboard"],
[NSURL URLWithString:@"ws://192.168.1.66:8795/eve/dashboard"],
];
NSArray<NSURL *> *dashboardURLs = EVEConfiguredURLs(@"EVE_DASHBOARD_URLS");
self.dashboardClient = [[EVEDashboardClient alloc] initWithURLs:dashboardURLs delegate:self];
[self.dashboardClient connect];
NSArray<NSURL *> *cameraURLs = @[
[NSURL URLWithString:@"ws://127.0.0.1:8793/eve/camera"],
[NSURL URLWithString:@"ws://192.168.1.66:8793/eve/camera"],
];
NSArray<NSURL *> *micURLs = @[
[NSURL URLWithString:@"ws://127.0.0.1:8794/eve/mic"],
[NSURL URLWithString:@"ws://192.168.1.66:8794/eve/mic"],
];
NSArray<NSURL *> *cameraURLs = EVEConfiguredURLs(@"EVE_CAMERA_URLS");
NSArray<NSURL *> *micURLs = EVEConfiguredURLs(@"EVE_MIC_URLS");
self.cameraUplink = [[EVESensorUplinkClient alloc] initWithURLs:cameraURLs label:@"camera"];
self.micUplink = [[EVESensorUplinkClient alloc] initWithURLs:micURLs label:@"mic"];
[self.cameraUplink connect];
@ -1257,6 +1281,27 @@ static NSData *EVEMediaObservation(NSString *observationId,
return [self renderSurfaceRail:element props:props children:children depth:depth];
}
if ([kind isEqualToString:@"surface.slot"]) {
NSArray *slots = [element[@"embeddedDocuments"] isKindOfClass:NSArray.class] ? element[@"embeddedDocuments"] : @[];
NSDictionary *slot = slots.count > 0 && [slots[0] isKindOfClass:NSDictionary.class] ? slots[0] : @{};
NSString *slotId = [props[@"slotId"] isKindOfClass:NSString.class] ? props[@"slotId"] : ([slot[@"slotId"] isKindOfClass:NSString.class] ? slot[@"slotId"] : @"");
NSString *documentId = [props[@"documentId"] isKindOfClass:NSString.class] ? props[@"documentId"] : ([slot[@"documentId"] isKindOfClass:NSString.class] ? slot[@"documentId"] : @"");
NSString *schemaId = [props[@"schemaId"] isKindOfClass:NSString.class] ? props[@"schemaId"] : ([slot[@"schemaId"] isKindOfClass:NSString.class] ? slot[@"schemaId"] : @"");
NSString *presentationKind = [props[@"presentationKind"] isKindOfClass:NSString.class] ? props[@"presentationKind"] : ([slot[@"presentationKind"] isKindOfClass:NSString.class] ? slot[@"presentationKind"] : @"");
NSString *title = presentationKind.length > 0 ? presentationKind : (slotId.length > 0 ? slotId : @"embedded surface");
NSString *detail = [@[documentId ?: @"", schemaId ?: @""] componentsJoinedByString:@"\n"];
UILabel *label = [self surfaceLabelWithText:[NSString stringWithFormat:@"%@\n%@", title.uppercaseString, detail]
size:11.0
weight:UIFontWeightMedium
color:[self fensalirPrimaryTextColor]];
label.accessibilityIdentifier = [NSString stringWithFormat:@"embeddedDocuments:%@:%@:%@:%@", slotId ?: @"", documentId ?: @"", schemaId ?: @"", presentationKind ?: @""];
label.layer.borderWidth = 1.0;
label.layer.borderColor = [self fensalirAccentSoftColor].CGColor;
label.layer.cornerRadius = 5.0;
label.backgroundColor = [UIColor colorWithRed:0.014 green:0.050 blue:0.052 alpha:0.92];
return label;
}
if ([kind isEqualToString:@"avatar"]) {
return [self renderSurfaceAvatar:element props:props];
}

View file

@ -51,8 +51,6 @@ import org.gamecult.cultmesh.eve.EveDashboardUiElement
import org.gamecult.cultmesh.eve.EveMediaObservationDocument
import org.gamecult.cultmesh.eve.EveSensorObservationDocument
import java.net.URI
import java.net.HttpURLConnection
import java.net.URL
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
@ -61,13 +59,18 @@ import java.util.concurrent.Executors
import org.json.JSONObject
class MainActivity : Activity(), SensorEventListener {
companion object {
const val EXTRA_DASHBOARD_URI = "org.gamecult.eve.DASHBOARD_URI"
const val EXTRA_SENSOR_URI = "org.gamecult.eve.SENSOR_URI"
}
private val mesh = CultMeshNode()
private val main = Handler(Looper.getMainLooper())
private val workers = Executors.newCachedThreadPool()
private val deviceId = "periwinkle"
private val clientId = "eve-android-periwinkle"
private val dashboardUri = URI.create("ws://192.168.1.66:8795/eve/deck/cultmesh")
private val sensorUri = URI.create("ws://192.168.1.66:8796/eve/periwinkle")
private var dashboardUri: URI? = null
private var sensorUri: URI? = null
private var sensorManager: SensorManager? = null
private var dashboardSocket: CultNetWebSocketClient? = null
@ -102,6 +105,8 @@ class MainActivity : Activity(), SensorEventListener {
setContentView(buildParityFixtureUi())
return
}
dashboardUri = resolveDashboardUri(intent.getStringExtra(EXTRA_DASHBOARD_URI))
sensorUri = resolveConfiguredUri(intent.getStringExtra(EXTRA_SENSOR_URI))
sensorManager = getSystemService(SENSOR_SERVICE) as? SensorManager
setContentView(buildUi())
startSensors()
@ -197,6 +202,7 @@ class MainActivity : Activity(), SensorEventListener {
addView(renderCultUiNode(child, values, tokens), stageParams(child))
}
}
"surface.slot" -> embeddedSurfaceSlot(node, props, tokens)
"control.button" -> label(props.optString("label", ""), 14f, tokenColor(tokens, "colorText", 0xfff6f1e2.toInt()), true).apply {
gravity = Gravity.CENTER
setPadding(dp(14), dp(12), dp(14), dp(12))
@ -322,6 +328,34 @@ class MainActivity : Activity(), SensorEventListener {
}
}
private fun embeddedSurfaceSlot(node: JSONObject, props: JSONObject, tokens: JSONObject): View {
val slots = node.optJSONArray("embeddedDocuments")
val slot = slots?.optJSONObject(0)
val slotId = props.optString("slotId", slot?.optString("slotId", "") ?: "")
val documentId = props.optString("documentId", slot?.optString("documentId", "") ?: "")
val schemaId = props.optString("schemaId", slot?.optString("schemaId", "") ?: "")
val presentationKind = props.optString("presentationKind", slot?.optString("presentationKind", "") ?: "")
val title = presentationKind.ifBlank { slotId.ifBlank { "embedded surface" } }
val detail = listOf(documentId, schemaId)
.filter { it.isNotBlank() }
.joinToString("\n")
return LinearLayout(this).apply {
orientation = LinearLayout.VERTICAL
setPadding(dp(12), dp(10), dp(12), dp(10))
background = android.graphics.drawable.GradientDrawable().apply {
setColor(tokenColor(tokens, "colorPanel", Color.rgb(7, 25, 24)))
setStroke(dp(1), tokenColor(tokens, "colorAccent", 0xffff8a2a.toInt()))
cornerRadius = dp(6).toFloat()
}
addView(label(title, 12f, tokenColor(tokens, "colorAccent", 0xffff8a2a.toInt()), true))
if (detail.isNotBlank()) {
addView(label(detail, 11f, tokenColor(tokens, "colorMuted", 0xffd3bb7f.toInt()), false))
}
contentDescription = "embeddedDocuments:$slotId:$documentId:$schemaId:$presentationKind"
}
}
private fun stageParams(node: JSONObject): FrameLayout.LayoutParams {
val kind = node.optString("kind", "")
val props = node.optJSONObject("props") ?: JSONObject()
@ -430,9 +464,9 @@ class MainActivity : Activity(), SensorEventListener {
scroll.addView(root, ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT))
root.addView(label("Eve / Periwinkle", 18f, 0xff8efcff.toInt(), true))
root.addView(label("CultMesh dashboard and sensor edge", 11f, 0xffb7c7c7.toInt(), false))
brokerText = card("CultMesh broker\nconnecting $dashboardUri")
brokerText = card("CultMesh broker\n${dashboardUriText()}")
selectedText = card("selection\nwaiting for dashboard state")
sensorText = card("CultMesh sensors\nconnecting $sensorUri")
sensorText = card("CultMesh sensors\n${sensorUriText()}")
touchText = card("touch surface\nwaiting for operator input")
mediaText = card("media sensors\nwaiting for camera/mic permissions")
providerPicker = Spinner(this).apply {
@ -471,7 +505,6 @@ class MainActivity : Activity(), SensorEventListener {
root.addView(mediaText)
root.addView(touchText)
root.addView(card("contract\nPeriwinkle consumes mimir.eve_dashboard_state.v1, sends mimir.eve_dashboard_command.v1, and publishes mimir.eve_sensor_observation.v1 plus mimir.eve_media_observation.v1. Mimir accepts meaning; Android renders and observes."))
refreshProviderCatalog()
return scroll
}
@ -496,11 +529,16 @@ class MainActivity : Activity(), SensorEventListener {
}
private fun connectDashboard() {
val uri = dashboardUri
if (uri == null) {
postBroker("CultMesh broker\nno dashboard endpoint configured\nwaiting for Odin/CultMesh discovery")
return
}
workers.execute {
while (!Thread.currentThread().isInterrupted) {
try {
dashboardSocket = mesh.connect(dashboardUri)
postBroker("CultMesh broker\nconnected $dashboardUri\ndocument=mimir.eve_dashboard_state.v1")
dashboardSocket = mesh.connect(uri)
postBroker("CultMesh broker\nconnected $uri\ndocument=mimir.eve_dashboard_state.v1")
while (!Thread.currentThread().isInterrupted) {
val frame: CultNetFrame = dashboardSocket!!.readFrame()
if (frame.opcode == 0x8) error("dashboard closed")
@ -511,22 +549,45 @@ class MainActivity : Activity(), SensorEventListener {
}
}
} catch (ex: Exception) {
postBroker("CultMesh broker\nwaiting for Mimir\n${ex.javaClass.simpleName}: ${ex.message}")
postBroker("CultMesh broker\nwaiting for configured CultMesh dashboard\n${ex.javaClass.simpleName}: ${ex.message}")
Thread.sleep(2000)
}
}
}
}
private fun resolveDashboardUri(value: String?): URI? = resolveConfiguredUri(value)
private fun resolveConfiguredUri(value: String?): URI? {
val text = value?.trim().orEmpty()
if (text.isBlank()) return null
return try {
URI.create(text)
} catch (_: IllegalArgumentException) {
null
}
}
private fun dashboardUriText(): String =
dashboardUri?.let { "connecting $it" } ?: "no dashboard endpoint configured"
private fun sensorUriText(): String =
sensorUri?.let { "connecting $it" } ?: "no sensor endpoint configured"
private fun connectSensorUplink() {
val uri = sensorUri
if (uri == null) {
postSensor("CultMesh sensors\nno sensor endpoint configured\nwaiting for Odin/CultMesh discovery")
return
}
workers.execute {
while (!Thread.currentThread().isInterrupted) {
try {
sensorSocket = mesh.connect(sensorUri)
postSensor("CultMesh sensors\nconnected $sensorUri\ndocument=mimir.eve_sensor_observation.v1")
sensorSocket = mesh.connect(uri)
postSensor("CultMesh sensors\nconnected $uri\ndocument=mimir.eve_sensor_observation.v1")
return@execute
} catch (ex: Exception) {
postSensor("CultMesh sensors\nwaiting for receiver\n${ex.javaClass.simpleName}: ${ex.message}")
postSensor("CultMesh sensors\nwaiting for configured CultMesh sensor receiver\n${ex.javaClass.simpleName}: ${ex.message}")
Thread.sleep(2000)
}
}
@ -605,47 +666,6 @@ class MainActivity : Activity(), SensorEventListener {
}
}
private fun refreshProviderCatalog() {
workers.execute {
runCatching {
val url = URL("http://${dashboardUri.host}:${dashboardUri.port}/eve/deck/providers")
val connection = (url.openConnection() as HttpURLConnection).apply {
connectTimeout = 1500
readTimeout = 1500
}
val raw = connection.inputStream.bufferedReader().use { it.readText() }
val providers = JSONObject(raw).optJSONArray("providers") ?: return@runCatching
val nodes = mutableListOf<EveDashboardNodeSnapshot>()
for (index in 0 until providers.length()) {
val provider = providers.getJSONObject(index)
val providerId = provider.optString("id", "")
if (providerId.isBlank()) continue
nodes += EveDashboardNodeSnapshot(
id = "provider-${providerId.replace('.', '-')}",
label = provider.optString("title", providerId),
kind = "dashboard-provider",
visible = true,
x = 0.0,
y = 0.0,
z = 0.0,
rotation = 0.0,
scale = 1.0,
width = 0.0,
height = 0.0,
health = "ok",
providerId = providerId,
command = "open-provider",
endpoint = provider.optString("endpoint", "").ifBlank { null },
)
}
main.post {
if (nodes.isNotEmpty()) providerPickerNodes = nodes
latestState?.let { renderProviderPicker(it) }
}
}
}
}
private fun renderElement(element: EveDashboardUiElement, depth: Int): View {
element.metric?.let { metric ->
return LinearLayout(this).apply {

View file

@ -9,6 +9,8 @@ Current smoke target:
- `CultUiInspectorSurface`
- custom-painted slider track/fill/thumb
- `EmbeddedSurfaceDemo`, which verifies `surface.slot` /
`embeddedDocuments` parsing and rendering from the shared fixture
- golden PNG output through `flutter test --update-goldens`
Run from the repository root:

View file

@ -104,6 +104,7 @@ class EveNode {
required this.props,
required this.layout,
required this.children,
required this.embeddedDocuments,
});
final String id;
@ -111,6 +112,7 @@ class EveNode {
final Map<String, dynamic> props;
final Map<String, dynamic> layout;
final List<EveNode> children;
final List<EveEmbeddedDocumentSlot> embeddedDocuments;
factory EveNode.fromJson(Map<String, dynamic> json) {
return EveNode(
@ -121,6 +123,32 @@ class EveNode {
children: ((json['children'] as List<dynamic>?) ?? const [])
.map((child) => EveNode.fromJson(child as Map<String, dynamic>))
.toList(),
embeddedDocuments: ((json['embeddedDocuments'] as List<dynamic>?) ?? const [])
.map((slot) => EveEmbeddedDocumentSlot.fromJson(slot as Map<String, dynamic>))
.toList(),
);
}
}
class EveEmbeddedDocumentSlot {
EveEmbeddedDocumentSlot({
required this.slotId,
required this.documentId,
required this.schemaId,
required this.presentationKind,
});
final String slotId;
final String documentId;
final String schemaId;
final String presentationKind;
factory EveEmbeddedDocumentSlot.fromJson(Map<String, dynamic> json) {
return EveEmbeddedDocumentSlot(
slotId: (json['slotId'] ?? '').toString(),
documentId: (json['documentId'] ?? '').toString(),
schemaId: (json['schemaId'] ?? '').toString(),
presentationKind: (json['presentationKind'] ?? '').toString(),
);
}
}
@ -250,11 +278,36 @@ class EveNodeView extends StatelessWidget {
return _dialogue();
case 'rail.actions':
return _actions();
case 'surface.slot':
return _surfaceSlot();
default:
return _fallback();
}
}
Widget _surfaceSlot() {
final slot = node.embeddedDocuments.firstOrNull;
final label = node.props['presentationKind']?.toString() ??
slot?.presentationKind ??
node.props['slotId']?.toString() ??
slot?.slotId ??
'embedded surface';
return DecoratedBox(
decoration: BoxDecoration(
color: state.tokens.panelAlt.withValues(alpha: 0.65),
border: Border.all(color: state.tokens.accent.withValues(alpha: 0.45)),
borderRadius: BorderRadius.circular(6),
),
child: Padding(
padding: const EdgeInsets.all(12),
child: Text(
label,
style: TextStyle(color: state.tokens.muted, fontFamily: 'EveParity', fontSize: 12),
),
),
);
}
Widget _vnStage() {
final background = node.children.where((child) => child.kind == 'image.background').firstOrNull;
final sceneChildren = node.children.where((child) {

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

View file

@ -1,3 +1,6 @@
import 'dart:convert';
import 'dart:io';
import 'package:eve_parity/main.dart';
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
@ -39,6 +42,40 @@ void main() {
);
});
}
test('parses embedded document slots for CultUI nested surfaces', () {
final node = EveNode.fromJson(const {
'id': 'test.slot',
'kind': 'surface.slot',
'props': {'presentationKind': 'inventory.dropdown'},
'embeddedDocuments': [
{
'slotId': 'inventory.dropdown',
'documentId': 'cultmesh://test/dropdown',
'schemaId': 'gamecult.eve.surface.v1',
'presentationKind': 'inventory.dropdown',
}
],
});
expect(node.embeddedDocuments, hasLength(1));
expect(node.embeddedDocuments.single.slotId, 'inventory.dropdown');
expect(node.embeddedDocuments.single.documentId, 'cultmesh://test/dropdown');
});
test('embedded_surface_fixture_contract', () {
final raw = File('../../web/fixtures/cultui-embedded-surface.json').readAsStringSync();
final json = jsonDecode(raw) as Map<String, dynamic>;
final surface = json['surface'] as Map<String, dynamic>;
final root = EveNode.fromJson(surface['root'] as Map<String, dynamic>);
final slot = root.children.singleWhere((node) => node.kind == 'surface.slot');
expect(json['schema'], 'gamecult.eve.surface.v1');
expect(slot.embeddedDocuments, hasLength(1));
expect(slot.embeddedDocuments.single.slotId, 'inventory.dropdown');
expect(slot.embeddedDocuments.single.schemaId, 'gamecult.eve.surface.v1');
expect(slot.embeddedDocuments.single.presentationKind, 'inventory.dropdown');
});
}
const fixtureId = String.fromEnvironment('EVE_PARITY_FIXTURE', defaultValue: 'cultui-inspector');

View file

@ -40,3 +40,13 @@ test("preserves slider skin anatomy as retained control parts", () => {
assert.equal(skin.children[3].props.name, "thumb");
assert.equal(skin.children[3].props.bleed, 3);
});
test("surface JSON can declare embedded CultMesh document slots", () => {
const state = JSON.parse(readFileSync(new URL("./fixtures/cultui-embedded-surface.json", import.meta.url), "utf8"));
const slot = state.surface.root.children[1];
assert.equal(slot.kind, "surface.slot");
assert.equal(slot.embeddedDocuments.length, 1);
assert.equal(slot.embeddedDocuments[0].slotId, "inventory.dropdown");
assert.equal(slot.embeddedDocuments[0].documentId, "cultmesh://demo/inventory/dropdown");
});

View file

@ -212,11 +212,12 @@ function delay(ms) {
function renderSurface(state, source) {
surfaceId.textContent = state.providerId || "surface unknown";
surfaceVersion.textContent = `v${state.version ?? "?"}`;
const providerId = state.providerId || currentProvider?.providerId || "";
renderEveSurface(state, app, {
body: document.body,
clientId: liveHermodr ? "hermodr.browser" : "browser.reference",
commandSink: publishCommandIntent,
documentResolver: liveHermodr ? resolveHermodrDocument : undefined,
documentResolver: liveHermodr ? createHermodrDocumentResolver(providerId) : undefined,
assetUrlResolver: liveHermodr ? resolveHermodrAssetUrl : undefined,
provider: currentProvider,
source,
@ -224,15 +225,17 @@ function renderSurface(state, source) {
});
}
async function resolveHermodrDocument(request) {
const providerId = currentProvider?.providerId || "";
const params = new URLSearchParams();
params.set("documentId", request.documentId);
if (request.schemaId) params.set("schemaId", request.schemaId);
if (request.slotId) params.set("slotId", request.slotId);
const response = await fetch(`/hermodr/document/${encodeURIComponent(providerId)}?${params.toString()}`, { cache: "no-store" });
if (!response.ok) throw new Error(await response.text());
return response.json();
function createHermodrDocumentResolver(providerId) {
return async function resolveHermodrDocument(request) {
if (!providerId) return undefined;
const params = new URLSearchParams();
params.set("documentId", request.documentId);
if (request.schemaId) params.set("schemaId", request.schemaId);
if (request.slotId) params.set("slotId", request.slotId);
const response = await fetch(`/hermodr/document/${encodeURIComponent(providerId)}?${params.toString()}`, { cache: "no-store" });
if (!response.ok) throw new Error(await response.text());
return response.json();
};
}
function resolveHermodrAssetUrl(uri) {